Unverified Commit 577301d9 authored by Jay Guo's avatar Jay Guo Committed by Artem Barger
Browse files

[FAB-13967] Polling, instead of waiting in AfterEach



If something in AfterEach hangs till test timeout, the
coredump produced might covers up the actual assertion
failure in test body. This CR fixes this in etcdraft UT.

This CR also adds LongEventualTimeout to some places in
etcdraft chain_test to prevent flakes due to slow wal sync.

Change-Id: I585e59e5eb587f9e9eb082c5eb8f681141b16e55
Signed-off-by: default avatarJay Guo <guojiannan1101@gmail.com>
parent 316db769
......@@ -43,7 +43,7 @@ import (
const (
interval = time.Second
LongEventualTimeout = 5 * time.Second
LongEventualTimeout = 10 * time.Second
ELECTION_TICK = 2
HEARTBEAT_TICK = 1
)
......@@ -190,6 +190,7 @@ var _ = Describe("Chain", func() {
AfterEach(func() {
chain.Halt()
Eventually(chain.Errored, LongEventualTimeout).Should(BeClosed())
os.RemoveAll(dataDir)
})
......@@ -361,7 +362,7 @@ var _ = Describe("Chain", func() {
errorC := chain.Errored()
Expect(errorC).NotTo(BeClosed())
chain.Halt()
Eventually(errorC).Should(BeClosed())
Eventually(errorC, LongEventualTimeout).Should(BeClosed())
})
Describe("Config updates", func() {
......@@ -479,7 +480,7 @@ var _ = Describe("Chain", func() {
It("should be able to create a channel", func() {
err := chain.Configure(configEnv, configSeq)
Expect(err).NotTo(HaveOccurred())
Eventually(support.WriteConfigBlockCallCount).Should(Equal(1))
Eventually(support.WriteConfigBlockCallCount, LongEventualTimeout).Should(Equal(1))
})
})
}) // Context block for type A config
......@@ -1779,7 +1780,7 @@ var _ = Describe("Chain", func() {
network.exec(
func(c *chain) {
Eventually(func() int { return c.support.WriteBlockCallCount() }, LongEventualTimeout).Should(Equal(1))
Eventually(c.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(1))
})
By("respect batch timeout")
......@@ -1792,7 +1793,7 @@ var _ = Describe("Chain", func() {
c1.clock.WaitForNWatchersAndIncrement(timeout, 2)
network.exec(
func(c *chain) {
Eventually(func() int { return c.support.WriteBlockCallCount() }, LongEventualTimeout).Should(Equal(2))
Eventually(c.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
})
})
......@@ -1817,7 +1818,7 @@ var _ = Describe("Chain", func() {
c1.clock.WaitForNWatchersAndIncrement(timeout, 2)
network.exec(
func(c *chain) {
Eventually(func() int { return c.support.WriteBlockCallCount() }, LongEventualTimeout).Should(Equal(2))
Eventually(c.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
})
})
......@@ -1848,9 +1849,9 @@ var _ = Describe("Chain", func() {
network.connect(1)
c1.clock.Increment(interval)
Eventually(doneProp).Should(BeClosed())
Eventually(doneProp, LongEventualTimeout).Should(BeClosed())
network.exec(func(c *chain) {
Eventually(c.support.WriteBlockCallCount).Should(Equal(2))
Eventually(c.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
})
})
......@@ -1877,16 +1878,16 @@ var _ = Describe("Chain", func() {
network.elect(2)
Expect(c3.Order(env, 0)).To(Succeed())
Eventually(c1.support.WriteBlockCallCount).Should(Equal(0))
Eventually(c2.support.WriteBlockCallCount).Should(Equal(1))
Eventually(c3.support.WriteBlockCallCount).Should(Equal(1))
Eventually(c1.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(0))
Eventually(c2.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(1))
Eventually(c3.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(1))
network.connect(1)
c2.clock.Increment(interval)
Eventually(doneProp).Should(BeClosed())
Eventually(doneProp, LongEventualTimeout).Should(BeClosed())
network.exec(func(c *chain) {
Eventually(c.support.WriteBlockCallCount).Should(Equal(2))
Eventually(c.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
})
})
})
......@@ -1967,18 +1968,26 @@ var _ = Describe("Chain", func() {
network.disconnect(1)
c1.cutter.CutNext = true
for i := 0; i < 10; i++ {
for i := 0; i < 3; i++ {
err := c1.Order(env, 0)
Expect(err).NotTo(HaveOccurred())
}
Consistently(c1.support.WriteBlockCallCount).Should(Equal(0))
network.connect(1)
c1.clock.Increment(interval)
network.exec(
func(c *chain) {
Eventually(func() int { return c.support.WriteBlockCallCount() }, LongEventualTimeout).Should(Equal(10))
})
// keep ticking leader until all the blocks are replicated and committed.
// Otherwise, if we only tick once, and leader was still preserving block
// data to disk due to long wal sync, this tick may be dropped and cannot
// trigger retransmission of messages upon reconnection.
Eventually(func() int {
c1.clock.Increment(interval)
return c1.support.WriteBlockCallCount()
}, LongEventualTimeout).Should(Equal(3))
Eventually(c2.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(3))
Eventually(c3.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(3))
})
It("new leader should wait for in-fight blocks to commit before accepting new env", func() {
......@@ -2174,7 +2183,7 @@ var _ = Describe("Chain", func() {
network.exec(
func(c *chain) {
Eventually(func() int { return c.support.WriteBlockCallCount() }, LongEventualTimeout).Should(Equal(2))
Eventually(c.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
})
Eventually(c1.opts.MemoryStorage.FirstIndex, LongEventualTimeout).Should(BeNumerically(">", i))
......@@ -2186,10 +2195,10 @@ var _ = Describe("Chain", func() {
network.exec(
func(c *chain) {
Eventually(func() int { return c.support.WriteBlockCallCount() }, LongEventualTimeout).Should(Equal(3))
Eventually(c.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(3))
})
Eventually(c1.opts.MemoryStorage.FirstIndex).Should(BeNumerically(">", i))
Eventually(c1.opts.MemoryStorage.FirstIndex, LongEventualTimeout).Should(BeNumerically(">", i))
})
It("lagged node can catch up using snapshot", func() {
......@@ -2289,9 +2298,9 @@ var _ = Describe("Chain", func() {
err := c1.Order(env, 0)
Expect(err).NotTo(HaveOccurred())
Eventually(c1.support.WriteBlockCallCount).Should(Equal(1))
Eventually(c2.support.WriteBlockCallCount).Should(Equal(0))
Eventually(c3.support.WriteBlockCallCount).Should(Equal(1))
Eventually(c1.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(1))
Eventually(c2.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(0))
Eventually(c3.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(1))
network.connect(2)
......@@ -2772,8 +2781,8 @@ func (n *network) start(ids ...uint64) {
Eventually(func() error {
_, err := n.chains[id].storage.Entries(1, 1, 1)
return err
}).ShouldNot(HaveOccurred())
Eventually(n.chains[id].WaitReady).ShouldNot(HaveOccurred())
}, LongEventualTimeout).ShouldNot(HaveOccurred())
Eventually(n.chains[id].WaitReady, LongEventualTimeout).ShouldNot(HaveOccurred())
}
}
......@@ -2788,7 +2797,7 @@ func (n *network) stop(ids ...uint64) {
for _, id := range nodes {
c := n.chains[id]
c.Halt()
<-c.Errored()
Eventually(c.Errored).Should(BeClosed())
select {
case <-c.stopped:
default:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment