Commit d87e72c0 authored by Jay Guo's avatar Jay Guo
Browse files

FAB-14745 more harsh snapshot integration test



This CR alter the snapshot integration test to cover the case
where even lagged node can still help forming a minimum quorum
and start catching up with snapshot.

Change-Id: I97f91c311938c35a6a287237f48146020c799496
Signed-off-by: default avatarJay Guo <guojiannan1101@gmail.com>
parent c4432943
......@@ -148,8 +148,14 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
When("an orderer is behind the latest snapshot on leader", func() {
It("catches up using the block stored in snapshot", func() {
network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, 33000, components)
// Steps:
// - start o2 & o3
// - send several transactions so snapshot is created
// - kill o2 & o3, so that entries prior to snapshot are not in memory upon restart
// - start o1 & o2
// - assert that o1 can catch up with o2 using snapshot
network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, 33000, components)
o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
peer = network.Peer("Org1", "peer1")
......@@ -166,11 +172,13 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
ordererGroup := grouper.NewParallel(syscall.SIGTERM, orderers)
peerGroup := network.PeerGroupRunner()
By("Starting 2/3 of cluster")
ordererProc = ifrit.Invoke(ordererGroup)
Eventually(ordererProc.Ready()).Should(BeClosed())
peerProc = ifrit.Invoke(peerGroup)
Eventually(peerProc.Ready()).Should(BeClosed())
By("Creating channel and submitting several transactions to take snapshot")
network.CreateAndJoinChannel(o2, "testchannel")
nwo.DeployChaincode(network, "testchannel", o2, chaincode)
......@@ -188,33 +196,36 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
return len(files)
}).Should(Equal(5)) // snapshot interval is 1 KB, every block triggers snapshot
By("Killing orderers so they don't have blocks prior to latest snapshot in the memory")
ordererProc.Signal(syscall.SIGKILL)
Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive())
o1Runner := network.OrdererRunner(o1)
By("Starting lagged orderer and one of up-to-date orderers")
orderers = grouper.Members{
{Name: o1.ID(), Runner: network.OrdererRunner(o1)},
{Name: o2.ID(), Runner: network.OrdererRunner(o2)},
{Name: o3.ID(), Runner: network.OrdererRunner(o3)},
}
ordererGroup = grouper.NewParallel(syscall.SIGTERM, orderers)
ordererProc = ifrit.Invoke(ordererGroup)
Eventually(ordererProc.Ready()).Should(BeClosed())
o1Proc = ifrit.Invoke(o1Runner)
Eventually(o1Proc.Ready()).Should(BeClosed())
o1SnapDir := path.Join(network.RootDir, "orderers", o1.ID(), "etcdraft", "snapshot")
By("Asserting that orderer1 has snapshot dir for both system and application channel")
Eventually(func() int {
files, err := ioutil.ReadDir(o1SnapDir)
Expect(err).NotTo(HaveOccurred())
return len(files)
}, network.EventuallyTimeout).Should(Equal(2))
By("Asserting that orderer1 receives and persists snapshot")
Eventually(func() int {
files, err := ioutil.ReadDir(path.Join(o1SnapDir, "testchannel"))
Expect(err).NotTo(HaveOccurred())
return len(files)
}, network.EventuallyTimeout).Should(Equal(1))
By("Asserting cluster is still functional")
RunInvoke(network, o1, peer, "testchannel")
Eventually(func() int {
return RunQuery(network, o1, peer, "testchannel")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment