FAB-14745 more harsh snapshot integration test

This CR alter the snapshot integration test to cover the case
where even lagged node can still help forming a minimum quorum
and start catching up with snapshot.

Change-Id: I97f91c311938c35a6a287237f48146020c799496
Signed-off-by: Jay Guo <guojiannan1101@gmail.com>

integration/e2e/cft_test.go

@@ -148,8 +148,14 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 
 	When("an orderer is behind the latest snapshot on leader", func() {
 		It("catches up using the block stored in snapshot", func() {
-			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, 33000, components)
+			// Steps:
+			// - start o2 & o3
+			// - send several transactions so snapshot is created
+			// - kill o2 & o3, so that entries prior to snapshot are not in memory upon restart
+			// - start o1 & o2
+			// - assert that o1 can catch up with o2 using snapshot
 
+			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, 33000, components)
 			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
 
 			peer = network.Peer("Org1", "peer1")
@@ -166,11 +172,13 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 			ordererGroup := grouper.NewParallel(syscall.SIGTERM, orderers)
 			peerGroup := network.PeerGroupRunner()
 
+			By("Starting 2/3 of cluster")
 			ordererProc = ifrit.Invoke(ordererGroup)
 			Eventually(ordererProc.Ready()).Should(BeClosed())
 			peerProc = ifrit.Invoke(peerGroup)
 			Eventually(peerProc.Ready()).Should(BeClosed())
 
+			By("Creating channel and submitting several transactions to take snapshot")
 			network.CreateAndJoinChannel(o2, "testchannel")
 			nwo.DeployChaincode(network, "testchannel", o2, chaincode)
 
@@ -188,33 +196,36 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 				return len(files)
 			}).Should(Equal(5)) // snapshot interval is 1 KB, every block triggers snapshot
 
+			By("Killing orderers so they don't have blocks prior to latest snapshot in the memory")
 			ordererProc.Signal(syscall.SIGKILL)
 			Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive())
 
-			o1Runner := network.OrdererRunner(o1)
+			By("Starting lagged orderer and one of up-to-date orderers")
 			orderers = grouper.Members{
+				{Name: o1.ID(), Runner: network.OrdererRunner(o1)},
 				{Name: o2.ID(), Runner: network.OrdererRunner(o2)},
-				{Name: o3.ID(), Runner: network.OrdererRunner(o3)},
 			}
 			ordererGroup = grouper.NewParallel(syscall.SIGTERM, orderers)
 			ordererProc = ifrit.Invoke(ordererGroup)
 			Eventually(ordererProc.Ready()).Should(BeClosed())
 
-			o1Proc = ifrit.Invoke(o1Runner)
-			Eventually(o1Proc.Ready()).Should(BeClosed())
-
 			o1SnapDir := path.Join(network.RootDir, "orderers", o1.ID(), "etcdraft", "snapshot")
+
+			By("Asserting that orderer1 has snapshot dir for both system and application channel")
 			Eventually(func() int {
 				files, err := ioutil.ReadDir(o1SnapDir)
 				Expect(err).NotTo(HaveOccurred())
 				return len(files)
 			}, network.EventuallyTimeout).Should(Equal(2))
+
+			By("Asserting that orderer1 receives and persists snapshot")
 			Eventually(func() int {
 				files, err := ioutil.ReadDir(path.Join(o1SnapDir, "testchannel"))
 				Expect(err).NotTo(HaveOccurred())
 				return len(files)
 			}, network.EventuallyTimeout).Should(Equal(1))
 
+			By("Asserting cluster is still functional")
 			RunInvoke(network, o1, peer, "testchannel")
 			Eventually(func() int {
 				return RunQuery(network, o1, peer, "testchannel")