[FAB-11918] Add CFT integration test.

Change-Id: I23e4ef5c764dc980f1e100cfc26ffe2988ce6f6e
Signed-off-by: Jay Guo <guojiannan1101@gmail.com>

integration/e2e/cft_test.go

+/*
+Copyright IBM Corp All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package e2e
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	docker "github.com/fsouza/go-dockerclient"
+	"github.com/hyperledger/fabric/integration/nwo"
+	"github.com/hyperledger/fabric/integration/nwo/commands"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+	"github.com/tedsuo/ifrit"
+	"github.com/tedsuo/ifrit/grouper"
+)
+
+var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
+	var (
+		testDir   string
+		client    *docker.Client
+		network   *nwo.Network
+		chaincode nwo.Chaincode
+
+		networkProc, o1Proc ifrit.Process
+	)
+
+	BeforeEach(func() {
+		var err error
+		testDir, err = ioutil.TempDir("", "e2e")
+		Expect(err).NotTo(HaveOccurred())
+
+		client, err = docker.NewClientFromEnv()
+		Expect(err).NotTo(HaveOccurred())
+
+		chaincode = nwo.Chaincode{
+			Name:    "mycc",
+			Version: "0.0",
+			Path:    "github.com/hyperledger/fabric/integration/chaincode/simple/cmd",
+			Ctor:    `{"Args":["init","a","100","b","200"]}`,
+			Policy:  `AND ('Org1MSP.member','Org2MSP.member')`,
+		}
+	})
+
+	AfterEach(func() {
+		if o1Proc != nil {
+			o1Proc.Signal(syscall.SIGTERM)
+			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+		}
+		if networkProc != nil {
+			networkProc.Signal(syscall.SIGTERM)
+			Eventually(networkProc.Wait(), network.EventuallyTimeout).Should(Receive())
+		}
+		if network != nil {
+			network.Cleanup()
+		}
+		os.RemoveAll(testDir)
+	})
+
+	When("orderer stops and restarts", func() {
+		It("keeps network up and running", func() {
+			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, 33000, components)
+
+			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
+			p := network.Peer("Org1", "peer1")
+			blockFile1 := filepath.Join(testDir, "newest_orderer1_block.pb")
+			blockFile2 := filepath.Join(testDir, "newest_orderer2_block.pb")
+
+			network.GenerateConfigTree()
+			network.Bootstrap()
+
+			o1Runner := network.OrdererRunner(o1)
+			orderers := grouper.Members{
+				{Name: o2.ID(), Runner: network.OrdererRunner(o2)},
+				{Name: o3.ID(), Runner: network.OrdererRunner(o3)},
+			}
+			ordererGroup := grouper.NewParallel(syscall.SIGTERM, orderers)
+			peerGroup := network.PeerGroupRunner()
+
+			networkRunner := grouper.NewOrdered(
+				syscall.SIGTERM,
+				grouper.Members{
+					{Name: "orderers", Runner: ordererGroup},
+					{Name: "peers", Runner: peerGroup},
+				},
+			)
+			o1Proc = ifrit.Invoke(o1Runner)
+			networkProc = ifrit.Invoke(networkRunner)
+			Eventually(o1Proc.Ready()).Should(BeClosed())
+			Eventually(networkProc.Ready()).Should(BeClosed())
+
+			By("performing operation with orderer1")
+			network.CreateAndJoinChannel(o1, "testchannel")
+
+			By("killing orderer1")
+			o1Proc.Signal(syscall.SIGKILL)
+			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137")))
+
+			By("performing operations with running orderer")
+			nwo.DeployChaincode(network, "testchannel", o2, chaincode)
+
+			By("restarting orderer1")
+			o1Runner = network.OrdererRunner(o1)
+			o1Proc = ifrit.Invoke(o1Runner)
+			Eventually(o1Proc.Ready()).Should(BeClosed())
+
+			By("executing transaction with restarted orderer")
+			RunQueryInvokeQuery(network, o1, p, "testchannel")
+
+			fetchLatestBlock := func(targetOrderer *nwo.Orderer, blockFile string) {
+				c := commands.ChannelFetch{
+					ChannelID:  "testchannel",
+					Block:      "newest",
+					OutputFile: blockFile,
+				}
+				if targetOrderer != nil {
+					c.Orderer = network.OrdererAddress(targetOrderer, nwo.ListenPort)
+				}
+				sess, err := network.PeerAdminSession(p, c)
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
+			}
+
+			fetchLatestBlock(o1, blockFile1)
+			fetchLatestBlock(o2, blockFile2)
+			b1 := nwo.UnmarshalBlockFromFile(blockFile1)
+			b2 := nwo.UnmarshalBlockFromFile(blockFile2)
+			Expect(b1.Header.Bytes()).To(Equal(b2.Header.Bytes()))
+		})
+	})
+})