FAB-13264 consensus migration: kafka2raft green path #1

This is the first of four (1/4) sub-tasks that focus on the
"green" path of consensus-type migration from Kafka to Raft. 

By "green" we mean that there are no failures or aborts along
the way. The flow of the green path and the changes made in
these 4 tasks are described below. The 4 sub-tasks are staged
in a way that minimizes dependencies between them.

In this sub-task we introduce changes to the 
orderer/common/bootstrap package (see details below).
In essence, Just before the last config block of the system
channel (COMMIT) is written to the ledger, the bootstrap file
(a.k.a "genesis.block", do not confuse with the first block of
the ledger) is swapped with the last block of the system
channel. This sub-task extends package orderer/common/bootstrap
to support this functionality.

See respective JIRA item for further details.

The "green" path for migration is the following:

1. Start with a Kafka-based ordering service
2. Send a config update tx (START-TX) on the system channel that:
 - Has ConsensusType.MigrationState=START
 - This will disable the creation of new channels
 - This will disable the processing of normal (standard channel) transactions
3. Wait until the START-TX is committed and get the block height H of that tx
4. Send a config update tx (CONTEXT-TX) on each of the standard channels that:
 - Has ConsensusType.MigrationState=CONTEXT
 - Has ConsensusType.MigrationContext=H
 - Has ConsensusType.Type="etcdraft"
 - Has ConsensusType.Metadata=<a marshaled etcdraft metadata: Consenters,
   Options, etc>
5. Send a config update tx (COMMIT-TX) on the system channel that:
 - Has ConsensusType.MigrationState=COMMIT
 - Has ConsensusType.MigrationContext=H
 - Has ConsensusType.Type="etcdraft"
 - Has ConsensusType.Metadata=<a marshaled etcdraft metadata: Consenters,
   Options, etc>
 - The metadata should be the same as for the standard channels, with the same
   precautions.
 - If committed successfully, no further configuration will be possible
6. Restart each orderer
 - The orderer will bootstrap into an etcdraft mode
 - Each channel will form a cluster
 - Normal transactions can resume now
7. In order to configure the channels (system or standard), make sure that
   the first
   config update tx (on any given channel) after migration has:
 - Has ConsensusType.MigrationState=NONE
 - Has ConsensusType.MigrationS=NONE
 - In addition to other changes to the channel's config.

Change-Id: Iccd146bb7260bafa4e4d8c4ee457d2ac19f5a642
Signed-off-by: Yoav Tock <tock@il.ibm.com>

orderer/common/bootstrap/bootstrap.go

-/*
-Copyright IBM Corp. 2016 All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-                 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+// Copyright IBM Corp. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 package bootstrap
 
@@ -26,3 +13,26 @@ type Helper interface {
 	// the ledger (be it reading from the filesystem, generating it, etc.)
 	GenesisBlock() *ab.Block
 }
+
+// Replacer provides the ability to to replace the current genesis block used
+// for bootstrapping with the supplied block. It is used during consensus-type
+// migration in order to replace the original genesis block used for
+// bootstrapping with the latest config block of the system channel, which
+// contains the new consensus-type. This will ensure the instantiation of the
+// correct consenter type when the server restarts.
+type Replacer interface {
+	// ReplaceGenesisBlockFile should first copy the current file to a backup
+	// file: <genesis-file-name> => <genesis-file-name>.bak
+	// and then overwrite the original file with the content of the given block.
+	// If something goes wrong during migration, the original file could be
+	// restored from the backup.
+	// An error is returned if the operation was not completed successfully.
+	ReplaceGenesisBlockFile(block *ab.Block) error
+
+	// CheckReadWrite checks whether the current file is readable and writable,
+	// because if it is not, there is no point in attempting to replace. This
+	// check is performed at the beginning of the consensus-type migration
+	// process.
+	// An error is returned if the file is not readable and writable.
+	CheckReadWrite() error
+}

orderer/common/bootstrap/file/bootstrap.go

-/*
-Copyright IBM Corp. 2016 All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-                 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+// Copyright IBM Corp. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 package file
 
 import (
-	"fmt"
+	"io"
 	"io/ioutil"
+	"os"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/hyperledger/fabric/orderer/common/bootstrap"
 	cb "github.com/hyperledger/fabric/protos/common"
+	"github.com/pkg/errors"
 )
 
 type fileBootstrapper struct {
 	GenesisBlockFile string
 }
 
-// New returns a new static bootstrap helper
+// New returns a new static bootstrap helper.
 func New(fileName string) bootstrap.Helper {
 	return &fileBootstrapper{
 		GenesisBlockFile: fileName,
 	}
 }
 
-// GenesisBlock returns the genesis block to be used for bootstrapping
+// NewReplacer returns a new bootstrap replacer.
+func NewReplacer(fileName string) bootstrap.Replacer {
+	return &fileBootstrapper{
+		GenesisBlockFile: fileName,
+	}
+}
+
+// GenesisBlock returns the genesis block to be used for bootstrapping.
 func (b *fileBootstrapper) GenesisBlock() *cb.Block {
 	bootstrapFile, fileErr := ioutil.ReadFile(b.GenesisBlockFile)
 	if fileErr != nil {
-		panic(fmt.Errorf("Unable to bootstrap orderer. Error reading genesis block file: %v", fileErr))
+		panic(errors.Errorf("unable to bootstrap orderer. Error reading genesis block file: %v", fileErr))
 	}
 	genesisBlock := &cb.Block{}
 	unmarshallErr := proto.Unmarshal(bootstrapFile, genesisBlock)
 	if unmarshallErr != nil {
-		panic(fmt.Errorf("Unable to bootstrap orderer. Error unmarshalling genesis block: %v", unmarshallErr))
+		panic(errors.Errorf("unable to bootstrap orderer. Error unmarshalling genesis block: %v", unmarshallErr))
 
 	}
 	return genesisBlock
 } // GenesisBlock
+
+// ReplaceGenesisBlockFile creates a backup of the genesis block file, and then replaces
+// it with the content of the given block.
+// This is used during consensus-type migration in order to generate a bootstrap file that
+// specifies the new consensus-type.
+func (b *fileBootstrapper) ReplaceGenesisBlockFile(block *cb.Block) error {
+	buff, marshalErr := proto.Marshal(block)
+	if marshalErr != nil {
+		return errors.Wrap(marshalErr, "could not marshal block into a []byte")
+	}
+
+	genFileStat, statErr := os.Stat(b.GenesisBlockFile)
+	if statErr != nil {
+		return errors.Wrapf(statErr, "could not get the os.Stat of the genesis block file: %s", b.GenesisBlockFile)
+	}
+
+	if !genFileStat.Mode().IsRegular() {
+		return errors.Errorf("genesis block file: %s, is not a regular file", b.GenesisBlockFile)
+	}
+
+	backupFile := b.GenesisBlockFile + ".bak"
+	if err := backupGenesisFile(b.GenesisBlockFile, backupFile); err != nil {
+		return errors.Wrapf(err, "could not copy genesis block file (%s) into backup file: %s",
+			b.GenesisBlockFile, backupFile)
+	}
+
+	if err := ioutil.WriteFile(b.GenesisBlockFile, buff, genFileStat.Mode()); err != nil {
+		return errors.Wrapf(err, "could not write new genesis block into file: %s; use backup if necessary: %s",
+			b.GenesisBlockFile, backupFile)
+	}
+
+	return nil
+}
+
+func backupGenesisFile(src, dst string) error {
+	source, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+
+	_, err = io.Copy(destination, source)
+	return err
+}
+
+func (b *fileBootstrapper) CheckReadWrite() error {
+	genFileStat, statErr := os.Stat(b.GenesisBlockFile)
+	if statErr != nil {
+		return errors.Wrapf(statErr, "could not get the os.Stat of the genesis block file: %s", b.GenesisBlockFile)
+	}
+
+	if !genFileStat.Mode().IsRegular() {
+		return errors.Errorf("genesis block file: %s, is not a regular file", b.GenesisBlockFile)
+	}
+
+	genFile, openErr := os.OpenFile(b.GenesisBlockFile, os.O_RDWR, genFileStat.Mode().Perm())
+	if openErr != nil {
+		if os.IsPermission(openErr) {
+			return errors.Wrapf(openErr, "genesis block file: %s, cannot be opened for read-write, check permissions", b.GenesisBlockFile)
+		} else {
+			return errors.Wrapf(openErr, "genesis block file: %s, cannot be opened for read-write", b.GenesisBlockFile)
+		}
+	}
+	genFile.Close()
+
+	return nil
+}

orderer/common/bootstrap/file/bootstrap_test.go

-/*
-Copyright IBM Corp. 2016 All Rights Reserved.
+// Copyright IBM Corp. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-                 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package file
+package file_test
 
 import (
-	"bytes"
+	"io/ioutil"
 	"os"
+	"path"
 	"testing"
 
 	"github.com/golang/protobuf/proto"
+	bootfile "github.com/hyperledger/fabric/orderer/common/bootstrap/file"
 	cb "github.com/hyperledger/fabric/protos/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-const file = "./abc"
+const file = "abc.genesis"
+const fileBak = file + ".bak"
+const fileFake = file + ".fake"
 
-func TestNoFile(t *testing.T) {
-	defer func() {
-		if r := recover(); r == nil {
-			t.Errorf("TestNoFile should have panicked")
-		}
-	}()
+func TestGenesisBlock(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "unittest")
+	assert.NoErrorf(t, err, "generate temporary test dir")
+	defer os.RemoveAll(testDir)
 
-	helper := New(file)
-	_ = helper.GenesisBlock()
+	testFile := path.Join(testDir, file)
 
-} // TestNoFile
+	testFileFake := path.Join(testDir, fileFake)
 
-func TestBadBlock(t *testing.T) {
-	defer func() {
-		if r := recover(); r == nil {
-			t.Errorf("TestBadBlock should have panicked")
-		}
-	}()
+	t.Run("Bad - No file", func(t *testing.T) {
+		assert.Panics(t, func() {
+			helper := bootfile.New(testFileFake)
+			_ = helper.GenesisBlock()
+		}, "No file")
+	})
 
-	testFile, _ := os.Create(file)
-	defer os.Remove(file)
-	testFile.Write([]byte("abc"))
-	testFile.Close()
-	helper := New(file)
-	_ = helper.GenesisBlock()
-} // TestBadBlock
+	t.Run("Bad - Malformed Block", func(t *testing.T) {
+		testFileHandle, err := os.Create(testFile)
+		assert.NoErrorf(t, err, "generate temporary test file: %s", file)
+		defer os.Remove(testFile)
+		testFileHandle.Write([]byte("abc"))
+		testFileHandle.Close()
 
-func TestGenesisBlock(t *testing.T) {
-	defer func() {
-		if r := recover(); r != nil {
-			t.Errorf("TestGenesisBlock: unexpected panic")
+		assert.Panics(t, func() {
+			helper := bootfile.New(testFile)
+			_ = helper.GenesisBlock()
+		}, "Malformed Block")
+	})
+
+	t.Run("Correct flow", func(t *testing.T) {
+		//The original block and file
+		expectedNumber := uint64(0)
+		expectedBytes := []byte("abc")
+		expectedHash := []byte(nil)
+
+		header := &cb.BlockHeader{
+			Number:       expectedNumber,
+			PreviousHash: expectedHash,
+			DataHash:     expectedBytes,
+		}
+		data := &cb.BlockData{
+			Data: [][]byte{expectedBytes},
+		}
+		expectedDataLen := len(data.Data)
+		metadata := &cb.BlockMetadata{
+			Metadata: [][]byte{expectedBytes},
 		}
-	}()
+		expectedMetaLen := len(metadata.Metadata)
+		block := &cb.Block{
+			Header:   header,
+			Data:     data,
+			Metadata: metadata,
+		}
+		marshalledBlock, _ := proto.Marshal(block)
+		testFileHandle, err := os.Create(testFile)
+		assert.NoErrorf(t, err, "generate temporary test file: %s", file)
+		defer os.Remove(testFile)
+		testFileHandle.Write(marshalledBlock)
+		testFileHandle.Close()
+
+		helper := bootfile.New(testFile)
+		outBlock := helper.GenesisBlock()
+
+		outHeader := outBlock.Header
+		assert.Equal(t, expectedNumber, outHeader.Number, "block header Number not read correctly")
+		assert.Equal(t, expectedHash, outHeader.PreviousHash, "block header PreviousHash not read correctly")
+		assert.Equal(t, expectedBytes, outHeader.DataHash, "block header DataHash not read correctly")
 
+		outData := outBlock.Data
+		assert.Equal(t, expectedDataLen, len(outData.Data), "block len(data) not read correctly")
+		assert.Equal(t, expectedBytes, outData.Data[0], "block data not read correctly")
+
+		outMeta := outBlock.Metadata
+		assert.Equal(t, expectedMetaLen, len(outMeta.Metadata), "block len(Metadata) not read correctly")
+		assert.Equal(t, expectedBytes, outMeta.Metadata[0], "block Metadata not read correctly")
+	})
+}
+
+func TestReplaceGenesisBlockFile(t *testing.T) {
+	//The original block and file
+	expectedNumber := uint64(0)
+	expectedBytes := []byte("abc")
+	expectedHash := []byte(nil)
 	header := &cb.BlockHeader{
-		Number:       0,
-		PreviousHash: nil,
-		DataHash:     []byte("abc"),
+		Number:       expectedNumber,
+		PreviousHash: expectedHash,
+		DataHash:     expectedBytes,
 	}
 	data := &cb.BlockData{
-		Data: [][]byte{[]byte("abc")},
+		Data: [][]byte{expectedBytes},
 	}
 	metadata := &cb.BlockMetadata{
-		Metadata: [][]byte{[]byte("abc")},
+		Metadata: [][]byte{expectedBytes},
 	}
 	block := &cb.Block{
 		Header:   header,
@@ -79,24 +121,148 @@ func TestGenesisBlock(t *testing.T) {
 	}
 	marshalledBlock, _ := proto.Marshal(block)
 
-	testFile, _ := os.Create(file)
-	defer os.Remove(file)
-	testFile.Write(marshalledBlock)
-	testFile.Close()
+	testDir, err := ioutil.TempDir("", "unittest")
+	assert.NoErrorf(t, err, "generate temporary test dir")
+	defer os.RemoveAll(testDir)
+
+	testFile := path.Join(testDir, file)
+	testFileHandle, err := os.Create(testFile)
+	assert.NoErrorf(t, err, "generate temporary test file: %s", file)
 
-	helper := New(file)
-	outBlock := helper.GenesisBlock()
+	testFileHandle.Write(marshalledBlock)
+	testFileHandle.Close()
 
-	outHeader := outBlock.Header
-	if outHeader.Number != 0 || outHeader.PreviousHash != nil || !bytes.Equal(outHeader.DataHash, []byte("abc")) {
-		t.Errorf("block header not read correctly. Got %+v\n . Should have been %+v\n", outHeader, header)
+	testFileBak := path.Join(testDir, fileBak)
+	testFileFake := path.Join(testDir, fileFake)
+
+	// The new block
+	expectedNumber2 := uint64(1)
+	expectedBytes2 := []byte("def")
+	expectedHash2 := []byte(nil)
+
+	header2 := &cb.BlockHeader{
+		Number:       expectedNumber2,
+		PreviousHash: expectedHash2,
+		DataHash:     expectedBytes2,
+	}
+	data2 := &cb.BlockData{
+		Data: [][]byte{expectedBytes2},
 	}
-	outData := outBlock.Data
-	if len(outData.Data) != 1 && !bytes.Equal(outData.Data[0], []byte("abc")) {
-		t.Errorf("block data not read correctly. Got %+v\n . Should have been %+v\n", outData, data)
+	expectedDataLen2 := len(data2.Data)
+	metadata2 := &cb.BlockMetadata{
+		Metadata: [][]byte{expectedBytes2},
 	}
-	outMeta := outBlock.Metadata
-	if len(outMeta.Metadata) != 1 && !bytes.Equal(outMeta.Metadata[0], []byte("abc")) {
-		t.Errorf("Metadata data not read correctly. Got %+v\n . Should have been %+v\n", outMeta, metadata)
+	expectedMetaLen2 := len(metadata2.Metadata)
+	block2 := &cb.Block{
+		Header:   header2,
+		Data:     data2,
+		Metadata: metadata2,
 	}
-} // TestGenesisBlock
+
+	t.Run("Good", func(t *testing.T) {
+		replacer := bootfile.NewReplacer(testFile)
+		errWr := replacer.CheckReadWrite()
+		require.NoErrorf(t, errWr, "Failed to verify writable: %s", testFile)
+
+		errRep := replacer.ReplaceGenesisBlockFile(block2)
+		defer os.Remove(testFileBak)
+		require.NoErrorf(t, errRep, "Failed to replace: %s", testFile)
+
+		helper := bootfile.New(testFile)
+		outBlock := helper.GenesisBlock()
+
+		outHeader := outBlock.Header
+		assert.Equal(t, expectedNumber2, outHeader.Number, "block header Number not read correctly.")
+		assert.Equal(t, []uint8([]byte(nil)), outHeader.PreviousHash, "block header PreviousHash not read correctly.")
+		assert.Equal(t, expectedBytes2, outHeader.DataHash, "block header DataHash not read correctly.")
+
+		outData := outBlock.Data
+		assert.Equal(t, expectedDataLen2, len(outData.Data), "block len(data) not read correctly.")
+		assert.Equal(t, expectedBytes2, outData.Data[0], "block data not read correctly.")
+
+		outMeta := outBlock.Metadata
+		assert.Equal(t, expectedMetaLen2, len(outMeta.Metadata), "block len(Metadata) not read correctly.")
+		assert.Equal(t, expectedBytes2, outMeta.Metadata[0], "block Metadata not read correctly.")
+	})
+
+	t.Run("Bad - No original", func(t *testing.T) {
+		replacer := bootfile.NewReplacer(testFileFake)
+		errWr := replacer.CheckReadWrite()
+		assert.Error(t, errWr, "no such file")
+		assert.Contains(t, errWr.Error(), "no such file or directory")
+
+		errRep := replacer.ReplaceGenesisBlockFile(block2)
+		assert.Error(t, errRep, "no such file")
+		assert.Contains(t, errRep.Error(), "no such file or directory")
+	})
+
+	t.Run("Bad - Not a regular file", func(t *testing.T) {
+		replacer := bootfile.NewReplacer(testDir)
+		errWr := replacer.CheckReadWrite()
+		assert.Error(t, errWr, "not a regular file")
+		assert.Contains(t, errWr.Error(), "not a regular file")
+
+		errRep := replacer.ReplaceGenesisBlockFile(block2)
+		assert.Error(t, errRep, "not a regular file")
+		assert.Contains(t, errRep.Error(), "not a regular file")
+	})
+
+	t.Run("Bad - backup not writable", func(t *testing.T) {
+		replacer := bootfile.NewReplacer(testFile)
+
+		_, err := os.Create(testFileBak)
+		defer os.Remove(testFileBak)
+		assert.NoErrorf(t, err, "Failed to create backup")
+		err = os.Chmod(testFileBak, 0400)
+		assert.NoErrorf(t, err, "Failed to change permission on backup")
+
+		err = replacer.ReplaceGenesisBlockFile(block2)
+		assert.Errorf(t, err, "Fail to replace, backup")
+		assert.Contains(t, err.Error(), "permission denied")
+		assert.Contains(t, err.Error(), "could not copy genesis block file")
+
+		err = os.Chmod(testFileBak, 0600)
+		assert.NoErrorf(t, err, "Failed to restore permission on backup")
+	})
+
+	t.Run("Bad - source not writable", func(t *testing.T) {
+		replacer := bootfile.NewReplacer(testFile)
+
+		errC := os.Chmod(testFile, 0400)
+		assert.NoErrorf(t, errC, "Failed to change permission on origin")
+
+		errWr := replacer.CheckReadWrite()
+		assert.Error(t, errWr, "not writable")
+		assert.Contains(t, errWr.Error(), "permission denied")
+		assert.Contains(t, errWr.Error(), "cannot be opened for read-write, check permissions")
+
+		errRep := replacer.ReplaceGenesisBlockFile(block2)
+		assert.Errorf(t, errRep, "Fail to replace, unwritable origin")
+		assert.Contains(t, errRep.Error(), "permission denied")
+		assert.Contains(t, errRep.Error(), "could not write new genesis block into file")
+		assert.Contains(t, errRep.Error(), "use backup if necessary")
+
+		err = os.Chmod(testFile, 0600)
+		assert.NoErrorf(t, err, "Failed to restore permission, origin")
+	})
+
+	t.Run("Bad - source not readable", func(t *testing.T) {
+		replacer := bootfile.NewReplacer(testFile)
+
+		errC := os.Chmod(testFile, 0200)
+		assert.NoErrorf(t, errC, "Failed to change permission on origin")
+
+		errWr := replacer.CheckReadWrite()
+		assert.Error(t, errWr, "not writable")
+		assert.Contains(t, errWr.Error(), "permission denied")
+		assert.Contains(t, errWr.Error(), "cannot be opened for read-write, check permissions")
+
+		errRep := replacer.ReplaceGenesisBlockFile(block2)
+		assert.Errorf(t, errRep, "Fail to replace, unwritable origin")
+		assert.Contains(t, errRep.Error(), "permission denied")
+		assert.Contains(t, errRep.Error(), "could not copy genesis block file")
+
+		err = os.Chmod(testFile, 0600)
+		assert.NoErrorf(t, err, "Failed to restore permission, origin")
+	})
+}