Commit 2a16532c authored by manish's avatar manish
Browse files

Move Blockstorage code under /fabric/common package

https://jira.hyperledger.org/browse/FAB-2022



This changes introduced by this CR
- Moves the block storage code from package
core/ledger/blkstorage to common/ledger/blkstorage

- Splits the ledger_interface.go so as to move common interfaces
and data type to common/ledger package

- Moves some of the util functions to common/ledger package

- Moves core/ledger/ordererledger package to orderer/ledger/fsledger
orderer folks can futher rename/refactor this as seems suitable to them

Change-Id: I759e16f00dc2ec9bb62196121083cf48eae76948
Signed-off-by: default avatarmanish <manish.sethi@gmail.com>
parent e67a2382
......@@ -16,7 +16,7 @@ limitations under the License.
package version
import "github.com/hyperledger/fabric/core/ledger/util"
import "github.com/hyperledger/fabric/common/ledger/util"
// Height represents the height of a transaction in blockchain
type Height struct {
......
......@@ -19,7 +19,7 @@ package version
import (
"testing"
"github.com/hyperledger/fabric/core/ledger/testutil"
"github.com/hyperledger/fabric/common/ledger/testutil"
)
func TestVersionSerialization(t *testing.T) {
......
......@@ -17,32 +17,10 @@ limitations under the License.
package ledger
import (
commonledger "github.com/hyperledger/fabric/common/ledger"
"github.com/hyperledger/fabric/protos/common"
pb "github.com/hyperledger/fabric/protos/peer"
)
// Ledger captures the methods that are common across the 'PeerLedger', 'OrdererLedger', and 'ValidatedLedger'
type Ledger interface {
// GetBlockchainInfo returns basic info about blockchain
GetBlockchainInfo() (*pb.BlockchainInfo, error)
// GetBlockByNumber returns block at a given height
// blockNumber of math.MaxUint64 will return last block
GetBlockByNumber(blockNumber uint64) (*common.Block, error)
// GetBlocksIterator returns an iterator that starts from `startBlockNumber`(inclusive).
// The iterator is a blocking iterator i.e., it blocks till the next block gets available in the ledger
// ResultsIterator contains type BlockHolder
GetBlocksIterator(startBlockNumber uint64) (ResultsIterator, error)
// Close closes the ledger
Close()
// Commit adds a new block
Commit(block *common.Block) error
}
// OrdererLedger implements methods required by 'orderer ledger'
type OrdererLedger interface {
Ledger
}
// PeerLedgerProvider provides handle to ledger instances
type PeerLedgerProvider interface {
// Create creates a new ledger with a given unique id
......@@ -57,24 +35,10 @@ type PeerLedgerProvider interface {
Close()
}
// OrdererLedgerProvider provides handle to raw ledger instances
type OrdererLedgerProvider interface {
// Create creates a new ledger with a given unique id
Create(ledgerID string) (OrdererLedger, error)
// Open opens an already created ledger
Open(ledgerID string) (OrdererLedger, error)
// Exists tells whether the ledger with given id exits
Exists(ledgerID string) (bool, error)
// List lists the ids of the existing ledgers
List() ([]string, error)
// Close closes the ValidatedLedgerProvider
Close()
}
// PeerLedger differs from the OrdererLedger in that PeerLedger locally maintain a bitmask
// that tells apart valid transactions from invalid ones
type PeerLedger interface {
Ledger
commonledger.Ledger
// GetTransactionByID retrieves a transaction by id
GetTransactionByID(txID string) (*common.Envelope, error)
// GetBlockByHash returns a block given it's hash
......@@ -92,13 +56,13 @@ type PeerLedger interface {
// Any synchronization should be performed at the implementation level if required
NewHistoryQueryExecutor() (HistoryQueryExecutor, error)
//Prune prunes the blocks/transactions that satisfy the given policy
Prune(policy PrunePolicy) error
Prune(policy commonledger.PrunePolicy) error
}
// ValidatedLedger represents the 'final ledger' after filtering out invalid transactions from PeerLedger.
// Post-v1
type ValidatedLedger interface {
Ledger
commonledger.Ledger
}
// QueryExecutor executes the queries
......@@ -116,10 +80,10 @@ type QueryExecutor interface {
// and an empty endKey refers to the last available key. For scanning all the keys, both the startKey and the endKey
// can be supplied as empty strings. However, a full scan shuold be used judiciously for performance reasons.
// The returned ResultsIterator contains results of type *KV
GetStateRangeScanIterator(namespace string, startKey string, endKey string) (ResultsIterator, error)
GetStateRangeScanIterator(namespace string, startKey string, endKey string) (commonledger.ResultsIterator, error)
// ExecuteQuery executes the given query and returns an iterator that contains results of type specific to the underlying data store.
// Only used for state databases that support query
ExecuteQuery(query string) (ResultsIterator, error)
ExecuteQuery(query string) (commonledger.ResultsIterator, error)
// Done releases resources occupied by the QueryExecutor
Done()
}
......@@ -127,7 +91,7 @@ type QueryExecutor interface {
// HistoryQueryExecutor executes the history queries
type HistoryQueryExecutor interface {
// GetHistoryForKey retrieves the history of values for a key.
GetHistoryForKey(namespace string, key string) (ResultsIterator, error)
GetHistoryForKey(namespace string, key string) (commonledger.ResultsIterator, error)
}
// TxSimulator simulates a transaction on a consistent snapshot of the 'as recent state as possible'
......@@ -153,18 +117,6 @@ type TxSimulator interface {
GetTxSimulationResults() ([]byte, error)
}
// ResultsIterator - an iterator for query result set
type ResultsIterator interface {
// Next returns the next item in the result set. The `QueryResult` is expected to be nil when
// the iterator gets exhausted
Next() (QueryResult, error)
// Close releases resources occupied by the iterator
Close()
}
// QueryResult - a general interface for supporting different types of query results. Actual types differ for different queries
type QueryResult interface{}
// KV - QueryResult for KV-based datamodel. Holds a key and corresponding value. A nil value indicates a non-existent key.
type KV struct {
Key string
......@@ -184,13 +136,3 @@ type QueryRecord struct {
Key string
Record []byte
}
// BlockHolder holds block returned by the iterator in GetBlocksIterator.
// The sole purpose of this holder is to avoid desrialization if block is desired in raw bytes form (e.g., for transfer)
type BlockHolder interface {
GetBlock() *common.Block
GetBlockBytes() []byte
}
// PrunePolicy - a general interface for supporting different pruning policies
type PrunePolicy interface{}
......@@ -19,7 +19,8 @@ package ledgerconfig
import (
"testing"
"github.com/hyperledger/fabric/core/ledger/testutil"
"github.com/hyperledger/fabric/common/ledger/testutil"
ledgertestutil "github.com/hyperledger/fabric/core/ledger/testutil"
"github.com/spf13/viper"
)
......@@ -30,7 +31,7 @@ func TestIsCouchDBEnabledDefault(t *testing.T) {
// If the ledger test are run with CouchDb enabled, need to provide a mechanism
// To let this test run but still test default values.
if IsCouchDBEnabled() == true {
testutil.ResetConfigToDefaultValues()
ledgertestutil.ResetConfigToDefaultValues()
defer viper.Set("ledger.state.stateDatabase", "CouchDB")
}
defaultValue := IsCouchDBEnabled()
......@@ -39,7 +40,7 @@ func TestIsCouchDBEnabledDefault(t *testing.T) {
func TestIsCouchDBEnabled(t *testing.T) {
setUpCoreYAMLConfig()
defer testutil.ResetConfigToDefaultValues()
defer ledgertestutil.ResetConfigToDefaultValues()
viper.Set("ledger.state.stateDatabase", "CouchDB")
updatedValue := IsCouchDBEnabled()
testutil.AssertEquals(t, updatedValue, true) //test config returns true
......@@ -47,7 +48,7 @@ func TestIsCouchDBEnabled(t *testing.T) {
func TestGetCouchDBDefinition(t *testing.T) {
setUpCoreYAMLConfig()
defer testutil.ResetConfigToDefaultValues()
defer ledgertestutil.ResetConfigToDefaultValues()
viper.Set("ledger.state.stateDatabase", "CouchDB")
couchDBDef := GetCouchDBDefinition()
testutil.AssertEquals(t, couchDBDef.URL, "127.0.0.1:5984")
......@@ -63,7 +64,7 @@ func TestIsHistoryDBEnabledDefault(t *testing.T) {
func TestIsHistoryDBEnabledTrue(t *testing.T) {
setUpCoreYAMLConfig()
defer testutil.ResetConfigToDefaultValues()
defer ledgertestutil.ResetConfigToDefaultValues()
viper.Set("ledger.state.historyDatabase", true)
updatedValue := IsHistoryDBEnabled()
testutil.AssertEquals(t, updatedValue, true) //test config returns true
......@@ -71,7 +72,7 @@ func TestIsHistoryDBEnabledTrue(t *testing.T) {
func TestIsHistoryDBEnabledFalse(t *testing.T) {
setUpCoreYAMLConfig()
defer testutil.ResetConfigToDefaultValues()
defer ledgertestutil.ResetConfigToDefaultValues()
viper.Set("ledger.state.historyDatabase", false)
updatedValue := IsHistoryDBEnabled()
testutil.AssertEquals(t, updatedValue, false) //test config returns false
......@@ -79,5 +80,5 @@ func TestIsHistoryDBEnabledFalse(t *testing.T) {
func setUpCoreYAMLConfig() {
//call a helper method to load the core.yaml
testutil.SetupCoreYAMLConfig("./../../../peer")
ledgertestutil.SetupCoreYAMLConfig("./../../../peer")
}
......@@ -22,8 +22,8 @@ import (
"os"
"github.com/hyperledger/fabric/common/ledger/testutil"
"github.com/hyperledger/fabric/core/ledger"
"github.com/hyperledger/fabric/core/ledger/testutil"
"github.com/spf13/viper"
)
......
......@@ -17,18 +17,13 @@ limitations under the License.
package testutil
import (
"crypto/rand"
"flag"
"fmt"
mathRand "math/rand"
"reflect"
"regexp"
"runtime"
"strings"
"testing"
"time"
"github.com/hyperledger/fabric/common/util"
"github.com/op/go-logging"
"github.com/spf13/viper"
)
......@@ -108,150 +103,3 @@ func ParseTestParams() []string {
paramsArray := regex.Split(*testParams, -1)
return paramsArray
}
// AssertNil varifies that the value is nil
func AssertNil(t testing.TB, value interface{}) {
if !isNil(value) {
t.Fatalf("Value not nil. value=[%#v]\n %s", value, getCallerInfo())
}
}
// AssertNotNil varifies that the value is not nil
func AssertNotNil(t testing.TB, value interface{}) {
if isNil(value) {
t.Fatalf("Values is nil. %s", getCallerInfo())
}
}
// AssertSame varifies that the two values are same
func AssertSame(t testing.TB, actual interface{}, expected interface{}) {
t.Logf("%s: AssertSame [%#v] and [%#v]", getCallerInfo(), actual, expected)
if actual != expected {
t.Fatalf("Values actual=[%#v] and expected=[%#v] do not point to same object. %s", actual, expected, getCallerInfo())
}
}
// AssertEquals varifies that the two values are equal
func AssertEquals(t testing.TB, actual interface{}, expected interface{}) {
t.Logf("%s: AssertEquals [%#v] and [%#v]", getCallerInfo(), actual, expected)
if expected == nil && isNil(actual) {
return
}
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("Values are not equal.\n Actual=[%#v], \n Expected=[%#v]\n %s", actual, expected, getCallerInfo())
}
}
// AssertNotEquals varifies that the two values are not equal
func AssertNotEquals(t testing.TB, actual interface{}, expected interface{}) {
if reflect.DeepEqual(actual, expected) {
t.Fatalf("Values are not supposed to be equal. Actual=[%#v], Expected=[%#v]\n %s", actual, expected, getCallerInfo())
}
}
// AssertError varifies that the err is not nil
func AssertError(t testing.TB, err error, message string) {
if err == nil {
t.Fatalf("%s\n %s", message, getCallerInfo())
}
}
// AssertNoError varifies that the err is nil
func AssertNoError(t testing.TB, err error, message string) {
if err != nil {
t.Fatalf("%s - Error: %s\n %s", message, err, getCallerInfo())
}
}
// AssertContains varifies that the slice contains the value
func AssertContains(t testing.TB, slice interface{}, value interface{}) {
if reflect.TypeOf(slice).Kind() != reflect.Slice && reflect.TypeOf(slice).Kind() != reflect.Array {
t.Fatalf("Type of argument 'slice' is expected to be a slice/array, found =[%s]\n %s", reflect.TypeOf(slice), getCallerInfo())
}
if !Contains(slice, value) {
t.Fatalf("Expected value [%s] not found in slice %s\n %s", value, slice, getCallerInfo())
}
}
// AssertContainsAll varifies that sliceActual is a superset of sliceExpected
func AssertContainsAll(t testing.TB, sliceActual interface{}, sliceExpected interface{}) {
if reflect.TypeOf(sliceActual).Kind() != reflect.Slice && reflect.TypeOf(sliceActual).Kind() != reflect.Array {
t.Fatalf("Type of argument 'sliceActual' is expected to be a slice/array, found =[%s]\n %s", reflect.TypeOf(sliceActual), getCallerInfo())
}
if reflect.TypeOf(sliceExpected).Kind() != reflect.Slice && reflect.TypeOf(sliceExpected).Kind() != reflect.Array {
t.Fatalf("Type of argument 'sliceExpected' is expected to be a slice/array, found =[%s]\n %s", reflect.TypeOf(sliceExpected), getCallerInfo())
}
array := reflect.ValueOf(sliceExpected)
for i := 0; i < array.Len(); i++ {
element := array.Index(i).Interface()
if !Contains(sliceActual, element) {
t.Fatalf("Expected value [%s] not found in slice %s\n %s", element, sliceActual, getCallerInfo())
}
}
}
// AssertPanic varifies that a panic is raised during a test
func AssertPanic(t testing.TB, msg string) {
x := recover()
if x == nil {
t.Fatal(msg)
} else {
t.Logf("A panic was caught successfully. Actual msg = %s", x)
}
}
// ComputeCryptoHash computes crypto hash for testing
func ComputeCryptoHash(content ...[]byte) []byte {
return util.ComputeCryptoHash(AppendAll(content...))
}
// AppendAll combines the bytes from different []byte into one []byte
func AppendAll(content ...[]byte) []byte {
combinedContent := []byte{}
for _, b := range content {
combinedContent = append(combinedContent, b...)
}
return combinedContent
}
// GenerateID generates a uuid
func GenerateID(t *testing.T) string {
return util.GenerateUUID()
}
// ConstructRandomBytes constructs random bytes of given size
func ConstructRandomBytes(t testing.TB, size int) []byte {
value := make([]byte, size)
_, err := rand.Read(value)
if err != nil {
t.Fatalf("Error while generating random bytes: %s", err)
}
return value
}
// Contains returns true iff the `value` is present in the `slice`
func Contains(slice interface{}, value interface{}) bool {
array := reflect.ValueOf(slice)
for i := 0; i < array.Len(); i++ {
element := array.Index(i).Interface()
if value == element || reflect.DeepEqual(element, value) {
return true
}
}
return false
}
func isNil(in interface{}) bool {
return in == nil || reflect.ValueOf(in).IsNil() || (reflect.TypeOf(in).Kind() == reflect.Slice && reflect.ValueOf(in).Len() == 0)
}
func getCallerInfo() string {
_, file, line, ok := runtime.Caller(2)
if !ok {
return "Could not retrieve caller's info"
}
return fmt.Sprintf("CallerInfo = [%s:%d]", file, line)
}
......@@ -21,8 +21,9 @@ import (
"fmt"
"testing"
"github.com/hyperledger/fabric/common/ledger/testutil"
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
"github.com/hyperledger/fabric/core/ledger/testutil"
ledgertestutil "github.com/hyperledger/fabric/core/ledger/testutil"
)
//Basic setup to test couch
......@@ -54,7 +55,7 @@ var assetJSON = []byte(`{"asset_name":"marble1","color":"blue","size":"35","owne
func TestDBConnectionDef(t *testing.T) {
//call a helper method to load the core.yaml
testutil.SetupCoreYAMLConfig("./../../../../peer")
ledgertestutil.SetupCoreYAMLConfig("./../../../../peer")
//create a new connection
_, err := CreateConnectionDefinition(connectURL, "", "")
......
......@@ -20,15 +20,16 @@ import (
"fmt"
"testing"
"github.com/hyperledger/fabric/common/ledger/testutil"
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
"github.com/hyperledger/fabric/core/ledger/testutil"
ledgertestutil "github.com/hyperledger/fabric/core/ledger/testutil"
)
//Unit test of couch db util functionality
func TestCreateCouchDBConnectionAndDB(t *testing.T) {
//call a helper method to load the core.yaml
testutil.SetupCoreYAMLConfig("./../../../../peer")
ledgertestutil.SetupCoreYAMLConfig("./../../../../peer")
if ledgerconfig.IsCouchDBEnabled() == true {
......
......@@ -17,52 +17,10 @@ limitations under the License.
package util
import (
"encoding/binary"
"fmt"
"reflect"
"sort"
"github.com/golang/protobuf/proto"
)
// EncodeOrderPreservingVarUint64 returns a byte-representation for a uint64 number such that
// all zero-bits starting bytes are trimmed in order to reduce the length of the array
// For preserving the order in a default bytes-comparison, first byte contains the number of remaining bytes.
// The presence of first byte also allows to use the returned bytes as part of other larger byte array such as a
// composite-key representation in db
func EncodeOrderPreservingVarUint64(number uint64) []byte {
bytes := make([]byte, 8)
binary.BigEndian.PutUint64(bytes, number)
startingIndex := 0
size := 0
for i, b := range bytes {
if b != 0x00 {
startingIndex = i
size = 8 - i
break
}
}
sizeBytes := proto.EncodeVarint(uint64(size))
if len(sizeBytes) > 1 {
panic(fmt.Errorf("[]sizeBytes should not be more than one byte because the max number it needs to hold is 8. size=%d", size))
}
encodedBytes := make([]byte, size+1)
encodedBytes[0] = sizeBytes[0]
copy(encodedBytes[1:], bytes[startingIndex:])
return encodedBytes
}
// DecodeOrderPreservingVarUint64 decodes the number from the bytes obtained from method 'EncodeOrderPreservingVarUint64'.
// Also, returns the number of bytes that are consumed in the process
func DecodeOrderPreservingVarUint64(bytes []byte) (uint64, int) {
s, _ := proto.DecodeVarint(bytes)
size := int(s)
decodedBytes := make([]byte, 8)
copy(decodedBytes[8-size:], bytes[1:size+1])
numBytesConsumed := size + 1
return binary.BigEndian.Uint64(decodedBytes), numBytesConsumed
}
// GetSortedKeys returns the keys of the map in a sorted order. This function assumes that the keys are string
func GetSortedKeys(m interface{}) []string {
mapVal := reflect.ValueOf(m)
......
......@@ -24,6 +24,7 @@ import (
"github.com/op/go-logging"
"github.com/spf13/viper"
commonledger "github.com/hyperledger/fabric/common/ledger"
"github.com/hyperledger/fabric/core/chaincode/shim"
"github.com/hyperledger/fabric/core/ledger"
"github.com/hyperledger/fabric/core/peer"
......@@ -118,7 +119,7 @@ func getQueryResult(vledger ledger.PeerLedger, query []byte) (res pb.Response) {
}
qstring := string(query)
var qexe ledger.QueryExecutor
var ri ledger.ResultsIterator
var ri commonledger.ResultsIterator
var err error
// We install a recover() to gain control in 2 cases
......@@ -149,7 +150,7 @@ func getQueryResult(vledger ledger.PeerLedger, query []byte) (res pb.Response) {
var buffer bytes.Buffer
buffer.WriteString("[")
var qresult ledger.QueryResult
var qresult commonledger.QueryResult
bArrayMemberAlreadyWritten := false
qresult, err = ri.Next()
for r := 0; qresult != nil && err == nil && r < limit; r++ {
......
......@@ -14,55 +14,54 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package ordererledger
package fsledger
import (
"errors"
"github.com/hyperledger/fabric/core/ledger"
"github.com/hyperledger/fabric/core/ledger/blkstorage"
"github.com/hyperledger/fabric/common/ledger"
"github.com/hyperledger/fabric/common/ledger/blkstorage"
"github.com/hyperledger/fabric/protos/common"
pb "github.com/hyperledger/fabric/protos/peer"
)
const (
fileSegmentSize = 64 * 1024 * 1024
)
// fsBasedOrdererLedger - an orderer ledger implementation that persists blocks on filesystem based store
type fsBasedOrdererLedger struct {
// fsLedger - an orderer ledger implementation that persists blocks on filesystem based store
type fsLedger struct {
blockStore blkstorage.BlockStore
}
// GetBlockchainInfo returns basic info about blockchain
func (rl *fsBasedOrdererLedger) GetBlockchainInfo() (*pb.BlockchainInfo, error) {
return rl.blockStore.GetBlockchainInfo()
func (l *fsLedger) GetBlockchainInfo() (*common.BlockchainInfo, error) {
return l.blockStore.GetBlockchainInfo()
}
// GetBlockByNumber returns block at a given height
func (rl *fsBasedOrdererLedger) GetBlockByNumber(blockNumber uint64) (*common.Block, error) {
return rl.blockStore.RetrieveBlockByNumber(blockNumber)
func (l *fsLedger) GetBlockByNumber(blockNumber uint64) (*common.Block, error) {
return l.blockStore.RetrieveBlockByNumber(blockNumber)
}
// GetBlocksIterator returns an iterator that starts from `startBlockNumber`(inclusive).
// The iterator is a blocking iterator i.e., it blocks till the next block gets available in the ledger
// ResultsIterator contains type BlockHolder
func (rl *fsBasedOrdererLedger) GetBlocksIterator(startBlockNumber uint64) (ledger.ResultsIterator, error) {
return rl.blockStore.RetrieveBlocks(startBlockNumber)
func (l *fsLedger) GetBlocksIterator(startBlockNumber uint64) (ledger.ResultsIterator, error) {
return l.blockStore.RetrieveBlocks(startBlockNumber)
}
//Prune prunes the blocks/transactions that satisfy the given policy
func (rl *fsBasedOrdererLedger) Prune(policy ledger.PrunePolicy) error {
func (l *fsLedger) Prune(policy ledger.PrunePolicy) error {
return errors.New("Not yet implemented")
}
// Close closes the ledger
func (rl *fsBasedOrdererLedger) Close() {
rl.blockStore.Shutdown()
func (l *fsLedger) Close() {
l.blockStore.Shutdown()
}
// Commit adds a new block
func (rl *fsBasedOrdererLedger) Commit(block *common.Block) error {
return rl.blockStore.AddBlock(block)
func (l *fsLedger) Commit(block *common.Block) error {
return l.blockStore.AddBlock(block)
}
......@@ -14,58 +14,57 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package ordererledger
package fsledger
import (
"github.com/hyperledger/fabric/core/ledger"
"github.com/hyperledger/fabric/core/ledger/blkstorage"
"github.com/hyperledger/fabric/core/ledger/blkstorage/fsblkstorage"
"github.com/hyperledger/fabric/common/ledger/blkstorage"
"github.com/hyperledger/fabric/common/ledger/blkstorage/fsblkstorage"
)
// FSBasedOrdererLedgerProvider impements interface ledger.OrdererLedgerProvider
type FSBasedOrdererLedgerProvider struct {
// Provider impements interface OrdererLedgerProvider
type Provider struct {
blkStoreProvider blkstorage.BlockStoreProvider
}
// NewFSBasedOrdererLedgerProvider construct a new filesystem based orderer ledger provider. Only one instance should be created
func NewFSBasedOrdererLedgerProvider(conf *fsblkstorage.Conf) *FSBasedOrdererLedgerProvider {
// NewProvider construct a new filesystem based orderer ledger provider. Only one instance should be created
func NewProvider(conf *fsblkstorage.Conf) *Provider {
attr