store 16 blocks and use Snappy compression

Author: dustinxie

blockchain/block/blockstore.go

@@ -9,14 +9,13 @@ package block
 import (
 	"github.com/golang/protobuf/proto"
 
-	"github.com/iotexproject/iotex-proto/golang/iotexapi"
 	"github.com/iotexproject/iotex-proto/golang/iotextypes"
 
 	"github.com/iotexproject/iotex-core/action"
 )
 
 type (
-	// Store defines block and receipts
+	// Store defines block storage schema
 	Store struct {
 		Block    *Block
 		Receipts []*action.Receipt
@@ -29,19 +28,19 @@ func (in *Store) Serialize() ([]byte, error) {
 }
 
 // ToProto converts to proto message
-func (in *Store) ToProto() *iotexapi.BlockInfo {
+func (in *Store) ToProto() *iotextypes.BlockStore {
 	receipts := []*iotextypes.Receipt{}
 	for _, r := range in.Receipts {
 		receipts = append(receipts, r.ConvertToReceiptPb())
 	}
-	return &iotexapi.BlockInfo{
+	return &iotextypes.BlockStore{
 		Block:    in.Block.ConvertToBlockPb(),
 		Receipts: receipts,
 	}
 }
 
 // FromProto converts from proto message
-func (in *Store) FromProto(pb *iotexapi.BlockInfo) error {
+func (in *Store) FromProto(pb *iotextypes.BlockStore) error {
 	in.Block = &Block{}
 	if err := in.Block.ConvertFromBlockPb(pb.Block); err != nil {
 		return err
@@ -62,9 +61,18 @@ func (in *Store) FromProto(pb *iotexapi.BlockInfo) error {
 
 // Deserialize parses the byte stream into Store
 func (in *Store) Deserialize(buf []byte) error {
-	pbInfo := &iotexapi.BlockInfo{}
-	if err := proto.Unmarshal(buf, pbInfo); err != nil {
+	pbStore := &iotextypes.BlockStore{}
+	if err := proto.Unmarshal(buf, pbStore); err != nil {
 		return err
 	}
-	return in.FromProto(pbInfo)
+	return in.FromProto(pbStore)
+}
+
+// DeserializeBlockStoresPb decode byte stream into BlockStores pb message
+func DeserializeBlockStoresPb(buf []byte) (*iotextypes.BlockStores, error) {
+	pbStores := &iotextypes.BlockStores{}
+	if err := proto.Unmarshal(buf, pbStores); err != nil {
+		return nil, err
+	}
+	return pbStores, nil
 }

blockchain/blockdao/blockdao_test.go

@@ -433,11 +433,11 @@ func createTestBlockDAO(inMemory, legacy, compressBlock bool, cfg config.DB) (Bl
 		}
 	} else {
 		cfg.CompressData = compressBlock
-		file, err := newFileDAONew(db.NewBoltDB(cfg), 1, cfg)
+		file, err := newFileDAOv2(db.NewBoltDB(cfg), 1, cfg)
 		if err != nil {
 			return nil, err
 		}
-		fileDAO, err = createFileDAO(nil, map[uint64]FileDAONew{1: file})
+		fileDAO, err = createFileDAO(nil, map[uint64]FileDAO{1: file})
 		if err != nil {
 			return nil, err
 		}

blockchain/blockdao/filedao.go

@@ -64,7 +64,7 @@ type (
 		lifecycle lifecycle.Lifecycle
 		currFd    FileDAO
 		legacyFd  FileDAO
-		newFd     map[uint64]FileDAONew
+		v2Fd      map[uint64]FileDAO
 	}
 )
 
@@ -103,7 +103,7 @@ func (fd *fileDAO) GetBlockHash(height uint64) (hash.Hash256, error) {
 		h   hash.Hash256
 		err error
 	)
-	for _, file := range fd.newFd {
+	for _, file := range fd.v2Fd {
 		if h, err = file.GetBlockHash(height); err == nil {
 			return h, nil
 		}
@@ -119,7 +119,7 @@ func (fd *fileDAO) GetBlockHeight(hash hash.Hash256) (uint64, error) {
 		height uint64
 		err    error
 	)
-	for _, file := range fd.newFd {
+	for _, file := range fd.v2Fd {
 		if height, err = file.GetBlockHeight(hash); err == nil {
 			return height, nil
 		}
@@ -135,7 +135,7 @@ func (fd *fileDAO) GetBlock(hash hash.Hash256) (*block.Block, error) {
 		blk *block.Block
 		err error
 	)
-	for _, file := range fd.newFd {
+	for _, file := range fd.v2Fd {
 		if blk, err = file.GetBlock(hash); err == nil {
 			return blk, nil
 		}
@@ -151,7 +151,7 @@ func (fd *fileDAO) GetBlockByHeight(height uint64) (*block.Block, error) {
 		blk *block.Block
 		err error
 	)
-	for _, file := range fd.newFd {
+	for _, file := range fd.v2Fd {
 		if blk, err = file.GetBlockByHeight(height); err == nil {
 			return blk, nil
 		}
@@ -167,7 +167,7 @@ func (fd *fileDAO) GetReceipts(height uint64) ([]*action.Receipt, error) {
 		receipts []*action.Receipt
 		err      error
 	)
-	for _, file := range fd.newFd {
+	for _, file := range fd.v2Fd {
 		if receipts, err = file.GetReceipts(height); err == nil {
 			return receipts, nil
 		}
@@ -188,7 +188,7 @@ func (fd *fileDAO) TransactionLogs(height uint64) (*iotextypes.TransactionLogs,
 		log *iotextypes.TransactionLogs
 		err error
 	)
-	for _, file := range fd.newFd {
+	for _, file := range fd.v2Fd {
 		if log, err = file.TransactionLogs(height); err == nil {
 			return log, nil
 		}
@@ -214,10 +214,10 @@ func (fd *fileDAO) DeleteTipBlock() error {
 	return fd.currFd.DeleteTipBlock()
 }
 
-func createFileDAO(legacy FileDAO, newFile map[uint64]FileDAONew) (FileDAO, error) {
+func createFileDAO(legacy FileDAO, newFile map[uint64]FileDAO) (FileDAO, error) {
 	fileDAO := &fileDAO{
 		legacyFd: legacy,
-		newFd:    newFile,
+		v2Fd:     newFile,
 	}
 
 	var (

blockchain/blockdao/filedao_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func TestChecksumNamespaceAndKeys(t *testing.T) {
-	require := require.New(t)
+	r := require.New(t)
 
 	a := []hash.Hash256{
 		// filedao
@@ -34,16 +34,18 @@ func TestChecksumNamespaceAndKeys(t *testing.T) {
 		hash.BytesToHash256([]byte(receiptsNS)),
 		hash.BytesToHash256(heightPrefix),
 		hash.BytesToHash256(heightToFileBucket),
-		// filedao_new
+		// filedao_v2
 		hash.BytesToHash256([]byte{_normal}),
 		hash.BytesToHash256([]byte{_compressed}),
+		hash.BytesToHash256([]byte{blockStoreBatchSize}),
 		hash.BytesToHash256([]byte(hashDataNS)),
 		hash.BytesToHash256([]byte(blockDataNS)),
+		hash.BytesToHash256([]byte(stagingDataNS)),
 		hash.BytesToHash256(bottomHeightKey),
 	}
 
 	checksum := crypto.NewMerkleTree(a)
-	require.NotNil(checksum)
+	r.NotNil(checksum)
 	h := checksum.HashTree()
-	require.Equal("1cc352fff5fc29d8ac7dcc186cce7b9e7d87c41ceeba41993b96b8a9566facaa", hex.EncodeToString(h[:]))
+	r.Equal("f584356aa8ad1da8cb803162f06c96ff04e4478ad8f4a2667d44cee3e79f9d94", hex.EncodeToString(h[:]))
 }