Skip to content

Commit f370920

Browse files
dustinxiedustinxie
authored
add FileTip to avoid writing entire header to chain db (iotexproject#2455)
1 parent 78d0c17 commit f370920

File tree

6 files changed

+213
-117
lines changed

6 files changed

+213
-117
lines changed

‎blockchain/filedao/filedao_header.go

+51-17
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,17 @@ type (
2323
Compressor string
2424
BlockStoreSize uint64
2525
Start uint64
26-
End uint64
27-
TopHash hash.Hash256
26+
}
27+
28+
// FileTip is tip info of chain
29+
FileTip struct {
30+
Height uint64
31+
Hash hash.Hash256
2832
}
2933
)
3034

31-
// ReadFileHeader reads header from KVStore
32-
func ReadFileHeader(kv db.KVStore, ns string, key []byte) (*FileHeader, error) {
35+
// ReadHeader reads header from KVStore
36+
func ReadHeader(kv db.KVStore, ns string, key []byte) (*FileHeader, error) {
3337
value, err := kv.Get(ns, key)
3438
if err != nil {
3539
return nil, errors.Wrap(err, "file header not exist")
@@ -57,19 +61,15 @@ func (h *FileHeader) toProto() *headerpb.FileHeader {
5761
Compressor: h.Compressor,
5862
BlockStoreSize: h.BlockStoreSize,
5963
Start: h.Start,
60-
End: h.End,
61-
TopHash: h.TopHash[:],
6264
}
6365
}
6466

65-
func fromProto(pb *headerpb.FileHeader) *FileHeader {
67+
func fromProtoFileHeader(pb *headerpb.FileHeader) *FileHeader {
6668
return &FileHeader{
6769
Version: pb.Version,
6870
Compressor: pb.Compressor,
6971
BlockStoreSize: pb.BlockStoreSize,
7072
Start: pb.Start,
71-
End: pb.End,
72-
TopHash: hash.BytesToHash256(pb.TopHash),
7373
}
7474
}
7575

@@ -79,17 +79,51 @@ func DeserializeFileHeader(buf []byte) (*FileHeader, error) {
7979
if err := proto.Unmarshal(buf, pbHeader); err != nil {
8080
return nil, err
8181
}
82-
return fromProto(pbHeader), nil
82+
return fromProtoFileHeader(pbHeader), nil
8383
}
8484

85-
func (h *FileHeader) createHeaderBytes(height uint64, hash hash.Hash256) ([]byte, error) {
86-
currHeight := h.End
87-
h.End = height
88-
h.TopHash = hash
89-
ser, err := h.Serialize()
85+
// ReadTip reads tip from KVStore
86+
func ReadTip(kv db.KVStore, ns string, key []byte) (*FileTip, error) {
87+
value, err := kv.Get(ns, key)
9088
if err != nil {
89+
return nil, errors.Wrap(err, "file header not exist")
90+
}
91+
return DeserializeFileTip(value)
92+
}
93+
94+
// WriteTip writes tip to KVStore
95+
func WriteTip(kv db.KVStore, ns string, key []byte, tip *FileTip) error {
96+
ser, err := tip.Serialize()
97+
if err != nil {
98+
return err
99+
}
100+
return kv.Put(ns, key, ser)
101+
}
102+
103+
// Serialize serializes FileTip to byte-stream
104+
func (t *FileTip) Serialize() ([]byte, error) {
105+
return proto.Marshal(t.toProto())
106+
}
107+
108+
func (t *FileTip) toProto() *headerpb.FileTip {
109+
return &headerpb.FileTip{
110+
Height: t.Height,
111+
Hash: t.Hash[:],
112+
}
113+
}
114+
115+
func fromProtoFileTip(pb *headerpb.FileTip) *FileTip {
116+
return &FileTip{
117+
Height: pb.Height,
118+
Hash: hash.BytesToHash256(pb.Hash),
119+
}
120+
}
121+
122+
// DeserializeFileTip deserializes byte-stream to FileTip
123+
func DeserializeFileTip(buf []byte) (*FileTip, error) {
124+
pbTip := &headerpb.FileTip{}
125+
if err := proto.Unmarshal(buf, pbTip); err != nil {
91126
return nil, err
92127
}
93-
h.End = currHeight
94-
return ser, nil
128+
return fromProtoFileTip(pbTip), nil
95129
}

‎blockchain/filedao/filedao_header_test.go

+26-24
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package filedao
22

33
import (
4-
"bytes"
54
"context"
65
"testing"
76

@@ -15,42 +14,33 @@ import (
1514
"github.com/iotexproject/iotex-core/testutil"
1615
)
1716

18-
func TestHeaderProto(t *testing.T) {
17+
func TestFileProto(t *testing.T) {
1918
r := require.New(t)
2019

21-
sh := hash.Hash256{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
2220
h := &FileHeader{
2321
Version: FileV2,
2422
Compressor: "test",
2523
BlockStoreSize: 32,
2624
Start: 3,
27-
End: 1003,
28-
TopHash: sh,
2925
}
3026
ser, err := h.Serialize()
3127
r.NoError(err)
32-
3328
h1, err := DeserializeFileHeader(ser)
3429
r.NoError(err)
3530
r.Equal(h, h1)
3631

37-
for _, v := range []struct {
38-
height uint64
39-
hash hash.Hash256
40-
equal bool
41-
}{
42-
{h.End + 1, sh, false},
43-
{h.End, hash.ZeroHash256, false},
44-
{h.End, sh, true},
45-
} {
46-
ser1, err := h.createHeaderBytes(v.height, v.hash)
47-
r.NoError(err)
48-
r.EqualValues(1003, h.End)
49-
r.Equal(v.equal, bytes.Compare(ser1, ser) == 0)
32+
c := &FileTip{
33+
Height: 1003,
34+
Hash: hash.Hash256{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
5035
}
36+
ser, err = c.Serialize()
37+
r.NoError(err)
38+
c1, err := DeserializeFileTip(ser)
39+
r.NoError(err)
40+
r.Equal(c, c1)
5141
}
5242

53-
func TestFileHeader(t *testing.T) {
43+
func TestFileReadWrite(t *testing.T) {
5444
testHeader := func(kv db.KVStore, t *testing.T) {
5545
r := require.New(t)
5646
r.NotNil(kv)
@@ -59,7 +49,7 @@ func TestFileHeader(t *testing.T) {
5949
r.NoError(kv.Start(ctx))
6050
defer kv.Stop(ctx)
6151

62-
h, err := ReadFileHeader(kv, headerDataNs, fileHeaderKey)
52+
h, err := ReadHeader(kv, headerDataNs, fileHeaderKey)
6353
r.Equal(db.ErrNotExist, errors.Cause(err))
6454
r.Nil(h)
6555

@@ -68,14 +58,26 @@ func TestFileHeader(t *testing.T) {
6858
Compressor: "test",
6959
BlockStoreSize: 32,
7060
Start: 3,
71-
End: 1003,
72-
TopHash: hash.Hash256{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
7361
}
7462
r.NoError(WriteHeader(kv, headerDataNs, fileHeaderKey, h))
7563

76-
h1, err := ReadFileHeader(kv, headerDataNs, fileHeaderKey)
64+
h1, err := ReadHeader(kv, headerDataNs, fileHeaderKey)
7765
r.NoError(err)
7866
r.Equal(h, h1)
67+
68+
c, err := ReadTip(kv, headerDataNs, topHeightKey)
69+
r.Equal(db.ErrNotExist, errors.Cause(err))
70+
r.Nil(c)
71+
72+
c = &FileTip{
73+
Height: 1003,
74+
Hash: hash.Hash256{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
75+
}
76+
r.NoError(WriteTip(kv, headerDataNs, topHeightKey, c))
77+
78+
c1, err := ReadTip(kv, headerDataNs, topHeightKey)
79+
r.NoError(err)
80+
r.Equal(c, c1)
7981
}
8082

8183
r := require.New(t)

‎blockchain/filedao/filedao_v2.go

+51-46
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,16 @@ var (
4141

4242
type (
4343
fileDAOv2 struct {
44-
compressor string
45-
storeBatch uint64 // number of blocks to be stored together as a unit
46-
bottomHeight uint64 // height of first block in this DB file
47-
header *FileHeader
48-
cfg config.DB
49-
blkBuffer *stagingBuffer
50-
blkCache *cache.ThreadSafeLruCache
51-
kvStore db.KVStore
52-
batch batch.KVStoreBatch
53-
hashStore db.CountingIndex // store block hash
54-
blkStore db.CountingIndex // store raw blocks
55-
sysStore db.CountingIndex // store transaction log
44+
header *FileHeader
45+
tip *FileTip
46+
cfg config.DB
47+
blkBuffer *stagingBuffer
48+
blkCache *cache.ThreadSafeLruCache
49+
kvStore db.KVStore
50+
batch batch.KVStoreBatch
51+
hashStore db.CountingIndex // store block hash
52+
blkStore db.CountingIndex // store raw blocks
53+
sysStore db.CountingIndex // store transaction log
5654
}
5755
)
5856

@@ -63,52 +61,58 @@ func NewFileDAOv2(kvStore db.KVStore, bottom uint64, cfg config.DB) (FileDAO, er
6361
}
6462

6563
fd := fileDAOv2{
66-
compressor: cfg.Compressor,
67-
storeBatch: uint64(cfg.BlockStoreBatchSize),
68-
bottomHeight: bottom,
69-
cfg: cfg,
70-
blkCache: cache.NewThreadSafeLruCache(16),
71-
kvStore: kvStore,
72-
batch: batch.NewBatch(),
64+
header: &FileHeader{
65+
Version: FileV2,
66+
Compressor: cfg.Compressor,
67+
BlockStoreSize: uint64(cfg.BlockStoreBatchSize),
68+
Start: bottom,
69+
},
70+
tip: &FileTip{
71+
Height: bottom - 1,
72+
},
73+
cfg: cfg,
74+
blkCache: cache.NewThreadSafeLruCache(16),
75+
kvStore: kvStore,
76+
batch: batch.NewBatch(),
7377
}
7478
return &fd, nil
7579
}
7680

7781
// openFileDAOv2 opens an existing v2 file
7882
func openFileDAOv2(kvStore db.KVStore, cfg config.DB) (FileDAO, error) {
7983
fd := fileDAOv2{
80-
compressor: cfg.Compressor,
81-
cfg: cfg,
82-
blkCache: cache.NewThreadSafeLruCache(16),
83-
kvStore: kvStore,
84-
batch: batch.NewBatch(),
84+
cfg: cfg,
85+
blkCache: cache.NewThreadSafeLruCache(16),
86+
kvStore: kvStore,
87+
batch: batch.NewBatch(),
8588
}
8689
return &fd, nil
8790
}
8891

8992
func (fd *fileDAOv2) Start(ctx context.Context) error {
90-
var err error
91-
if err = fd.kvStore.Start(ctx); err != nil {
93+
if err := fd.kvStore.Start(ctx); err != nil {
9294
return err
9395
}
9496

9597
// check file header
96-
fd.header, err = ReadFileHeader(fd.kvStore, headerDataNs, fileHeaderKey)
98+
header, err := ReadHeader(fd.kvStore, headerDataNs, fileHeaderKey)
9799
if err != nil {
98100
if errors.Cause(err) != db.ErrBucketNotExist && errors.Cause(err) != db.ErrNotExist {
99101
return errors.Wrap(err, "failed to get file header")
100102
}
101-
// write file header
102-
fd.header = &FileHeader{
103-
Version: FileV2,
104-
Compressor: fd.cfg.Compressor,
105-
BlockStoreSize: fd.storeBatch,
106-
Start: fd.bottomHeight,
107-
End: fd.bottomHeight - 1,
108-
}
103+
// write file header and tip
109104
if err = WriteHeader(fd.kvStore, headerDataNs, fileHeaderKey, fd.header); err != nil {
110105
return err
111106
}
107+
if err = WriteTip(fd.kvStore, headerDataNs, topHeightKey, fd.tip); err != nil {
108+
return err
109+
}
110+
} else {
111+
fd.header = header
112+
// read file tip
113+
if fd.tip, err = ReadTip(fd.kvStore, headerDataNs, topHeightKey); err != nil {
114+
return err
115+
}
112116
}
113117

114118
// create counting index for hash, blk, and transaction log
@@ -134,19 +138,20 @@ func (fd *fileDAOv2) Stop(ctx context.Context) error {
134138
}
135139

136140
func (fd *fileDAOv2) Height() (uint64, error) {
137-
h, err := ReadFileHeader(fd.kvStore, headerDataNs, fileHeaderKey)
141+
var err error
142+
fd.tip, err = ReadTip(fd.kvStore, headerDataNs, topHeightKey)
138143
if err != nil {
139144
return 0, errors.Wrap(err, "failed to get top height")
140145
}
141-
return h.End, nil
146+
return fd.tip.Height, nil
142147
}
143148

144149
func (fd *fileDAOv2) Bottom() (uint64, error) {
145150
return fd.header.Start, nil
146151
}
147152

148153
func (fd *fileDAOv2) ContainsHeight(height uint64) bool {
149-
return fd.header.Start <= height && height <= fd.header.End
154+
return fd.header.Start <= height && height <= fd.tip.Height
150155
}
151156

152157
func (fd *fileDAOv2) GetBlockHash(height uint64) (hash.Hash256, error) {
@@ -208,15 +213,15 @@ func (fd *fileDAOv2) TransactionLogs(height uint64) (*iotextypes.TransactionLogs
208213
if err != nil {
209214
return nil, errors.Wrapf(err, "failed to get transaction log at height %d", height)
210215
}
211-
value, err = decompBytes(value, fd.compressor)
216+
value, err = decompBytes(value, fd.header.Compressor)
212217
if err != nil {
213218
return nil, errors.Wrapf(err, "failed to get transaction log at height %d", height)
214219
}
215220
return block.DeserializeSystemLogPb(value)
216221
}
217222

218223
func (fd *fileDAOv2) PutBlock(_ context.Context, blk *block.Block) error {
219-
if blk.Height() != fd.header.End+1 {
224+
if blk.Height() != fd.tip.Height+1 {
220225
return ErrInvalidTipHeight
221226
}
222227

@@ -238,7 +243,7 @@ func (fd *fileDAOv2) PutBlock(_ context.Context, blk *block.Block) error {
238243
if err := fd.kvStore.WriteBatch(fd.batch); err != nil {
239244
return errors.Wrapf(err, "failed to put block at height %d", blk.Height())
240245
}
241-
fd.header.End = blk.Height()
246+
fd.tip.Height = blk.Height()
242247
return nil
243248
}
244249

@@ -269,11 +274,11 @@ func (fd *fileDAOv2) DeleteTipBlock() error {
269274
}
270275

271276
// delete hash -> height mapping
272-
header, err := ReadFileHeader(fd.kvStore, headerDataNs, fileHeaderKey)
277+
fd.tip, err = ReadTip(fd.kvStore, headerDataNs, topHeightKey)
273278
if err != nil {
274279
return err
275280
}
276-
fd.batch.Delete(blockHashHeightMappingNS, hashKey(header.TopHash), "failed to delete hash -> height mapping")
281+
fd.batch.Delete(blockHashHeightMappingNS, hashKey(fd.tip.Hash), "failed to delete hash -> height mapping")
277282

278283
// update file header
279284
h := hash.ZeroHash256
@@ -283,15 +288,15 @@ func (fd *fileDAOv2) DeleteTipBlock() error {
283288
return err
284289
}
285290
}
286-
ser, err := fd.header.createHeaderBytes(height-1, h)
291+
ser, err := (&FileTip{Height: height - 1, Hash: h}).Serialize()
287292
if err != nil {
288293
return err
289294
}
290-
fd.batch.Put(headerDataNs, fileHeaderKey, ser, "failed to put file header")
295+
fd.batch.Put(headerDataNs, topHeightKey, ser, "failed to put file tip")
291296

292297
if err := fd.kvStore.WriteBatch(fd.batch); err != nil {
293298
return err
294299
}
295-
fd.header.End = height - 1
300+
fd.tip.Height = height - 1
296301
return nil
297302
}

0 commit comments

Comments
 (0)