Skip to content

Commit 3713e8f

Browse files
dustinxiedustinxie
committed
[filedao] check legacy and V2 files in cfg.DbPath (iotexproject#1817)
1 parent d874c5d commit 3713e8f

13 files changed

+790
-266
lines changed

‎blockchain/blockdao/blockdao_test.go

+5-13
Original file line numberDiff line numberDiff line change
@@ -425,28 +425,20 @@ func createTestBlockDAO(inMemory, legacy bool, compressBlock string, cfg config.
425425
return NewBlockDAOInMemForTest(nil, cfg), nil
426426
}
427427

428-
var fileDAO filedao.FileDAO
429428
if legacy {
430429
file, err := filedao.NewFileDAOLegacy(compressBlock != "", cfg)
431430
if err != nil {
432431
return nil, err
433432
}
434-
fileDAO, err = filedao.CreateFileDAO(file, nil)
435-
if err != nil {
436-
return nil, err
437-
}
438-
} else {
439-
cfg.Compressor = compressBlock
440-
file, err := filedao.NewFileDAOv2(1, cfg)
441-
if err != nil {
442-
return nil, err
443-
}
444-
fileDAO, err = filedao.CreateFileDAO(nil, map[uint64]filedao.FileDAO{1: file})
433+
fileDAO, err := filedao.CreateFileDAO(file, nil, cfg)
445434
if err != nil {
446435
return nil, err
447436
}
437+
return createBlockDAO(fileDAO, nil, cfg), nil
448438
}
449-
return createBlockDAO(fileDAO, nil, cfg), nil
439+
440+
cfg.Compressor = compressBlock
441+
return NewBlockDAO(nil, false, cfg), nil
450442
}
451443

452444
func BenchmarkBlockCache(b *testing.B) {

‎blockchain/filedao/filedao.go

+113-78
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package filedao
88

99
import (
1010
"context"
11+
"fmt"
1112

1213
"github.com/pkg/errors"
1314
"go.uber.org/zap"
@@ -18,7 +19,6 @@ import (
1819
"github.com/iotexproject/iotex-core/action"
1920
"github.com/iotexproject/iotex-core/blockchain/block"
2021
"github.com/iotexproject/iotex-core/config"
21-
"github.com/iotexproject/iotex-core/pkg/lifecycle"
2222
"github.com/iotexproject/iotex-core/pkg/log"
2323
)
2424

@@ -35,6 +35,8 @@ var (
3535

3636
// vars
3737
var (
38+
ErrFileNotExist = errors.New("file does not exist")
39+
ErrFileInvalid = errors.New("file format is not valid")
3840
ErrNotSupported = errors.New("feature not supported")
3941
ErrAlreadyExist = errors.New("block already exist")
4042
ErrInvalidTipHeight = errors.New("invalid tip height")
@@ -60,20 +62,42 @@ type (
6062

6163
// fileDAO implements FileDAO
6264
fileDAO struct {
63-
lifecycle lifecycle.Lifecycle
64-
currFd FileDAO
65-
legacyFd FileDAO
66-
v2Fd map[uint64]FileDAO
65+
currFd FileDAO
66+
legacyFd FileDAO
67+
v2Fd FileV2Manager // a collection of v2 db files
6768
}
6869
)
6970

7071
// NewFileDAO creates an instance of FileDAO
7172
func NewFileDAO(compressLegacy bool, cfg config.DB) (FileDAO, error) {
72-
legacyFd, err := NewFileDAOLegacy(compressLegacy, cfg)
73-
if err != nil {
73+
header, v2Files, err := checkChainDBFiles(cfg)
74+
if err == ErrFileInvalid {
7475
return nil, err
7576
}
76-
return CreateFileDAO(legacyFd, nil)
77+
78+
if err == ErrFileNotExist {
79+
// start new chain db using v2 format
80+
if err := createNewV2File(1, cfg); err != nil {
81+
return nil, err
82+
}
83+
return CreateFileDAO(nil, []string{cfg.DbPath}, cfg)
84+
}
85+
86+
switch header.Version {
87+
case FileLegacyMaster:
88+
// default file is legacy format
89+
legacyFd, err := NewFileDAOLegacy(compressLegacy, cfg)
90+
if err != nil {
91+
return nil, err
92+
}
93+
return CreateFileDAO(legacyFd, v2Files, cfg)
94+
case FileV2:
95+
// default file is v2 format, add it to filenames
96+
v2Files = append(v2Files, cfg.DbPath)
97+
return CreateFileDAO(nil, v2Files, cfg)
98+
default:
99+
panic(fmt.Errorf("corrupted file version: %s", header.Version))
100+
}
77101
}
78102

79103
// NewFileDAOInMemForTest creates an in-memory FileDAO for testing
@@ -82,35 +106,59 @@ func NewFileDAOInMemForTest(cfg config.DB) (FileDAO, error) {
82106
if err != nil {
83107
return nil, err
84108
}
85-
return CreateFileDAO(legacyFd, nil)
109+
return CreateFileDAO(legacyFd, nil, cfg)
86110
}
87111

88112
func (fd *fileDAO) Start(ctx context.Context) error {
89-
return fd.lifecycle.OnStart(ctx)
113+
if fd.legacyFd != nil {
114+
if err := fd.legacyFd.Start(ctx); err != nil {
115+
return err
116+
}
117+
}
118+
if fd.v2Fd != nil {
119+
if err := fd.v2Fd.Start(ctx); err != nil {
120+
return err
121+
}
122+
}
123+
124+
if fd.v2Fd != nil {
125+
fd.currFd = fd.v2Fd.TopFd()
126+
} else {
127+
fd.currFd = fd.legacyFd
128+
}
129+
return nil
90130
}
91131

92132
func (fd *fileDAO) Stop(ctx context.Context) error {
93-
return fd.lifecycle.OnStop(ctx)
133+
if fd.legacyFd != nil {
134+
if err := fd.legacyFd.Stop(ctx); err != nil {
135+
return err
136+
}
137+
}
138+
if fd.v2Fd != nil {
139+
return fd.v2Fd.Stop(ctx)
140+
}
141+
return nil
94142
}
95143

96144
func (fd *fileDAO) Height() (uint64, error) {
97145
return fd.currFd.Height()
98146
}
99147

100148
func (fd *fileDAO) GetBlockHash(height uint64) (hash.Hash256, error) {
101-
var (
102-
h hash.Hash256
103-
err error
104-
)
105-
for _, file := range fd.v2Fd {
106-
if h, err = file.GetBlockHash(height); err == nil {
107-
return h, nil
149+
if fd.v2Fd != nil {
150+
if height == 0 {
151+
return hash.ZeroHash256, nil
152+
}
153+
if v2 := fd.v2Fd.FileDAOByHeight(height); v2 != nil {
154+
return v2.GetBlockHash(height)
108155
}
109156
}
157+
110158
if fd.legacyFd != nil {
111159
return fd.legacyFd.GetBlockHash(height)
112160
}
113-
return hash.ZeroHash256, err
161+
return hash.ZeroHash256, ErrNotSupported
114162
}
115163

116164
func (fd *fileDAO) GetBlockHeight(hash hash.Hash256) (uint64, error) {
@@ -119,10 +167,11 @@ func (fd *fileDAO) GetBlockHeight(hash hash.Hash256) (uint64, error) {
119167
err error
120168
)
121169
for _, file := range fd.v2Fd {
122-
if height, err = file.GetBlockHeight(hash); err == nil {
170+
if height, err = file.fd.GetBlockHeight(hash); err == nil {
123171
return height, nil
124172
}
125173
}
174+
126175
if fd.legacyFd != nil {
127176
return fd.legacyFd.GetBlockHeight(hash)
128177
}
@@ -135,46 +184,41 @@ func (fd *fileDAO) GetBlock(hash hash.Hash256) (*block.Block, error) {
135184
err error
136185
)
137186
for _, file := range fd.v2Fd {
138-
if blk, err = file.GetBlock(hash); err == nil {
187+
if blk, err = file.fd.GetBlock(hash); err == nil {
139188
return blk, nil
140189
}
141190
}
191+
142192
if fd.legacyFd != nil {
143193
return fd.legacyFd.GetBlock(hash)
144194
}
145195
return nil, err
146196
}
147197

148198
func (fd *fileDAO) GetBlockByHeight(height uint64) (*block.Block, error) {
149-
var (
150-
blk *block.Block
151-
err error
152-
)
153-
for _, file := range fd.v2Fd {
154-
if blk, err = file.GetBlockByHeight(height); err == nil {
155-
return blk, nil
199+
if fd.v2Fd != nil {
200+
if v2 := fd.v2Fd.FileDAOByHeight(height); v2 != nil {
201+
return v2.GetBlockByHeight(height)
156202
}
157203
}
204+
158205
if fd.legacyFd != nil {
159206
return fd.legacyFd.GetBlockByHeight(height)
160207
}
161-
return nil, err
208+
return nil, ErrNotSupported
162209
}
163210

164211
func (fd *fileDAO) GetReceipts(height uint64) ([]*action.Receipt, error) {
165-
var (
166-
receipts []*action.Receipt
167-
err error
168-
)
169-
for _, file := range fd.v2Fd {
170-
if receipts, err = file.GetReceipts(height); err == nil {
171-
return receipts, nil
212+
if fd.v2Fd != nil {
213+
if v2 := fd.v2Fd.FileDAOByHeight(height); v2 != nil {
214+
return v2.GetReceipts(height)
172215
}
173216
}
217+
174218
if fd.legacyFd != nil {
175219
return fd.legacyFd.GetReceipts(height)
176220
}
177-
return nil, err
221+
return nil, ErrNotSupported
178222
}
179223

180224
func (fd *fileDAO) ContainsTransactionLog() bool {
@@ -183,19 +227,16 @@ func (fd *fileDAO) ContainsTransactionLog() bool {
183227
}
184228

185229
func (fd *fileDAO) TransactionLogs(height uint64) (*iotextypes.TransactionLogs, error) {
186-
var (
187-
log *iotextypes.TransactionLogs
188-
err error
189-
)
190-
for _, file := range fd.v2Fd {
191-
if log, err = file.TransactionLogs(height); err == nil {
192-
return log, nil
230+
if fd.v2Fd != nil {
231+
if v2 := fd.v2Fd.FileDAOByHeight(height); v2 != nil {
232+
return v2.TransactionLogs(height)
193233
}
194234
}
235+
195236
if fd.legacyFd != nil {
196237
return fd.legacyFd.TransactionLogs(height)
197238
}
198-
return nil, err
239+
return nil, ErrNotSupported
199240
}
200241

201242
func (fd *fileDAO) PutBlock(ctx context.Context, blk *block.Block) error {
@@ -214,44 +255,38 @@ func (fd *fileDAO) DeleteTipBlock() error {
214255
}
215256

216257
// CreateFileDAO creates FileDAO from legacy and new files
217-
func CreateFileDAO(legacy FileDAO, newFile map[uint64]FileDAO) (FileDAO, error) {
218-
fileDAO := &fileDAO{
219-
legacyFd: legacy,
220-
v2Fd: newFile,
258+
func CreateFileDAO(legacy FileDAO, v2Files []string, cfg config.DB) (FileDAO, error) {
259+
if legacy == nil && len(v2Files) == 0 {
260+
return nil, ErrNotSupported
221261
}
222262

223-
var (
224-
tipHeight uint64
225-
currFd FileDAO
226-
)
227-
228-
// find the file with highest start height
229-
for start, fd := range newFile {
230-
if currFd == nil {
231-
currFd = fd
232-
tipHeight = start
233-
} else {
234-
if start > tipHeight {
235-
currFd = fd
236-
tipHeight = start
237-
}
238-
}
239-
fileDAO.lifecycle.Add(fd)
240-
}
241-
if legacy != nil {
242-
fileDAO.lifecycle.Add(legacy)
243-
if currFd == nil {
244-
currFd = legacy
263+
var v2Fd FileV2Manager
264+
if len(v2Files) > 0 {
265+
fds := make([]*fileDAOv2, len(v2Files))
266+
for i, name := range v2Files {
267+
cfg.DbPath = name
268+
fds[i] = openFileDAOv2(cfg)
245269
}
270+
v2Fd = NewFileV2Manager(fds)
246271
}
247272

248-
if currFd == nil {
249-
return nil, errors.New("failed to find valid chain db file")
250-
}
251-
fileDAO.currFd = currFd
252-
return fileDAO, nil
273+
return &fileDAO{
274+
legacyFd: legacy,
275+
v2Fd: v2Fd,
276+
}, nil
253277
}
254278

255-
func hashKey(h hash.Hash256) []byte {
256-
return append(hashPrefix, h[:]...)
279+
// createNewV2File creates a new v2 chain db file
280+
func createNewV2File(start uint64, cfg config.DB) error {
281+
v2, err := newFileDAOv2(start, cfg)
282+
if err != nil {
283+
return err
284+
}
285+
286+
// calling Start() will write the header
287+
ctx := context.Background()
288+
if err := v2.Start(ctx); err != nil {
289+
return err
290+
}
291+
return v2.Stop(ctx)
257292
}

‎blockchain/filedao/filedao_header.go

+24
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,14 @@ import (
1616
"github.com/iotexproject/iotex-core/db"
1717
)
1818

19+
// constants
20+
const (
21+
FileLegacyMaster = "V1-master"
22+
FileLegacyAuxiliary = "V1-aux"
23+
FileV2 = "V2"
24+
FileAll = "All"
25+
)
26+
1927
type (
2028
// FileHeader is header of chain db file
2129
FileHeader struct {
@@ -32,6 +40,22 @@ type (
3240
}
3341
)
3442

43+
// ReadHeaderLegacy reads header from KVStore
44+
func ReadHeaderLegacy(kv *db.BoltDB) (*FileHeader, error) {
45+
// legacy file has these 6 buckets
46+
if !kv.BucketExists(receiptsNS) || !kv.BucketExists(blockHeaderNS) ||
47+
!kv.BucketExists(blockBodyNS) || !kv.BucketExists(blockFooterNS) {
48+
return nil, ErrFileInvalid
49+
}
50+
51+
// check tip height stored in master file
52+
_, err := getValueMustBe8Bytes(kv, blockNS, topHeightKey)
53+
if err == nil && kv.BucketExists(blockHashHeightMappingNS) {
54+
return &FileHeader{Version: FileLegacyMaster}, nil
55+
}
56+
return &FileHeader{Version: FileLegacyAuxiliary}, nil
57+
}
58+
3559
// ReadHeaderV2 reads header from KVStore
3660
func ReadHeaderV2(kv db.KVStore) (*FileHeader, error) {
3761
value, err := kv.Get(headerDataNs, fileHeaderKey)

0 commit comments

Comments
 (0)