From c49ad654582ea095778817e5afb7644effcc5323 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Tue, 18 Jun 2019 21:13:31 +0800 Subject: [PATCH 1/2] fix state_sync according to time_lock --- store/statesync_helper.go | 49 +++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/store/statesync_helper.go b/store/statesync_helper.go index 277f0fbc9..f5704e5e7 100644 --- a/store/statesync_helper.go +++ b/store/statesync_helper.go @@ -70,8 +70,6 @@ type StateSyncHelper struct { incompleteChunks map[int64][]incompleteChunkItem // node idx -> incomplete chunk items, for caching incomplete nodes temporally prefixNodeDBs []PrefixNodeDB - storeKeys []sdk.StoreKey - reloadingMtx sync.RWMutex // guard below fields to make sure no concurrent load snapshot and response snapshot, and they should be updated atomically snapshotManager *snapshot.SnapshotManager @@ -88,10 +86,23 @@ func NewStateSyncHelper( helper.commitMS = cms helper.cdc = cdc - kvStores := cms.GetCommitKVStores() + helper.SnapshotHeights = make(chan int64, snapshotWorkingQueueSize) + helper.HeightsToDelete = make(chan int64, snapshotToRemoveQueueSize) + + return &helper +} + +// not all key in cms is committed +// for example the BEP9 timelock store upgrade will not commit the newly added store until upgrade height +func (helper *StateSyncHelper) getCommitedSortedStoreKeys() []sdk.StoreKey { + kvStores := helper.commitMS.GetCommitKVStores() names := make([]string, 0, len(kvStores)) nameToKey := make(map[string]sdk.StoreKey, len(kvStores)) - for key, store := range cms.GetCommitKVStores() { + for key, store := range helper.commitMS.GetCommitKVStores() { + if !sdk.ShouldCommitStore(key.Name()) { + continue + } + switch store.(type) { case *IavlStore: nameToKey[key.Name()] = key @@ -101,14 +112,11 @@ func NewStateSyncHelper( } } sort.Strings(names) + storeKeys := make([]sdk.StoreKey, 0, len(names)) for _, name := range names { - helper.storeKeys = append(helper.storeKeys, nameToKey[name]) + storeKeys = append(storeKeys, nameToKey[name]) } - - helper.SnapshotHeights = make(chan int64, snapshotWorkingQueueSize) - helper.HeightsToDelete = make(chan int64, snapshotToRemoveQueueSize) - - return &helper + return storeKeys } // Split Init method and NewStateSyncHelper for snapshot command @@ -129,11 +137,14 @@ func (helper *StateSyncHelper) Init(lastBreatheBlockHeight int64) { func (helper *StateSyncHelper) StartRecovery(manifest *abci.Manifest) error { helper.logger.Info("start recovery") + sdk.UpgradeMgr.SetHeight(manifest.Height) + storeKeys := helper.getCommitedSortedStoreKeys() + helper.manifest = manifest - helper.stateSyncStoreInfos = make([]StoreInfo, 0, len(helper.storeKeys)) + helper.stateSyncStoreInfos = make([]StoreInfo, 0, len(storeKeys)) helper.hashesToIdx = make(map[abci.SHA256Sum]int, len(manifest.AppStateHashes)) helper.incompleteChunks = make(map[int64][]incompleteChunkItem, 0) - helper.prefixNodeDBs = make([]PrefixNodeDB, 0, len(helper.storeKeys)) + helper.prefixNodeDBs = make([]PrefixNodeDB, 0, len(storeKeys)) idxOfChunk := 0 for _, h := range manifest.AppStateHashes { @@ -141,19 +152,19 @@ func (helper *StateSyncHelper) StartRecovery(manifest *abci.Manifest) error { idxOfChunk++ } - if len(manifest.NumKeys) != len(helper.storeKeys) { - return fmt.Errorf("sub store count in manifest %d does not match local %d", len(manifest.NumKeys), len(helper.storeKeys)) + if len(manifest.NumKeys) != len(storeKeys) { + return fmt.Errorf("sub store count in manifest %d does not match local %d", len(manifest.NumKeys), len(storeKeys)) } var startIdxForEachStore int64 for idx, numOfKeys := range manifest.NumKeys { - db := dbm.NewPrefixDB(helper.db, []byte("s/k:"+helper.storeKeys[idx].Name()+"/")) + db := dbm.NewPrefixDB(helper.db, []byte("s/k:"+storeKeys[idx].Name()+"/")) nodeDB := iavl.NewNodeDB(db, 10000) helper.prefixNodeDBs = append(helper.prefixNodeDBs, PrefixNodeDB{ startIdxForEachStore, startIdxForEachStore + numOfKeys, - helper.storeKeys[idx].Name(), + storeKeys[idx].Name(), nodeDB}) startIdxForEachStore += numOfKeys } @@ -378,6 +389,8 @@ func (helper *StateSyncHelper) takeSnapshotImpl(height int64, retry int) { return } + storeKeys := helper.getCommitedSortedStoreKeys() + failed := true for failed { failed = false @@ -395,11 +408,11 @@ func (helper *StateSyncHelper) takeSnapshotImpl(height int64, retry int) { } totalKeys := int64(0) - numKeys := make([]int64, 0, len(helper.storeKeys)) + numKeys := make([]int64, 0, len(storeKeys)) currChunkNodes := make([][]byte, 0, 40000) // one account leaf node is around 100 bytes according to testnet experiment, non-leaf node should be less, 40000 should be a bit less than 4M var currStartIdx int64 var currChunkTotalBytes int - for _, key := range helper.storeKeys { + for _, key := range storeKeys { var currStoreKeys int64 store := helper.commitMS.GetKVStore(key) // TODO: use Iterator method of store interface, no longer rely on implementation of KVStore From 99f7fee840dc14ecdbe989068e716841e386798d Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 19 Jun 2019 13:34:37 +0800 Subject: [PATCH 2/2] clean according to review comments --- Gopkg.lock | 4 ++-- Gopkg.toml | 3 +-- store/statesync_helper.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 0d40ab598..7f0e956a2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -445,12 +445,12 @@ version = "v0.14.1" [[projects]] - branch = "add_set_version" digest = "1:1d88f1320e234cdf1627b9025a5b565bceb597795e76db163af5e298cc715cae" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "df2579cea9dde3c1b79dec98893e38766576f386" + revision = "" + version = "v0.12.0-binance.1" source = "github.com/binance-chain/bnc-tendermint-iavl" [[projects]] diff --git a/Gopkg.toml b/Gopkg.toml index bab9f00c5..cd61489ff 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -50,8 +50,7 @@ [[override]] name = "github.com/tendermint/iavl" source = "github.com/binance-chain/bnc-tendermint-iavl" - branch = "add_set_version" -# version = "=v0.12.0-binance.0" + version = "=v0.12.0-binance.1" [[override]] name = "github.com/tendermint/tendermint" diff --git a/store/statesync_helper.go b/store/statesync_helper.go index f5704e5e7..1e6845d69 100644 --- a/store/statesync_helper.go +++ b/store/statesync_helper.go @@ -98,7 +98,7 @@ func (helper *StateSyncHelper) getCommitedSortedStoreKeys() []sdk.StoreKey { kvStores := helper.commitMS.GetCommitKVStores() names := make([]string, 0, len(kvStores)) nameToKey := make(map[string]sdk.StoreKey, len(kvStores)) - for key, store := range helper.commitMS.GetCommitKVStores() { + for key, store := range kvStores { if !sdk.ShouldCommitStore(key.Name()) { continue }