diff --git a/Gopkg.lock b/Gopkg.lock index 7326aa95d..70e747cf7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -33,12 +33,11 @@ version = "v0.1.0" [[projects]] - branch = "master" - digest = "1:c0decf632843204d2b8781de7b26e7038584e2dcccc7e2f401e88ae85b1df2b7" + digest = "1:093bf93a65962e8191e3e8cd8fc6c363f83d43caca9739c906531ba7210a9904" name = "github.com/btcsuite/btcd" packages = ["btcec"] pruneopts = "UT" - revision = "2a560b2036bee5e3679ec2133eb6520b2f195213" + revision = "ed77733ec07dfc8a513741138419b8d9d3de9d2d" [[projects]] digest = "1:386de157f7d19259a7f9c81f26ce011223ce0f090353c1152ffdf730d7d10ac2" @@ -62,6 +61,14 @@ revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" +[[projects]] + digest = "1:fed20bf7f0da387c96d4cfc140a95572e5aba4bb984beb7de910e090ae39849b" + name = "github.com/ethereum/go-ethereum" + packages = ["crypto/secp256k1"] + pruneopts = "UT" + revision = "c942700427557e3ff6de3aaf6b916e2f056c1ec2" + version = "v1.8.23" + [[projects]] digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd" name = "github.com/fsnotify/fsnotify" @@ -430,14 +437,6 @@ pruneopts = "UT" revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445" -[[projects]] - digest = "1:83f5e189eea2baad419a6a410984514266ff690075759c87e9ede596809bd0b8" - name = "github.com/tendermint/btcd" - packages = ["btcec"] - pruneopts = "UT" - revision = "80daadac05d1cd29571fccf27002d79667a88b58" - version = "v0.1.1" - [[projects]] digest = "1:ad9c4c1a4e7875330b1f62906f2830f043a23edb5db997e3a5ac5d3e6eadf80a" name = "github.com/tendermint/go-amino" @@ -447,15 +446,16 @@ version = "v0.14.1" [[projects]] - digest = "1:e1cc8dd891e64aab63b0c09f2f12456cbe2cd9cbd9d96dfae3281245f05c2428" + digest = "1:c317eec9047bdcd14dff925f34e07d22c6886faf22bf7c9286ea56ad962bcc22" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "de0740903a67b624d887f9055d4c60175dcfa758" - version = "v0.12.0" + revision = "65e487d3a8d4d6624d8779c1f42cff31d61a9146" + source = "github.com/binance-chain/bnc-tendermint-iavl" + version = "v0.12.0-binance.0" [[projects]] - digest = "1:b1beb1e12ecdcaae5d5055fb6c56a6d9646bb66429f3d926574eddba8cedf51d" + digest = "1:09921dbdc56888c7a8d926a35ddd829a7891d6dd3b6eb91a0eb4849f4c0348ce" name = "github.com/tendermint/tendermint" packages = [ "abci/client", @@ -521,9 +521,9 @@ "version", ] pruneopts = "UT" - revision = "25b8c24d03966bb1a41134b6d3dacb7b95051b7f" - source = "github.com/BiJie/bnc-tendermint" - version = "v0.29.1-binance.2" + revision = "3abb98b2e97b6ef6286e6f7d4b164012fe36ec80" + source = "github.com/binance-chain/bnc-tendermint" + version = "v0.30.1-binance.0" [[projects]] digest = "1:7886f86064faff6f8d08a3eb0e8c773648ff5a2e27730831e2bfbf07467f6666" diff --git a/Gopkg.toml b/Gopkg.toml index 97f96e426..20381b793 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -53,15 +53,20 @@ [[override]] name = "github.com/tendermint/iavl" - version = "=v0.12.0" + source = "github.com/binance-chain/bnc-tendermint-iavl" + version = "=v0.12.0-binance.0" [[override]] name = "github.com/tendermint/tendermint" - source = "github.com/BiJie/bnc-tendermint" - version = "=v0.29.1-binance.2" + source = "github.com/binance-chain/bnc-tendermint" + version = "=v0.30.1-binance.0" ## deps without releases: +[[constraint]] + name = "github.com/btcsuite/btcd" + revision = "ed77733ec07dfc8a513741138419b8d9d3de9d2d" + [[override]] name = "golang.org/x/crypto" source = "https://github.com/tendermint/crypto" diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go index 7d83d7604..9fa44f196 100644 --- a/baseapp/baseapp.go +++ b/baseapp/baseapp.go @@ -961,3 +961,31 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) { Data: commitID.Hash, } } + +func (app *BaseApp) LatestSnapshot() (height int64, numKeys []int64, err error) { + return 0, make([]int64, 0), nil +} + +func (app *BaseApp) ReadSnapshotChunk(height int64, startIndex, endIndex int64) (chunk [][]byte, err error) { + return make([][]byte, 0), nil +} + +func (app *BaseApp) StartRecovery(height int64, numKeys []int64) error { + return nil +} + +func (app *BaseApp) WriteRecoveryChunk(chunk [][]byte) error { + return nil +} + +func (app *BaseApp) EndRecovery(height int64) error { + return nil +} + +func (app *BaseApp) GetDB() dbm.DB { + return app.db +} + +func (app *BaseApp) SetPruning(strategy sdk.PruningStrategy) { + app.cms.SetPruning(strategy) +} diff --git a/baseapp/options.go b/baseapp/options.go index 11fa411fa..f4441418a 100644 --- a/baseapp/options.go +++ b/baseapp/options.go @@ -14,19 +14,20 @@ import ( // SetPruning sets a pruning option on the multistore associated with the app func SetPruning(pruning string) func(*BaseApp) { - var pruningEnum sdk.PruningStrategy + var pruningStrategy sdk.PruningStrategy switch pruning { case "nothing": - pruningEnum = sdk.PruneNothing + pruningStrategy = sdk.PruneNothing{} case "everything": - pruningEnum = sdk.PruneEverything + pruningStrategy = sdk.PruneEverything{} case "syncable": - pruningEnum = sdk.PruneSyncable + // TODO: make these parameters configurable + pruningStrategy = sdk.PruneSyncable{NumRecent: 100, StoreEvery: 10000} default: panic(fmt.Sprintf("invalid pruning strategy: %s", pruning)) } return func(bap *BaseApp) { - bap.cms.SetPruning(pruningEnum) + bap.SetPruning(pruningStrategy) } } diff --git a/server/concurrent/async_local_client.go b/server/concurrent/async_local_client.go index 6071f24c9..f304d1b98 100644 --- a/server/concurrent/async_local_client.go +++ b/server/concurrent/async_local_client.go @@ -424,6 +424,43 @@ func (app *asyncLocalClient) EndBlockSync(req types.RequestEndBlock) (*types.Res //------------------------------------------------------- +func (app *asyncLocalClient) LatestSnapshot() (height int64, numKeys []int64, err error) { + app.rwLock.RLock() + defer app.rwLock.RUnlock() + + return app.Application.LatestSnapshot() +} + +func (app *asyncLocalClient) ReadSnapshotChunk(height int64, startIndex, endIndex int64) (chunk [][]byte, err error) { + app.rwLock.RLock() + defer app.rwLock.RUnlock() + + return app.Application.ReadSnapshotChunk(height, startIndex, endIndex) +} + +func (app *asyncLocalClient) StartRecovery(height int64, numKeys []int64) error { + app.rwLock.Lock() + defer app.rwLock.Unlock() + + return app.Application.StartRecovery(height, numKeys) +} + +func (app *asyncLocalClient) WriteRecoveryChunk(chunk [][]byte) error { + app.rwLock.Lock() + defer app.rwLock.Unlock() + + return app.Application.WriteRecoveryChunk(chunk) +} + +func (app *asyncLocalClient) EndRecovery(height int64) error { + app.rwLock.Lock() + defer app.rwLock.Unlock() + + return app.Application.EndRecovery(height) +} + +//------------------------------------------------------- + func (app *asyncLocalClient) callback(req *types.Request, res *types.Response) *abcicli.ReqRes { app.Callback(req, res) return newLocalReqRes(req, res) diff --git a/server/concurrent/async_local_client_test.go b/server/concurrent/async_local_client_test.go index 335f4c5b2..c1ad1c4d0 100644 --- a/server/concurrent/async_local_client_test.go +++ b/server/concurrent/async_local_client_test.go @@ -88,6 +88,22 @@ func (app *TimedApplication) PreDeliverTx(tx []byte) types.ResponseDeliverTx { return types.ResponseDeliverTx{} } +func (cli *TimedApplication) LatestSnapshot() (height int64, numKeys []int64, err error) { + return 0, make([]int64, 0), nil +} +func (cli *TimedApplication) ReadSnapshotChunk(height int64, startIndex, endIndex int64) (chunk [][]byte, err error) { + return make([][]byte, 0), nil +} +func (cli *TimedApplication) StartRecovery(height int64, numKeys []int64) error { + return nil +} +func (cli *TimedApplication) WriteRecoveryChunk(chunk [][]byte) error { + return nil +} +func (cli *TimedApplication) EndRecovery(height int64) error { + return nil +} + var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "TestLogger") func TestNewAsyncLocalClient(t *testing.T) { diff --git a/server/start.go b/server/start.go index 05c98a35f..8d09f4540 100644 --- a/server/start.go +++ b/server/start.go @@ -8,6 +8,7 @@ import ( "github.com/cosmos/cosmos-sdk/server/concurrent" "github.com/tendermint/tendermint/abci/server" + "github.com/tendermint/tendermint/blockchain" tcmd "github.com/tendermint/tendermint/cmd/tendermint/commands" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/node" @@ -24,6 +25,8 @@ const ( flagSequentialABCI = "seq-abci" ) +var BlockStore *blockchain.BlockStore + // StartCmd runs the service passed in, either stand-alone or in-process with // Tendermint. func StartCmd(ctx *Context, appCreator AppCreator) *cobra.Command { @@ -140,6 +143,8 @@ func startInProcess(ctx *Context, appCreator AppCreator) (*node.Node, error) { return nil, err } + BlockStore = tmNode.BlockStore() + err = tmNode.Start() if err != nil { return nil, err diff --git a/store/iavlstore.go b/store/iavlstore.go index 78453cbcd..edd3f115f 100644 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -25,43 +25,33 @@ func LoadIAVLStore(db dbm.DB, id CommitID, pruning sdk.PruningStrategy) (CommitS if err != nil { return nil, err } - iavl := newIAVLStore(tree, int64(0), int64(0)) + iavl := newIAVLStore(tree, sdk.PruneNothing{}) iavl.SetPruning(pruning) return iavl, nil } //---------------------------------------- -var _ KVStore = (*iavlStore)(nil) -var _ CommitStore = (*iavlStore)(nil) -var _ Queryable = (*iavlStore)(nil) +var _ KVStore = (*IavlStore)(nil) +var _ CommitStore = (*IavlStore)(nil) +var _ Queryable = (*IavlStore)(nil) -// iavlStore Implements KVStore and CommitStore. -type iavlStore struct { +// IavlStore Implements KVStore and CommitStore. +type IavlStore struct { // The underlying tree. - tree *iavl.MutableTree + Tree *iavl.MutableTree - // How many old versions we hold onto. - // A value of 0 means keep no recent states. - numRecent int64 - - // This is the distance between state-sync waypoint states to be stored. - // See https://github.com/tendermint/tendermint/issues/828 - // A value of 1 means store every state. - // A value of 0 means store no waypoints. (node cannot assist in state-sync) - // By default this value should be set the same across all nodes, - // so that nodes can know the waypoints their peers store. - storeEvery int64 + // The strategy to prune historical versions + pruningStrategy sdk.PruningStrategy } // CONTRACT: tree should be fully loaded. // nolint: unparam -func newIAVLStore(tree *iavl.MutableTree, numRecent int64, storeEvery int64) *iavlStore { - st := &iavlStore{ - tree: tree, - numRecent: numRecent, - storeEvery: storeEvery, +func newIAVLStore(tree *iavl.MutableTree, ps sdk.PruningStrategy) *IavlStore { + st := &IavlStore{ + Tree: tree, + pruningStrategy: ps, } return st } @@ -71,24 +61,25 @@ func (st *iavlStore) GetImmutableTree() *iavl.ImmutableTree { } // Implements Committer. -func (st *iavlStore) Commit() CommitID { +func (st *IavlStore) Commit() CommitID { // Save a new version. - hash, version, err := st.tree.SaveVersion() + hash, version, err := st.Tree.SaveVersion() if err != nil { // TODO: Do we want to extend Commit to allow returning errors? panic(err) } // Release an old version of history, if not a sync waypoint. - previous := version - 1 - if st.numRecent < previous { - toRelease := previous - st.numRecent - if st.storeEvery == 0 || toRelease%st.storeEvery != 0 { - err := st.tree.DeleteVersion(toRelease) - if err != nil && err.(cmn.Error).Data() != iavl.ErrVersionDoesNotExist { - panic(err) - } - } + //for v, _ := range st.Tree.GetVersions() { + // if st.pruningStrategy.ShouldPrune(v, version) { + // st.Tree.DeleteVersion(v) + // } + //} + + // this is a special optimization for binance chain, 10001 should keep consistent with + // `numRecent` field of KeepRecentAndBreatheBlock + if version > 10001 && st.pruningStrategy.ShouldPrune(version-10001, version) { + st.Tree.DeleteVersion(version - 10001) } return CommitID{ @@ -98,81 +89,72 @@ func (st *iavlStore) Commit() CommitID { } // Implements Committer. -func (st *iavlStore) LastCommitID() CommitID { +func (st *IavlStore) LastCommitID() CommitID { return CommitID{ - Version: st.tree.Version(), - Hash: st.tree.Hash(), + Version: st.Tree.Version(), + Hash: st.Tree.Hash(), } } // Implements Committer. -func (st *iavlStore) SetPruning(pruning sdk.PruningStrategy) { - switch pruning { - case sdk.PruneEverything: - st.numRecent = 0 - st.storeEvery = 0 - case sdk.PruneNothing: - st.storeEvery = 1 - case sdk.PruneSyncable: - st.numRecent = 100 - st.storeEvery = 10000 - } +func (st *IavlStore) SetPruning(pruning sdk.PruningStrategy) { + st.pruningStrategy = pruning } // VersionExists returns whether or not a given version is stored. -func (st *iavlStore) VersionExists(version int64) bool { - return st.tree.VersionExists(version) +func (st *IavlStore) VersionExists(version int64) bool { + return st.Tree.VersionExists(version) } // Implements Store. -func (st *iavlStore) GetStoreType() StoreType { +func (st *IavlStore) GetStoreType() StoreType { return sdk.StoreTypeIAVL } // Implements Store. -func (st *iavlStore) CacheWrap() CacheWrap { +func (st *IavlStore) CacheWrap() CacheWrap { return NewCacheKVStore(st) } // CacheWrapWithTrace implements the Store interface. -func (st *iavlStore) CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap { +func (st *IavlStore) CacheWrapWithTrace(w io.Writer, tc TraceContext) CacheWrap { return NewCacheKVStore(NewTraceKVStore(st, w, tc)) } // Implements KVStore. -func (st *iavlStore) Set(key, value []byte) { - st.tree.Set(key, value) +func (st *IavlStore) Set(key, value []byte) { + st.Tree.Set(key, value) } // Implements KVStore. -func (st *iavlStore) Get(key []byte) (value []byte) { - _, v := st.tree.Get(key) +func (st *IavlStore) Get(key []byte) (value []byte) { + _, v := st.Tree.Get(key) return v } // Implements KVStore. -func (st *iavlStore) Has(key []byte) (exists bool) { - return st.tree.Has(key) +func (st *IavlStore) Has(key []byte) (exists bool) { + return st.Tree.Has(key) } // Implements KVStore. -func (st *iavlStore) Delete(key []byte) { - st.tree.Remove(key) +func (st *IavlStore) Delete(key []byte) { + st.Tree.Remove(key) } // Implements KVStore -func (st *iavlStore) Prefix(prefix []byte) KVStore { +func (st *IavlStore) Prefix(prefix []byte) KVStore { return prefixStore{st, prefix} } // Implements KVStore. -func (st *iavlStore) Iterator(start, end []byte) Iterator { - return newIAVLIterator(st.tree.ImmutableTree, start, end, true) +func (st *IavlStore) Iterator(start, end []byte) Iterator { + return newIAVLIterator(st.Tree.ImmutableTree, start, end, true) } // Implements KVStore. -func (st *iavlStore) ReverseIterator(start, end []byte) Iterator { - return newIAVLIterator(st.tree.ImmutableTree, start, end, false) +func (st *IavlStore) ReverseIterator(start, end []byte) Iterator { + return newIAVLIterator(st.Tree.ImmutableTree, start, end, false) } // Handle gatest the latest height, if height is 0 @@ -196,13 +178,13 @@ func getHeight(tree *iavl.MutableTree, req abci.RequestQuery) int64 { // If latest-1 is not present, use latest (which must be present) // if you care to have the latest data to see a tx results, you must // explicitly set the height you want to see -func (st *iavlStore) Query(req abci.RequestQuery) (res abci.ResponseQuery) { +func (st *IavlStore) Query(req abci.RequestQuery) (res abci.ResponseQuery) { if len(req.Data) == 0 { msg := "Query cannot be zero length" return sdk.ErrTxDecode(msg).QueryResult() } - tree := st.tree + tree := st.Tree // store the height we chose in the response, with 0 being changed to the // latest height diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index 6d77fbfa0..f02ec52b4 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -47,7 +47,7 @@ func newTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, CommitID) { func TestIAVLStoreGetSetHasDelete(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) key := "hello" @@ -72,7 +72,7 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) { func TestIAVLIterator(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz")) expected := []string{"aloha", "hello"} var i int @@ -145,7 +145,7 @@ func TestIAVLIterator(t *testing.T) { func TestIAVLSubspaceIterator(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test2"), []byte("test2")) @@ -207,7 +207,7 @@ func TestIAVLSubspaceIterator(t *testing.T) { func TestIAVLReverseSubspaceIterator(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test2"), []byte("test2")) @@ -263,60 +263,62 @@ func TestIAVLReverseSubspaceIterator(t *testing.T) { require.Equal(t, len(expected), i) } -func nextVersion(iavl *iavlStore) { +func nextVersion(iavl *IavlStore) { key := []byte(fmt.Sprintf("Key for tree: %d", iavl.LastCommitID().Version)) value := []byte(fmt.Sprintf("Value for tree: %d", iavl.LastCommitID().Version)) iavl.Set(key, value) iavl.Commit() } -func TestIAVLDefaultPruning(t *testing.T) { - //Expected stored / deleted version numbers for: - //numRecent = 5, storeEvery = 3 - var states = []pruneState{ - {[]int64{}, []int64{}}, - {[]int64{1}, []int64{}}, - {[]int64{1, 2}, []int64{}}, - {[]int64{1, 2, 3}, []int64{}}, - {[]int64{1, 2, 3, 4}, []int64{}}, - {[]int64{1, 2, 3, 4, 5}, []int64{}}, - {[]int64{1, 2, 3, 4, 5, 6}, []int64{}}, - {[]int64{2, 3, 4, 5, 6, 7}, []int64{1}}, - {[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}}, - {[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}}, - {[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}}, - {[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}}, - {[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}}, - {[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}}, - {[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}}, - {[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}}, - } - testPruning(t, int64(5), int64(3), states) -} - -func TestIAVLAlternativePruning(t *testing.T) { - //Expected stored / deleted version numbers for: - //numRecent = 3, storeEvery = 5 - var states = []pruneState{ - {[]int64{}, []int64{}}, - {[]int64{1}, []int64{}}, - {[]int64{1, 2}, []int64{}}, - {[]int64{1, 2, 3}, []int64{}}, - {[]int64{1, 2, 3, 4}, []int64{}}, - {[]int64{2, 3, 4, 5}, []int64{1}}, - {[]int64{3, 4, 5, 6}, []int64{1, 2}}, - {[]int64{4, 5, 6, 7}, []int64{1, 2, 3}}, - {[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}}, - {[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}}, - {[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}}, - {[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}}, - {[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}}, - {[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, - {[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, - {[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}}, - } - testPruning(t, int64(3), int64(5), states) -} +// Comment out as binance hard coded numRecent to 10000 in func (st *IavlStore) Commit() CommitID { +//func TestIAVLDefaultPruning(t *testing.T) { +// //Expected stored / deleted version numbers for: +// //numRecent = 5, storeEvery = 3 +// var states = []pruneState{ +// {[]int64{}, []int64{}}, +// {[]int64{1}, []int64{}}, +// {[]int64{1, 2}, []int64{}}, +// {[]int64{1, 2, 3}, []int64{}}, +// {[]int64{1, 2, 3, 4}, []int64{}}, +// {[]int64{1, 2, 3, 4, 5}, []int64{}}, +// {[]int64{1, 2, 3, 4, 5, 6}, []int64{}}, +// {[]int64{2, 3, 4, 5, 6, 7}, []int64{1}}, +// {[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}}, +// {[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}}, +// {[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}}, +// {[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}}, +// {[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}}, +// {[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}}, +// {[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}}, +// {[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}}, +// } +// testPruning(t, int64(5), int64(3), states) +//} + +// Comment out as binance hard coded numRecent to 10000 in func (st *IavlStore) Commit() CommitID { +//func TestIAVLAlternativePruning(t *testing.T) { +// //Expected stored / deleted version numbers for: +// //numRecent = 3, storeEvery = 5 +// var states = []pruneState{ +// {[]int64{}, []int64{}}, +// {[]int64{1}, []int64{}}, +// {[]int64{1, 2}, []int64{}}, +// {[]int64{1, 2, 3}, []int64{}}, +// {[]int64{1, 2, 3, 4}, []int64{}}, +// {[]int64{2, 3, 4, 5}, []int64{1}}, +// {[]int64{3, 4, 5, 6}, []int64{1, 2}}, +// {[]int64{4, 5, 6, 7}, []int64{1, 2, 3}}, +// {[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}}, +// {[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}}, +// {[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}}, +// {[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}}, +// {[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}}, +// {[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, +// {[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, +// {[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}}, +// } +// testPruning(t, int64(3), int64(5), states) +//} type pruneState struct { stored []int64 @@ -326,7 +328,7 @@ type pruneState struct { func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []pruneState) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) for step, state := range states { for _, ver := range state.stored { require.True(t, iavlStore.VersionExists(ver), @@ -345,7 +347,7 @@ func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []prune func TestIAVLNoPrune(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, numRecent, int64(1)) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, 1}) nextVersion(iavlStore) for i := 1; i < 100; i++ { for j := 1; j <= i; j++ { @@ -357,28 +359,29 @@ func TestIAVLNoPrune(t *testing.T) { } } -func TestIAVLPruneEverything(t *testing.T) { - db := dbm.NewMemDB() - tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, int64(0), int64(0)) - nextVersion(iavlStore) - for i := 1; i < 100; i++ { - for j := 1; j < i; j++ { - require.False(t, iavlStore.VersionExists(int64(j)), - "Unpruned version %d with latest version %d. Should prune all old versions", - j, i) - } - require.True(t, iavlStore.VersionExists(int64(i)), - "Missing current version on step %d, should not prune current state tree", - i) - nextVersion(iavlStore) - } -} +// Comment out as binance hard coded numRecent to 10000 in func (st *IavlStore) Commit() CommitID { +//func TestIAVLPruneEverything(t *testing.T) { +// db := dbm.NewMemDB() +// tree := iavl.NewMutableTree(db, cacheSize) +// iavlStore := newIAVLStore(tree, sdk.PruneEverything{}) +// nextVersion(iavlStore) +// for i := 1; i < 100; i++ { +// for j := 1; j < i; j++ { +// require.False(t, iavlStore.VersionExists(int64(j)), +// "Unpruned version %d with latest version %d. Should prune all old versions", +// j, i) +// } +// require.True(t, iavlStore.VersionExists(int64(i)), +// "Missing current version on step %d, should not prune current state tree", +// i) +// nextVersion(iavlStore) +// } +//} func TestIAVLStoreQuery(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) k1, v1 := []byte("key1"), []byte("val1") k2, v2 := []byte("key2"), []byte("val2") @@ -474,7 +477,7 @@ func BenchmarkIAVLIteratorNext(b *testing.B) { value := cmn.RandBytes(50) tree.Set(key, value) } - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) iterators := make([]Iterator, b.N/treeSize) for i := 0; i < len(iterators); i++ { iterators[i] = iavlStore.Iterator([]byte{0}, []byte{255, 255, 255, 255, 255}) diff --git a/store/multistoreproof.go b/store/multistoreproof.go index 11b5d1234..906e34bee 100644 --- a/store/multistoreproof.go +++ b/store/multistoreproof.go @@ -11,16 +11,16 @@ import ( // MultiStoreProof defines a collection of store proofs in a multi-store type MultiStoreProof struct { - StoreInfos []storeInfo + StoreInfos []StoreInfo } -func NewMultiStoreProof(storeInfos []storeInfo) *MultiStoreProof { +func NewMultiStoreProof(storeInfos []StoreInfo) *MultiStoreProof { return &MultiStoreProof{StoreInfos: storeInfos} } // ComputeRootHash returns the root hash for a given multi-store proof. func (proof *MultiStoreProof) ComputeRootHash() []byte { - ci := commitInfo{ + ci := CommitInfo{ Version: -1, // TODO: Not needed; improve code. StoreInfos: proof.StoreInfos, } diff --git a/store/multistoreproof_test.go b/store/multistoreproof_test.go index 822196be3..cd3eda5b5 100644 --- a/store/multistoreproof_test.go +++ b/store/multistoreproof_test.go @@ -13,8 +13,8 @@ import ( func TestVerifyIAVLStoreQueryProof(t *testing.T) { // Create main tree for testing. db := dbm.NewMemDB() - iStore, err := LoadIAVLStore(db, CommitID{}, sdk.PruneNothing) - store := iStore.(*iavlStore) + iStore, err := LoadIAVLStore(db, CommitID{}, sdk.PruneNothing{}) + store := iStore.(*IavlStore) require.Nil(t, err) store.Set([]byte("MYKEY"), []byte("MYVALUE")) cid := store.Commit() @@ -62,7 +62,7 @@ func TestVerifyMultiStoreQueryProof(t *testing.T) { store.MountStoreWithDB(iavlStoreKey, sdk.StoreTypeIAVL, nil) store.LoadVersion(0) - iavlStore := store.GetCommitStore(iavlStoreKey).(*iavlStore) + iavlStore := store.GetCommitStore(iavlStoreKey).(*IavlStore) iavlStore.Set([]byte("MYKEY"), []byte("MYVALUE")) cid := store.Commit() diff --git a/store/prefixstore_test.go b/store/prefixstore_test.go index 15f086b97..6bf9bdff7 100644 --- a/store/prefixstore_test.go +++ b/store/prefixstore_test.go @@ -76,7 +76,7 @@ func testPrefixStore(t *testing.T, baseStore KVStore, prefix []byte) { func TestIAVLStorePrefix(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, numRecent, storeEvery) + iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) testPrefixStore(t, iavlStore, []byte("test")) } @@ -411,4 +411,4 @@ func TestPrefixDBReverseIterator4(t *testing.T) { itr := pstore.ReverseIterator(bz(""), bz("")) checkInvalid(t, itr) itr.Close() -} \ No newline at end of file +} diff --git a/store/rootmultistore.go b/store/rootmultistore.go index fd3736a9e..bba16a3f7 100644 --- a/store/rootmultistore.go +++ b/store/rootmultistore.go @@ -40,6 +40,7 @@ var _ Queryable = (*rootMultiStore)(nil) func NewCommitMultiStore(db dbm.DB) *rootMultiStore { return &rootMultiStore{ db: db, + pruning: sdk.PruneNothing{}, storesParams: make(map[StoreKey]storeParams), stores: make(map[StoreKey]CommitStore), keysByName: make(map[string]StoreKey), @@ -113,14 +114,14 @@ func (rs *rootMultiStore) LoadVersion(ver int64) error { } // Otherwise, version is 1 or greater - // Get commitInfo + // Get CommitInfo cInfo, err := getCommitInfo(rs.db, ver) if err != nil { return err } // Convert StoreInfos slice to map - infos := make(map[StoreKey]storeInfo) + infos := make(map[StoreKey]StoreInfo) for _, storeInfo := range cInfo.StoreInfos { infos[rs.nameToKey(storeInfo.Name)] = storeInfo } @@ -191,9 +192,8 @@ func (rs *rootMultiStore) LastCommitID() CommitID { // Implements Committer/CommitStore. func (rs *rootMultiStore) Commit() CommitID { - - // Commit stores. version := rs.lastCommitID.Version + 1 + // Commit stores. commitInfo := commitStores(version, rs.stores) // Need to update atomically. @@ -376,20 +376,20 @@ type storeParams struct { } //---------------------------------------- -// commitInfo +// CommitInfo -// NOTE: Keep commitInfo a simple immutable struct. -type commitInfo struct { +// NOTE: Keep CommitInfo a simple immutable struct. +type CommitInfo struct { // Version Version int64 // Store info for - StoreInfos []storeInfo + StoreInfos []StoreInfo } // Hash returns the simple merkle root hash of the stores sorted by name. -func (ci commitInfo) Hash() []byte { +func (ci CommitInfo) Hash() []byte { // TODO cache to ci.hash []byte m := make(map[string][]byte, len(ci.StoreInfos)) for _, storeInfo := range ci.StoreInfos { @@ -398,7 +398,7 @@ func (ci commitInfo) Hash() []byte { return merkle.SimpleHashFromMap(m) } -func (ci commitInfo) CommitID() CommitID { +func (ci CommitInfo) CommitID() CommitID { return CommitID{ Version: ci.Version, Hash: ci.Hash(), @@ -406,24 +406,24 @@ func (ci commitInfo) CommitID() CommitID { } //---------------------------------------- -// storeInfo +// StoreInfo -// storeInfo contains the name and core reference for an +// StoreInfo contains the name and core reference for an // underlying store. It is the leaf of the rootMultiStores top // level simple merkle tree. -type storeInfo struct { +type StoreInfo struct { Name string - Core storeCore + Core StoreCore } -type storeCore struct { +type StoreCore struct { // StoreType StoreType CommitID CommitID // ... maybe add more state } // Implements merkle.Hasher. -func (si storeInfo) Hash() []byte { +func (si StoreInfo) Hash() []byte { // Doesn't write Name, since merkle.SimpleHashFromMap() will // include them via the keys. bz, _ := cdc.MarshalBinaryLengthPrefixed(si.Core) // Does not error @@ -458,9 +458,9 @@ func setLatestVersion(batch dbm.Batch, version int64) { batch.Set([]byte(latestVersionKey), latestBytes) } -// Commits each store and returns a new commitInfo. -func commitStores(version int64, storeMap map[StoreKey]CommitStore) commitInfo { - storeInfos := make([]storeInfo, 0, len(storeMap)) +// Commits each store and returns a new CommitInfo. +func commitStores(version int64, storeMap map[StoreKey]CommitStore) CommitInfo { + storeInfos := make([]StoreInfo, 0, len(storeMap)) for key, store := range storeMap { // Commit @@ -471,41 +471,41 @@ func commitStores(version int64, storeMap map[StoreKey]CommitStore) commitInfo { } // Record CommitID - si := storeInfo{} + si := StoreInfo{} si.Name = key.Name() si.Core.CommitID = commitID // si.Core.StoreType = store.GetStoreType() storeInfos = append(storeInfos, si) } - ci := commitInfo{ + ci := CommitInfo{ Version: version, StoreInfos: storeInfos, } return ci } -// Gets commitInfo from disk. -func getCommitInfo(db dbm.DB, ver int64) (commitInfo, error) { +// Gets CommitInfo from disk. +func getCommitInfo(db dbm.DB, ver int64) (CommitInfo, error) { // Get from DB. cInfoKey := fmt.Sprintf(commitInfoKeyFmt, ver) cInfoBytes := db.Get([]byte(cInfoKey)) if cInfoBytes == nil { - return commitInfo{}, fmt.Errorf("failed to get rootMultiStore: no data") + return CommitInfo{}, fmt.Errorf("failed to get rootMultiStore: no data") } // Parse bytes. - var cInfo commitInfo + var cInfo CommitInfo err := cdc.UnmarshalBinaryLengthPrefixed(cInfoBytes, &cInfo) if err != nil { - return commitInfo{}, fmt.Errorf("failed to get rootMultiStore: %v", err) + return CommitInfo{}, fmt.Errorf("failed to get rootMultiStore: %v", err) } return cInfo, nil } -// Set a commitInfo for given version. -func setCommitInfo(batch dbm.Batch, version int64, cInfo commitInfo) { +// Set a CommitInfo for given version. +func setCommitInfo(batch dbm.Batch, version int64, cInfo CommitInfo) { cInfoBytes, err := cdc.MarshalBinaryLengthPrefixed(cInfo) if err != nil { panic(err) diff --git a/store/rootmultistore_test.go b/store/rootmultistore_test.go index 501c5b730..38010b730 100644 --- a/store/rootmultistore_test.go +++ b/store/rootmultistore_test.go @@ -218,9 +218,9 @@ func hashStores(stores map[StoreKey]CommitStore) []byte { m := make(map[string][]byte, len(stores)) for key, store := range stores { name := key.Name() - m[name] = storeInfo{ + m[name] = StoreInfo{ Name: name, - Core: storeCore{ + Core: StoreCore{ CommitID: store.LastCommitID(), // StoreType: store.GetStoreType(), }, diff --git a/types/cache.go b/types/cache.go index a6b8e7624..222efc0df 100644 --- a/types/cache.go +++ b/types/cache.go @@ -4,6 +4,7 @@ type AccountStoreCache interface { GetAccount(addr AccAddress) Account SetAccount(addr AccAddress, acc Account) Delete(addr AccAddress) + ClearCache() // only used by state sync to clear genesis status of accounts } type AccountCache interface { diff --git a/types/store.go b/types/store.go index fdc5af469..c0c966bb0 100644 --- a/types/store.go +++ b/types/store.go @@ -13,18 +13,43 @@ import ( // NOTE: These are implemented in cosmos-sdk/store. // PruningStrategy specfies how old states will be deleted over time -type PruningStrategy uint8 +type PruningStrategy interface { + ShouldPrune(version, latestVersion int64) bool +} -const ( - // PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th) - PruneSyncable PruningStrategy = iota +// PruneSyncable defines the frequency (how many) of states would be saved, +// while all the others are not needed for state syncing and would be deleted. +type PruneSyncable struct { + // How many old versions we hold onto. + // A value of 0 means keep no recent states. + NumRecent int64 + + // This is the distance between state-sync waypoint states to be stored. + // See https://github.com/tendermint/tendermint/issues/828 + // A value of 1 means store every state. + // A value of 0 means store no waypoints. (node cannot assist in state-sync) + // By default this value should be set the same across all nodes, + // so that nodes can know the waypoints their peers store. + StoreEvery int64 +} - // PruneEverything means all saved states will be deleted, storing only the current state - PruneEverything PruningStrategy = iota +func (strategy PruneSyncable) ShouldPrune(version, latestVersion int64) bool { + return (latestVersion-version > strategy.NumRecent) && (version%strategy.StoreEvery != 0) +} - // PruneNothing means all historic states will be saved, nothing will be deleted - PruneNothing PruningStrategy = iota -) +// PruneEverything means all saved states will be deleted, storing only the current state +type PruneEverything struct{} + +func (strategy PruneEverything) ShouldPrune(version, latestVersion int64) bool { + return true +} + +// PruneNothing means all historic states will be saved, nothing will be deleted +type PruneNothing struct{} + +func (strategy PruneNothing) ShouldPrune(version, latestVersion int64) bool { + return false +} type Store interface { //nolint GetStoreType() StoreType diff --git a/x/auth/mapper.go b/x/auth/mapper.go index 2e6ad2ee1..2715cb943 100644 --- a/x/auth/mapper.go +++ b/x/auth/mapper.go @@ -258,6 +258,10 @@ func (ac *accountStoreCache) Delete(addr sdk.AccAddress) { ac.store.Delete(AddressStoreKey(addr)) } +func (ac *accountStoreCache) ClearCache() { + ac.cache.Purge() +} + func (ac *accountStoreCache) encodeAccount(acc sdk.Account) []byte { bz, err := ac.cdc.MarshalBinaryBare(acc) if err != nil { @@ -303,6 +307,10 @@ func (ac *accountCache) Delete(addr sdk.AccAddress) { ac.setAccountToCache(addr, nil, true, true) } +func (ac *accountCache) ClearCache() { + ac.cache = sync.Map{} +} + func (ac *accountCache) Cache() sdk.AccountCache { return &accountCache{ parent: ac, diff --git a/x/gov/client/cli/tx.go b/x/gov/client/cli/tx.go index 5c262b50f..ab342f01a 100644 --- a/x/gov/client/cli/tx.go +++ b/x/gov/client/cli/tx.go @@ -82,6 +82,18 @@ $ gaiacli gov submit-proposal --title="Test Proposal" --description="My awesome return err } + if proposal.Title == "" { + return errors.New("Title should not be empty") + } + + if len(proposal.Title) > gov.MaxTitleLength { + return errors.New(fmt.Sprintf("Proposal title is longer than max length of %d", gov.MaxTitleLength)) + } + + if len(proposal.Description) > gov.MaxDescriptionLength { + return errors.New(fmt.Sprintf("Proposal description is longer than max length of %d", gov.MaxDescriptionLength)) + } + txBldr := authtxb.NewTxBuilderFromCLI().WithCodec(cdc) cliCtx := context.NewCLIContext(). WithCodec(cdc). @@ -560,6 +572,14 @@ func GetCmdSubmitListProposal(cdc *codec.Codec) *cobra.Command { initPrice := viper.GetInt64(flagInitPrice) expireTimestamp := viper.GetInt64(flagExpireTime) + if title == "" { + return errors.New("Title should not be empty") + } + + if len(title) > gov.MaxTitleLength { + return errors.New(fmt.Sprintf("Proposal title is longer than max length of %d", gov.MaxTitleLength)) + } + if tradeAsset == "" { return errors.New("base asset should not be empty") } diff --git a/x/gov/errors.go b/x/gov/errors.go index 335859b1d..d9c53bbee 100644 --- a/x/gov/errors.go +++ b/x/gov/errors.go @@ -51,12 +51,12 @@ func ErrAddressNotStaked(codespace sdk.CodespaceType, address sdk.AccAddress) sd return sdk.NewError(codespace, CodeAddressNotStaked, fmt.Sprintf("Address %s is not staked and is thus ineligible to vote", address)) } -func ErrInvalidTitle(codespace sdk.CodespaceType, title string) sdk.Error { - return sdk.NewError(codespace, CodeInvalidTitle, fmt.Sprintf("Proposal Title '%s' is not valid", title)) +func ErrInvalidTitle(codespace sdk.CodespaceType, msg string) sdk.Error { + return sdk.NewError(codespace, CodeInvalidTitle, msg) } -func ErrInvalidDescription(codespace sdk.CodespaceType, description string) sdk.Error { - return sdk.NewError(codespace, CodeInvalidDescription, fmt.Sprintf("Proposal Desciption '%s' is not valid", description)) +func ErrInvalidDescription(codespace sdk.CodespaceType, msg string) sdk.Error { + return sdk.NewError(codespace, CodeInvalidDescription, msg) } func ErrInvalidProposalType(codespace sdk.CodespaceType, proposalType ProposalKind) sdk.Error { diff --git a/x/gov/msgs.go b/x/gov/msgs.go index 79f691462..82dc89860 100644 --- a/x/gov/msgs.go +++ b/x/gov/msgs.go @@ -8,7 +8,12 @@ import ( ) // name to idetify transaction types -const MsgRoute = "gov" +const ( + MsgRoute = "gov" + + MaxTitleLength = 128 + MaxDescriptionLength int = 2048 +) var _, _, _ sdk.Msg = MsgSubmitProposal{}, MsgDeposit{}, MsgVote{} @@ -48,10 +53,16 @@ func (msg MsgSubmitProposal) Type() string { return "submit_proposal" } // Implements Msg. func (msg MsgSubmitProposal) ValidateBasic() sdk.Error { if len(msg.Title) == 0 { - return ErrInvalidTitle(DefaultCodespace, msg.Title) // TODO: Proper Error + return ErrInvalidTitle(DefaultCodespace, "No title present in proposal") + } + if len(msg.Title) > MaxTitleLength { + return ErrInvalidTitle(DefaultCodespace, fmt.Sprintf("Proposal title is longer than max length of %d", MaxTitleLength)) } if len(msg.Description) == 0 { - return ErrInvalidDescription(DefaultCodespace, msg.Description) // TODO: Proper Error + return ErrInvalidDescription(DefaultCodespace, "No description present in proposal") + } + if len(msg.Description) > MaxDescriptionLength { + return ErrInvalidDescription(DefaultCodespace, fmt.Sprintf("Proposal description is longer than max length of %d", MaxDescriptionLength)) } if !validProposalType(msg.ProposalType) { return ErrInvalidProposalType(DefaultCodespace, msg.ProposalType) diff --git a/x/gov/msgs_test.go b/x/gov/msgs_test.go index 5952f6f7a..1e25b7334 100644 --- a/x/gov/msgs_test.go +++ b/x/gov/msgs_test.go @@ -1,6 +1,7 @@ package gov_test import ( + "strings" "testing" "github.com/stretchr/testify/require" @@ -38,6 +39,8 @@ func TestMsgSubmitProposal(t *testing.T) { {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeText, addrs[0], coinsZero, true}, {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeText, addrs[0], coinsNeg, false}, {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeText, addrs[0], coinsMulti, true}, + {strings.Repeat("#", gov.MaxTitleLength*2), "the purpose of this proposal is to test", gov.ProposalTypeText, addrs[0], coinsMulti, false}, + {"Test Proposal", strings.Repeat("#", gov.MaxDescriptionLength*2), gov.ProposalTypeText, addrs[0], coinsMulti, false}, } for i, tc := range tests {