diff --git a/Gopkg.lock b/Gopkg.lock index 70e747cf7..334a9d37e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,14 +1,6 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. -[[projects]] - branch = "master" - digest = "1:7736fc6da04620727f8f3aa2ced8d77be8e074a302820937aa5993848c769b27" - name = "github.com/ZondaX/hid-go" - packages = ["."] - pruneopts = "UT" - revision = "48b08affede2cea076a3cf13b2e3f72ed262b743" - [[projects]] digest = "1:09a7f74eb6bb3c0f14d8926610c87f569c5cff68e978d30e9a3540aeb626fdf0" name = "github.com/bartekn/go-bip39" @@ -61,14 +53,6 @@ revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" -[[projects]] - digest = "1:fed20bf7f0da387c96d4cfc140a95572e5aba4bb984beb7de910e090ae39849b" - name = "github.com/ethereum/go-ethereum" - packages = ["crypto/secp256k1"] - pruneopts = "UT" - revision = "c942700427557e3ff6de3aaf6b916e2f056c1ec2" - version = "v1.8.23" - [[projects]] digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd" name = "github.com/fsnotify/fsnotify" @@ -110,7 +94,7 @@ version = "v1.8.0" [[projects]] - digest = "1:35621fe20f140f05a0c4ef662c26c0ab4ee50bca78aa30fe87d33120bd28165e" + digest = "1:95e1006e41c641abd2f365dfa0f1213c04da294e7cd5f0bf983af234b775db64" name = "github.com/gogo/protobuf" packages = [ "gogoproto", @@ -121,11 +105,11 @@ "types", ] pruneopts = "UT" - revision = "636bf0302bc95575d69441b25a2603156ffdddf1" - version = "v1.1.1" + revision = "ba06b47c162d49f2af050fb4c75bcbc86a159d5c" + version = "v1.2.1" [[projects]] - digest = "1:17fe264ee908afc795734e8c4e63db2accabaf57326dbf21763a7d6b86096260" + digest = "1:239c4c7fd2159585454003d9be7207167970194216193a8a210b8d29576f19c9" name = "github.com/golang/protobuf" packages = [ "proto", @@ -135,8 +119,8 @@ "ptypes/timestamp", ] pruneopts = "UT" - revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" - version = "v1.1.0" + revision = "b5d812f8a3706043e23a9cd5babf2e5423744d30" + version = "v1.3.1" [[projects]] branch = "master" @@ -208,12 +192,12 @@ version = "v1.0" [[projects]] - branch = "master" - digest = "1:39b27d1381a30421f9813967a5866fba35dc1d4df43a6eefe3b7a5444cb07214" + digest = "1:a74b5a8e34ee5843cd6e65f698f3e75614f812ff170c2243425d75bc091e9af2" name = "github.com/jmhodges/levigo" packages = ["."] pruneopts = "UT" - revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" + revision = "853d788c5c416eaaee5b044570784a96c7a26975" + version = "v1.0.0" [[projects]] branch = "master" @@ -223,6 +207,14 @@ pruneopts = "UT" revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" +[[projects]] + digest = "1:b18a269f11ff51135d6f82987dbb53288f4d66098a6639b429f4f494a910155b" + name = "github.com/libp2p/go-buffer-pool" + packages = ["."] + pruneopts = "UT" + revision = "c4a5988a1e475884367015e1a2d0bd5fa4c491f4" + version = "v0.0.2" + [[projects]] digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7" name = "github.com/magiconair/properties" @@ -437,6 +429,13 @@ pruneopts = "UT" revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445" +[[projects]] + digest = "1:605b6546f3f43745695298ec2d342d3e952b6d91cdf9f349bea9315f677d759f" + name = "github.com/tendermint/btcd" + packages = ["btcec"] + pruneopts = "UT" + revision = "e5840949ff4fff0c56f9b6a541e22b63581ea9df" + [[projects]] digest = "1:ad9c4c1a4e7875330b1f62906f2830f043a23edb5db997e3a5ac5d3e6eadf80a" name = "github.com/tendermint/go-amino" @@ -446,16 +445,16 @@ version = "v0.14.1" [[projects]] - digest = "1:c317eec9047bdcd14dff925f34e07d22c6886faf22bf7c9286ea56ad962bcc22" + digest = "1:1ac12d8dc0930588ad4082b0c186c4a602a17dee5e74869c0237b212ed4e5fee" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "65e487d3a8d4d6624d8779c1f42cff31d61a9146" + revision = "b84252441ae02d158b54f62138a929365fd1d4ee" source = "github.com/binance-chain/bnc-tendermint-iavl" - version = "v0.12.0-binance.0" + version = "v0.12.0-binance.1" [[projects]] - digest = "1:09921dbdc56888c7a8d926a35ddd829a7891d6dd3b6eb91a0eb4849f4c0348ce" + digest = "1:dc2205db501620ad7c2be28dec030e1c953ecb05b0010c26271e4f832c171a2e" name = "github.com/tendermint/tendermint" packages = [ "abci/client", @@ -477,6 +476,7 @@ "crypto/multisig", "crypto/multisig/bitarray", "crypto/secp256k1", + "crypto/secp256k1/internal/secp256k1", "crypto/tmhash", "crypto/xsalsa20symmetric", "evidence", @@ -490,6 +490,7 @@ "libs/events", "libs/fail", "libs/flowrate", + "libs/gopool", "libs/log", "libs/pubsub", "libs/pubsub/query", @@ -512,7 +513,11 @@ "rpc/lib/client", "rpc/lib/server", "rpc/lib/types", + "snapshot", "state", + "state/blockindex", + "state/blockindex/kv", + "state/blockindex/null", "state/txindex", "state/txindex/kv", "state/txindex/null", @@ -521,17 +526,35 @@ "version", ] pruneopts = "UT" - revision = "3abb98b2e97b6ef6286e6f7d4b164012fe36ec80" + revision = "9e594f505df07ce1d4067918dbe90d4df7bae8bc" source = "github.com/binance-chain/bnc-tendermint" - version = "v0.30.1-binance.0" + version = "v0.31.5-binance.0" + +[[projects]] + digest = "1:b73f5e117bc7c6e8fc47128f20db48a873324ad5cfeeebfc505e85c58682b5e4" + name = "github.com/zondax/hid" + packages = ["."] + pruneopts = "T" + revision = "302fd402163c34626286195dfa9adac758334acc" + version = "v0.9.0" [[projects]] - digest = "1:7886f86064faff6f8d08a3eb0e8c773648ff5a2e27730831e2bfbf07467f6666" - name = "github.com/zondax/ledger-goclient" + digest = "1:c95f97a7ebc86127478375e2f8e8b49f096e92b79e49e70a570862eda4b18f5e" + name = "github.com/zondax/ledger-cosmos-go" packages = ["."] pruneopts = "UT" - revision = "58598458c11bc0ad1c1b8dac3dc3e11eaf270b79" - version = "v0.1.0" + revision = "fee2574758229322031b8b8d5c5f8197d121b952" + source = "https://github.com/binance-chain/ledger-cosmos-go" + version = "v0.9.9-binance.1" + +[[projects]] + digest = "1:11b3a00fab978e4edbad3322b9e024ea9031c7714f41dd9e92ac70f27d9adbe7" + name = "github.com/zondax/ledger-go" + packages = ["."] + pruneopts = "UT" + revision = "6d4c92579518a79bb50589cc1c35995f610832f5" + source = "https://github.com/binance-chain/ledger-go" + version = "v0.9.1" [[projects]] digest = "1:6f6dc6060c4e9ba73cf28aa88f12a69a030d3d19d518ef8e931879eaa099628d" @@ -679,6 +702,7 @@ "github.com/stretchr/testify/assert", "github.com/stretchr/testify/require", "github.com/syndtr/goleveldb/leveldb/opt", + "github.com/tendermint/btcd/btcec", "github.com/tendermint/go-amino", "github.com/tendermint/iavl", "github.com/tendermint/tendermint/abci/client", @@ -713,9 +737,11 @@ "github.com/tendermint/tendermint/rpc/core/types", "github.com/tendermint/tendermint/rpc/lib/client", "github.com/tendermint/tendermint/rpc/lib/server", + "github.com/tendermint/tendermint/snapshot", + "github.com/tendermint/tendermint/state", "github.com/tendermint/tendermint/types", "github.com/tendermint/tendermint/version", - "github.com/zondax/ledger-goclient", + "github.com/zondax/ledger-cosmos-go", "golang.org/x/crypto/bcrypt", ] solver-name = "gps-cdcl" diff --git a/Gopkg.toml b/Gopkg.toml index 20381b793..5ce34eacf 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -23,10 +23,6 @@ name = "github.com/bgentry/speakeasy" version = "~0.1.0" -[[override]] - name = "github.com/golang/protobuf" - version = "=1.1.0" - [[constraint]] name = "github.com/mattn/go-isatty" version = "~0.0.3" @@ -54,12 +50,12 @@ [[override]] name = "github.com/tendermint/iavl" source = "github.com/binance-chain/bnc-tendermint-iavl" - version = "=v0.12.0-binance.0" + version = "=v0.12.0-binance.1" [[override]] name = "github.com/tendermint/tendermint" source = "github.com/binance-chain/bnc-tendermint" - version = "=v0.30.1-binance.0" + version = "=v0.31.5-binance.0" ## deps without releases: @@ -76,9 +72,18 @@ name = "github.com/cosmos/go-bip39" revision = "52158e4697b87de16ed390e1bdaf813e581008fa" +[[override]] + name = "github.com/zondax/ledger-go" + source = "https://github.com/binance-chain/ledger-go" + version = "v0.9.1" + [[constraint]] - name = "github.com/zondax/ledger-goclient" - version = "=v0.1.0" + name = "github.com/zondax/ledger-cosmos-go" + source = "https://github.com/binance-chain/ledger-cosmos-go" + version = "v0.9.9-binance.1" + [[prune.project]] + name = "github.com/zondax/hid" + unused-packages = false ## transitive deps, with releases: diff --git a/Makefile b/Makefile index ec02f1402..cd8883c4e 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,10 @@ godocs: ######################################## ### Testing -test: test_unit +test: + make test_unit + make test_race + make test_sim_modules test_cli: @go test -count 1 -p 1 `go list github.com/cosmos/cosmos-sdk/cmd/gaia/cli_test` -tags=cli_test diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go index b90569a0c..f96caed7c 100644 --- a/baseapp/baseapp.go +++ b/baseapp/baseapp.go @@ -16,6 +16,7 @@ import ( cmn "github.com/tendermint/tendermint/libs/common" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/snapshot" tmtypes "github.com/tendermint/tendermint/types" "github.com/cosmos/cosmos-sdk/codec" @@ -34,6 +35,8 @@ var dbHeaderKey = []byte("header") const ( // we pass txHash of current handling message via context so that we can publish it as metadata of Msg TxHashKey = "txHash" + // we pass txSrc of current handling message via context so that we can publish it as metadata of Msg + TxSourceKey = "txSrc" //this number should be around the size of the transactions in a block, TODO: configurable TxMsgCacheSize = 4000 ) @@ -74,6 +77,9 @@ type BaseApp struct { txMsgCache *lru.Cache Pool *sdk.Pool + // Snapshot for state sync related fields + StateSyncHelper *store.StateSyncHelper // manage state sync related status + // flag for sealing sealed bool } @@ -692,6 +698,11 @@ func validateBasicTxMsgs(msgs []sdk.Msg) sdk.Error { } for _, msg := range msgs { + if !sdk.IsMsgTypeSupported(msg.Type()) { + return sdk.ErrMsgNotSupported(fmt.Sprintf("msg type(%s) is not supported before height %d", + msg.Type(), sdk.UpgradeMgr.GetMsgTypeHeight(msg.Type()))) + } + // Validate the Msg. err := msg.ValidateBasic() if err != nil { @@ -726,7 +737,7 @@ func (app *BaseApp) getContextWithCache(mode sdk.RunTxMode, txBytes []byte, txHa } // Iterates through msgs and executes them -func (app *BaseApp) runMsgs(ctx sdk.Context, msgs []sdk.Msg, txHash string, mode sdk.RunTxMode) (result sdk.Result) { +func (app *BaseApp) runMsgs(ctx sdk.Context, msgs []sdk.Msg, mode sdk.RunTxMode) (result sdk.Result) { // accumulate results logs := make([]string, 0, len(msgs)) var data []byte // NOTE: we just append them all (?!) @@ -740,7 +751,7 @@ func (app *BaseApp) runMsgs(ctx sdk.Context, msgs []sdk.Msg, txHash string, mode return sdk.ErrUnknownRequest("Unrecognized Msg type: " + msgRoute).Result() } - msgResult := handler(ctx.WithValue(TxHashKey, txHash).WithRunTxMode(mode), msg) + msgResult := handler(ctx.WithRunTxMode(mode), msg) msgResult.Tags = append(msgResult.Tags, sdk.MakeTag("action", []byte(msg.Type()))) // Append Data and Tags @@ -830,7 +841,14 @@ func (app *BaseApp) RunTx(mode sdk.RunTxMode, txBytes []byte, tx sdk.Tx, txHash } } - result = app.runMsgs(ctx, msgs, txHash, mode) + var txSrc int64 + if stdTx, ok := tx.(auth.StdTx); ok { + txSrc = stdTx.GetSource() + } + result = app.runMsgs( + ctx.WithValue(TxHashKey, txHash).WithValue(TxSourceKey, txSrc), + msgs, + mode) if mode == sdk.RunTxModeSimulate { return @@ -884,7 +902,11 @@ func (app *BaseApp) ReRunTx(txBytes []byte, tx sdk.Tx) (result sdk.Result) { } var msgs = tx.GetMsgs() - result = app.runMsgs(ctx, msgs, txHash, mode) + var txSrc int64 + if stdTx, ok := tx.(auth.StdTx); ok { + txSrc = stdTx.GetSource() + } + result = app.runMsgs(ctx.WithValue(TxHashKey, txHash).WithValue(TxSourceKey, txSrc), msgs, mode) // only update state if all messages pass if result.IsOK() { @@ -943,23 +965,29 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) { } } -func (app *BaseApp) LatestSnapshot() (height int64, numKeys []int64, err error) { - return 0, make([]int64, 0), nil -} - -func (app *BaseApp) ReadSnapshotChunk(height int64, startIndex, endIndex int64) (chunk [][]byte, err error) { - return make([][]byte, 0), nil +func (app *BaseApp) StartRecovery(manifest *abci.Manifest) error { + return app.StateSyncHelper.StartRecovery(manifest) } -func (app *BaseApp) StartRecovery(height int64, numKeys []int64) error { - return nil -} +func (app *BaseApp) WriteRecoveryChunk(hash abci.SHA256Sum, chunk *abci.AppStateChunk, isComplete bool) error { + if err := app.StateSyncHelper.WriteRecoveryChunk(hash, chunk, isComplete); err != nil { + return err + } -func (app *BaseApp) WriteRecoveryChunk(chunk [][]byte) error { - return nil -} + if isComplete { + // load into memory from db + if err := app.LoadCMSLatestVersion(); err != nil { + return err + } + stores := app.GetCommitMultiStore() + commitId := stores.LastCommitID() + hashHex := fmt.Sprintf("%X", commitId.Hash) + app.Logger.Info("commit by state reactor", "version", commitId.Version, "hash", hashHex) -func (app *BaseApp) EndRecovery(height int64) error { + // simulate we just "Commit()" :P + app.SetCheckState(abci.Header{Height: snapshot.Manager().RestorationManifest.Height}) + app.DeliverState = nil + } return nil } diff --git a/baseapp/helpers.go b/baseapp/helpers.go index cc965316c..5708906bf 100644 --- a/baseapp/helpers.go +++ b/baseapp/helpers.go @@ -2,8 +2,6 @@ package baseapp import ( sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/tendermint/tendermint/abci/server" - abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -25,28 +23,3 @@ func (app *BaseApp) Deliver(tx sdk.Tx) (result sdk.Result) { txHash := cmn.HexBytes(tmhash.Sum(nil)).String() return app.RunTx(sdk.RunTxModeDeliver, nil, tx, txHash) } - -// RunForever - BasecoinApp execution and cleanup -func RunForever(app abci.Application) { - - // Start the ABCI server - srv, err := server.NewServer("0.0.0.0:26658", "socket", app) - if err != nil { - cmn.Exit(err.Error()) - return - } - err = srv.Start() - if err != nil { - cmn.Exit(err.Error()) - return - } - - // Wait forever - cmn.TrapSignal(func() { - // Cleanup - err := srv.Stop() - if err != nil { - cmn.Exit(err.Error()) - } - }) -} diff --git a/baseapp/options.go b/baseapp/options.go index f4441418a..11fa411fa 100644 --- a/baseapp/options.go +++ b/baseapp/options.go @@ -14,20 +14,19 @@ import ( // SetPruning sets a pruning option on the multistore associated with the app func SetPruning(pruning string) func(*BaseApp) { - var pruningStrategy sdk.PruningStrategy + var pruningEnum sdk.PruningStrategy switch pruning { case "nothing": - pruningStrategy = sdk.PruneNothing{} + pruningEnum = sdk.PruneNothing case "everything": - pruningStrategy = sdk.PruneEverything{} + pruningEnum = sdk.PruneEverything case "syncable": - // TODO: make these parameters configurable - pruningStrategy = sdk.PruneSyncable{NumRecent: 100, StoreEvery: 10000} + pruningEnum = sdk.PruneSyncable default: panic(fmt.Sprintf("invalid pruning strategy: %s", pruning)) } return func(bap *BaseApp) { - bap.SetPruning(pruningStrategy) + bap.cms.SetPruning(pruningEnum) } } diff --git a/client/context/context.go b/client/context/context.go index e8cfd4264..6b74730cb 100644 --- a/client/context/context.go +++ b/client/context/context.go @@ -26,7 +26,8 @@ import ( const ctxAccStoreName = "acc" var ( - verifier tmlite.Verifier + verifier tmlite.Verifier + verifierHome string ) // CLIContext implements a typical CLI context created in SDK modules for @@ -46,6 +47,7 @@ type CLIContext struct { JSON bool PrintResponse bool Verifier tmlite.Verifier + VerifierHome string DryRun bool GenerateOnly bool fromAddress types.AccAddress @@ -67,8 +69,9 @@ func NewCLIContext() CLIContext { fromAddress, fromName := fromFields(from) // We need to use a single verifier for all contexts - if verifier == nil { + if verifier == nil || verifierHome != viper.GetString(cli.HomeFlag) { verifier = createVerifier() + verifierHome = viper.GetString(cli.HomeFlag) } return CLIContext{ diff --git a/client/context/query.go b/client/context/query.go index 7e4c237f8..c5852aec8 100644 --- a/client/context/query.go +++ b/client/context/query.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/spf13/viper" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/merkle" cmn "github.com/tendermint/tendermint/libs/common" @@ -13,6 +14,7 @@ import ( rpcclient "github.com/tendermint/tendermint/rpc/client" tmtypes "github.com/tendermint/tendermint/types" + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/store" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth" @@ -113,6 +115,9 @@ func (ctx CLIContext) GetAccountSequence(address []byte) (int64, error) { // EnsureAccountExists ensures that an account exists for a given context. An // error is returned if it does not. func (ctx CLIContext) EnsureAccountExists() error { + if viper.GetBool(client.FlagOffline) { + return nil + } addr, err := ctx.GetFromAddress() if err != nil { return err @@ -220,9 +225,17 @@ func (ctx CLIContext) verifyProof(queryPath string, resp abci.ResponseQuery) err kp = kp.AppendKey([]byte(storeName), merkle.KeyEncodingURL) kp = kp.AppendKey(resp.Key, merkle.KeyEncodingURL) + if resp.Value == nil { + err = prt.VerifyAbsence(resp.Proof, commit.Header.AppHash, kp.String()) + if err != nil { + return errors.Wrap(err, "failed to prove merkle absence proof") + } + return nil + } + err = prt.VerifyValue(resp.Proof, commit.Header.AppHash, kp.String(), resp.Value) if err != nil { - return errors.Wrap(err, "failed to prove merkle proof") + return errors.Wrap(err, "failed to prove merkle existence proof") } return nil diff --git a/client/flags.go b/client/flags.go index d9dfc3f4f..8ffad7754 100644 --- a/client/flags.go +++ b/client/flags.go @@ -22,6 +22,7 @@ const ( FlagJson = "json" FlagPrintResponse = "print-response" FlagDryRun = "dry-run" + FlagOffline = "offline" FlagGenerateOnly = "generate-only" FlagIndentResponse = "indent" ) @@ -66,6 +67,7 @@ func PostCommands(cmds ...*cobra.Command) []*cobra.Command { c.Flags().Bool(FlagPrintResponse, true, "return tx response (only works with async = false)") c.Flags().Bool(FlagTrustNode, true, "Trust connected full node (don't verify proofs for responses)") c.Flags().Bool(FlagDryRun, false, "ignore the perform a simulation of a transaction, but don't broadcast it") + c.Flags().Bool(FlagOffline, false, "Offline mode. Do not query blockchain data") c.Flags().Bool(FlagGenerateOnly, false, "build an unsigned transaction and write it to STDOUT") viper.BindPFlag(FlagTrustNode, c.Flags().Lookup(FlagTrustNode)) viper.BindPFlag(FlagUseLedger, c.Flags().Lookup(FlagUseLedger)) diff --git a/client/keys/add.go b/client/keys/add.go index 6b6e38117..5dd8910eb 100644 --- a/client/keys/add.go +++ b/client/keys/add.go @@ -91,7 +91,7 @@ func runAddCmd(cmd *cobra.Command, args []string) error { if viper.GetBool(client.FlagUseLedger) { account := uint32(viper.GetInt(flagAccount)) index := uint32(viper.GetInt(flagIndex)) - path := ccrypto.DerivationPath{44, 118, account, 0, index} + path := ccrypto.DerivationPath{44, 714, account, 0, index} algo := keys.SigningAlgo(viper.GetString(flagType)) info, err := kb.CreateLedger(name, path, algo) if err != nil { diff --git a/client/lcd/root.go b/client/lcd/root.go index 10faadd1a..dc94a2904 100644 --- a/client/lcd/root.go +++ b/client/lcd/root.go @@ -12,6 +12,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/rpc" "github.com/cosmos/cosmos-sdk/client/tx" "github.com/cosmos/cosmos-sdk/codec" + "github.com/cosmos/cosmos-sdk/server" auth "github.com/cosmos/cosmos-sdk/x/auth/client/rest" bank "github.com/cosmos/cosmos-sdk/x/bank/client/rest" gov "github.com/cosmos/cosmos-sdk/x/gov/client/rest" @@ -21,7 +22,6 @@ import ( "github.com/rakyll/statik/fs" "github.com/spf13/cobra" "github.com/spf13/viper" - cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" tmserver "github.com/tendermint/tendermint/rpc/lib/server" ) @@ -58,12 +58,13 @@ func ServeCommand(cdc *codec.Codec) *cobra.Command { var listener net.Listener var fingerprint string if viper.GetBool(flagInsecure) { - listener, err := tmserver.Listen(listenAddr, tmserver.Config{MaxOpenConnections: maxOpen}) + cfg := &tmserver.Config{MaxOpenConnections: maxOpen} + listener, err := tmserver.Listen(listenAddr, cfg) if err != nil { return err } go func() { - if err = tmserver.StartHTTPServer(listener, handler, logger); err != nil { + if err = tmserver.StartHTTPServer(listener, handler, logger, cfg); err != nil { panic(err) } }() @@ -91,12 +92,13 @@ func ServeCommand(cdc *codec.Codec) *cobra.Command { } defer cleanupFunc() } - listener, err := tmserver.Listen(listenAddr, tmserver.Config{MaxOpenConnections: maxOpen}) + cfg := &tmserver.Config{MaxOpenConnections: maxOpen} + listener, err := tmserver.Listen(listenAddr, cfg) if err != nil { return err } go func() { - if err = tmserver.StartHTTPServer(listener, handler, logger); err != nil { + if err = tmserver.StartHTTPAndTLSServer(listener, handler, certFile, keyFile, logger, cfg); err != nil { panic(err) } }() @@ -105,7 +107,7 @@ func ServeCommand(cdc *codec.Codec) *cobra.Command { logger.Info("REST server started") // wait forever and cleanup - cmn.TrapSignal(func() { + server.TrapSignal(func() { defer cleanupFunc() err := listener.Close() logger.Error("error closing listener", "err", err) diff --git a/client/lcd/test_helpers.go b/client/lcd/test_helpers.go index 77929dcc2..7aa6960a2 100644 --- a/client/lcd/test_helpers.go +++ b/client/lcd/test_helpers.go @@ -264,8 +264,7 @@ func InitializeTestLCD( // XXX: Need to set this so LCD knows the tendermint node address! viper.Set(client.FlagNode, config.RPC.ListenAddress) viper.Set(client.FlagChainID, genDoc.ChainID) - // TODO Set to false once the upstream Tendermint proof verification issue is fixed. - viper.Set(client.FlagTrustNode, true) + viper.Set(client.FlagTrustNode, false) dir, err := ioutil.TempDir("", "lcd_test") require.NoError(t, err) viper.Set(cli.HomeFlag, dir) @@ -334,12 +333,12 @@ func startTM( // // NOTE: This causes the thread to block. func startLCD(logger log.Logger, listenAddr string, cdc *codec.Codec) (net.Listener, error) { - listener, err := tmrpc.Listen(listenAddr, tmrpc.Config{}) + listener, err := tmrpc.Listen(listenAddr, &tmrpc.Config{}) if err != nil { return nil, err } - go tmrpc.StartHTTPServer(listener, createHandler(cdc), logger) + go tmrpc.StartHTTPServer(listener, createHandler(cdc), logger, &tmrpc.Config{}) return listener, nil } diff --git a/client/utils/utils.go b/client/utils/utils.go index 25dec7e01..0de57a3b0 100644 --- a/client/utils/utils.go +++ b/client/utils/utils.go @@ -5,6 +5,9 @@ import ( "fmt" "os" + "github.com/spf13/viper" + + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/context" "github.com/cosmos/cosmos-sdk/client/keys" "github.com/cosmos/cosmos-sdk/codec" @@ -56,7 +59,7 @@ func CompleteAndBroadcastTxCli(txBldr authtxb.TxBuilder, cliCtx context.CLIConte func simulateMsgs(txBldr authtxb.TxBuilder, cliCtx context.CLIContext, name, passphrase string, msgs []sdk.Msg) error { txBytes, err := txBldr.BuildAndSign(name, passphrase, msgs) if err != nil { - return err + return err } // run a simulation (via /app/simulate query) @@ -88,8 +91,9 @@ func printTxResult(result sdk.Result) { // PrintUnsignedStdTx builds an unsigned StdTx and prints it to os.Stdout. // Don't perform online validation or lookups if offline is true. -func PrintUnsignedStdTx(txBldr authtxb.TxBuilder, cliCtx context.CLIContext, msgs []sdk.Msg, offline bool) (err error) { +func PrintUnsignedStdTx(txBldr authtxb.TxBuilder, cliCtx context.CLIContext, msgs []sdk.Msg) (err error) { var stdTx auth.StdTx + offline := viper.GetBool(client.FlagOffline) if offline { stdTx, err = buildUnsignedStdTxOffline(txBldr, msgs) } else { @@ -169,7 +173,7 @@ func prepareTxBuilder(txBldr authtxb.TxBuilder, cliCtx context.CLIContext) (auth // TODO: (ref #1903) Allow for user supplied account number without // automatically doing a manual lookup. - if txBldr.AccountNumber == 0 { + if txBldr.AccountNumber == 0 && !viper.GetBool(client.FlagOffline) { accNum, err := cliCtx.GetAccountNumber(from) if err != nil { return txBldr, err @@ -179,7 +183,7 @@ func prepareTxBuilder(txBldr authtxb.TxBuilder, cliCtx context.CLIContext) (auth // TODO: (ref #1903) Allow for user supplied account sequence without // automatically doing a manual lookup. - if txBldr.Sequence == 0 { + if txBldr.Sequence == 0 && !viper.GetBool(client.FlagOffline) { accSeq, err := cliCtx.GetAccountSequence(from) if err != nil { return txBldr, err diff --git a/crypto/ledger.go b/crypto/ledger.go index b9aa65b56..36b5646b8 100644 --- a/crypto/ledger.go +++ b/crypto/ledger.go @@ -3,7 +3,7 @@ package crypto import ( - ledger "github.com/zondax/ledger-goclient" + ledger "github.com/zondax/ledger-cosmos-go" ) // If ledger support (build tag) has been enabled, which implies a CGO dependency, @@ -11,7 +11,7 @@ import ( // device at runtime or returning an error. func init() { discoverLedger = func() (LedgerSECP256K1, error) { - device, err := ledger.FindLedger() + device, err := ledger.FindLedgerCosmosUserApp() if err != nil { return nil, err } diff --git a/crypto/ledger_secp256k1.go b/crypto/ledger_secp256k1.go index ff05d31ba..9d968af5c 100644 --- a/crypto/ledger_secp256k1.go +++ b/crypto/ledger_secp256k1.go @@ -1,11 +1,17 @@ package crypto import ( + "bufio" "fmt" + "os" + "strings" - "github.com/pkg/errors" + "github.com/btcsuite/btcd/btcec" + sdk "github.com/cosmos/cosmos-sdk/types" + ledgergo "github.com/zondax/ledger-cosmos-go" - secp256k1 "github.com/btcsuite/btcd/btcec" + "github.com/pkg/errors" + tmbtcec "github.com/tendermint/btcd/btcec" tmcrypto "github.com/tendermint/tendermint/crypto" tmsecp256k1 "github.com/tendermint/tendermint/crypto/secp256k1" ) @@ -29,7 +35,9 @@ type ( // the SECP256K1 scheme. LedgerSECP256K1 interface { GetPublicKeySECP256K1([]uint32) ([]byte, error) + ShowAddressSECP256K1([]uint32, string) error SignSECP256K1([]uint32, []byte) ([]byte, error) + GetVersion() (*ledgergo.VersionInfo, error) } // PrivKeyLedgerSecp256k1 implements PrivKey, calling the ledger nano we @@ -117,12 +125,43 @@ func (pkl PrivKeyLedgerSecp256k1) Equals(other tmcrypto.PrivKey) bool { // an error, so this should only trigger if the private key is held in memory // for a while before use. func (pkl PrivKeyLedgerSecp256k1) Sign(msg []byte) ([]byte, error) { + ledgerAppVersion, err := pkl.ledger.GetVersion() + if err != nil { + return nil, err + } + if ledgerAppVersion.Major > 1 || ledgerAppVersion.Major == 1 && ledgerAppVersion.Minor >= 1 { + fmt.Print(fmt.Sprintf("Please confirm if address displayed on ledger is identical to %s (yes/no)?", sdk.AccAddress(pkl.CachedPubKey.Address()).String())) + err = pkl.ledger.ShowAddressSECP256K1(pkl.Path, sdk.GetConfig().GetBech32AccountAddrPrefix()) + if err != nil { + return nil, err + } + + buf, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return nil, err + } + confirm := strings.ToLower(strings.TrimSpace(buf)) + if confirm != "y" && confirm != "yes" { + return nil, fmt.Errorf("ledger account doesn't match") + } + } + fmt.Println("Please verify the transaction data on ledger") + sig, err := pkl.signLedgerSecp256k1(msg) if err != nil { return nil, err } - return sig, nil + return convertDERtoBER(sig) +} + +func convertDERtoBER(signatureDER []byte) ([]byte, error) { + sigDER, err := btcec.ParseDERSignature(signatureDER[:], btcec.S256()) + if err != nil { + return nil, err + } + sigBER := tmbtcec.Signature{R: sigDER.R, S: sigDER.S} + return sigBER.Serialize(), nil } // getPubKey reads the pubkey the ledger itself @@ -150,7 +189,7 @@ func (pkl PrivKeyLedgerSecp256k1) pubkeyLedgerSecp256k1() (pub tmcrypto.PubKey, var pk tmsecp256k1.PubKeySecp256k1 // re-serialize in the 33-byte compressed format - cmp, err := secp256k1.ParsePubKey(key[:], secp256k1.S256()) + cmp, err := btcec.ParsePubKey(key[:], btcec.S256()) if err != nil { return nil, fmt.Errorf("error parsing public key: %v", err) } diff --git a/server/concurrent/async_local_client.go b/server/concurrent/async_local_client.go index f304d1b98..478fe5c33 100644 --- a/server/concurrent/async_local_client.go +++ b/server/concurrent/async_local_client.go @@ -424,39 +424,18 @@ func (app *asyncLocalClient) EndBlockSync(req types.RequestEndBlock) (*types.Res //------------------------------------------------------- -func (app *asyncLocalClient) LatestSnapshot() (height int64, numKeys []int64, err error) { - app.rwLock.RLock() - defer app.rwLock.RUnlock() - - return app.Application.LatestSnapshot() -} - -func (app *asyncLocalClient) ReadSnapshotChunk(height int64, startIndex, endIndex int64) (chunk [][]byte, err error) { - app.rwLock.RLock() - defer app.rwLock.RUnlock() - - return app.Application.ReadSnapshotChunk(height, startIndex, endIndex) -} - -func (app *asyncLocalClient) StartRecovery(height int64, numKeys []int64) error { - app.rwLock.Lock() - defer app.rwLock.Unlock() - - return app.Application.StartRecovery(height, numKeys) -} - -func (app *asyncLocalClient) WriteRecoveryChunk(chunk [][]byte) error { +func (app *asyncLocalClient) StartRecovery(manifest *types.Manifest) error { app.rwLock.Lock() defer app.rwLock.Unlock() - return app.Application.WriteRecoveryChunk(chunk) + return app.Application.StartRecovery(manifest) } -func (app *asyncLocalClient) EndRecovery(height int64) error { +func (app *asyncLocalClient) WriteRecoveryChunk(hash types.SHA256Sum, chunk *types.AppStateChunk, isComplete bool) error { app.rwLock.Lock() defer app.rwLock.Unlock() - return app.Application.EndRecovery(height) + return app.Application.WriteRecoveryChunk(hash, chunk, isComplete) } //------------------------------------------------------- diff --git a/server/concurrent/async_local_client_test.go b/server/concurrent/async_local_client_test.go index c1ad1c4d0..2977806bd 100644 --- a/server/concurrent/async_local_client_test.go +++ b/server/concurrent/async_local_client_test.go @@ -88,19 +88,10 @@ func (app *TimedApplication) PreDeliverTx(tx []byte) types.ResponseDeliverTx { return types.ResponseDeliverTx{} } -func (cli *TimedApplication) LatestSnapshot() (height int64, numKeys []int64, err error) { - return 0, make([]int64, 0), nil -} -func (cli *TimedApplication) ReadSnapshotChunk(height int64, startIndex, endIndex int64) (chunk [][]byte, err error) { - return make([][]byte, 0), nil -} -func (cli *TimedApplication) StartRecovery(height int64, numKeys []int64) error { - return nil -} -func (cli *TimedApplication) WriteRecoveryChunk(chunk [][]byte) error { +func (cli *TimedApplication) StartRecovery(manifest *types.Manifest) error { return nil } -func (cli *TimedApplication) EndRecovery(height int64) error { +func (cli *TimedApplication) WriteRecoveryChunk(hash types.SHA256Sum, chunk *types.AppStateChunk, isComplete bool) error { return nil } diff --git a/server/mock/store.go b/server/mock/store.go index 2d4b0155a..906238248 100644 --- a/server/mock/store.go +++ b/server/mock/store.go @@ -14,6 +14,10 @@ type multiStore struct { kv map[sdk.StoreKey]kvStore } +func (ms multiStore) SetVersion(version int64) { + panic("not implemented") +} + func (ms multiStore) CacheMultiStore() sdk.CacheMultiStore { panic("not implemented") } @@ -58,6 +62,10 @@ func (ms multiStore) GetCommitKVStore(key sdk.StoreKey) sdk.CommitKVStore { panic("not implemented") } +func (ms multiStore) GetCommitKVStores() map[sdk.StoreKey]sdk.CommitKVStore { + panic("not implemented") +} + func (ms multiStore) GetCommitStore(key sdk.StoreKey) sdk.CommitStore { panic("not implemented") } diff --git a/server/start.go b/server/start.go index 9012bc198..13a084edb 100644 --- a/server/start.go +++ b/server/start.go @@ -87,13 +87,14 @@ func startStandAlone(ctx *Context, appCreator AppCreator) error { } // wait forever - cmn.TrapSignal(func() { + cmn.TrapSignal(ctx.Logger, func() { // cleanup err = svr.Stop() if err != nil { cmn.Exit(err.Error()) } }) + select {} return nil } diff --git a/store/iavlstore.go b/store/iavlstore.go index b636720a7..1b2b21fab 100644 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -25,7 +25,7 @@ func LoadIAVLStore(db dbm.DB, id CommitID, pruning sdk.PruningStrategy) (CommitS if err != nil { return nil, err } - iavl := newIAVLStore(tree, sdk.PruneNothing{}) + iavl := newIAVLStore(tree, int64(0), int64(0)) iavl.SetPruning(pruning) return iavl, nil } @@ -42,16 +42,26 @@ type IavlStore struct { // The underlying tree. Tree *iavl.MutableTree - // The strategy to prune historical versions - pruningStrategy sdk.PruningStrategy + // How many old versions we hold onto. + // A value of 0 means keep no recent states. + numRecent int64 + + // This is the distance between state-sync waypoint states to be stored. + // See https://github.com/tendermint/tendermint/issues/828 + // A value of 1 means store every state. + // A value of 0 means store no waypoints. (node cannot assist in state-sync) + // By default this value should be set the same across all nodes, + // so that nodes can know the waypoints their peers store. + storeEvery int64 } // CONTRACT: tree should be fully loaded. // nolint: unparam -func newIAVLStore(tree *iavl.MutableTree, ps sdk.PruningStrategy) *IavlStore { +func newIAVLStore(tree *iavl.MutableTree, numRecent int64, storeEvery int64) *IavlStore { st := &IavlStore{ - Tree: tree, - pruningStrategy: ps, + Tree: tree, + numRecent: numRecent, + storeEvery: storeEvery, } return st } @@ -60,6 +70,10 @@ func (st *IavlStore) GetImmutableTree() *iavl.ImmutableTree { return st.Tree.ImmutableTree } +func (st *IavlStore) SetVersion(version int64) { + st.Tree.SetVersion(version) +} + // Implements Committer. func (st *IavlStore) Commit() CommitID { // Save a new version. @@ -70,16 +84,15 @@ func (st *IavlStore) Commit() CommitID { } // Release an old version of history, if not a sync waypoint. - //for v, _ := range st.Tree.GetVersions() { - // if st.pruningStrategy.ShouldPrune(v, version) { - // st.Tree.DeleteVersion(v) - // } - //} - - // this is a special optimization for binance chain, 10001 should keep consistent with - // `numRecent` field of KeepRecentAndBreatheBlock - if version > 10001 && st.pruningStrategy.ShouldPrune(version-10001, version) { - st.Tree.DeleteVersion(version - 10001) + previous := version - 1 + if st.numRecent < previous { + toRelease := previous - st.numRecent + if st.storeEvery == 0 || toRelease%st.storeEvery != 0 { + err := st.Tree.DeleteVersion(toRelease) + if err != nil && err.(cmn.Error).Data() != iavl.ErrVersionDoesNotExist { + panic(err) + } + } } return CommitID{ @@ -98,7 +111,16 @@ func (st *IavlStore) LastCommitID() CommitID { // Implements Committer. func (st *IavlStore) SetPruning(pruning sdk.PruningStrategy) { - st.pruningStrategy = pruning + switch pruning { + case sdk.PruneEverything: + st.numRecent = 0 + st.storeEvery = 0 + case sdk.PruneNothing: + st.storeEvery = 1 + case sdk.PruneSyncable: + st.numRecent = 10000 // fork github.com/cosmos/cosmos-sdk/blob/9a16e2675f392b083dd1074ff92ff1f9fbda750d/store/types/pruning.go#L34 + st.storeEvery = 10000 + } } // VersionExists returns whether or not a given version is stored. @@ -204,8 +226,21 @@ func (st *IavlStore) Query(req abci.RequestQuery) (res abci.ResponseQuery) { res.Log = err.Error() break } - res.Value = value - res.Proof = &merkle.Proof{Ops: []merkle.ProofOp{iavl.NewIAVLValueOp(key, proof).ProofOp()}} + if proof == nil { + // Proof == nil implies that the store is empty. + if value != nil { + panic("unexpected value for an empty proof") + } + } + if value != nil { + // value was found + res.Value = value + res.Proof = &merkle.Proof{Ops: []merkle.ProofOp{iavl.NewIAVLValueOp(key, proof).ProofOp()}} + } else { + // value wasn't found + res.Value = nil + res.Proof = &merkle.Proof{Ops: []merkle.ProofOp{iavl.NewIAVLAbsenceOp(key, proof).ProofOp()}} + } } else { _, res.Value = tree.GetVersioned(key, res.Height) } diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index f02ec52b4..3f853ec30 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -47,7 +47,7 @@ func newTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, CommitID) { func TestIAVLStoreGetSetHasDelete(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) key := "hello" @@ -72,7 +72,7 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) { func TestIAVLIterator(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz")) expected := []string{"aloha", "hello"} var i int @@ -145,7 +145,7 @@ func TestIAVLIterator(t *testing.T) { func TestIAVLSubspaceIterator(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test2"), []byte("test2")) @@ -207,7 +207,7 @@ func TestIAVLSubspaceIterator(t *testing.T) { func TestIAVLReverseSubspaceIterator(t *testing.T) { db := dbm.NewMemDB() tree, _ := newTree(t, db) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) iavlStore.Set([]byte("test1"), []byte("test1")) iavlStore.Set([]byte("test2"), []byte("test2")) @@ -270,55 +270,53 @@ func nextVersion(iavl *IavlStore) { iavl.Commit() } -// Comment out as binance hard coded numRecent to 10000 in func (st *IavlStore) Commit() CommitID { -//func TestIAVLDefaultPruning(t *testing.T) { -// //Expected stored / deleted version numbers for: -// //numRecent = 5, storeEvery = 3 -// var states = []pruneState{ -// {[]int64{}, []int64{}}, -// {[]int64{1}, []int64{}}, -// {[]int64{1, 2}, []int64{}}, -// {[]int64{1, 2, 3}, []int64{}}, -// {[]int64{1, 2, 3, 4}, []int64{}}, -// {[]int64{1, 2, 3, 4, 5}, []int64{}}, -// {[]int64{1, 2, 3, 4, 5, 6}, []int64{}}, -// {[]int64{2, 3, 4, 5, 6, 7}, []int64{1}}, -// {[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}}, -// {[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}}, -// {[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}}, -// {[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}}, -// {[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}}, -// {[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}}, -// {[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}}, -// {[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}}, -// } -// testPruning(t, int64(5), int64(3), states) -//} - -// Comment out as binance hard coded numRecent to 10000 in func (st *IavlStore) Commit() CommitID { -//func TestIAVLAlternativePruning(t *testing.T) { -// //Expected stored / deleted version numbers for: -// //numRecent = 3, storeEvery = 5 -// var states = []pruneState{ -// {[]int64{}, []int64{}}, -// {[]int64{1}, []int64{}}, -// {[]int64{1, 2}, []int64{}}, -// {[]int64{1, 2, 3}, []int64{}}, -// {[]int64{1, 2, 3, 4}, []int64{}}, -// {[]int64{2, 3, 4, 5}, []int64{1}}, -// {[]int64{3, 4, 5, 6}, []int64{1, 2}}, -// {[]int64{4, 5, 6, 7}, []int64{1, 2, 3}}, -// {[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}}, -// {[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}}, -// {[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}}, -// {[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}}, -// {[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}}, -// {[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, -// {[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, -// {[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}}, -// } -// testPruning(t, int64(3), int64(5), states) -//} +func TestIAVLDefaultPruning(t *testing.T) { + //Expected stored / deleted version numbers for: + //numRecent = 5, storeEvery = 3 + var states = []pruneState{ + {[]int64{}, []int64{}}, + {[]int64{1}, []int64{}}, + {[]int64{1, 2}, []int64{}}, + {[]int64{1, 2, 3}, []int64{}}, + {[]int64{1, 2, 3, 4}, []int64{}}, + {[]int64{1, 2, 3, 4, 5}, []int64{}}, + {[]int64{1, 2, 3, 4, 5, 6}, []int64{}}, + {[]int64{2, 3, 4, 5, 6, 7}, []int64{1}}, + {[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}}, + {[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}}, + {[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}}, + {[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}}, + {[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}}, + {[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}}, + {[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}}, + {[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}}, + } + testPruning(t, int64(5), int64(3), states) +} + +func TestIAVLAlternativePruning(t *testing.T) { + //Expected stored / deleted version numbers for: + //numRecent = 3, storeEvery = 5 + var states = []pruneState{ + {[]int64{}, []int64{}}, + {[]int64{1}, []int64{}}, + {[]int64{1, 2}, []int64{}}, + {[]int64{1, 2, 3}, []int64{}}, + {[]int64{1, 2, 3, 4}, []int64{}}, + {[]int64{2, 3, 4, 5}, []int64{1}}, + {[]int64{3, 4, 5, 6}, []int64{1, 2}}, + {[]int64{4, 5, 6, 7}, []int64{1, 2, 3}}, + {[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}}, + {[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}}, + {[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}}, + {[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}}, + {[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}}, + {[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, + {[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}}, + {[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}}, + } + testPruning(t, int64(3), int64(5), states) +} type pruneState struct { stored []int64 @@ -328,7 +326,7 @@ type pruneState struct { func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []pruneState) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) for step, state := range states { for _, ver := range state.stored { require.True(t, iavlStore.VersionExists(ver), @@ -347,7 +345,7 @@ func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []prune func TestIAVLNoPrune(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, 1}) + iavlStore := newIAVLStore(tree, numRecent, int64(1)) nextVersion(iavlStore) for i := 1; i < 100; i++ { for j := 1; j <= i; j++ { @@ -359,29 +357,28 @@ func TestIAVLNoPrune(t *testing.T) { } } -// Comment out as binance hard coded numRecent to 10000 in func (st *IavlStore) Commit() CommitID { -//func TestIAVLPruneEverything(t *testing.T) { -// db := dbm.NewMemDB() -// tree := iavl.NewMutableTree(db, cacheSize) -// iavlStore := newIAVLStore(tree, sdk.PruneEverything{}) -// nextVersion(iavlStore) -// for i := 1; i < 100; i++ { -// for j := 1; j < i; j++ { -// require.False(t, iavlStore.VersionExists(int64(j)), -// "Unpruned version %d with latest version %d. Should prune all old versions", -// j, i) -// } -// require.True(t, iavlStore.VersionExists(int64(i)), -// "Missing current version on step %d, should not prune current state tree", -// i) -// nextVersion(iavlStore) -// } -//} +func TestIAVLPruneEverything(t *testing.T) { + db := dbm.NewMemDB() + tree := iavl.NewMutableTree(db, cacheSize) + iavlStore := newIAVLStore(tree, int64(0), int64(0)) + nextVersion(iavlStore) + for i := 1; i < 100; i++ { + for j := 1; j < i; j++ { + require.False(t, iavlStore.VersionExists(int64(j)), + "Unpruned version %d with latest version %d. Should prune all old versions", + j, i) + } + require.True(t, iavlStore.VersionExists(int64(i)), + "Missing current version on step %d, should not prune current state tree", + i) + nextVersion(iavlStore) + } +} func TestIAVLStoreQuery(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) k1, v1 := []byte("key1"), []byte("val1") k2, v2 := []byte("key2"), []byte("val2") @@ -477,7 +474,7 @@ func BenchmarkIAVLIteratorNext(b *testing.B) { value := cmn.RandBytes(50) tree.Set(key, value) } - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) iterators := make([]Iterator, b.N/treeSize) for i := 0; i < len(iterators); i++ { iterators[i] = iavlStore.Iterator([]byte{0}, []byte{255, 255, 255, 255, 255}) diff --git a/store/multistoreproof_test.go b/store/multistoreproof_test.go index cd3eda5b5..521da6f2a 100644 --- a/store/multistoreproof_test.go +++ b/store/multistoreproof_test.go @@ -13,7 +13,7 @@ import ( func TestVerifyIAVLStoreQueryProof(t *testing.T) { // Create main tree for testing. db := dbm.NewMemDB() - iStore, err := LoadIAVLStore(db, CommitID{}, sdk.PruneNothing{}) + iStore, err := LoadIAVLStore(db, CommitID{}, sdk.PruneNothing) store := iStore.(*IavlStore) require.Nil(t, err) store.Set([]byte("MYKEY"), []byte("MYVALUE")) diff --git a/store/prefixstore_test.go b/store/prefixstore_test.go index 6bf9bdff7..79eeefccd 100644 --- a/store/prefixstore_test.go +++ b/store/prefixstore_test.go @@ -76,7 +76,7 @@ func testPrefixStore(t *testing.T, baseStore KVStore, prefix []byte) { func TestIAVLStorePrefix(t *testing.T) { db := dbm.NewMemDB() tree := iavl.NewMutableTree(db, cacheSize) - iavlStore := newIAVLStore(tree, sdk.PruneSyncable{numRecent, storeEvery}) + iavlStore := newIAVLStore(tree, numRecent, storeEvery) testPrefixStore(t, iavlStore, []byte("test")) } diff --git a/store/rootmultistore.go b/store/rootmultistore.go index bba16a3f7..781a3be1a 100644 --- a/store/rootmultistore.go +++ b/store/rootmultistore.go @@ -40,7 +40,6 @@ var _ Queryable = (*rootMultiStore)(nil) func NewCommitMultiStore(db dbm.DB) *rootMultiStore { return &rootMultiStore{ db: db, - pruning: sdk.PruneNothing{}, storesParams: make(map[StoreKey]storeParams), stores: make(map[StoreKey]CommitStore), keysByName: make(map[string]StoreKey), @@ -89,6 +88,16 @@ func (rs *rootMultiStore) GetCommitKVStore(key StoreKey) CommitKVStore { return rs.stores[key].(CommitKVStore) } +func (rs *rootMultiStore) GetCommitKVStores() map[StoreKey]CommitKVStore { + res := make(map[StoreKey]CommitKVStore) + for key, store := range rs.stores { + if s, ok := store.(CommitKVStore); ok { + res[key] = s + } + } + return res +} + // Implements CommitMultiStore. func (rs *rootMultiStore) LoadLatestVersion() error { ver := getLatestVersion(rs.db) @@ -190,6 +199,9 @@ func (rs *rootMultiStore) LastCommitID() CommitID { return rs.lastCommitID } +// Implements Committer/CommitStore. +func (rs *rootMultiStore) SetVersion(version int64) {} + // Implements Committer/CommitStore. func (rs *rootMultiStore) Commit() CommitID { version := rs.lastCommitID.Version + 1 @@ -463,6 +475,15 @@ func commitStores(version int64, storeMap map[StoreKey]CommitStore) CommitInfo { storeInfos := make([]StoreInfo, 0, len(storeMap)) for key, store := range storeMap { + if !sdk.ShouldCommitStore(key.Name()) { + continue + } + + // set version for store to commit, just to keep the same as the other stores + if sdk.ShouldSetStoreVersion(key.Name()) { + store.SetVersion(version - 1) + } + // Commit commitID := store.Commit() diff --git a/store/statesync_helper.go b/store/statesync_helper.go new file mode 100644 index 000000000..1e6845d69 --- /dev/null +++ b/store/statesync_helper.go @@ -0,0 +1,484 @@ +package store + +import ( + "bytes" + "fmt" + "runtime/debug" + "sort" + "sync" + "time" + + "github.com/tendermint/iavl" + abci "github.com/tendermint/tendermint/abci/types" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/snapshot" + sm "github.com/tendermint/tendermint/state" + + "github.com/cosmos/cosmos-sdk/codec" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +const ( + snapshotWorkingQueueSize = 5 + snapshotToRemoveQueueSize = 5 + snapshotRetry = 5 +) + +type incompleteChunkItem struct { + chunkIdx int + completeness uint8 + nodePart []byte +} + +type chunkItemSorter struct { + chunkItems []incompleteChunkItem +} + +func (cis *chunkItemSorter) Len() int { + return len(cis.chunkItems) +} + +func (cis *chunkItemSorter) Swap(i, j int) { + cis.chunkItems[i], cis.chunkItems[j] = cis.chunkItems[j], cis.chunkItems[i] +} + +func (cis *chunkItemSorter) Less(i, j int) bool { + return cis.chunkItems[i].chunkIdx < cis.chunkItems[j].chunkIdx +} + +type PrefixNodeDB struct { + startIdxInclusive int64 + endIdxExclusive int64 + storeName string + *iavl.NodeDB +} + +type StateSyncHelper struct { + SnapshotHeights chan int64 + HeightsToDelete chan int64 + + logger log.Logger + commitMS sdk.CommitMultiStore + db dbm.DB + cdc *codec.Codec + + manifest *abci.Manifest + stateSyncStoreInfos []StoreInfo + + hashesToIdx map[abci.SHA256Sum]int // chunkhash -> idx in manifest + incompleteChunks map[int64][]incompleteChunkItem // node idx -> incomplete chunk items, for caching incomplete nodes temporally + prefixNodeDBs []PrefixNodeDB + + reloadingMtx sync.RWMutex // guard below fields to make sure no concurrent load snapshot and response snapshot, and they should be updated atomically + + snapshotManager *snapshot.SnapshotManager +} + +func NewStateSyncHelper( + logger log.Logger, + db dbm.DB, + cms sdk.CommitMultiStore, + cdc *codec.Codec) *StateSyncHelper { + var helper StateSyncHelper + helper.logger = logger + helper.db = db + helper.commitMS = cms + helper.cdc = cdc + + helper.SnapshotHeights = make(chan int64, snapshotWorkingQueueSize) + helper.HeightsToDelete = make(chan int64, snapshotToRemoveQueueSize) + + return &helper +} + +// not all key in cms is committed +// for example the BEP9 timelock store upgrade will not commit the newly added store until upgrade height +func (helper *StateSyncHelper) getCommitedSortedStoreKeys() []sdk.StoreKey { + kvStores := helper.commitMS.GetCommitKVStores() + names := make([]string, 0, len(kvStores)) + nameToKey := make(map[string]sdk.StoreKey, len(kvStores)) + for key, store := range kvStores { + if !sdk.ShouldCommitStore(key.Name()) { + continue + } + + switch store.(type) { + case *IavlStore: + nameToKey[key.Name()] = key + names = append(names, key.Name()) + default: + // deliberately do nothing other store type doesn't effect app hash + } + } + sort.Strings(names) + storeKeys := make([]sdk.StoreKey, 0, len(names)) + for _, name := range names { + storeKeys = append(storeKeys, nameToKey[name]) + } + return storeKeys +} + +// Split Init method and NewStateSyncHelper for snapshot command +func (helper *StateSyncHelper) Init(lastBreatheBlockHeight int64) { + go helper.ReloadSnapshotRoutine(lastBreatheBlockHeight, 0) + go func() { + for height := range helper.SnapshotHeights { + helper.ReloadSnapshotRoutine(height, snapshotRetry) + } + }() + go func() { + for height := range helper.HeightsToDelete { + helper.DeleteSnapshot(height) + } + }() +} + +func (helper *StateSyncHelper) StartRecovery(manifest *abci.Manifest) error { + helper.logger.Info("start recovery") + + sdk.UpgradeMgr.SetHeight(manifest.Height) + storeKeys := helper.getCommitedSortedStoreKeys() + + helper.manifest = manifest + helper.stateSyncStoreInfos = make([]StoreInfo, 0, len(storeKeys)) + helper.hashesToIdx = make(map[abci.SHA256Sum]int, len(manifest.AppStateHashes)) + helper.incompleteChunks = make(map[int64][]incompleteChunkItem, 0) + helper.prefixNodeDBs = make([]PrefixNodeDB, 0, len(storeKeys)) + + idxOfChunk := 0 + for _, h := range manifest.AppStateHashes { + helper.hashesToIdx[h] = idxOfChunk + idxOfChunk++ + } + + if len(manifest.NumKeys) != len(storeKeys) { + return fmt.Errorf("sub store count in manifest %d does not match local %d", len(manifest.NumKeys), len(storeKeys)) + } + + var startIdxForEachStore int64 + for idx, numOfKeys := range manifest.NumKeys { + db := dbm.NewPrefixDB(helper.db, []byte("s/k:"+storeKeys[idx].Name()+"/")) + nodeDB := iavl.NewNodeDB(db, 10000) + helper.prefixNodeDBs = append(helper.prefixNodeDBs, + PrefixNodeDB{ + startIdxForEachStore, + startIdxForEachStore + numOfKeys, + storeKeys[idx].Name(), + nodeDB}) + startIdxForEachStore += numOfKeys + } + + return nil +} + +func (helper *StateSyncHelper) WriteRecoveryChunk(hash abci.SHA256Sum, chunk *abci.AppStateChunk, isComplete bool) (err error) { + helper.reloadingMtx.Lock() + defer helper.reloadingMtx.Unlock() + + if chunk != nil { + numOfNodes := len(chunk.Nodes) + nodes := make([]*iavl.Node, 0, numOfNodes) + + helper.logger.Info("start write recovery chunk", "isComplete", isComplete, "hash", fmt.Sprintf("%x", hash), "startIdx", chunk.StartIdx, "numOfNodes", numOfNodes, "chunkCompletion", chunk.Completeness) + + switch chunk.Completeness { + case abci.Complete: // chunk is independent and complete + for idx := 0; idx < numOfNodes; idx++ { + node, _ := iavl.MakeNode(chunk.Nodes[idx]) + iavl.Hash(node) + nodes = append(nodes, node) + } + case abci.InComplete_First: + for idx := 0; idx < numOfNodes-1; idx++ { + if node, err := iavl.MakeNode(chunk.Nodes[idx]); err == nil { + iavl.Hash(node) + nodes = append(nodes, node) + } else { + return err + } + } + nodeIdx := chunk.StartIdx + int64(numOfNodes-1) + helper.incompleteChunks[nodeIdx] = append(helper.incompleteChunks[nodeIdx], + incompleteChunkItem{ + helper.hashesToIdx[hash], + chunk.Completeness, + chunk.Nodes[numOfNodes-1]}) + case abci.InComplete_Mid, abci.InComplete_Last: + if numOfNodes != 1 { + helper.logger.Error("incomplete chunk should has only one node", "hash", hash, "startIdx", chunk.StartIdx, "completeness", chunk.Completeness, "numOfNodes", numOfNodes) + } + + helper.incompleteChunks[chunk.StartIdx] = append(helper.incompleteChunks[chunk.StartIdx], incompleteChunkItem{helper.hashesToIdx[hash], chunk.Completeness, chunk.Nodes[0]}) + default: + helper.logger.Error("unknown completeness status", "hash", hash, "startIdx", chunk.StartIdx, "completeness", chunk.Completeness, "numOfNodes", numOfNodes) + } + + // write complete nodes right now + for idx, node := range nodes { + nodeIdx := chunk.StartIdx + int64(idx) + helper.saveNode(nodeIdx, node) + } + + helper.logger.Info("finished write recovery chunk", "isComplete", isComplete, "hash", fmt.Sprintf("%x", hash), "startIdx", chunk.StartIdx, "numOfNodes", numOfNodes, "chunkCompletion", chunk.Completeness) + } + + if isComplete { + err = helper.finishCompleteChunkWrite() + } + + return err +} + +func (helper *StateSyncHelper) DeleteSnapshot(height int64) error { + err := snapshot.ManagerAt(height).Delete() + helper.logger.Info("deleted snapshot", "height", height, "err", err) + return err +} + +func (helper *StateSyncHelper) finishCompleteChunkWrite() error { + helper.prepareEmptyStores() + if err := helper.saveIncompleteChunks(); err != nil { + return err + } + if err := helper.commitDB(); err != nil { + return err + } + + return nil +} + +func (helper *StateSyncHelper) prepareEmptyStores() { + for _, nodeDB := range helper.prefixNodeDBs { + if nodeDB.endIdxExclusive == nodeDB.startIdxInclusive { + nodeDB.NodeDB.SaveEmptyRoot(helper.manifest.Height, true) + helper.stateSyncStoreInfos = append(helper.stateSyncStoreInfos, StoreInfo{ + Name: nodeDB.storeName, + Core: StoreCore{ + CommitID: CommitID{ + Version: helper.manifest.Height, + Hash: nil, + }, + }, + }) + } + } +} + +func (helper *StateSyncHelper) saveIncompleteChunks() error { + for nodeIdx, chunkItems := range helper.incompleteChunks { + helper.logger.Debug("processing incomplete node", "nodeIdx", nodeIdx) + // sort and check chunkItems are valid + sort.Sort(&chunkItemSorter{chunkItems}) + + expectedNodeParts := chunkItems[len(chunkItems)-1].chunkIdx - chunkItems[0].chunkIdx + 1 + if expectedNodeParts != len(chunkItems) { + return fmt.Errorf("node parts are not complete, should be %d, but have %d, nodeIdx: %d", expectedNodeParts, len(chunkItems), nodeIdx) + } + + var completeNode bytes.Buffer + for idx, chunkItem := range chunkItems { + if idx == 0 { + if chunkItem.completeness != abci.InComplete_First { + return fmt.Errorf("first node part containing chunk's completeness %d is wrong, should be %d, nodeIdx: %d", chunkItem.completeness, abci.InComplete_First, nodeIdx) + } + } else if idx == len(chunkItems)-1 { + if chunkItem.completeness != abci.InComplete_Last { + return fmt.Errorf("last node part containing chunk's completeness %d is wrong, should be %d, nodeIdx: %d", chunkItem.completeness, abci.InComplete_Last, nodeIdx) + } + } else { + if chunkItem.completeness != abci.InComplete_Mid { + return fmt.Errorf("middle node part containing chunk's completeness %d is wrong, should be %d, nodeIdx: %d", chunkItem.completeness, abci.InComplete_Mid, nodeIdx) + } + } + completeNode.Write(chunkItem.nodePart) + } + + if node, err := iavl.MakeNode(completeNode.Bytes()); err == nil { + iavl.Hash(node) + helper.saveNode(nodeIdx, node) + } else { + return err + } + } + return nil +} + +func (helper *StateSyncHelper) commitDB() error { + height := helper.manifest.Height + + // TODO: revisit would it be problem commit too late? would there be memory or performance issue? + // probably we need commit as soon as store is complete + for _, db := range helper.prefixNodeDBs { + db.NodeDB.Commit() + } + + // simulate setLatestversion key + batch := helper.db.NewBatch() + latestBytes, _ := helper.cdc.MarshalBinaryLengthPrefixed(height) // Does not error + batch.Set([]byte("s/latest"), latestBytes) + + ci := CommitInfo{ + Version: height, + StoreInfos: helper.stateSyncStoreInfos, + } + if cInfoBytes, err := helper.cdc.MarshalBinaryLengthPrefixed(ci); err != nil { + return err + } else { + cInfoKey := fmt.Sprintf("s/%d", height) + batch.Set([]byte(cInfoKey), cInfoBytes) + batch.WriteSync() + return nil + } +} + +func (helper *StateSyncHelper) saveNode(nodeIdx int64, node *iavl.Node) { + for _, nodeDB := range helper.prefixNodeDBs { + if nodeIdx < nodeDB.endIdxExclusive { + if nodeIdx == nodeDB.startIdxInclusive { + nodeDB.NodeDB.SaveRoot(node, helper.manifest.Height, true) + rootHash := iavl.Hash(node) + helper.stateSyncStoreInfos = append(helper.stateSyncStoreInfos, StoreInfo{ + Name: nodeDB.storeName, + Core: StoreCore{ + CommitID: CommitID{ + Version: helper.manifest.Height, + Hash: rootHash, + }, + }, + }) + helper.logger.Info("save root hash", "store", nodeDB.storeName, "hash", fmt.Sprintf("%X", rootHash)) + } + iavl.Hash(node) + nodeDB.NodeDB.SaveNode(node) + helper.logger.Debug("saved node to store", "nodeIdx", nodeIdx, "store", nodeDB.storeName) + break + } + } +} + +// the method might take quite a while, BETTER to be called concurrently +// so we only do it once a day after breathe block +func (helper StateSyncHelper) ReloadSnapshotRoutine(height int64, retry int) { + helper.reloadingMtx.Lock() + defer helper.reloadingMtx.Unlock() + + helper.takeSnapshotImpl(height, retry) +} + +func (helper *StateSyncHelper) takeSnapshotImpl(height int64, retry int) { + defer func() { + if r := recover(); r != nil { + log := fmt.Sprintf("recovered: %v\nstack:\n%v", r, string(debug.Stack())) + helper.logger.Error("failed loading latest snapshot", "err", log) + } + }() + helper.logger.Info("reload latest snapshot", "height", height) + + for { + if mgr := snapshot.ManagerAt(height); mgr != nil { + helper.snapshotManager = mgr + break + } else { + helper.logger.Debug("waiting base snapshot manager is initialized") + time.Sleep(100 * time.Millisecond) + } + } + + if helper.snapshotManager.IsFinalized() { + return + } + + storeKeys := helper.getCommitedSortedStoreKeys() + + failed := true + for failed { + failed = false + if state := sm.LoadStateForHeight(helper.snapshotManager.GetStateDB(), height); state == nil { + helper.logger.Info("expected state has not committed yet", "height", height) + failed = true + time.Sleep(1 * time.Second) // Endblocker has notified this reload snapshot, + // wait for 1 sec after commit finish + if retry > 0 { + retry-- + continue + } else { + return + } + } + + totalKeys := int64(0) + numKeys := make([]int64, 0, len(storeKeys)) + currChunkNodes := make([][]byte, 0, 40000) // one account leaf node is around 100 bytes according to testnet experiment, non-leaf node should be less, 40000 should be a bit less than 4M + var currStartIdx int64 + var currChunkTotalBytes int + for _, key := range storeKeys { + var currStoreKeys int64 + store := helper.commitMS.GetKVStore(key) + // TODO: use Iterator method of store interface, no longer rely on implementation of KVStore + // as we only append storeKeys for IavlStore at constructor, so this type assertion should never fail + mutableTree := store.(*IavlStore).Tree + if tree, err := mutableTree.GetImmutable(height); err == nil { + tree.IterateFirst(func(nodeBytes []byte) { + nodeBytesLength := len(nodeBytes) + + if currChunkTotalBytes+nodeBytesLength <= abci.ChunkPayloadMaxBytes { + currChunkNodes = append(currChunkNodes, nodeBytes) + currChunkTotalBytes += nodeBytesLength + } else { + helper.finalizeAppStateChunk(currStartIdx, abci.Complete, currChunkNodes) + currStartIdx += int64(len(currChunkNodes)) + currChunkNodes = currChunkNodes[:0] + currChunkTotalBytes = 0 + + // One chunk should have AT MOST one incomplete node + // For a large node, we at most waste one chunk (the last finalized one) + if nodeBytesLength > abci.ChunkPayloadMaxBytes { + firstPart := nodeBytes[:abci.ChunkPayloadMaxBytes] + currChunkNodes = append(currChunkNodes, firstPart) + helper.finalizeAppStateChunk(currStartIdx, abci.InComplete_First, currChunkNodes) + + startCutIdx := len(firstPart) + for ; startCutIdx+abci.ChunkPayloadMaxBytes < nodeBytesLength; startCutIdx += abci.ChunkPayloadMaxBytes { + helper.finalizeAppStateChunk(totalKeys+currStoreKeys, abci.InComplete_Mid, [][]byte{nodeBytes[startCutIdx : startCutIdx+abci.ChunkPayloadMaxBytes]}) + } + + lastPart := nodeBytes[startCutIdx:] + helper.finalizeAppStateChunk(totalKeys+currStoreKeys, abci.InComplete_Last, [][]byte{lastPart}) + + currStartIdx = totalKeys + currStoreKeys + 1 + currChunkNodes = currChunkNodes[:0] + currChunkTotalBytes = 0 + } else { + currChunkNodes = append(currChunkNodes, nodeBytes) + currChunkTotalBytes += nodeBytesLength + } + } + + currStoreKeys++ + }) + helper.logger.Info("snapshoted a substore", "storeName", key, "numOfKeys", currStoreKeys) + } else { + helper.logger.Error("failed to load immutable tree", "err", err, "store", key) + return + } + totalKeys += currStoreKeys + numKeys = append(numKeys, currStoreKeys) + } + + if !failed { + if len(currChunkNodes) > 0 { + helper.finalizeAppStateChunk(currStartIdx, abci.Complete, currChunkNodes) + } + if err := helper.snapshotManager.SelfFinalize(numKeys); err == nil { + helper.logger.Info("finish read snapshot chunk", "height", height, "keys", totalKeys) + } else { + helper.logger.Error("failed read snapshot chunk", "height", height, "keys", totalKeys, "err", err) + } + } + } +} + +func (helper *StateSyncHelper) finalizeAppStateChunk(startIdx int64, completeness uint8, nodes [][]byte) error { + return helper.snapshotManager.WriteAppStateChunk(&abci.AppStateChunk{startIdx, completeness, nodes}) +} diff --git a/store/transientstore.go b/store/transientstore.go index d3364e45f..588fe5732 100644 --- a/store/transientstore.go +++ b/store/transientstore.go @@ -1,8 +1,9 @@ package store import ( - sdk "github.com/cosmos/cosmos-sdk/types" dbm "github.com/tendermint/tendermint/libs/db" + + sdk "github.com/cosmos/cosmos-sdk/types" ) var _ KVStore = (*transientStore)(nil) @@ -33,6 +34,9 @@ func (ts *transientStore) LastCommitID() (id CommitID) { return } +// Implements CommitStore +func (ts *transientStore) SetVersion(version int64) {} + // Implements KVStore func (ts *transientStore) Prefix(prefix []byte) KVStore { return prefixStore{ts, prefix} diff --git a/types/errors.go b/types/errors.go index 387217b84..1544c44fb 100644 --- a/types/errors.go +++ b/types/errors.go @@ -4,10 +4,10 @@ import ( "fmt" "strings" - "github.com/cosmos/cosmos-sdk/codec" + abci "github.com/tendermint/tendermint/abci/types" cmn "github.com/tendermint/tendermint/libs/common" - abci "github.com/tendermint/tendermint/abci/types" + "github.com/cosmos/cosmos-sdk/codec" ) // ABCICodeType - combined codetype / codespace @@ -56,6 +56,7 @@ const ( CodeInvalidCoins CodeType = 11 CodeMemoTooLarge CodeType = 12 CodeInsufficientFee CodeType = 13 + CodeMsgNotSupported CodeType = 14 // CodespaceRoot is a codespace for error codes in this file only. // Notice that 0 is an "unset" codespace, which can be overridden with @@ -100,6 +101,8 @@ func CodeToDefaultMsg(code CodeType) string { return "memo too large" case CodeInsufficientFee: return "insufficient fee" + case CodeMsgNotSupported: + return "msg not supported" default: return unknownCodeMsg(code) } @@ -146,6 +149,9 @@ func ErrInvalidCoins(msg string) Error { func ErrMemoTooLarge(msg string) Error { return newErrorWithRootCodespace(CodeMemoTooLarge, msg) } +func ErrMsgNotSupported(msg string) Error { + return newErrorWithRootCodespace(CodeMsgNotSupported, msg) +} //---------------------------------------- // Error & sdkError diff --git a/types/store.go b/types/store.go index c0c966bb0..90b70f787 100644 --- a/types/store.go +++ b/types/store.go @@ -13,43 +13,18 @@ import ( // NOTE: These are implemented in cosmos-sdk/store. // PruningStrategy specfies how old states will be deleted over time -type PruningStrategy interface { - ShouldPrune(version, latestVersion int64) bool -} - -// PruneSyncable defines the frequency (how many) of states would be saved, -// while all the others are not needed for state syncing and would be deleted. -type PruneSyncable struct { - // How many old versions we hold onto. - // A value of 0 means keep no recent states. - NumRecent int64 - - // This is the distance between state-sync waypoint states to be stored. - // See https://github.com/tendermint/tendermint/issues/828 - // A value of 1 means store every state. - // A value of 0 means store no waypoints. (node cannot assist in state-sync) - // By default this value should be set the same across all nodes, - // so that nodes can know the waypoints their peers store. - StoreEvery int64 -} - -func (strategy PruneSyncable) ShouldPrune(version, latestVersion int64) bool { - return (latestVersion-version > strategy.NumRecent) && (version%strategy.StoreEvery != 0) -} - -// PruneEverything means all saved states will be deleted, storing only the current state -type PruneEverything struct{} +type PruningStrategy uint8 -func (strategy PruneEverything) ShouldPrune(version, latestVersion int64) bool { - return true -} +const ( + // PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th) + PruneSyncable PruningStrategy = iota -// PruneNothing means all historic states will be saved, nothing will be deleted -type PruneNothing struct{} + // PruneEverything means all saved states will be deleted, storing only the current state + PruneEverything PruningStrategy = iota -func (strategy PruneNothing) ShouldPrune(version, latestVersion int64) bool { - return false -} + // PruneNothing means all historic states will be saved, nothing will be deleted + PruneNothing PruningStrategy = iota +) type Store interface { //nolint GetStoreType() StoreType @@ -61,6 +36,7 @@ type Committer interface { Commit() CommitID LastCommitID() CommitID SetPruning(PruningStrategy) + SetVersion(version int64) } // Stores of MultiStore must implement CommitStore. @@ -133,6 +109,8 @@ type CommitMultiStore interface { // Panics on a nil key. GetCommitKVStore(key StoreKey) CommitKVStore + GetCommitKVStores() map[StoreKey]CommitKVStore + // Load the latest persisted version. Called once after all // calls to Mount*Store() are complete. LoadLatestVersion() error diff --git a/types/upgrade.go b/types/upgrade.go index f9aa8dcc8..263870973 100644 --- a/types/upgrade.go +++ b/types/upgrade.go @@ -5,13 +5,13 @@ import "fmt" var UpgradeMgr = NewUpgradeManager(UpgradeConfig{}) var MainNetConfig = UpgradeConfig{ - HeightMap: map[string]int64{ - - }, + HeightMap: map[string]int64{}, } type UpgradeConfig struct { HeightMap map[string]int64 + StoreKeyMap map[string]int64 + MsgTypeMap map[string]int64 BeginBlockers map[int64][]func(ctx Context) } @@ -82,6 +82,52 @@ func (mgr *UpgradeManager) GetUpgradeHeight(name string) int64 { return mgr.Config.HeightMap[name] } +func (mgr *UpgradeManager) RegisterStoreKeys(upgradeName string, storeKeyNames ...string) { + height := mgr.GetUpgradeHeight(upgradeName) + if height == 0 { + panic(fmt.Errorf("no UpgradeHeight found for %s", upgradeName)) + } + + if mgr.Config.StoreKeyMap == nil { + mgr.Config.StoreKeyMap = map[string]int64{} + } + + for _, storeKeyName := range storeKeyNames { + mgr.Config.StoreKeyMap[storeKeyName] = height + } +} + +func (mgr *UpgradeManager) RegisterMsgTypes(upgradeName string, msgTypes ...string) { + height := mgr.GetUpgradeHeight(upgradeName) + if height == 0 { + panic(fmt.Errorf("no UpgradeHeight found for %s", upgradeName)) + } + + if mgr.Config.MsgTypeMap == nil { + mgr.Config.MsgTypeMap = map[string]int64{} + } + + for _, msgType := range msgTypes { + mgr.Config.MsgTypeMap[msgType] = height + } +} + +func (mgr *UpgradeManager) GetStoreKeyHeight(storeKeyName string) int64 { + if mgr.Config.StoreKeyMap == nil { + return 0 + } + + return mgr.Config.StoreKeyMap[storeKeyName] +} + +func (mgr *UpgradeManager) GetMsgTypeHeight(msgType string) int64 { + if mgr.Config.MsgTypeMap == nil { + return 0 + } + + return mgr.Config.MsgTypeMap[msgType] +} + func IsUpgradeHeight(name string) bool { upgradeHeight := UpgradeMgr.GetUpgradeHeight(name) if upgradeHeight == 0 { @@ -100,6 +146,33 @@ func IsUpgrade(name string) bool { return UpgradeMgr.GetHeight() >= upgradeHeight } +func ShouldCommitStore(storeKeyName string) bool { + storeKeyHeight := UpgradeMgr.GetStoreKeyHeight(storeKeyName) + if storeKeyHeight == 0 { + return true + } + + return UpgradeMgr.GetHeight() >= storeKeyHeight +} + +func ShouldSetStoreVersion(storeKeyName string) bool { + storeKeyHeight := UpgradeMgr.GetStoreKeyHeight(storeKeyName) + if storeKeyHeight == 0 { + return false + } + + return UpgradeMgr.GetHeight() == storeKeyHeight +} + +func IsMsgTypeSupported(msgType string) bool { + msgTypeHeight := UpgradeMgr.GetMsgTypeHeight(msgType) + if msgTypeHeight == 0 { + return true + } + + return UpgradeMgr.GetHeight() >= msgTypeHeight +} + func Upgrade(name string, before func(), in func(), after func()) { // if no special logic for the UpgradeHeight, than apply the `after` logic if in == nil { diff --git a/types/upgrade_test.go b/types/upgrade_test.go index c1946d048..26d262fef 100644 --- a/types/upgrade_test.go +++ b/types/upgrade_test.go @@ -7,6 +7,8 @@ import ( ) const UpgradeTest = "upgradeTest" +const StoreKeyNameTest = "storeKeyTest" +const MsgTypeTest = "msgTypeTest" func TestUpgrade(t *testing.T) { UpgradeMgr = NewUpgradeManager(UpgradeConfig{}) @@ -64,3 +66,111 @@ func TestUpgrade(t *testing.T) { require.Equal(t, tc.heightResult, IsUpgradeHeight(UpgradeTest)) } } + +func TestStoreKey(t *testing.T) { + UpgradeMgr = NewUpgradeManager(UpgradeConfig{}) + + type testCase struct { + config UpgradeConfig + height int64 + shouldCommit bool + } + + testCases := []testCase{ + { + config: UpgradeConfig{ + HeightMap: map[string]int64{}, + }, + height: 10000, + shouldCommit: true, + }, + { + config: UpgradeConfig{ + HeightMap: map[string]int64{ + UpgradeTest: 545000, + }, + }, + height: 10000, + shouldCommit: false, + }, { + config: UpgradeConfig{ + HeightMap: map[string]int64{ + UpgradeTest: 545000, + }, + }, + height: 545000, + shouldCommit: true, + }, { + config: UpgradeConfig{ + HeightMap: map[string]int64{ + UpgradeTest: 545000, + }, + }, + height: 545001, + shouldCommit: true, + }, + } + + for _, tc := range testCases { + UpgradeMgr.AddConfig(tc.config) + if UpgradeMgr.GetUpgradeHeight(UpgradeTest) != 0 { + UpgradeMgr.RegisterStoreKeys(UpgradeTest, StoreKeyNameTest) + } + UpgradeMgr.SetHeight(tc.height) + require.Equal(t, tc.shouldCommit, ShouldCommitStore(StoreKeyNameTest)) + } +} + +func TestMsgType(t *testing.T) { + UpgradeMgr = NewUpgradeManager(UpgradeConfig{}) + + type testCase struct { + config UpgradeConfig + height int64 + isSupported bool + } + + testCases := []testCase{ + { + config: UpgradeConfig{ + HeightMap: map[string]int64{}, + }, + height: 10000, + isSupported: true, + }, + { + config: UpgradeConfig{ + HeightMap: map[string]int64{ + UpgradeTest: 545000, + }, + }, + height: 10000, + isSupported: false, + }, { + config: UpgradeConfig{ + HeightMap: map[string]int64{ + UpgradeTest: 545000, + }, + }, + height: 545000, + isSupported: true, + }, { + config: UpgradeConfig{ + HeightMap: map[string]int64{ + UpgradeTest: 545000, + }, + }, + height: 545001, + isSupported: true, + }, + } + + for _, tc := range testCases { + UpgradeMgr.AddConfig(tc.config) + if UpgradeMgr.GetUpgradeHeight(UpgradeTest) != 0 { + UpgradeMgr.RegisterMsgTypes(UpgradeTest, MsgTypeTest) + } + UpgradeMgr.SetHeight(tc.height) + require.Equal(t, tc.isSupported, IsMsgTypeSupported(MsgTypeTest)) + } +} diff --git a/x/auth/client/cli/sign.go b/x/auth/client/cli/sign.go index 67c64dc2a..93408ae2b 100644 --- a/x/auth/client/cli/sign.go +++ b/x/auth/client/cli/sign.go @@ -39,7 +39,6 @@ recommended to set such parameters manually.`, cmd.Flags().String(client.FlagName, "", "Name of private key with which to sign") cmd.Flags().Bool(flagAppend, true, "Append the signature to the existing ones. If disabled, old signatures would be overwritten") cmd.Flags().Bool(flagPrintSigs, false, "Print the addresses that must sign the transaction and those who have already signed it, then exit") - cmd.Flags().Bool(flagOffline, false, "Offline mode. Do not query local cache.") return cmd } @@ -58,6 +57,9 @@ func makeSignCmd(cdc *amino.Codec, decoder auth.AccountDecoder) func(cmd *cobra. name := viper.GetString(client.FlagName) cliCtx := context.NewCLIContext().WithCodec(cdc).WithAccountDecoder(decoder) txBldr := authtxb.NewTxBuilderFromCLI() + if len(txBldr.ChainID) == 0 { + return fmt.Errorf("chain-id is missing") + } newTx, err := utils.SignStdTx(txBldr, cliCtx, name, stdTx, viper.GetBool(flagAppend), viper.GetBool(flagOffline)) if err != nil { diff --git a/x/bank/client/cli/sendtx.go b/x/bank/client/cli/sendtx.go index c72ed6fcb..c8abc3f64 100644 --- a/x/bank/client/cli/sendtx.go +++ b/x/bank/client/cli/sendtx.go @@ -15,8 +15,9 @@ import ( ) const ( - flagTo = "to" - flagAmount = "amount" + flagTo = "to" + flagAmount = "amount" + flagOffline = "offline" ) // SendTxCmd will create a send tx and sign it with the given key. @@ -53,20 +54,22 @@ func SendTxCmd(cdc *codec.Codec) *cobra.Command { return err } - account, err := cliCtx.GetAccount(from) - if err != nil { - return err - } + if !viper.GetBool(flagOffline) { + account, err := cliCtx.GetAccount(from) + if err != nil { + return err + } - // ensure account has enough coins - if !account.GetCoins().IsGTE(coins) { - return errors.Errorf("Address %s doesn't have enough coins to pay for this transaction.", from) + // ensure account has enough coins + if !account.GetCoins().IsGTE(coins) { + return errors.Errorf("Address %s doesn't have enough coins to pay for this transaction.", from) + } } // build and sign the transaction, then broadcast to Tendermint msg := client.CreateMsg(from, to, coins) if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) diff --git a/x/gov/client/cli/commands.go b/x/gov/client/cli/commands.go index 447c12003..84e03dd86 100644 --- a/x/gov/client/cli/commands.go +++ b/x/gov/client/cli/commands.go @@ -22,6 +22,7 @@ func AddCommands(cmd *cobra.Command, cdc *codec.Codec) { GetCmdDeposit(cdc), GetCmdSubmitProposal(cdc), GetCmdSubmitListProposal(cdc), + GetCmdSubmitDelistProposal(cdc), GetCmdVote(cdc), )..., ) diff --git a/x/gov/client/cli/tx.go b/x/gov/client/cli/tx.go index 373ab5360..a2b1e35d8 100644 --- a/x/gov/client/cli/tx.go +++ b/x/gov/client/cli/tx.go @@ -25,6 +25,7 @@ const ( flagProposalID = "proposal-id" flagTitle = "title" flagDescription = "description" + flagJustification = "justification" flagProposalType = "type" flagVotingPeriod = "voting-period" flagDeposit = "deposit" @@ -134,7 +135,7 @@ $ CLI gov submit-proposal --title="Test Proposal" --description="My awesome prop } if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // Build and sign the transaction, then broadcast to Tendermint @@ -216,7 +217,7 @@ func GetCmdDeposit(cdc *codec.Codec) *cobra.Command { } if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // Build and sign the transaction, then broadcast to a Tendermint @@ -262,7 +263,7 @@ func GetCmdVote(cdc *codec.Codec) *cobra.Command { } if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } fmt.Printf("Vote[Voter:%s,ProposalID:%d,Option:%s]", @@ -651,6 +652,10 @@ func GetCmdSubmitListProposal(cdc *codec.Codec) *cobra.Command { return err } + if cliCtx.GenerateOnly { + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) + } + cliCtx.PrintResponse = true return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) }, @@ -667,3 +672,97 @@ func GetCmdSubmitListProposal(cdc *codec.Codec) *cobra.Command { return cmd } + +// GetCmdSubmitDelistProposal implements submitting a delist proposal transaction command. +func GetCmdSubmitDelistProposal(cdc *codec.Codec) *cobra.Command { + cmd := &cobra.Command{ + Use: "submit-delist-proposal", + Short: "Submit a delist proposal along with an initial deposit", + RunE: func(cmd *cobra.Command, args []string) error { + txBldr := authtxb.NewTxBuilderFromCLI().WithCodec(cdc) + cliCtx := context.NewCLIContext(). + WithCodec(cdc). + WithAccountDecoder(authcmd.GetAccountDecoder(cdc)) + + title := viper.GetString(flagTitle) + justification := viper.GetString(flagJustification) + initialDeposit := viper.GetString(flagDeposit) + baseAsset := viper.GetString(flagBaseAsset) + quoteAsset := viper.GetString(flagQuoteAsset) + votingPeriodInSeconds := viper.GetInt64(flagVotingPeriod) + + if title == "" { + return errors.New("Title should not be empty") + } + + if len(title) > gov.MaxTitleLength { + return errors.New(fmt.Sprintf("Proposal title is longer than max length of %d", gov.MaxTitleLength)) + } + + if baseAsset == "" { + return errors.New("base asset should not be empty") + } + + if quoteAsset == "" { + return errors.New("quote asset should not be empty") + } + + if justification == "" { + return errors.New("justification should not be empty") + } + + if votingPeriodInSeconds <= 0 { + return errors.New("voting period should be positive") + } + + votingPeriod := time.Duration(votingPeriodInSeconds) * time.Second + if votingPeriod > gov.MaxVotingPeriod { + return fmt.Errorf("voting period should be less than %d seconds", gov.MaxVotingPeriod/time.Second) + } + + fromAddr, err := cliCtx.GetFromAddress() + if err != nil { + return err + } + + amount, err := sdk.ParseCoins(initialDeposit) + if err != nil { + return err + } + + delistParams := gov.DelistTradingPairParams{ + BaseAssetSymbol: baseAsset, + QuoteAssetSymbol: quoteAsset, + Justification: justification, + IsExecuted: false, + } + + delistParamsBz, err := json.Marshal(delistParams) + if err != nil { + return err + } + msg := gov.NewMsgSubmitProposal(title, string(delistParamsBz), gov.ProposalTypeDelistTradingPair, fromAddr, amount, votingPeriod) + + err = msg.ValidateBasic() + if err != nil { + return err + } + + if cliCtx.GenerateOnly { + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) + } + + cliCtx.PrintResponse = true + return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) + }, + } + + cmd.Flags().String(flagTitle, "", "title of proposal") + cmd.Flags().String(flagJustification, "", "justification of delist trading pair") + cmd.Flags().Int64(flagVotingPeriod, 7*24*60*60, "voting period in seconds") + cmd.Flags().String(flagDeposit, "", "deposit of proposal") + cmd.Flags().String(flagBaseAsset, "", "base asset symbol") + cmd.Flags().String(flagQuoteAsset, "", "quote asset symbol") + + return cmd +} diff --git a/x/gov/msgs.go b/x/gov/msgs.go index 390200b41..c35293d45 100644 --- a/x/gov/msgs.go +++ b/x/gov/msgs.go @@ -27,6 +27,14 @@ type ListTradingPairParams struct { ExpireTime time.Time `json:"expire_time"` // expire time } +//----------------------------------------------------------- +type DelistTradingPairParams struct { + BaseAssetSymbol string `json:"base_asset_symbol"` // base asset symbol + QuoteAssetSymbol string `json:"quote_asset_symbol"` // quote asset symbol + Justification string `json:"justification"` // justification + IsExecuted bool `json:"is_executed"` // is this proposal executed +} + //----------------------------------------------------------- // MsgSubmitProposal type MsgSubmitProposal struct { diff --git a/x/gov/msgs_test.go b/x/gov/msgs_test.go index 5563f7b80..d0d58166e 100644 --- a/x/gov/msgs_test.go +++ b/x/gov/msgs_test.go @@ -36,7 +36,7 @@ func TestMsgSubmitProposal(t *testing.T) { {"Test Proposal", "", gov.ProposalTypeText, addrs[0], coinsPos, 1000 * time.Second, false}, {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeParameterChange, addrs[0], coinsPos, 1000 * time.Second, true}, {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeSoftwareUpgrade, addrs[0], coinsPos, 1000 * time.Second, true}, - {"Test Proposal", "the purpose of this proposal is to test", 0x08, addrs[0], coinsPos, 1, false}, + {"Test Proposal", "the purpose of this proposal is to test", 0x10, addrs[0], coinsPos, 1, false}, {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeText, sdk.AccAddress{}, coinsPos, 1000 * time.Second, false}, {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeText, addrs[0], coinsZero, 1000 * time.Second, true}, {"Test Proposal", "the purpose of this proposal is to test", gov.ProposalTypeText, addrs[0], coinsNeg, 1000 * time.Second, false}, diff --git a/x/gov/proposals.go b/x/gov/proposals.go index e7a1fbf26..a28f0383f 100644 --- a/x/gov/proposals.go +++ b/x/gov/proposals.go @@ -126,9 +126,10 @@ const ( ProposalTypeSoftwareUpgrade ProposalKind = 0x03 ProposalTypeListTradingPair ProposalKind = 0x04 // ProposalTypeFeeChange belongs to ProposalTypeParameterChange. We use this to make it easily to distinguish。 - ProposalTypeFeeChange ProposalKind = 0x05 - ProposalTypeCreateValidator ProposalKind = 0x06 - ProposalTypeRemoveValidator ProposalKind = 0x07 + ProposalTypeFeeChange ProposalKind = 0x05 + ProposalTypeCreateValidator ProposalKind = 0x06 + ProposalTypeRemoveValidator ProposalKind = 0x07 + ProposalTypeDelistTradingPair ProposalKind = 0x08 ) // String to proposalType byte. Returns ff if invalid. @@ -148,6 +149,8 @@ func ProposalTypeFromString(str string) (ProposalKind, error) { return ProposalTypeCreateValidator, nil case "RemoveValidator": return ProposalTypeRemoveValidator, nil + case "DelistTradingPair": + return ProposalTypeDelistTradingPair, nil default: return ProposalKind(0xff), errors.Errorf("'%s' is not a valid proposal type", str) } @@ -161,7 +164,8 @@ func validProposalType(pt ProposalKind) bool { pt == ProposalTypeListTradingPair || pt == ProposalTypeFeeChange || pt == ProposalTypeCreateValidator || - pt == ProposalTypeRemoveValidator { + pt == ProposalTypeRemoveValidator || + pt == ProposalTypeDelistTradingPair { return true } return false @@ -216,6 +220,8 @@ func (pt ProposalKind) String() string { return "CreateValidator" case ProposalTypeRemoveValidator: return "RemoveValidator" + case ProposalTypeDelistTradingPair: + return "DelistTradingPair" default: return "" } diff --git a/x/gov/simulation/sim_test.go b/x/gov/simulation/sim_test.go index f19b16482..e7cdbf1ae 100644 --- a/x/gov/simulation/sim_test.go +++ b/x/gov/simulation/sim_test.go @@ -30,7 +30,7 @@ func TestGovWithRandomMessages(t *testing.T) { paramKey := sdk.NewKVStoreKey("params") paramTKey := sdk.NewTransientStoreKey("transient_params") paramKeeper := params.NewKeeper(mapp.Cdc, paramKey, paramTKey) - stakeKeeper := stake.NewKeeper(mapp.Cdc, stakeKey, stakeTKey, bankKeeper, paramKeeper.Subspace(stake.DefaultParamspace), stake.DefaultCodespace) + stakeKeeper := stake.NewKeeper(mapp.Cdc, stakeKey, stakeTKey, bankKeeper, nil, paramKeeper.Subspace(stake.DefaultParamspace), stake.DefaultCodespace) govKey := sdk.NewKVStoreKey("gov") govKeeper := gov.NewKeeper(mapp.Cdc, govKey, paramKeeper, paramKeeper.Subspace(gov.DefaultParamSpace), bankKeeper, stakeKeeper, gov.DefaultCodespace, &sdk.Pool{}) mapp.Router().AddRoute("gov", gov.NewHandler(govKeeper)) diff --git a/x/ibc/client/cli/ibctx.go b/x/ibc/client/cli/ibctx.go index be6dfc940..3b024d9f9 100644 --- a/x/ibc/client/cli/ibctx.go +++ b/x/ibc/client/cli/ibctx.go @@ -42,7 +42,7 @@ func IBCTransferCmd(cdc *codec.Codec) *cobra.Command { return err } if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) diff --git a/x/slashing/client/cli/tx.go b/x/slashing/client/cli/tx.go index f70be1871..a3508f880 100644 --- a/x/slashing/client/cli/tx.go +++ b/x/slashing/client/cli/tx.go @@ -31,7 +31,7 @@ func GetCmdUnjail(cdc *codec.Codec) *cobra.Command { msg := slashing.NewMsgUnjail(sdk.ValAddress(valAddr)) if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) }, diff --git a/x/stake/client/cli/flags.go b/x/stake/client/cli/flags.go index fc6834a2c..aeb38fc5b 100644 --- a/x/stake/client/cli/flags.go +++ b/x/stake/client/cli/flags.go @@ -27,6 +27,7 @@ const ( FlagCommissionMaxChangeRate = "commission-max-change-rate" FlagGenesisFormat = "genesis-format" + FlagOffline = "offline" FlagNodeID = "node-id" FlagIP = "ip" diff --git a/x/stake/client/cli/tx.go b/x/stake/client/cli/tx.go index 12f32c1f9..7f48d4a2c 100644 --- a/x/stake/client/cli/tx.go +++ b/x/stake/client/cli/tx.go @@ -100,7 +100,6 @@ func GetCmdCreateValidator(cdc *codec.Codec) *cobra.Command { } proposalId := viper.GetInt64(FlagProposalID) - if proposalId == -1 { depositStr := viper.GetString(FlagDeposit) if depositStr == "" { @@ -137,7 +136,9 @@ func GetCmdCreateValidator(cdc *codec.Codec) *cobra.Command { } if viper.GetBool(FlagGenesisFormat) || cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, true) + //Enable offline mode + viper.Set(FlagOffline, true) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // build and sign the transaction, then broadcast to Tendermint @@ -218,7 +219,7 @@ func GetCmdRemoveValidator(cdc *codec.Codec) *cobra.Command { } if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // build and sign the transaction, then broadcast to Tendermint return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) @@ -272,7 +273,7 @@ func GetCmdEditValidator(cdc *codec.Codec) *cobra.Command { msg := stake.NewMsgEditValidator(sdk.ValAddress(valAddr), description, newRate) if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // build and sign the transaction, then broadcast to Tendermint @@ -315,7 +316,7 @@ func GetCmdDelegate(cdc *codec.Codec) *cobra.Command { msg := stake.NewMsgDelegate(delAddr, valAddr, amount) if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // build and sign the transaction, then broadcast to Tendermint return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) @@ -385,7 +386,7 @@ func GetCmdBeginRedelegate(storeName string, cdc *codec.Codec) *cobra.Command { msg := stake.NewMsgBeginRedelegate(delAddr, valSrcAddr, valDstAddr, sharesAmount) if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // build and sign the transaction, then broadcast to Tendermint return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) @@ -448,7 +449,7 @@ func GetCmdBeginUnbonding(storeName string, cdc *codec.Codec) *cobra.Command { msg := stake.NewMsgBeginUnbonding(delAddr, valAddr, sharesAmount) if cliCtx.GenerateOnly { - return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}, false) + return utils.PrintUnsignedStdTx(txBldr, cliCtx, []sdk.Msg{msg}) } // build and sign the transaction, then broadcast to Tendermint return utils.CompleteAndBroadcastTxCli(txBldr, cliCtx, []sdk.Msg{msg}) diff --git a/x/stake/handler.go b/x/stake/handler.go index 6ede6c536..71fae4753 100644 --- a/x/stake/handler.go +++ b/x/stake/handler.go @@ -59,6 +59,7 @@ func NewStakeHandler(k Keeper) sdk.Handler { // Called every block, update validator set func EndBlocker(ctx sdk.Context, k keeper.Keeper) (ValidatorUpdates []abci.ValidatorUpdate, completedUnbondingDelegations []types.UnbondingDelegation) { endBlockerTags := sdk.EmptyTags() + logger := ctx.Logger().With("module", "stake") k.UnbondAllMatureValidatorQueue(ctx) @@ -66,10 +67,12 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper) (ValidatorUpdates []abci.Valid for _, dvPair := range matureUnbonds { ubd, found := k.GetUnbondingDelegation(ctx, dvPair.DelegatorAddr, dvPair.ValidatorAddr) if !found { + logger.Error("Failed to get unbonding delegation", "delegator_address",dvPair.DelegatorAddr.String(), "validator_address", dvPair.ValidatorAddr.String()) continue } err := k.CompleteUnbonding(ctx, dvPair.DelegatorAddr, dvPair.ValidatorAddr) if err != nil { + logger.Error(fmt.Sprintf("Failed to complete unbonding delegation: %s", err.Error()), "delegator_address",dvPair.DelegatorAddr.String(), "validator_address", dvPair.ValidatorAddr.String()) continue } completedUnbondingDelegations = append(completedUnbondingDelegations, ubd) @@ -84,6 +87,7 @@ func EndBlocker(ctx sdk.Context, k keeper.Keeper) (ValidatorUpdates []abci.Valid for _, dvvTriplet := range matureRedelegations { err := k.CompleteRedelegation(ctx, dvvTriplet.DelegatorAddr, dvvTriplet.ValidatorSrcAddr, dvvTriplet.ValidatorDstAddr) if err != nil { + logger.Error(fmt.Sprintf("Failed to complete redelegation: %s", err.Error()), "delegator_address",dvvTriplet.DelegatorAddr.String(), "source_validator_address", dvvTriplet.ValidatorSrcAddr.String(), "source_validator_address", dvvTriplet.ValidatorDstAddr.String()) continue } endBlockerTags.AppendTags(sdk.NewTags( diff --git a/x/stake/simulation/sim_test.go b/x/stake/simulation/sim_test.go index d9835e899..c2b029abd 100644 --- a/x/stake/simulation/sim_test.go +++ b/x/stake/simulation/sim_test.go @@ -33,11 +33,11 @@ func TestStakeWithRandomMessages(t *testing.T) { feeCollectionKeeper := auth.NewFeeCollectionKeeper(mapp.Cdc, feeKey) paramstore := params.NewKeeper(mapp.Cdc, paramsKey, paramsTKey) - stakeKeeper := stake.NewKeeper(mapp.Cdc, stakeKey, stakeTKey, bankKeeper, paramstore.Subspace(stake.DefaultParamspace), stake.DefaultCodespace) + stakeKeeper := stake.NewKeeper(mapp.Cdc, stakeKey, stakeTKey, bankKeeper, nil, paramstore.Subspace(stake.DefaultParamspace), stake.DefaultCodespace) distrKeeper := distribution.NewKeeper(mapp.Cdc, distrKey, paramstore.Subspace(distribution.DefaultParamspace), bankKeeper, stakeKeeper, feeCollectionKeeper, distribution.DefaultCodespace) mapp.Router().AddRoute("stake", stake.NewStakeHandler(stakeKeeper)) mapp.SetEndBlocker(func(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock { - validatorUpdates := stake.EndBlocker(ctx, stakeKeeper) + validatorUpdates, _ := stake.EndBlocker(ctx, stakeKeeper) return abci.ResponseEndBlock{ ValidatorUpdates: validatorUpdates, }