From 9405a05c78a32130236482f8614a9e1095fe25ee Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 07:32:08 +0200 Subject: [PATCH] feat: Add a cli cmd to prune old states according to current settings (backport #12742) (#13119) * feat: Add a cli cmd to prune old states according to current settings (#12742) * add PruningCmd and change PruneStores signature * the mimimum default pruning interval is 10 Co-authored-by: Marko (cherry picked from commit d874acee4cf1a32a9078967ed35807a8eb647609) * fix backport error Co-authored-by: adu-crypto <94821467+adu-crypto@users.noreply.github.com> Co-authored-by: adu --- CHANGELOG.md | 1 + client/pruning/main.go | 117 +++++++++++++++++++++++++++++++++++++++ simapp/simd/cmd/root.go | 4 +- store/rootmulti/store.go | 33 +++++++---- 4 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 client/pruning/main.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e380ab60a34..799f9e9d8daf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Features * (x/authz) [#13047](https://github.com/cosmos/cosmos-sdk/pull/13047) Add a GetAuthorization function to the keeper. +* (cli) [#12742](https://github.com/cosmos/cosmos-sdk/pull/12742) Add the `prune` CLI cmd to manually prune app store history versions based on the pruning options. ### Improvements diff --git a/client/pruning/main.go b/client/pruning/main.go new file mode 100644 index 000000000000..fb62dd21a050 --- /dev/null +++ b/client/pruning/main.go @@ -0,0 +1,117 @@ +package pruning + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cosmos/cosmos-sdk/client/flags" + pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types" + "github.com/cosmos/cosmos-sdk/server" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + "github.com/cosmos/cosmos-sdk/store/rootmulti" + "github.com/tendermint/tendermint/libs/log" + dbm "github.com/tendermint/tm-db" +) + +const FlagAppDBBackend = "app-db-backend" + +// PruningCmd prunes the sdk root multi store history versions based on the pruning options +// specified by command flags. +func PruningCmd(appCreator servertypes.AppCreator) *cobra.Command { + cmd := &cobra.Command{ + Use: "prune", + Short: "Prune app history states by keeping the recent heights and deleting old heights", + Long: `Prune app history states by keeping the recent heights and deleting old heights. + The pruning option is provided via the '--pruning' flag or alternatively with '--pruning-keep-recent' + + For '--pruning' the options are as follows: + + default: the last 362880 states are kept + nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node) + everything: 2 latest states will be kept + custom: allow pruning options to be manually specified through 'pruning-keep-recent'. + besides pruning options, database home directory and database backend type should also be specified via flags + '--home' and '--app-db-backend'. + valid app-db-backend type includes 'goleveldb', 'cleveldb', 'rocksdb', 'boltdb', and 'badgerdb'. + `, + Example: "prune --home './' --app-db-backend 'goleveldb' --pruning 'custom' --pruning-keep-recent 100", + RunE: func(cmd *cobra.Command, _ []string) error { + vp := viper.New() + + // Bind flags to the Context's Viper so we can get pruning options. + if err := vp.BindPFlags(cmd.Flags()); err != nil { + return err + } + pruningOptions, err := server.GetPruningOptionsFromFlags(vp) + if err != nil { + return err + } + fmt.Printf("get pruning options from command flags, strategy: %v, keep-recent: %v\n", + pruningOptions.Strategy, + pruningOptions.KeepRecent, + ) + + home := vp.GetString(flags.FlagHome) + db, err := openDB(home, server.GetAppDBBackend(vp)) + if err != nil { + return err + } + + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + app := appCreator(logger, db, nil, vp) + cms := app.CommitMultiStore() + + rootMultiStore, ok := cms.(*rootmulti.Store) + if !ok { + return fmt.Errorf("currently only support the pruning of rootmulti.Store type") + } + latestHeight := rootmulti.GetLatestVersion(db) + // valid heights should be greater than 0. + if latestHeight <= 0 { + return fmt.Errorf("the database has no valid heights to prune, the latest height: %v", latestHeight) + } + + var pruningHeights []int64 + for height := int64(1); height < latestHeight; height++ { + if height < latestHeight-int64(pruningOptions.KeepRecent) { + pruningHeights = append(pruningHeights, height) + } + } + if len(pruningHeights) == 0 { + fmt.Printf("no heights to prune\n") + return nil + } + fmt.Printf( + "pruning heights start from %v, end at %v\n", + pruningHeights[0], + pruningHeights[len(pruningHeights)-1], + ) + + err = rootMultiStore.PruneStores(false, pruningHeights) + if err != nil { + return err + } + fmt.Printf("successfully pruned the application root multi stores\n") + return nil + }, + } + + cmd.Flags().String(flags.FlagHome, "", "The database home directory") + cmd.Flags().String(FlagAppDBBackend, "", "The type of database for application and snapshots databases") + cmd.Flags().String(server.FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") + cmd.Flags().Uint64(server.FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(server.FlagPruningInterval, 10, + `Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom'), + this is not used by this command but kept for compatibility with the complete pruning options`) + + return cmd +} + +func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) { + dataDir := filepath.Join(rootDir, "data") + return dbm.NewDB("application", backendType, dataDir) +} diff --git a/simapp/simd/cmd/root.go b/simapp/simd/cmd/root.go index f659caab2b62..b04417d4d193 100644 --- a/simapp/simd/cmd/root.go +++ b/simapp/simd/cmd/root.go @@ -19,6 +19,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/debug" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/keys" + "github.com/cosmos/cosmos-sdk/client/pruning" "github.com/cosmos/cosmos-sdk/client/rpc" "github.com/cosmos/cosmos-sdk/server" serverconfig "github.com/cosmos/cosmos-sdk/server/config" @@ -156,6 +157,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { cfg := sdk.GetConfig() cfg.Seal() + a := appCreator{encodingConfig} rootCmd.AddCommand( genutilcli.InitCmd(simapp.ModuleBasics, simapp.DefaultNodeHome), genutilcli.CollectGenTxsCmd(banktypes.GenesisBalancesIterator{}, simapp.DefaultNodeHome), @@ -167,9 +169,9 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { NewTestnetCmd(simapp.ModuleBasics, banktypes.GenesisBalancesIterator{}), debug.Cmd(), config.Cmd(), + pruning.PruningCmd(a.newApp), ) - a := appCreator{encodingConfig} server.AddCommands(rootCmd, simapp.DefaultNodeHome, a.newApp, a.appExport, addModuleInitFlags) // add keybase, auxiliary RPC, query, and tx child commands diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 0da37a788a7d..0eafdf7af7d4 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -162,7 +162,7 @@ func (rs *Store) StoreKeysByName() map[string]types.StoreKey { // LoadLatestVersionAndUpgrade implements CommitMultiStore func (rs *Store) LoadLatestVersionAndUpgrade(upgrades *types.StoreUpgrades) error { - ver := getLatestVersion(rs.db) + ver := GetLatestVersion(rs.db) return rs.loadVersion(ver, upgrades) } @@ -173,7 +173,7 @@ func (rs *Store) LoadVersionAndUpgrade(ver int64, upgrades *types.StoreUpgrades) // LoadLatestVersion implements CommitMultiStore. func (rs *Store) LoadLatestVersion() error { - ver := getLatestVersion(rs.db) + ver := GetLatestVersion(rs.db) return rs.loadVersion(ver, nil) } @@ -391,7 +391,7 @@ func (rs *Store) ListeningEnabled(key types.StoreKey) bool { func (rs *Store) LastCommitID() types.CommitID { if rs.lastCommitInfo == nil { return types.CommitID{ - Version: getLatestVersion(rs.db), + Version: GetLatestVersion(rs.db), } } @@ -539,17 +539,28 @@ func (rs *Store) handlePruning(version int64) error { } rs.logger.Info("prune start", "height", version) defer rs.logger.Info("prune end", "height", version) - return rs.pruneStores() + return rs.PruneStores(true, nil) } -func (rs *Store) pruneStores() error { - pruningHeights, err := rs.pruningManager.GetFlushAndResetPruningHeights() - if err != nil { - return err +// PruneStores prunes the specific heights of the multi store. +// If clearPruningManager is true, the pruning manager will return the pruning heights, +// and they are appended to the pruningHeights to be pruned. +func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (err error) { + if clearPruningManager { + heights, err := rs.pruningManager.GetFlushAndResetPruningHeights() + if err != nil { + return err + } + + if len(heights) == 0 { + rs.logger.Debug("no heights to be pruned from pruning manager") + } + + pruningHeights = append(pruningHeights, heights...) } if len(pruningHeights) == 0 { - rs.logger.Debug("pruning skipped; no heights to prune") + rs.logger.Debug("no heights need to be pruned") return nil } @@ -689,7 +700,7 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error { if height == 0 { return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0") } - if height > uint64(getLatestVersion(rs.db)) { + if height > uint64(GetLatestVersion(rs.db)) { return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height) } @@ -972,7 +983,7 @@ type storeParams struct { initialVersion uint64 } -func getLatestVersion(db dbm.DB) int64 { +func GetLatestVersion(db dbm.DB) int64 { bz, err := db.Get([]byte(latestVersionKey)) if err != nil { panic(err)