From dd940e568bcfaa9d00388a9b499106c21a8ed30c Mon Sep 17 00:00:00 2001 From: Kartik Bhat Date: Wed, 30 Oct 2024 14:13:40 -0400 Subject: [PATCH] Archive Node Online Migration (#1863) * Archive Node Online Migration * Add migrate-iavl flag for online migration * Update sei cosmos * Add QMS for online migration (#1870) * Add QMS for online migration * Fix lint --------- Co-authored-by: kbhat1 * Add migrate-height flag to start cmd * Bump seidb * Reduce logging * Archive Migration doc * Update form * Update * Export metric * latest height * Update start module logic * Remove migrate SS command * Enable ss store * Update archive node migration docs * More migration doc updates * Update metrics in readme * Merge main * Remove dex * Add Faq section * Add requirements section * Systemd instructions * Minor add to readme * Add more examples for migration height * Bump sei cosmos * Update error log * Update command sc migration background --------- Co-authored-by: Yiming Zang <50607998+yzang2019@users.noreply.github.com> --- app/app.go | 7 +- app/seidb.go | 18 +- app/test_state_store.go | 20 ++ cmd/seid/cmd/root.go | 20 +- docs/migration/seidb_archive_migration.md | 212 ++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- tools/migration/cmd/cmd.go | 41 ++--- tools/migration/sc/migrator.go | 6 +- tools/migration/ss/migrator.go | 121 +++++++----- 10 files changed, 369 insertions(+), 82 deletions(-) create mode 100644 docs/migration/seidb_archive_migration.md diff --git a/app/app.go b/app/app.go index a159cfd56..00aa5fe32 100644 --- a/app/app.go +++ b/app/app.go @@ -368,6 +368,7 @@ type App struct { genesisImportConfig genesistypes.GenesisImportConfig + stateStore seidb.StateStore receiptStore seidb.StateStore } @@ -396,7 +397,7 @@ func New( cdc := encodingConfig.Amino interfaceRegistry := encodingConfig.InterfaceRegistry - bAppOptions := SetupSeiDB(logger, homePath, appOpts, baseAppOptions) + bAppOptions, stateStore := SetupSeiDB(logger, homePath, appOpts, baseAppOptions) bApp := baseapp.NewBaseApp(AppName, logger, db, encodingConfig.TxConfig.TxDecoder(), tmConfig, appOpts, bAppOptions...) bApp.SetCommitMultiStoreTracer(traceStore) @@ -429,6 +430,7 @@ func New( versionInfo: version.NewInfo(), metricCounter: &map[string]float32{}, encodingConfig: encodingConfig, + stateStore: stateStore, } for _, option := range appOptions { @@ -1064,6 +1066,9 @@ func (app *App) Name() string { return app.BaseApp.Name() } // GetBaseApp returns the base app of the application func (app App) GetBaseApp() *baseapp.BaseApp { return app.BaseApp } +// GetStateStore returns the state store of the application +func (app App) GetStateStore() seidb.StateStore { return app.stateStore } + // BeginBlocker application updates every begin block func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock { metrics.GaugeSeidVersionAndCommit(app.versionInfo.Version, app.versionInfo.GitCommit) diff --git a/app/seidb.go b/app/seidb.go index d51aadc6e..175d5c1a4 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -7,6 +7,7 @@ import ( servertypes "github.com/cosmos/cosmos-sdk/server/types" "github.com/cosmos/cosmos-sdk/storev2/rootmulti" "github.com/sei-protocol/sei-db/config" + seidb "github.com/sei-protocol/sei-db/ss/types" "github.com/spf13/cast" "github.com/tendermint/tendermint/libs/log" ) @@ -33,6 +34,8 @@ const ( // Other configs FlagSnapshotInterval = "state-sync.snapshot-interval" + FlagMigrateIAVL = "migrate-iavl" + FlagMigrateHeight = "migrate-height" ) func SetupSeiDB( @@ -40,11 +43,11 @@ func SetupSeiDB( homePath string, appOpts servertypes.AppOptions, baseAppOptions []func(*baseapp.BaseApp), -) []func(*baseapp.BaseApp) { +) ([]func(*baseapp.BaseApp), seidb.StateStore) { scEnabled := cast.ToBool(appOpts.Get(FlagSCEnable)) if !scEnabled { logger.Info("SeiDB is disabled, falling back to IAVL") - return baseAppOptions + return baseAppOptions, nil } logger.Info("SeiDB SC is enabled, running node with StoreV2 commit store") scConfig := parseSCConfigs(appOpts) @@ -56,14 +59,21 @@ func SetupSeiDB( // cms must be overridden before the other options, because they may use the cms, // make sure the cms aren't be overridden by the other options later on. - cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig, false) + cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig, cast.ToBool(appOpts.Get("migrate-iavl"))) + migrationEnabled := cast.ToBool(appOpts.Get(FlagMigrateIAVL)) + migrationHeight := cast.ToInt64(appOpts.Get(FlagMigrateHeight)) baseAppOptions = append([]func(*baseapp.BaseApp){ func(baseApp *baseapp.BaseApp) { + if migrationEnabled { + originalCMS := baseApp.CommitMultiStore() + baseApp.SetQueryMultiStore(originalCMS) + baseApp.SetMigrationHeight(migrationHeight) + } baseApp.SetCMS(cms) }, }, baseAppOptions...) - return baseAppOptions + return baseAppOptions, cms.GetStateStore() } func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig { diff --git a/app/test_state_store.go b/app/test_state_store.go index 15b4d63e1..a5b071145 100644 --- a/app/test_state_store.go +++ b/app/test_state_store.go @@ -244,6 +244,26 @@ func (s *InMemoryStateStore) RawImport(ch <-chan types.RawSnapshotNode) error { return nil } +func (s *InMemoryStateStore) SetLatestMigratedModule(module string) error { + // TODO: Add set call here + return nil +} + +func (s *InMemoryStateStore) GetLatestMigratedModule() (string, error) { + // TODO: Add get call here + return "", nil +} + +func (s *InMemoryStateStore) SetLatestMigratedKey(key []byte) error { + // TODO: Add set call here + return nil +} + +func (s *InMemoryStateStore) GetLatestMigratedKey() ([]byte, error) { + // TODO: Add get call here + return nil, nil +} + func (s *InMemoryStateStore) Prune(version int64) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/cmd/seid/cmd/root.go b/cmd/seid/cmd/root.go index df8ed1fe1..81c7f8203 100644 --- a/cmd/seid/cmd/root.go +++ b/cmd/seid/cmd/root.go @@ -38,6 +38,7 @@ import ( "github.com/sei-protocol/sei-chain/app/params" "github.com/sei-protocol/sei-chain/evmrpc" "github.com/sei-protocol/sei-chain/tools" + "github.com/sei-protocol/sei-chain/tools/migration/ss" "github.com/sei-protocol/sei-chain/x/evm/blocktest" "github.com/sei-protocol/sei-chain/x/evm/querier" "github.com/sei-protocol/sei-chain/x/evm/replay" @@ -220,6 +221,8 @@ func txCommand() *cobra.Command { func addModuleInitFlags(startCmd *cobra.Command) { crisis.AddModuleInitFlags(startCmd) + startCmd.Flags().Bool("migrate-iavl", false, "Run migration of IAVL data store to SeiDB State Store") + startCmd.Flags().Int64("migrate-height", 0, "Height at which to start the migration") } // newApp creates a new Cosmos SDK app @@ -266,7 +269,7 @@ func newApp( // This makes it such that the wasm VM gas converts to sdk gas at a 6.66x rate vs that of the previous multiplier wasmGasRegisterConfig.GasMultiplier = 21_000_000 - return app.New( + app := app.New( logger, db, traceStore, @@ -302,6 +305,21 @@ func newApp( baseapp.SetSnapshotDirectory(cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))), baseapp.SetOccEnabled(cast.ToBool(appOpts.Get(baseapp.FlagOccEnabled))), ) + + // Start migration if --migrate flag is set + if cast.ToBool(appOpts.Get("migrate-iavl")) { + go func() { + homeDir := cast.ToString(appOpts.Get(flags.FlagHome)) + stateStore := app.GetStateStore() + migrationHeight := cast.ToInt64(appOpts.Get("migrate-height")) + migrator := ss.NewMigrator(homeDir, db, stateStore) + if err := migrator.Migrate(migrationHeight, homeDir); err != nil { + panic(err) + } + }() + } + + return app } // appExport creates a new simapp (optionally at a given height) diff --git a/docs/migration/seidb_archive_migration.md b/docs/migration/seidb_archive_migration.md new file mode 100644 index 000000000..9d2747951 --- /dev/null +++ b/docs/migration/seidb_archive_migration.md @@ -0,0 +1,212 @@ +# SeiDB Archive Migration Guide + +## Overview +SeiDB is the next generation of chain storage in SeiV2. +One issue for running SeiDB on archive nodes is that we need to keep the full state of the chain, so we can't +state sync it and clear out previous iavl data. + +In order to run an archive node with SeiDB, we need to run a migration from iavl to sei-db. + +The overall process will work as follows: + +1. Update config to enable SeiDB (state committment + state store) +2. Stop the node and Run SC Migration +3. Note down MIGRATION_HEIGHT +4. Re start seid with `--migrate-iavl` enabled (migrating state store in background) +5. Verify migration at various sampled heights once state store is complete +6. Restart seid normally and verify node runs properly +7. Clear out iavl and restart seid normally, now only using SeiDB fully + + +## Requirements + +### Additional Disk Space + +Ensure you have at least 10TB of free disk space available for the migration process. This extra space is needed because during migration, both IAVL and SeiDB state stores will exist simultaneously. Note +that this is ONLY during the migration process, and will not be necessary after completion + + +## Migration Steps + +### Step 1: Add SeiDB Configurations +We can enable SeiDB by adding the following configs to app.toml file. +Usually you can find this file under ~/.sei/config/app.toml. +```bash +############################################################################# +### SeiDB Configuration ### +############################################################################# + +[state-commit] +# Enable defines if the SeiDB should be enabled to override existing IAVL db backend. +sc-enable = true + +# AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up +# performance, <=0 means synchronous commit. +sc-async-commit-buffer = 100 + +# SnapshotKeepRecent defines how many memiavl snapshots (beyond the latest one) to keep +# Recommend to set to 1 to make sure IBC relayers work. +sc-keep-recent = 1 + +# SnapshotInterval defines the number of blocks interval the memiavl snapshot is taken, default to 10000 blocks. +# Adjust based on your needs: +# Setting this too low could lead to lot of extra heavy disk IO +# Setting this too high could lead to slow restart +sc-snapshot-interval = 10000 + +# SnapshotWriterLimit defines the max concurrency for taking memiavl snapshot +sc-snapshot-writer-limit = 1 + +# CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000. +sc-cache-size = 100000 + +[state-store] +# Enable defines if the state-store should be enabled for historical queries. +# In order to use state-store, you need to make sure to enable state-commit at the same time. +# Validator nodes should turn this off. +# State sync node or full nodes should turn this on. +ss-enable = true + +# DBBackend defines the backend database used for state-store. +# Supported backends: pebbledb, rocksdb +# defaults to pebbledb (recommended) +ss-backend = "pebbledb" + +# AsyncWriteBuffer defines the async queue length for commits to be applied to State Store +# Set <= 0 for synchronous writes, which means commits also need to wait for data to be persisted in State Store. +# defaults to 100 +ss-async-write-buffer = 100 + +# KeepRecent defines the number of versions to keep in state store +# Setting it to 0 means keep everything, default to 100000 +ss-keep-recent = 0 + +# PruneIntervalSeconds defines the minimum interval in seconds + some random delay to trigger pruning. +# It is more efficient to trigger pruning less frequently with large interval. +# default to 600 seconds +ss-prune-interval = 600 + +# ImportNumWorkers defines the concurrency for state sync import +# defaults to 1 +ss-import-num-workers = 1 +``` + + +### Step 2: Stop the node and Run SC Migration + +```bash +systemctl stop seid +seid tools migrate-iavl --home-dir /root/.sei +``` + +You can also run this sc migration in the background: +```bash +seid tools migrate-iavl --home-dir /root/.sei > migrate-sc.log 2>&1 & +``` + +This may take a couple hours to run. You will see logs of form +`Start restoring SC store for height` + + +### Step 3: Note down MIGRATION_HEIGHT +Note down the latest height as outputted from the sc migration log. + +As an example the sc migration log would show: +``` +latest version: 111417590 +D[2024-10-29|15:26:03.657] Finished loading IAVL tree +D[2024-10-29|15:26:03.657] Finished loading IAVL tree +D[2024-10-29|15:26:03.657] Finished loading IAVL tree +``` + +Save that `latest version` (in example 111417590) as an env var $MIGRATION_HEIGHT. + +```bash +MIGRATION_HEIGHT=<> +``` + + +### Step 4: Restart seid with background SS migration + +If you are using systemd, make sure to update your service configuration to use this command. +Always be sure to run with these flags until migration is complete. +```bash +seid start --migrate-iavl --migrate-height $MIGRATION_HEIGHT --chain-id pacific-1 +``` + +Seid will run normally and the migration will run in the background. Data from iavl +will be written to SS and new writes will be directed at SS not iavl. + +You will see logs of form +`SeiDB Archive Migration: Iterating through %s module...` and +`SeiDB Archive Migration: Last 1,000,000 iterations took:...` + +NOTE: While this is running, any historical queries will be routed to iavl if +they are for a height BEFORE the migrate-height. Any queries on heights +AFTER the migrate-height will be routed to state store (pebbledb). + + +### Step 5: Verify State Store Migration after completion +Once State Store Migration is complete, you will see logs of form +`SeiDB Archive Migration: DB scanning completed. Total time taken:...` + +You DO NOT immediately need to do anything. Your node will continue to run +and will operate normally. However we added a verification tool that will iterate through +all keys in iavl at a specific height and verify they exist in State Store. + +You should run the following command for a selection of different heights +```bash +seid tools verify-migration --version $VERIFICATION_HEIGHT +``` + +This will output `Verification Succeeded` if the verification was successful. + + +### Step 6: Restart seid normally and verify node runs properly +Once the verification has completed, we can restart seid normally and verify +that the node operates. + +If you are using systemd, make sure to update your service configuration to use this command: +```bash +seid start --chain-id pacific-1 +``` + +Note how we are not using the `--migrate-iavl` and `--migration-height` flags. +We can let this run for a couple hours and verify node oeprates normally. + + +### Step 7: Clear out Iavl and restart seid +Once it has been confirmed that the node has been running normally, +we can proceed to clear out the iavl and restart seid normally. + +```bash +systemctl stop seid +rm -rf ~/.sei/data/application.db +seid start --chain-id pacific-1 +``` + + +## Metrics + +During the State Store Migration, there are exported metrics that are helpful to keep track of +the progress. + +`sei_migration_leaf_nodes_exported` keeps track of how many nodes have been exported from iavl. + +`sei_migration_nodes_imported` keeps track of how many nodes have been imported into SeiDB (pebbledb). + +Both of these metrics have a `module` label which indicates what module is currently being exported / imported. + + +## FAQ + +### Can the state store migration be stopped and restarted? + +The state store migration can be stopped and restarted at any time. The migration +process saves the latest `module` and `key` written to State Store (pebbledb) and will +automatically resume the migration from that latest key once restarted. + +All one needs to do is restart seid with the migration command as in step 4 +```bash +seid start --migrate-iavl --migrate-height $MIGRATION_HEIGHT --chain-id pacific-1 +``` diff --git a/go.mod b/go.mod index d02d3f8a3..9288ff948 100644 --- a/go.mod +++ b/go.mod @@ -351,7 +351,7 @@ replace ( github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.2 github.com/ethereum/go-ethereum => github.com/sei-protocol/go-ethereum v1.13.5-sei-23 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 - github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.44 + github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.45 // Latest goleveldb is broken, we have to stick to this version github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.3.9-0.20240926181940-e9348a908b27 diff --git a/go.sum b/go.sum index 1d4d9e7db..83baf47cc 100644 --- a/go.sum +++ b/go.sum @@ -1349,8 +1349,8 @@ github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQp github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8= github.com/sei-protocol/sei-cosmos v0.3.41 h1:w5ekTGC5J/7kxJhRkfdHk2KWOqi1zhic0gOXNm6W5vI= github.com/sei-protocol/sei-cosmos v0.3.41/go.mod h1:ZwWxF/69WlcLEn4BzVjPPToTFkE2sjPanU8PNNyKoOk= -github.com/sei-protocol/sei-db v0.0.44 h1:HMgcyDTQlmXdJysHJxmIo66EKeXn1CSQT9qXDnxjJgI= -github.com/sei-protocol/sei-db v0.0.44/go.mod h1:F/ZKZA8HJPcUzSZPA8yt6pfwlGriJ4RDR4eHKSGLStI= +github.com/sei-protocol/sei-db v0.0.45 h1:95ygzGFMyvaGwEUmzlKi8MxwXfbluoNzbaIjy9zOG6o= +github.com/sei-protocol/sei-db v0.0.45/go.mod h1:m5g7p0QeAS3dNJHIl28zQpzOgxQmvYqPb7t4hwgIOCA= github.com/sei-protocol/sei-iavl v0.2.0 h1:OisPjXiDT+oe+aeckzDEFgkZCYuUjHgs/PP8DPicN+I= github.com/sei-protocol/sei-iavl v0.2.0/go.mod h1:qRf8QYUPfrAO7K6VDB2B2l/N7K5L76OorioGBcJBIbw= github.com/sei-protocol/sei-ibc-go/v3 v3.3.2 h1:BaMZ6gjwqe3R/5dLmcJ1TkSZ3omcWy2TjaAZAeOJH44= diff --git a/tools/migration/cmd/cmd.go b/tools/migration/cmd/cmd.go index d1ff27a28..a29b6386d 100644 --- a/tools/migration/cmd/cmd.go +++ b/tools/migration/cmd/cmd.go @@ -7,7 +7,10 @@ import ( "github.com/cosmos/cosmos-sdk/store/rootmulti" "github.com/sei-protocol/sei-chain/tools/migration/sc" "github.com/sei-protocol/sei-chain/tools/migration/ss" + "github.com/sei-protocol/sei-db/config" + sstypes "github.com/sei-protocol/sei-db/ss" "github.com/spf13/cobra" + "github.com/tendermint/tendermint/libs/log" dbm "github.com/tendermint/tm-db" ) @@ -18,13 +21,11 @@ func MigrateCmd() *cobra.Command { Run: execute, } cmd.PersistentFlags().String("home-dir", "/root/.sei", "Sei home directory") - cmd.PersistentFlags().String("target-db", "", "Available options: [SS, SC]") return cmd } func execute(cmd *cobra.Command, _ []string) { homeDir, _ := cmd.Flags().GetString("home-dir") - target, _ := cmd.Flags().GetString("target-db") dataDir := filepath.Join(homeDir, "data") db, err := dbm.NewGoLevelDB("application", dataDir) if err != nil { @@ -32,16 +33,9 @@ func execute(cmd *cobra.Command, _ []string) { } latestVersion := rootmulti.GetLatestVersion(db) fmt.Printf("latest version: %d\n", latestVersion) - if target == "SS" { - if err = migrateSS(latestVersion, homeDir, db); err != nil { - panic(err) - } - } else if target == "SC" { - if err = migrateSC(latestVersion, homeDir, db); err != nil { - panic(err) - } - } else { - panic("Invalid target-db, either SS or SC should be provided") + + if err = migrateSC(latestVersion, homeDir, db); err != nil { + panic(err) } } @@ -50,16 +44,6 @@ func migrateSC(version int64, homeDir string, db dbm.DB) error { return migrator.Migrate(version) } -func migrateSS(version int64, homeDir string, db dbm.DB) error { - migrator := ss.NewMigrator(homeDir, db) - return migrator.Migrate(version, homeDir) -} - -func verifySS(version int64, homeDir string, db dbm.DB) error { - migrator := ss.NewMigrator(homeDir, db) - return migrator.Verify(version) -} - func VerifyMigrationCmd() *cobra.Command { cmd := &cobra.Command{ Use: "verify-migration", @@ -97,3 +81,16 @@ func verify(cmd *cobra.Command, _ []string) { fmt.Println("Verification Succeeded") } + +func verifySS(version int64, homeDir string, db dbm.DB) error { + ssConfig := config.DefaultStateStoreConfig() + ssConfig.Enable = true + + stateStore, err := sstypes.NewStateStore(log.NewNopLogger(), homeDir, ssConfig) + if err != nil { + return err + } + + migrator := ss.NewMigrator(homeDir, db, stateStore) + return migrator.Verify(version) +} diff --git a/tools/migration/sc/migrator.go b/tools/migration/sc/migrator.go index b4d66e85c..debb1ffb7 100644 --- a/tools/migration/sc/migrator.go +++ b/tools/migration/sc/migrator.go @@ -56,7 +56,7 @@ var Keys = sdk.NewKVStoreKeys( minttypes.StoreKey, distrtypes.StoreKey, slashingtypes.StoreKey, govtypes.StoreKey, paramstypes.StoreKey, ibchost.StoreKey, upgradetypes.StoreKey, feegrant.StoreKey, evidencetypes.StoreKey, ibctransfertypes.StoreKey, capabilitytypes.StoreKey, oracletypes.StoreKey, - evmtypes.StoreKey, wasm.StoreKey, epochmoduletypes.StoreKey, tokenfactorytypes.StoreKey, "dex", + evmtypes.StoreKey, wasm.StoreKey, epochmoduletypes.StoreKey, tokenfactorytypes.StoreKey, ) func NewMigrator(homeDir string, db dbm.DB) *Migrator { @@ -76,8 +76,8 @@ func NewMigrator(homeDir string, db dbm.DB) *Migrator { scConfig := config.DefaultStateCommitConfig() scConfig.Enable = true ssConfig := config.DefaultStateStoreConfig() - ssConfig.Enable = false - cmsV2 := rootmulti2.NewStore(homeDir, logger, scConfig, ssConfig, false) + ssConfig.Enable = true + cmsV2 := rootmulti2.NewStore(homeDir, logger, scConfig, ssConfig, true) for _, key := range Keys { cmsV2.MountStoreWithDB(key, sdk.StoreTypeIAVL, db) } diff --git a/tools/migration/ss/migrator.go b/tools/migration/ss/migrator.go index cb4e42bfc..e050d27ec 100644 --- a/tools/migration/ss/migrator.go +++ b/tools/migration/ss/migrator.go @@ -3,12 +3,11 @@ package ss import ( "bytes" "fmt" + "time" + "github.com/armon/go-metrics" "github.com/cosmos/iavl" - "github.com/sei-protocol/sei-db/config" - "github.com/sei-protocol/sei-db/ss" "github.com/sei-protocol/sei-db/ss/types" - "github.com/tendermint/tendermint/libs/log" dbm "github.com/tendermint/tm-db" ) @@ -26,16 +25,7 @@ var modules = []string{ "wasm", "aclaccesscontrol", "oracle", "epoch", "mint", "acc", "bank", "feegrant", "staking", "distribution", "slashing", "gov", "params", "ibc", "upgrade", "evidence", "transfer", "tokenfactory", } -func NewMigrator(homeDir string, db dbm.DB) *Migrator { - // TODO: Pass in more configs outside default, in particular ImportNumWorkers - ssConfig := config.DefaultStateStoreConfig() - ssConfig.Enable = true - - stateStore, err := ss.NewStateStore(log.NewNopLogger(), homeDir, ssConfig) - if err != nil { - panic(err) - } - +func NewMigrator(homeDir string, db dbm.DB, stateStore types.StateStore) *Migrator { return &Migrator{ iavlDB: db, stateStore: stateStore, @@ -43,36 +33,42 @@ func NewMigrator(homeDir string, db dbm.DB) *Migrator { } func (m *Migrator) Migrate(version int64, homeDir string) error { - // TODO: Read in capacity of this buffered channel as param ch := make(chan types.RawSnapshotNode, 1000) errCh := make(chan error, 2) - fmt.Println("Beginning Migration...") + // Get the latest key, if any, to resume from + latestKey, err := m.stateStore.GetLatestMigratedKey() + if err != nil { + return fmt.Errorf("failed to get latest key: %w", err) + } + + latestModule, err := m.stateStore.GetLatestMigratedModule() + if err != nil { + return fmt.Errorf("failed to get latest module: %w", err) + } + + fmt.Println("Starting migration...") - // Goroutine to iterate through iavl and export leaf nodes + // Goroutine to iterate through IAVL and export leaf nodes go func() { defer close(ch) - errCh <- ExportLeafNodes(m.iavlDB, ch) + errCh <- ExportLeafNodesFromKey(m.iavlDB, ch, latestKey, latestModule) }() + // Import nodes into PebbleDB go func() { errCh <- m.stateStore.RawImport(ch) }() - // Block on completion of both goroutines + // Block until both processes complete for i := 0; i < 2; i++ { if err := <-errCh; err != nil { return err } } - // Set latest version - err := m.stateStore.SetLatestVersion(version) - if err != nil { - return err - } - - return nil + // Set latest version in the database + return m.stateStore.SetLatestVersion(version) } func (m *Migrator) Verify(version int64) error { @@ -96,57 +92,77 @@ func (m *Migrator) Verify(version int64) error { return true } if !bytes.Equal(val, value) { - verifyErr = fmt.Errorf("verification error: value doesn't match for key %s", string(key)) + verifyErr = fmt.Errorf("verification error: value doesn't match for key %s. Expected %s, got %s", string(key), string(value), string(val)) } count++ - if count%10000 == 0 { - fmt.Printf("Verified %d keys in for module %s\n", count, module) + if count%1000000 == 0 { + fmt.Printf("SeiDB Archive Migration: Verified %d keys in for module %s\n", count, module) } return false }) if err != nil { - fmt.Printf("Failed to iterate the tree %s: %s\n", module, err.Error()) + fmt.Printf("SeiDB Archive Migration: Failed to iterate the tree %s: %s\n", module, err.Error()) return err } - fmt.Printf("Finished verifying module %s, total scanned: %d keys\n", module, count) + fmt.Printf("SeiDB Archive Migration:: Finished verifying module %s, total scanned: %d keys\n", module, count) } return verifyErr } -// Export leaf nodes of iavl -func ExportLeafNodes(db dbm.DB, ch chan<- types.RawSnapshotNode) error { - // Module by module, TODO: Potentially parallelize +func ExportLeafNodesFromKey(db dbm.DB, ch chan<- types.RawSnapshotNode, startKey []byte, startModule string) error { count := 0 leafNodeCount := 0 - fmt.Println("Scanning database and exporting leaf nodes...") + fmt.Println("SeiDB Archive Migration: Scanning database and exporting leaf nodes...") + + startTimeTotal := time.Now() // Start measuring total time + + var batchLeafNodeCount int + startModuleFound := startModule == "" // true if no start module specified for _, module := range modules { - fmt.Printf("Iterating through %s module...\n", module) + if !startModuleFound { + if module == startModule { + startModuleFound = true + } else { + continue + } + } + startTimeModule := time.Now() // Measure time for each module + fmt.Printf("SeiDB Archive Migration: Iterating through %s module...\n", module) - // Can't use the previous, have to create an inner prefixDB := dbm.NewPrefixDB(db, []byte(buildRawPrefix(module))) - itr, err := prefixDB.Iterator(nil, nil) + var itr dbm.Iterator + var err error + + // If there is a starting key, seek to it, otherwise start from the beginning + if startKey != nil && bytes.HasPrefix(startKey, []byte(buildRawPrefix(module))) { + itr, err = prefixDB.Iterator(startKey, nil) // Start from the latest key + } else { + itr, err = prefixDB.Iterator(nil, nil) // Start from the beginning + } + if err != nil { - fmt.Printf("error Export Leaf Nodes %+v\n", err) + fmt.Printf("SeiDB Archive Migration: Error creating iterator: %+v\n", err) return fmt.Errorf("failed to create iterator: %w", err) } defer itr.Close() + startTimeBatch := time.Now() // Measure time for every 10,000 iterations + for ; itr.Valid(); itr.Next() { value := bytes.Clone(itr.Value()) node, err := iavl.MakeNode(value) - if err != nil { - fmt.Printf("failed to make node err: %+v\n", err) + fmt.Printf("SeiDB Archive Migration: Failed to make node: %+v\n", err) return fmt.Errorf("failed to make node: %w", err) } - // leaf node + // Only export leaf nodes if node.GetHeight() == 0 { leafNodeCount++ + batchLeafNodeCount++ ch <- types.RawSnapshotNode{ - // TODO: Likely need to clone StoreKey: module, Key: node.GetNodeKey(), Value: node.GetValue(), @@ -155,21 +171,30 @@ func ExportLeafNodes(db dbm.DB, ch chan<- types.RawSnapshotNode) error { } count++ - if count%10000 == 0 { - fmt.Printf("Total scanned: %d, leaf nodes exported: %d\n", count, leafNodeCount) + if count%1000000 == 0 { + batchDuration := time.Since(startTimeBatch) + fmt.Printf("SeiDB Archive Migration: Last 1,000,000 iterations took: %v. Total scanned: %d, leaf nodes exported: %d\n", batchDuration, count, leafNodeCount) + metrics.IncrCounterWithLabels([]string{"sei", "migration", "leaf_nodes_exported"}, float32(batchLeafNodeCount), []metrics.Label{ + {Name: "module", Value: module}, + }) + + batchLeafNodeCount = 0 + startTimeBatch = time.Now() } } - fmt.Printf("Finished scanning module %s Total scanned: %d, leaf nodes exported: %d\n", module, count, leafNodeCount) - if err := itr.Error(); err != nil { - fmt.Printf("iterator error: %+v\n", err) + fmt.Printf("Iterator error: %+v\n", err) return fmt.Errorf("iterator error: %w", err) } + moduleDuration := time.Since(startTimeModule) + fmt.Printf("SeiDB Archive Migration: Finished scanning module %s. Time taken: %v. Total scanned: %d, leaf nodes exported: %d\n", module, moduleDuration, count, leafNodeCount) } - fmt.Printf("DB contains %d entries, exported %d leaf nodes\n", count, leafNodeCount) + totalDuration := time.Since(startTimeTotal) + fmt.Printf("SeiDB Archive Migration: DB scanning completed. Total time taken: %v. Total entries scanned: %d, leaf nodes exported: %d\n", totalDuration, count, leafNodeCount) + return nil }