Archive Node Online Migration (#1863)

* Archive Node Online Migration

* Add migrate-iavl flag for online migration

* Update sei cosmos

* Add QMS for online migration (#1870)

* Add QMS for online migration

* Fix lint

---------

Co-authored-by: kbhat1 <[email protected]>

* Add migrate-height flag to start cmd

* Bump seidb

* Reduce logging

* Archive Migration doc

* Update form

* Update

* Export metric

* latest height

* Update start module logic

* Remove migrate SS command

* Enable ss store

* Update archive node migration docs

* More migration doc updates

* Update metrics in readme

* Merge main

* Remove dex

* Add Faq section

* Add requirements section

* Systemd instructions

* Minor add to readme

* Add more examples for migration height

* Bump sei cosmos

* Update error log

* Update command sc migration background

---------

Co-authored-by: Yiming Zang <[email protected]>

app/app.go

@@ -368,6 +368,7 @@ type App struct {
 
 	genesisImportConfig genesistypes.GenesisImportConfig
 
+	stateStore   seidb.StateStore
 	receiptStore seidb.StateStore
 }
 
@@ -396,7 +397,7 @@ func New(
 	cdc := encodingConfig.Amino
 	interfaceRegistry := encodingConfig.InterfaceRegistry
 
-	bAppOptions := SetupSeiDB(logger, homePath, appOpts, baseAppOptions)
+	bAppOptions, stateStore := SetupSeiDB(logger, homePath, appOpts, baseAppOptions)
 
 	bApp := baseapp.NewBaseApp(AppName, logger, db, encodingConfig.TxConfig.TxDecoder(), tmConfig, appOpts, bAppOptions...)
 	bApp.SetCommitMultiStoreTracer(traceStore)
@@ -429,6 +430,7 @@ func New(
 		versionInfo:       version.NewInfo(),
 		metricCounter:     &map[string]float32{},
 		encodingConfig:    encodingConfig,
+		stateStore:        stateStore,
 	}
 
 	for _, option := range appOptions {
@@ -1064,6 +1066,9 @@ func (app *App) Name() string { return app.BaseApp.Name() }
 // GetBaseApp returns the base app of the application
 func (app App) GetBaseApp() *baseapp.BaseApp { return app.BaseApp }
 
+// GetStateStore returns the state store of the application
+func (app App) GetStateStore() seidb.StateStore { return app.stateStore }
+
 // BeginBlocker application updates every begin block
 func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock {
 	metrics.GaugeSeidVersionAndCommit(app.versionInfo.Version, app.versionInfo.GitCommit)

app/seidb.go

@@ -7,6 +7,7 @@ import (
 	servertypes "github.com/cosmos/cosmos-sdk/server/types"
 	"github.com/cosmos/cosmos-sdk/storev2/rootmulti"
 	"github.com/sei-protocol/sei-db/config"
+	seidb "github.com/sei-protocol/sei-db/ss/types"
 	"github.com/spf13/cast"
 	"github.com/tendermint/tendermint/libs/log"
 )
@@ -33,18 +34,20 @@ const (
 
 	// Other configs
 	FlagSnapshotInterval = "state-sync.snapshot-interval"
+	FlagMigrateIAVL      = "migrate-iavl"
+	FlagMigrateHeight    = "migrate-height"
 )
 
 func SetupSeiDB(
 	logger log.Logger,
 	homePath string,
 	appOpts servertypes.AppOptions,
 	baseAppOptions []func(*baseapp.BaseApp),
-) []func(*baseapp.BaseApp) {
+) ([]func(*baseapp.BaseApp), seidb.StateStore) {
 	scEnabled := cast.ToBool(appOpts.Get(FlagSCEnable))
 	if !scEnabled {
 		logger.Info("SeiDB is disabled, falling back to IAVL")
-		return baseAppOptions
+		return baseAppOptions, nil
 	}
 	logger.Info("SeiDB SC is enabled, running node with StoreV2 commit store")
 	scConfig := parseSCConfigs(appOpts)
@@ -56,14 +59,21 @@ func SetupSeiDB(
 
 	// cms must be overridden before the other options, because they may use the cms,
 	// make sure the cms aren't be overridden by the other options later on.
-	cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig, false)
+	cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig, cast.ToBool(appOpts.Get("migrate-iavl")))
+	migrationEnabled := cast.ToBool(appOpts.Get(FlagMigrateIAVL))
+	migrationHeight := cast.ToInt64(appOpts.Get(FlagMigrateHeight))
 	baseAppOptions = append([]func(*baseapp.BaseApp){
 		func(baseApp *baseapp.BaseApp) {
+			if migrationEnabled {
+				originalCMS := baseApp.CommitMultiStore()
+				baseApp.SetQueryMultiStore(originalCMS)
+				baseApp.SetMigrationHeight(migrationHeight)
+			}
 			baseApp.SetCMS(cms)
 		},
 	}, baseAppOptions...)
 
-	return baseAppOptions
+	return baseAppOptions, cms.GetStateStore()
 }
 
 func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {

app/test_state_store.go

@@ -244,6 +244,26 @@ func (s *InMemoryStateStore) RawImport(ch <-chan types.RawSnapshotNode) error {
 	return nil
 }
 
+func (s *InMemoryStateStore) SetLatestMigratedModule(module string) error {
+	// TODO: Add set call here
+	return nil
+}
+
+func (s *InMemoryStateStore) GetLatestMigratedModule() (string, error) {
+	// TODO: Add get call here
+	return "", nil
+}
+
+func (s *InMemoryStateStore) SetLatestMigratedKey(key []byte) error {
+	// TODO: Add set call here
+	return nil
+}
+
+func (s *InMemoryStateStore) GetLatestMigratedKey() ([]byte, error) {
+	// TODO: Add get call here
+	return nil, nil
+}
+
 func (s *InMemoryStateStore) Prune(version int64) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()

cmd/seid/cmd/root.go

@@ -38,6 +38,7 @@ import (
 	"github.com/sei-protocol/sei-chain/app/params"
 	"github.com/sei-protocol/sei-chain/evmrpc"
 	"github.com/sei-protocol/sei-chain/tools"
+	"github.com/sei-protocol/sei-chain/tools/migration/ss"
 	"github.com/sei-protocol/sei-chain/x/evm/blocktest"
 	"github.com/sei-protocol/sei-chain/x/evm/querier"
 	"github.com/sei-protocol/sei-chain/x/evm/replay"
@@ -220,6 +221,8 @@ func txCommand() *cobra.Command {
 
 func addModuleInitFlags(startCmd *cobra.Command) {
 	crisis.AddModuleInitFlags(startCmd)
+	startCmd.Flags().Bool("migrate-iavl", false, "Run migration of IAVL data store to SeiDB State Store")
+	startCmd.Flags().Int64("migrate-height", 0, "Height at which to start the migration")
 }
 
 // newApp creates a new Cosmos SDK app
@@ -266,7 +269,7 @@ func newApp(
 	// This makes it such that the wasm VM gas converts to sdk gas at a 6.66x rate vs that of the previous multiplier
 	wasmGasRegisterConfig.GasMultiplier = 21_000_000
 
-	return app.New(
+	app := app.New(
 		logger,
 		db,
 		traceStore,
@@ -302,6 +305,21 @@ func newApp(
 		baseapp.SetSnapshotDirectory(cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))),
 		baseapp.SetOccEnabled(cast.ToBool(appOpts.Get(baseapp.FlagOccEnabled))),
 	)
+
+	// Start migration if --migrate flag is set
+	if cast.ToBool(appOpts.Get("migrate-iavl")) {
+		go func() {
+			homeDir := cast.ToString(appOpts.Get(flags.FlagHome))
+			stateStore := app.GetStateStore()
+			migrationHeight := cast.ToInt64(appOpts.Get("migrate-height"))
+			migrator := ss.NewMigrator(homeDir, db, stateStore)
+			if err := migrator.Migrate(migrationHeight, homeDir); err != nil {
+				panic(err)
+			}
+		}()
+	}
+
+	return app
 }
 
 // appExport creates a new simapp (optionally at a given height)

docs/migration/seidb_archive_migration.md

@@ -0,0 +1,212 @@
+# SeiDB Archive Migration Guide
+
+## Overview
+SeiDB is the next generation of chain storage in SeiV2.
+One issue for running SeiDB on archive nodes is that we need to keep the full state of the chain, so we can't
+state sync it and clear out previous iavl data.
+
+In order to run an archive node with SeiDB, we need to run a migration from iavl to sei-db.
+
+The overall process will work as follows:
+
+1. Update config to enable SeiDB (state committment + state store)
+2. Stop the node and Run SC Migration
+3. Note down MIGRATION_HEIGHT
+4. Re start seid with `--migrate-iavl` enabled (migrating state store in background)
+5. Verify migration at various sampled heights once state store is complete
+6. Restart seid normally and verify node runs properly
+7. Clear out iavl and restart seid normally, now only using SeiDB fully
+
+
+## Requirements
+
+### Additional Disk Space
+
+Ensure you have at least 10TB of free disk space available for the migration process. This extra space is needed because during migration, both IAVL and SeiDB state stores will exist simultaneously. Note
+that this is ONLY during the migration process, and will not be necessary after completion
+
+
+## Migration Steps
+
+### Step 1: Add SeiDB Configurations
+We can enable SeiDB by adding the following configs to app.toml file.
+Usually you can find this file under ~/.sei/config/app.toml.
+```bash
+#############################################################################
+###                             SeiDB Configuration                       ###
+#############################################################################
+
+[state-commit]
+# Enable defines if the SeiDB should be enabled to override existing IAVL db backend.
+sc-enable = true
+
+# AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up
+# performance, <=0 means synchronous commit.
+sc-async-commit-buffer = 100
+
+# SnapshotKeepRecent defines how many memiavl snapshots (beyond the latest one) to keep
+# Recommend to set to 1 to make sure IBC relayers work.
+sc-keep-recent = 1
+
+# SnapshotInterval defines the number of blocks interval the memiavl snapshot is taken, default to 10000 blocks.
+# Adjust based on your needs:
+# Setting this too low could lead to lot of extra heavy disk IO
+# Setting this too high could lead to slow restart
+sc-snapshot-interval = 10000
+
+# SnapshotWriterLimit defines the max concurrency for taking memiavl snapshot
+sc-snapshot-writer-limit = 1
+
+# CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000.
+sc-cache-size = 100000
+
+[state-store]
+# Enable defines if the state-store should be enabled for historical queries.
+# In order to use state-store, you need to make sure to enable state-commit at the same time.
+# Validator nodes should turn this off.
+# State sync node or full nodes should turn this on.
+ss-enable = true
+
+# DBBackend defines the backend database used for state-store.
+# Supported backends: pebbledb, rocksdb
+# defaults to pebbledb (recommended)
+ss-backend = "pebbledb"
+
+# AsyncWriteBuffer defines the async queue length for commits to be applied to State Store
+# Set <= 0 for synchronous writes, which means commits also need to wait for data to be persisted in State Store.
+# defaults to 100
+ss-async-write-buffer = 100
+
+# KeepRecent defines the number of versions to keep in state store
+# Setting it to 0 means keep everything, default to 100000
+ss-keep-recent = 0
+
+# PruneIntervalSeconds defines the minimum interval in seconds + some random delay to trigger pruning.
+# It is more efficient to trigger pruning less frequently with large interval.
+# default to 600 seconds
+ss-prune-interval = 600
+
+# ImportNumWorkers defines the concurrency for state sync import
+# defaults to 1
+ss-import-num-workers = 1
+```
+
+
+### Step 2: Stop the node and Run SC Migration
+
+```bash
+systemctl stop seid
+seid tools migrate-iavl --home-dir /root/.sei
+```
+
+You can also run this sc migration in the background:
+```bash
+seid tools migrate-iavl --home-dir /root/.sei > migrate-sc.log 2>&1 &
+```
+
+This may take a couple hours to run. You will see logs of form
+`Start restoring SC store for height`
+
+
+### Step 3: Note down MIGRATION_HEIGHT
+Note down the latest height as outputted from the sc migration log. 
+
+As an example the sc migration log would show:
+```
+latest version: 111417590
+D[2024-10-29|15:26:03.657] Finished loading IAVL tree
+D[2024-10-29|15:26:03.657] Finished loading IAVL tree
+D[2024-10-29|15:26:03.657] Finished loading IAVL tree
+```
+
+Save that `latest version` (in example 111417590) as an env var $MIGRATION_HEIGHT.
+
+```bash
+MIGRATION_HEIGHT=<>
+```
+
+
+### Step 4: Restart seid with background SS migration
+
+If you are using systemd, make sure to update your service configuration to use this command.
+Always be sure to run with these flags until migration is complete.
+```bash
+seid start --migrate-iavl --migrate-height $MIGRATION_HEIGHT --chain-id pacific-1
+```
+
+Seid will run normally and the migration will run in the background. Data from iavl
+will be written to SS and new writes will be directed at SS not iavl.
+
+You will see logs of form 
+`SeiDB Archive Migration: Iterating through %s module...` and 
+`SeiDB Archive Migration: Last 1,000,000 iterations took:...`
+
+NOTE: While this is running, any historical queries will be routed to iavl if
+they are for a height BEFORE the migrate-height. Any queries on heights
+AFTER the migrate-height will be routed to state store (pebbledb).
+
+
+### Step 5: Verify State Store Migration after completion
+Once State Store Migration is complete, you will see logs of form
+`SeiDB Archive Migration: DB scanning completed. Total time taken:...`
+
+You DO NOT immediately need to do anything. Your node will continue to run
+and will operate normally. However we added a verification tool that will iterate through
+all keys in iavl at a specific height and verify they exist in State Store.
+
+You should run the following command for a selection of different heights
+```bash
+seid tools verify-migration --version $VERIFICATION_HEIGHT
+```
+
+This will output `Verification Succeeded` if the verification was successful.
+
+
+### Step 6: Restart seid normally and verify node runs properly
+Once the verification has completed, we can restart seid normally and verify
+that the node operates.
+
+If you are using systemd, make sure to update your service configuration to use this command:
+```bash
+seid start --chain-id pacific-1
+```
+
+Note how we are not using the `--migrate-iavl` and `--migration-height` flags.
+We can let this run for a couple hours and verify node oeprates normally.
+
+
+### Step 7: Clear out Iavl and restart seid
+Once it has been confirmed that the node has been running normally,
+we can proceed to clear out the iavl and restart seid normally.
+
+```bash
+systemctl stop seid
+rm -rf ~/.sei/data/application.db
+seid start --chain-id pacific-1
+```
+
+
+## Metrics
+
+During the State Store Migration, there are exported metrics that are helpful to keep track of
+the progress.
+
+`sei_migration_leaf_nodes_exported` keeps track of how many nodes have been exported from iavl.
+
+`sei_migration_nodes_imported` keeps track of how many nodes have been imported into SeiDB (pebbledb).
+
+Both of these metrics have a `module` label which indicates what module is currently being exported / imported.
+
+
+## FAQ
+
+### Can the state store migration be stopped and restarted?
+
+The state store migration can be stopped and restarted at any time. The migration
+process saves the latest `module` and `key` written to State Store (pebbledb) and will
+automatically resume the migration from that latest key once restarted.
+
+All one needs to do is restart seid with the migration command as in step 4
+```bash
+seid start --migrate-iavl --migrate-height $MIGRATION_HEIGHT --chain-id pacific-1
+```

go.mod

@@ -351,7 +351,7 @@ replace (
 	github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.2
 	github.com/ethereum/go-ethereum => github.com/sei-protocol/go-ethereum v1.13.5-sei-23
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
-	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.44
+	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.45
 	// Latest goleveldb is broken, we have to stick to this version
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.3.9-0.20240926181940-e9348a908b27

go.sum

@@ -1349,8 +1349,8 @@ github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQp
 github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
 github.com/sei-protocol/sei-cosmos v0.3.41 h1:w5ekTGC5J/7kxJhRkfdHk2KWOqi1zhic0gOXNm6W5vI=
 github.com/sei-protocol/sei-cosmos v0.3.41/go.mod h1:ZwWxF/69WlcLEn4BzVjPPToTFkE2sjPanU8PNNyKoOk=
-github.com/sei-protocol/sei-db v0.0.44 h1:HMgcyDTQlmXdJysHJxmIo66EKeXn1CSQT9qXDnxjJgI=
-github.com/sei-protocol/sei-db v0.0.44/go.mod h1:F/ZKZA8HJPcUzSZPA8yt6pfwlGriJ4RDR4eHKSGLStI=
+github.com/sei-protocol/sei-db v0.0.45 h1:95ygzGFMyvaGwEUmzlKi8MxwXfbluoNzbaIjy9zOG6o=
+github.com/sei-protocol/sei-db v0.0.45/go.mod h1:m5g7p0QeAS3dNJHIl28zQpzOgxQmvYqPb7t4hwgIOCA=
 github.com/sei-protocol/sei-iavl v0.2.0 h1:OisPjXiDT+oe+aeckzDEFgkZCYuUjHgs/PP8DPicN+I=
 github.com/sei-protocol/sei-iavl v0.2.0/go.mod h1:qRf8QYUPfrAO7K6VDB2B2l/N7K5L76OorioGBcJBIbw=
 github.com/sei-protocol/sei-ibc-go/v3 v3.3.2 h1:BaMZ6gjwqe3R/5dLmcJ1TkSZ3omcWy2TjaAZAeOJH44=