[SeiDB] Add setup logic and integration test for SeiDB (#1170)

* Add setup logic for seiDB

* Add integration test support for SeiDB

* Bump golang version for docker

* Bump version for cosmos

* Bump seidb version

* Fix integration test for distribution

* Fix oracle vote round up issue

* Fix goleveldb version

* Upload latest gringotts

* Fix deploy_timelocked_token_contract.sh

* Fix integration test

* Update go mod

* Bump tendermint version

.github/workflows/codeql.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/fuzz.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.18
+        go-version: 1.19
 
     - name: Fuzz Place Order Msg
       run: go test github.com/sei-protocol/sei-chain/x/dex/keeper/msgserver -fuzz FuzzPlaceOrders -fuzztime 30s

.github/workflows/golangci.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/integration-test.yml

@@ -109,7 +109,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Start 4 node docker cluster
         run: make clean && INVARIANT_CHECK_INTERVAL=10 ${{matrix.test.env}} make docker-cluster-start &

Makefile

@@ -150,6 +150,7 @@ run-rpc-node: build-rpc-node
 	-v $(PROJECT_HOME):/sei-protocol/sei-chain:Z \
 	-v $(PROJECT_HOME)/../sei-tendermint:/sei-protocol/sei-tendermint:Z \
     -v $(PROJECT_HOME)/../sei-cosmos:/sei-protocol/sei-cosmos:Z \
+    -v $(PROJECT_HOME)/../sei-db:/sei-protocol/sei-db:Z \
 	-v $(GO_PKG_PATH)/mod:/root/go/pkg/mod:Z \
 	-p 26668-26670:26656-26658 \
 	--platform linux/x86_64 \
@@ -163,6 +164,7 @@ run-rpc-node-skipbuild: build-rpc-node
 	-v $(PROJECT_HOME):/sei-protocol/sei-chain:Z \
 	-v $(PROJECT_HOME)/../sei-tendermint:/sei-protocol/sei-tendermint:Z \
     -v $(PROJECT_HOME)/../sei-cosmos:/sei-protocol/sei-cosmos:Z \
+    -v $(PROJECT_HOME)/../sei-db:/sei-protocol/sei-db:Z \
 	-v $(GO_PKG_PATH)/mod:/root/go/pkg/mod:Z \
 	-p 26668-26670:26656-26658 \
 	--platform linux/x86_64 \

app/app.go

@@ -375,7 +375,8 @@ func New(
 	cdc := encodingConfig.Amino
 	interfaceRegistry := encodingConfig.InterfaceRegistry
 
-	bApp := baseapp.NewBaseApp(AppName, logger, db, encodingConfig.TxConfig.TxDecoder(), tmConfig, appOpts, baseAppOptions...)
+	bAppOptions := SetupSeiDB(logger, homePath, appOpts, baseAppOptions)
+	bApp := baseapp.NewBaseApp(AppName, logger, db, encodingConfig.TxConfig.TxDecoder(), tmConfig, appOpts, bAppOptions...)
 	bApp.SetCommitMultiStoreTracer(traceStore)
 	bApp.SetVersion(version.Version)
 	bApp.SetInterfaceRegistry(interfaceRegistry)

app/seidb.go

@@ -0,0 +1,80 @@
+package app
+
+import (
+	"github.com/cosmos/cosmos-sdk/baseapp"
+	servertypes "github.com/cosmos/cosmos-sdk/server/types"
+	"github.com/cosmos/cosmos-sdk/storev2/rootmulti"
+	"github.com/sei-protocol/sei-db/config"
+	"github.com/spf13/cast"
+	"github.com/tendermint/tendermint/libs/log"
+)
+
+const (
+	FlagSCEnable              = "state-commit.sc-enable"
+	FlagSCDirectory           = "state-commit.sc-directory"
+	FlagSCAsyncCommitBuffer   = "state-commit.sc-async-commit-buffer"
+	FlagSCZeroCopy            = "state-commit.sc-zero-copy"
+	FlagSCSnapshotKeepRecent  = "state-commit.sc-keep-recent"
+	FlagSCSnapshotInterval    = "state-commit.sc-snapshot-interval"
+	FlagSCSnapshotWriterLimit = "state-commit.sc-snapshot-writer-limit"
+	FlagSCCacheSize           = "state-commit.sc-cache-size"
+
+	FlagSSEnable            = "state-store.ss-enable"
+	FlagSSDirectory         = "state-store.ss-db-directory"
+	FlagSSBackend           = "state-store.ss-backend"
+	FlagSSAsyncWriterBuffer = "state-store.ss-async-write-buffer"
+	FlagSSKeepRecent        = "state-store.ss-keep-recent"
+	FlagSSPruneInterval     = "state-store.ss-prune-interval"
+	FlagSSImportNumWorkers  = "state-store.ss-import-num-workers"
+)
+
+func SetupSeiDB(
+	logger log.Logger,
+	homePath string,
+	appOpts servertypes.AppOptions,
+	baseAppOptions []func(*baseapp.BaseApp),
+) []func(*baseapp.BaseApp) {
+	scEnabled := cast.ToBool(appOpts.Get(FlagSCEnable))
+	if !scEnabled {
+		return baseAppOptions
+	}
+	logger.Info("SeiDB is enabled, replacing with StoreV2 RMS")
+	scConfig := parseSCConfigs(appOpts)
+	ssConfig := parseSSConfigs(appOpts)
+
+	// cms must be overridden before the other options, because they may use the cms,
+	// make sure the cms aren't be overridden by the other options later on.
+	cms := rootmulti.NewStore(homePath, logger, scConfig, ssConfig)
+	baseAppOptions = append([]func(*baseapp.BaseApp){
+		func(baseApp *baseapp.BaseApp) {
+			baseApp.SetCMS(cms)
+		},
+	}, baseAppOptions...)
+
+	return baseAppOptions
+}
+
+func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {
+	return config.StateCommitConfig{
+		Enable:              cast.ToBool(appOpts.Get(FlagSCEnable)),
+		Directory:           cast.ToString(appOpts.Get(FlagSCDirectory)),
+		ZeroCopy:            cast.ToBool(appOpts.Get(FlagSCZeroCopy)),
+		AsyncCommitBuffer:   cast.ToInt(appOpts.Get(FlagSCAsyncCommitBuffer)),
+		SnapshotKeepRecent:  cast.ToUint32(appOpts.Get(FlagSCSnapshotKeepRecent)),
+		SnapshotInterval:    cast.ToUint32(appOpts.Get(FlagSCSnapshotInterval)),
+		SnapshotWriterLimit: cast.ToInt(appOpts.Get(FlagSCSnapshotWriterLimit)),
+		CacheSize:           cast.ToInt(appOpts.Get(FlagSCCacheSize)),
+	}
+}
+
+func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig {
+	return config.StateStoreConfig{
+		Enable:               cast.ToBool(appOpts.Get(FlagSSEnable)),
+		Backend:              cast.ToString(appOpts.Get(FlagSSBackend)),
+		AsyncWriteBuffer:     cast.ToInt(appOpts.Get(FlagSSAsyncWriterBuffer)),
+		KeepRecent:           cast.ToInt(appOpts.Get(FlagSSKeepRecent)),
+		PruneIntervalSeconds: cast.ToInt(appOpts.Get(FlagSSPruneInterval)),
+		ImportNumWorkers:     cast.ToInt(appOpts.Get(FlagSSImportNumWorkers)),
+		DBDirectory:          cast.ToString(appOpts.Get(FlagSSDirectory)),
+	}
+}

cmd/seid/cmd/root.go

@@ -34,7 +34,6 @@ import (
 	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
 	"github.com/cosmos/cosmos-sdk/x/crisis"
 	genutilcli "github.com/cosmos/cosmos-sdk/x/genutil/client/cli"
-	"github.com/cosmos/iavl"
 	"github.com/sei-protocol/sei-chain/app"
 	"github.com/sei-protocol/sei-chain/app/params"
 	"github.com/sei-protocol/sei-chain/tools"
@@ -293,12 +292,6 @@ func newApp(
 		baseapp.SetSnapshotInterval(cast.ToUint64(appOpts.Get(server.FlagStateSyncSnapshotInterval))),
 		baseapp.SetSnapshotKeepRecent(cast.ToUint32(appOpts.Get(server.FlagStateSyncSnapshotKeepRecent))),
 		baseapp.SetSnapshotDirectory(cast.ToString(appOpts.Get(server.FlagStateSyncSnapshotDir))),
-		baseapp.SetOrphanConfig(&iavl.Options{
-			SeparateOrphanStorage:       cast.ToBool(appOpts.Get(server.FlagSeparateOrphanStorage)),
-			SeparateOphanVersionsToKeep: cast.ToInt64(appOpts.Get(server.FlagSeparateOrphanVersionsToKeep)),
-			NumOrphansPerFile:           cast.ToInt(appOpts.Get(server.FlagNumOrphanPerFile)),
-			OrphanDirectory:             cast.ToString(appOpts.Get(server.FlagOrphanDirectory)),
-		}),
 	)
 }
 

docker/docker-compose.yml

@@ -19,6 +19,7 @@ services:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
       - "${PROJECT_HOME}/../sei-cosmos:/sei-protocol/sei-cosmos:Z"
+      - "${PROJECT_HOME}/../sei-db:/sei-protocol/sei-db:Z"
       - "${GO_PKG_PATH}/mod:/root/go/pkg/mod:Z"
     networks:
       localnet:
@@ -42,6 +43,7 @@ services:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
       - "${PROJECT_HOME}/../sei-cosmos:/sei-protocol/sei-cosmos:Z"
+      - "${PROJECT_HOME}/../sei-db:/sei-protocol/sei-db:Z"
       - "${GO_PKG_PATH}/mod:/root/go/pkg/mod:Z"
     networks:
       localnet:
@@ -65,6 +67,7 @@ services:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
       - "${PROJECT_HOME}/../sei-cosmos:/sei-protocol/sei-cosmos:Z"
+      - "${PROJECT_HOME}/../sei-db:/sei-protocol/sei-db:Z"
       - "${GO_PKG_PATH}/mod:/root/go/pkg/mod:Z"
     networks:
       localnet:
@@ -88,6 +91,7 @@ services:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
       - "${PROJECT_HOME}/../sei-cosmos:/sei-protocol/sei-cosmos:Z"
+      - "${PROJECT_HOME}/../sei-db:/sei-protocol/sei-db:Z"
       - "${GO_PKG_PATH}/mod:/root/go/pkg/mod:Z"
     networks:
       localnet:

docker/localnode/Dockerfile

@@ -1,7 +1,10 @@
 FROM ubuntu:latest
 RUN apt-get update && \
-    apt-get install -y make git golang jq python3 curl vim uuid-runtime
+    apt-get install -y make git wget build-essential jq python3 curl vim uuid-runtime
 RUN rm -rf build/generated
+RUN wget https://go.dev/dl/go1.20.6.linux-amd64.tar.gz
+RUN tar -xvf go1.20.6.linux-amd64.tar.gz
+RUN mv go /usr/local/
 SHELL ["/bin/bash", "-c"]
 
 

docker/localnode/config/app.toml

@@ -203,4 +203,68 @@ snapshot-directory = ""
 query_gas_limit = 300000
 # This is the number of wasm vm instances we keep cached in memory for speed-up
 # Warning: this is currently unstable and may lead to crashes, best to keep for 0 unless testing locally
-lru_size = 0
+lru_size = 0
+
+###############################################################################
+###                             SeiDB Configuration                       ###
+###############################################################################
+
+[state-commit]
+
+# Enable defines if the state-commit (memiavl) should be enabled to override existing IAVL db backend.
+sc-enable = true
+
+sc-directory = ""
+
+# ZeroCopy defines if memiavl should return slices pointing to mmap-ed buffers directly (zero-copy),
+# the zero-copied slices must not be retained beyond current block's execution.
+# the sdk address cache will be disabled if zero-copy is enabled.
+sc-zero-copy = false
+
+# AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up
+# performance, <=0 means synchronous commit.
+sc-async-commit-buffer = 100
+
+# SnapshotKeepRecent defines how many state-commit snapshots (besides the latest one) to keep
+# defaults to 1 to make sure ibc relayers work.
+sc-keep-recent = 1
+
+# SnapshotInterval defines the block interval the snapshot is taken, default to 10000 blocks.
+sc-snapshot-interval = 1000
+
+# SnapshotWriterLimit defines the max concurrency for taking commit store snapshot
+sc-snapshot-writer-limit = 2
+
+# CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000.
+sc-cache-size = 1000
+
+[state-store]
+
+# Enable defines if the state-store should be enabled for historical queries.
+# In order to use state-store, you need to make sure to enable state-commit at the same time
+ss-enable = true
+
+ss-db-directory = ""
+
+# DBBackend defines the backend database used for state-store.
+# Supported backends: pebbledb, rocksdb, sqlite
+# defaults to pebbledb (recommended)
+ss-backend = "pebbledb"
+
+# AsyncWriteBuffer defines the async queue length for commits to be applied to State Store
+# Set <= 0 for synchronous writes, which means commits also need to wait for data to be persisted in State Store.
+# defaults to 100
+ss-async-write-buffer = 100
+
+# KeepRecent defines the number of versions to keep in state store
+# Setting it to 0 means keep everything, default to 0
+ss-keep-recent = 10000
+
+# PruneIntervalSeconds defines the minimum interval in seconds + some random delay to trigger pruning.
+# It is more efficient to trigger pruning less frequently with large interval.
+# default to 600 seconds
+ss-prune-interval = 60
+
+# ImportNumWorkers defines the concurrency for state sync import
+# defaults to 1
+ss-import-num-workers = 1

docker/localnode/config/config.toml

@@ -46,7 +46,7 @@ db-backend = "goleveldb"
 db-dir = "data"
 
 # Output level for logging, including package level options
-log-level = "debug"
+log-level = "info"
 
 # Output format: 'plain' (colored text) or 'json'
 log-format = "plain"

docker/rpcnode/Dockerfile

@@ -1,7 +1,9 @@
 FROM ubuntu:latest
 RUN apt-get update && \
-    apt-get install -y make git golang jq python3 curl vim
-
+    apt-get install -y make git wget build-essential  jq python3 curl vim
+RUN wget https://go.dev/dl/go1.20.6.linux-amd64.tar.gz
+RUN tar -xvf go1.20.6.linux-amd64.tar.gz
+RUN mv go /usr/local/
 SHELL ["/bin/bash", "-c"]
 
 VOLUME [ "/sei-protocol" ]

docker/rpcnode/config/app.toml

@@ -200,4 +200,64 @@ snapshot-keep-recent = 2
 query_gas_limit = 300000
 # This is the number of wasm vm instances we keep cached in memory for speed-up
 # Warning: this is currently unstable and may lead to crashes, best to keep for 0 unless testing locally
-lru_size = 0
+lru_size = 0
+
+###############################################################################
+###                             SeiDB Configuration                       ###
+###############################################################################
+
+[state-commit]
+
+# Enable defines if the state-commit (memiavl) should be enabled to override existing IAVL db backend.
+sc-enable = true
+
+# ZeroCopy defines if memiavl should return slices pointing to mmap-ed buffers directly (zero-copy),
+# the zero-copied slices must not be retained beyond current block's execution.
+# the sdk address cache will be disabled if zero-copy is enabled.
+sc-zero-copy = false
+
+# AsyncCommitBuffer defines the size of asynchronous commit queue, this greatly improve block catching-up
+# performance, <=0 means synchronous commit.
+sc-async-commit-buffer = 100
+
+# SnapshotKeepRecent defines how many state-commit snapshots (besides the latest one) to keep
+# defaults to 1 to make sure ibc relayers work.
+sc-keep-recent = 1
+
+# SnapshotInterval defines the block interval the snapshot is taken, default to 10000 blocks.
+sc-snapshot-interval = 1000
+
+# SnapshotWriterLimit defines the max concurrency for taking commit store snapshot
+sc-snapshot-writer-limit = 2
+
+# CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000.
+sc-cache-size = 1000
+
+[state-store]
+
+# Enable defines if the state-store should be enabled for historical queries.
+# In order to use state-store, you need to make sure to enable state-commit at the same time
+ss-enable = true
+
+# DBBackend defines the backend database used for state-store.
+# Supported backends: pebbledb, rocksdb, sqlite
+# defaults to pebbledb (recommended)
+ss-backend = "pebbledb"
+
+# AsyncWriteBuffer defines the async queue length for commits to be applied to State Store
+# Set <= 0 for synchronous writes, which means commits also need to wait for data to be persisted in State Store.
+# defaults to 100
+ss-async-write-buffer = 100
+
+# KeepRecent defines the number of versions to keep in state store
+# Setting it to 0 means keep everything, default to 0
+ss-keep-recent = 1000
+
+# PruneIntervalSeconds defines the minimum interval in seconds + some random delay to trigger pruning.
+# It is more efficient to trigger pruning less frequently with large interval.
+# default to 600 seconds
+ss-prune-interval = 60
+
+# ImportNumWorkers defines the concurrency for state sync import
+# defaults to 1
+ss-import-num-workers = 1

docker/rpcnode/config/config.toml

@@ -46,7 +46,7 @@ db-backend = "goleveldb"
 db-dir = "data"
 
 # Output level for logging, including package level options
-log-level = "debug"
+log-level = "info"
 
 # Output format: 'plain' (colored text) or 'json'
 log-format = "plain"