From a4ea44998161f64b0e1373f8b78369567457846c Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 9 Apr 2024 15:22:36 +0800
Subject: [PATCH] Problem: rocksdb wal format sometimes not backward compatible

Solution:
- flush the wal before quit the node, make the rocksdb upgrade smooth.

Update CHANGELOG.md

Signed-off-by: yihuang <huang@crypto.com>

fix flush
---
 CHANGELOG.md                 |  6 ++++++
 app/app.go                   | 24 ++++++++++++++++--------
 versiondb/multistore.go      |  5 +++++
 versiondb/tsrocksdb/store.go | 10 ++++++++++
 versiondb/types.go           |  4 ++++
 5 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d63b8cf93e..4cafb11737 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## UNRELEASED
+
+### Improvements
+
+* (versiondb) [#1379](https://github.com/crypto-org-chain/cronos/pull/1379) Flush versiondb when graceful shutdown, make rocksdb upgrade smooth.
+
 *April 8, 2024*
 
 ## v1.2.0-rc1
diff --git a/app/app.go b/app/app.go
index 3e404344bd..1533139bd9 100644
--- a/app/app.go
+++ b/app/app.go
@@ -143,7 +143,6 @@ import (
 	// this line is used by starport scaffolding # stargate/app/moduleImport
 
 	memiavlstore "github.com/crypto-org-chain/cronos/store"
-	memiavlrootmulti "github.com/crypto-org-chain/cronos/store/rootmulti"
 	"github.com/crypto-org-chain/cronos/v2/x/cronos"
 	cronosclient "github.com/crypto-org-chain/cronos/v2/x/cronos/client"
 	cronoskeeper "github.com/crypto-org-chain/cronos/v2/x/cronos/keeper"
@@ -370,6 +369,8 @@ type App struct {
 
 	// module configurator
 	configurator module.Configurator
+
+	qms storetypes.MultiStore
 }
 
 // New returns a reference to an initialized chain.
@@ -860,10 +861,9 @@ func New(
 
 	// wire up the versiondb's `StreamingService` and `MultiStore`.
 	streamers := cast.ToStringSlice(appOpts.Get("store.streamers"))
-	var qms sdk.MultiStore
 	if slices.Contains(streamers, "versiondb") {
 		var err error
-		qms, err = app.setupVersionDB(homePath, keys, tkeys, memKeys)
+		app.qms, err = app.setupVersionDB(homePath, keys, tkeys, memKeys)
 		if err != nil {
 			panic(err)
 		}
@@ -900,8 +900,8 @@ func New(
 			tmos.Exit(err.Error())
 		}
 
-		if qms != nil {
-			v1 := qms.LatestVersion()
+		if app.qms != nil {
+			v1 := app.qms.LatestVersion()
 			v2 := app.LastBlockHeight()
 			if v1 > 0 && v1 < v2 {
 				// try to prevent gap being created in versiondb
@@ -1197,11 +1197,19 @@ func VerifyAddressFormat(bz []byte) error {
 
 // Close will be called in graceful shutdown in start cmd
 func (app *App) Close() error {
-	err := app.BaseApp.Close()
+	errs := []error{app.BaseApp.Close()}
+
+	// flush the versiondb
+	if closer, ok := app.qms.(io.Closer); ok {
+		errs = append(errs, closer.Close())
+	}
 
-	if cms, ok := app.CommitMultiStore().(*memiavlrootmulti.Store); ok {
-		return stderrors.Join(err, cms.Close())
+	// mainly to flush memiavl
+	if closer, ok := app.CommitMultiStore().(io.Closer); ok {
+		errs = append(errs, closer.Close())
 	}
 
+	err := stderrors.Join(errs...)
+	app.Logger().Info("Application gracefully shutdown", "error", err)
 	return err
 }
diff --git a/versiondb/multistore.go b/versiondb/multistore.go
index 878f6dd08e..8f61075dc7 100644
--- a/versiondb/multistore.go
+++ b/versiondb/multistore.go
@@ -145,3 +145,8 @@ func (s *MultiStore) LatestVersion() int64 {
 	}
 	return version
 }
+
+// Close will flush the versiondb
+func (s *MultiStore) Close() error {
+	return s.versionDB.Flush()
+}
diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go
index b927ea36b9..be34c74632 100644
--- a/versiondb/tsrocksdb/store.go
+++ b/versiondb/tsrocksdb/store.go
@@ -205,6 +205,16 @@ func (s Store) Import(version int64, ch <-chan versiondb.ImportEntry) error {
 	return s.SetLatestVersion(version)
 }
 
+func (s Store) Flush() error {
+	opts := grocksdb.NewDefaultFlushOptions()
+	defer opts.Destroy()
+
+	return errors.Join(
+		s.db.Flush(opts),
+		s.db.FlushCF(s.cfHandle, opts),
+	)
+}
+
 func newTSReadOptions(version *int64) *grocksdb.ReadOptions {
 	var ver uint64
 	if version == nil {
diff --git a/versiondb/types.go b/versiondb/types.go
index 46bbe88e9e..6c69fbbf94 100644
--- a/versiondb/types.go
+++ b/versiondb/types.go
@@ -21,6 +21,10 @@ type VersionStore interface {
 
 	// Import the initial state of the store
 	Import(version int64, ch <-chan ImportEntry) error
+
+	// Flush wal logs, and make the changes persistent,
+	// mainly for rocksdb version upgrade, sometimes the wal format is not compatible.
+	Flush() error
 }
 
 type ImportEntry struct {