go/storage: Make maximum in-memory storage cache size configurable

.changelog/2494.feature.md

@@ -0,0 +1,5 @@
+Make maximum in-memory cache size for runtime storage configurable.
+
+Previously the value of 64mb was always used as the size of the in-memory storage cache. This adds a
+new configuration parameter/command-line flag `--storage.max_cache_size` which configures the
+maximum size of the in-memory runtime storage cache.

go/oasis-node/cmd/debug/storage/export.go

@@ -195,6 +195,7 @@ func newDirectStorageBackend(dataDir string, namespace common.Namespace) (storag
 		DB:                dataDir,
 		ApplyLockLRUSlots: uint64(viper.GetInt(storage.CfgLRUSlots)),
 		Namespace:         namespace,
+		MaxCacheSize:      int64(viper.GetSizeInBytes(storage.CfgMaxCacheSize)),
 	}
 
 	b := strings.ToLower(viper.GetString(storage.CfgBackend))

go/storage/api/api.go

@@ -90,13 +90,17 @@ type Config struct {
 
 	// Namespace is the namespace contained within the database.
 	Namespace common.Namespace
+
+	// MaxCacheSize is the maximum in-memory cache size for the database.
+	MaxCacheSize int64
 }
 
 // ToNodeDB converts from a Config to a node DB Config.
 func (cfg *Config) ToNodeDB() *nodedb.Config {
 	return &nodedb.Config{
-		DB:        cfg.DB,
-		Namespace: cfg.Namespace,
+		DB:           cfg.DB,
+		Namespace:    cfg.Namespace,
+		MaxCacheSize: cfg.MaxCacheSize,
 	}
 }
 

go/storage/crashing_test.go

@@ -24,8 +24,9 @@ func TestCrashingBackendDoNotInterfere(t *testing.T) {
 
 	var (
 		cfg = api.Config{
-			Backend:   database.BackendNameBadgerDB,
-			Namespace: testNs,
+			Backend:      database.BackendNameBadgerDB,
+			Namespace:    testNs,
+			MaxCacheSize: 16 * 1024 * 1024,
 		}
 		err error
 	)

go/storage/database/database_test.go

@@ -36,6 +36,7 @@ func doTestImpl(t *testing.T, backend string) {
 			Backend:           backend,
 			ApplyLockLRUSlots: 100,
 			Namespace:         testNs,
+			MaxCacheSize:      16 * 1024 * 1024,
 		}
 		err error
 	)

go/storage/init.go

@@ -22,10 +22,15 @@ import (
 
 const (
 	// CfgBackend configures the storage backend flag.
-	CfgBackend      = "storage.backend"
-	cfgCrashEnabled = "storage.crash.enabled"
+	CfgBackend = "storage.backend"
+
 	// CfgLRUSlots configures the LRU apply lock slots.
-	CfgLRUSlots           = "storage.root_cache.apply_lock_lru_slots"
+	CfgLRUSlots = "storage.root_cache.apply_lock_lru_slots"
+
+	// CfgMaxCacheSize configures the maximum in-memory cache size.
+	CfgMaxCacheSize = "storage.max_cache_size"
+
+	cfgCrashEnabled       = "storage.crash.enabled"
 	cfgInsecureSkipChecks = "storage.debug.insecure_skip_checks"
 )
 
@@ -48,6 +53,7 @@ func New(
 		ApplyLockLRUSlots:  uint64(viper.GetInt(CfgLRUSlots)),
 		InsecureSkipChecks: viper.GetBool(cfgInsecureSkipChecks) && cmdFlags.DebugDontBlameOasis(),
 		Namespace:          namespace,
+		MaxCacheSize:       int64(viper.GetSizeInBytes(CfgMaxCacheSize)),
 	}
 
 	var (
@@ -80,6 +86,7 @@ func init() {
 	Flags.String(CfgBackend, database.BackendNameBadgerDB, "Storage backend")
 	Flags.Bool(cfgCrashEnabled, false, "Enable the crashing storage wrapper")
 	Flags.Int(CfgLRUSlots, 1000, "How many LRU slots to use for Apply call locks in the MKVS tree root cache")
+	Flags.String(CfgMaxCacheSize, "64mb", "Maximum in-memory cache size")
 
 	Flags.Bool(cfgInsecureSkipChecks, false, "INSECURE: Skip known root checks")
 

go/storage/mkvs/urkel/db/api/api.go

@@ -50,6 +50,9 @@ type Config struct {
 
 	// Namespace is the namespace contained within the database.
 	Namespace common.Namespace
+
+	// MaxCacheSize is the maximum in-memory cache size for the database.
+	MaxCacheSize int64
 }
 
 // NodeDB is the persistence layer used for persisting the in-memory tree.

go/storage/mkvs/urkel/db/badger/badger.go

@@ -127,8 +127,7 @@ func New(cfg *api.Config) (api.NodeDB, error) {
 	opts = opts.WithLogger(cmnBadger.NewLogAdapter(db.logger))
 	opts = opts.WithSyncWrites(!cfg.DebugNoFsync)
 	opts = opts.WithCompression(options.None)
-	// Reduce cache size to 64 MiB as the default is 1 GiB.
-	opts = opts.WithMaxCacheSize(64 * 1024 * 1024)
+	opts = opts.WithMaxCacheSize(cfg.MaxCacheSize)
 
 	var err error
 	if db.db, err = badger.Open(opts); err != nil {

go/storage/mkvs/urkel/interop/cmd/protocol_server.go

@@ -76,6 +76,7 @@ func doProtoServer(cmd *cobra.Command, args []string) {
 		Signer:             ident.NodeSigner,
 		ApplyLockLRUSlots:  1,
 		InsecureSkipChecks: false,
+		MaxCacheSize:       16 * 1024 * 1024,
 	}
 	backend, err := database.New(&storageCfg)
 	if err != nil {

go/storage/mkvs/urkel/urkel_test.go

@@ -1887,6 +1887,7 @@ func TestUrkelBadgerBackend(t *testing.T) {
 			DB:           dir,
 			DebugNoFsync: true,
 			Namespace:    testNs,
+			MaxCacheSize: 16 * 1024 * 1024,
 		})
 		require.NoError(t, err, "New")
 
@@ -1942,7 +1943,9 @@ func benchmarkInsertBatch(b *testing.B, numValues int, commit bool) {
 		require.NoError(b, err, "TempDir")
 		defer os.RemoveAll(dir)
 		ndb, err := badgerDb.New(&db.Config{
-			DB: dir,
+			DB:           dir,
+			Namespace:    testNs,
+			MaxCacheSize: 16 * 1024 * 1024,
 		})
 		require.NoError(b, err, "New")
 		tree := New(nil, ndb)
@@ -1975,12 +1978,6 @@ func generateKeyValuePairs() ([][]byte, [][]byte) {
 	return generateKeyValuePairsEx("", insertItems)
 }
 
-func init() {
-	var ns hash.Hash
-	ns.FromBytes([]byte("oasis urkel test ns"))
-	copy(testNs[:], ns[:])
-}
-
 func generateLongKeyValuePairs() ([][]byte, [][]byte) {
 	keys := make([][]byte, len(longKey))
 	values := make([][]byte, len(longKey))
@@ -2013,3 +2010,9 @@ func generatePopulatedTree(t *testing.T, ndb db.NodeDB) ([][]byte, [][]byte, nod
 	}
 	return keys, values, root, tree
 }
+
+func init() {
+	var ns hash.Hash
+	ns.FromBytes([]byte("oasis urkel test ns"))
+	copy(testNs[:], ns[:])
+}