From 067af88244fc99ddbdb26c98bdb89424f580bea0 Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Fri, 27 Dec 2019 17:32:19 +0100 Subject: [PATCH] go/storage: Make maximum in-memory storage cache size configurable --- .changelog/2494.feature.md | 5 +++++ go/oasis-node/cmd/debug/storage/export.go | 1 + go/storage/api/api.go | 8 ++++++-- go/storage/crashing_test.go | 5 +++-- go/storage/database/database_test.go | 1 + go/storage/init.go | 13 ++++++++++--- go/storage/mkvs/urkel/db/api/api.go | 3 +++ go/storage/mkvs/urkel/db/badger/badger.go | 3 +-- .../mkvs/urkel/interop/cmd/protocol_server.go | 1 + go/storage/mkvs/urkel/urkel_test.go | 17 ++++++++++------- 10 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 .changelog/2494.feature.md diff --git a/.changelog/2494.feature.md b/.changelog/2494.feature.md new file mode 100644 index 00000000000..15ceb61af98 --- /dev/null +++ b/.changelog/2494.feature.md @@ -0,0 +1,5 @@ +Make maximum in-memory cache size for runtime storage configurable. + +Previously the value of 64mb was always used as the size of the in-memory storage cache. This adds a +new configuration parameter/command-line flag `--storage.max_cache_size` which configures the +maximum size of the in-memory runtime storage cache. diff --git a/go/oasis-node/cmd/debug/storage/export.go b/go/oasis-node/cmd/debug/storage/export.go index c2ad051b19c..7aa23d6c0d3 100644 --- a/go/oasis-node/cmd/debug/storage/export.go +++ b/go/oasis-node/cmd/debug/storage/export.go @@ -195,6 +195,7 @@ func newDirectStorageBackend(dataDir string, namespace common.Namespace) (storag DB: dataDir, ApplyLockLRUSlots: uint64(viper.GetInt(storage.CfgLRUSlots)), Namespace: namespace, + MaxCacheSize: int64(viper.GetSizeInBytes(storage.CfgMaxCacheSize)), } b := strings.ToLower(viper.GetString(storage.CfgBackend)) diff --git a/go/storage/api/api.go b/go/storage/api/api.go index 6352753dfe9..cd7777d0b13 100644 --- a/go/storage/api/api.go +++ b/go/storage/api/api.go @@ -90,13 +90,17 @@ type Config struct { // Namespace is the namespace contained within the database. Namespace common.Namespace + + // MaxCacheSize is the maximum in-memory cache size for the database. + MaxCacheSize int64 } // ToNodeDB converts from a Config to a node DB Config. func (cfg *Config) ToNodeDB() *nodedb.Config { return &nodedb.Config{ - DB: cfg.DB, - Namespace: cfg.Namespace, + DB: cfg.DB, + Namespace: cfg.Namespace, + MaxCacheSize: cfg.MaxCacheSize, } } diff --git a/go/storage/crashing_test.go b/go/storage/crashing_test.go index f9b8b28dbba..0b7e158be5b 100644 --- a/go/storage/crashing_test.go +++ b/go/storage/crashing_test.go @@ -24,8 +24,9 @@ func TestCrashingBackendDoNotInterfere(t *testing.T) { var ( cfg = api.Config{ - Backend: database.BackendNameBadgerDB, - Namespace: testNs, + Backend: database.BackendNameBadgerDB, + Namespace: testNs, + MaxCacheSize: 16 * 1024 * 1024, } err error ) diff --git a/go/storage/database/database_test.go b/go/storage/database/database_test.go index 7f683e0790c..97dd854089f 100644 --- a/go/storage/database/database_test.go +++ b/go/storage/database/database_test.go @@ -36,6 +36,7 @@ func doTestImpl(t *testing.T, backend string) { Backend: backend, ApplyLockLRUSlots: 100, Namespace: testNs, + MaxCacheSize: 16 * 1024 * 1024, } err error ) diff --git a/go/storage/init.go b/go/storage/init.go index 24b4e063f45..ff09b1f84dc 100644 --- a/go/storage/init.go +++ b/go/storage/init.go @@ -22,10 +22,15 @@ import ( const ( // CfgBackend configures the storage backend flag. - CfgBackend = "storage.backend" - cfgCrashEnabled = "storage.crash.enabled" + CfgBackend = "storage.backend" + // CfgLRUSlots configures the LRU apply lock slots. - CfgLRUSlots = "storage.root_cache.apply_lock_lru_slots" + CfgLRUSlots = "storage.root_cache.apply_lock_lru_slots" + + // CfgMaxCacheSize configures the maximum in-memory cache size. + CfgMaxCacheSize = "storage.max_cache_size" + + cfgCrashEnabled = "storage.crash.enabled" cfgInsecureSkipChecks = "storage.debug.insecure_skip_checks" ) @@ -48,6 +53,7 @@ func New( ApplyLockLRUSlots: uint64(viper.GetInt(CfgLRUSlots)), InsecureSkipChecks: viper.GetBool(cfgInsecureSkipChecks) && cmdFlags.DebugDontBlameOasis(), Namespace: namespace, + MaxCacheSize: int64(viper.GetSizeInBytes(CfgMaxCacheSize)), } var ( @@ -80,6 +86,7 @@ func init() { Flags.String(CfgBackend, database.BackendNameBadgerDB, "Storage backend") Flags.Bool(cfgCrashEnabled, false, "Enable the crashing storage wrapper") Flags.Int(CfgLRUSlots, 1000, "How many LRU slots to use for Apply call locks in the MKVS tree root cache") + Flags.String(CfgMaxCacheSize, "64mb", "Maximum in-memory cache size") Flags.Bool(cfgInsecureSkipChecks, false, "INSECURE: Skip known root checks") diff --git a/go/storage/mkvs/urkel/db/api/api.go b/go/storage/mkvs/urkel/db/api/api.go index 4911dcfca40..06d5183dacd 100644 --- a/go/storage/mkvs/urkel/db/api/api.go +++ b/go/storage/mkvs/urkel/db/api/api.go @@ -50,6 +50,9 @@ type Config struct { // Namespace is the namespace contained within the database. Namespace common.Namespace + + // MaxCacheSize is the maximum in-memory cache size for the database. + MaxCacheSize int64 } // NodeDB is the persistence layer used for persisting the in-memory tree. diff --git a/go/storage/mkvs/urkel/db/badger/badger.go b/go/storage/mkvs/urkel/db/badger/badger.go index e03f738057b..f88afbdeb9c 100644 --- a/go/storage/mkvs/urkel/db/badger/badger.go +++ b/go/storage/mkvs/urkel/db/badger/badger.go @@ -127,8 +127,7 @@ func New(cfg *api.Config) (api.NodeDB, error) { opts = opts.WithLogger(cmnBadger.NewLogAdapter(db.logger)) opts = opts.WithSyncWrites(!cfg.DebugNoFsync) opts = opts.WithCompression(options.None) - // Reduce cache size to 64 MiB as the default is 1 GiB. - opts = opts.WithMaxCacheSize(64 * 1024 * 1024) + opts = opts.WithMaxCacheSize(cfg.MaxCacheSize) var err error if db.db, err = badger.Open(opts); err != nil { diff --git a/go/storage/mkvs/urkel/interop/cmd/protocol_server.go b/go/storage/mkvs/urkel/interop/cmd/protocol_server.go index cc73f31c600..5aaac37af66 100644 --- a/go/storage/mkvs/urkel/interop/cmd/protocol_server.go +++ b/go/storage/mkvs/urkel/interop/cmd/protocol_server.go @@ -76,6 +76,7 @@ func doProtoServer(cmd *cobra.Command, args []string) { Signer: ident.NodeSigner, ApplyLockLRUSlots: 1, InsecureSkipChecks: false, + MaxCacheSize: 16 * 1024 * 1024, } backend, err := database.New(&storageCfg) if err != nil { diff --git a/go/storage/mkvs/urkel/urkel_test.go b/go/storage/mkvs/urkel/urkel_test.go index c4b838d9c93..a7ee84263c7 100644 --- a/go/storage/mkvs/urkel/urkel_test.go +++ b/go/storage/mkvs/urkel/urkel_test.go @@ -1887,6 +1887,7 @@ func TestUrkelBadgerBackend(t *testing.T) { DB: dir, DebugNoFsync: true, Namespace: testNs, + MaxCacheSize: 16 * 1024 * 1024, }) require.NoError(t, err, "New") @@ -1942,7 +1943,9 @@ func benchmarkInsertBatch(b *testing.B, numValues int, commit bool) { require.NoError(b, err, "TempDir") defer os.RemoveAll(dir) ndb, err := badgerDb.New(&db.Config{ - DB: dir, + DB: dir, + Namespace: testNs, + MaxCacheSize: 16 * 1024 * 1024, }) require.NoError(b, err, "New") tree := New(nil, ndb) @@ -1975,12 +1978,6 @@ func generateKeyValuePairs() ([][]byte, [][]byte) { return generateKeyValuePairsEx("", insertItems) } -func init() { - var ns hash.Hash - ns.FromBytes([]byte("oasis urkel test ns")) - copy(testNs[:], ns[:]) -} - func generateLongKeyValuePairs() ([][]byte, [][]byte) { keys := make([][]byte, len(longKey)) values := make([][]byte, len(longKey)) @@ -2013,3 +2010,9 @@ func generatePopulatedTree(t *testing.T, ndb db.NodeDB) ([][]byte, [][]byte, nod } return keys, values, root, tree } + +func init() { + var ns hash.Hash + ns.FromBytes([]byte("oasis urkel test ns")) + copy(testNs[:], ns[:]) +}