From 5a388dfb3bab661099f56d43ec3188b40cc58ec4 Mon Sep 17 00:00:00 2001 From: ptrus Date: Thu, 27 Feb 2020 12:03:51 +0100 Subject: [PATCH] go/badger: enable truncate to recover from corrupted value log file --- .changelog/2732.bugfix.md | 4 ++++ go/common/persistent/persistent.go | 3 +++ go/consensus/tendermint/db/badger/badger.go | 3 +++ go/runtime/history/db.go | 3 +++ go/runtime/localstorage/localstorage.go | 3 +++ go/storage/mkvs/urkel/db/badger/badger.go | 3 +++ 6 files changed, 19 insertions(+) create mode 100644 .changelog/2732.bugfix.md diff --git a/.changelog/2732.bugfix.md b/.changelog/2732.bugfix.md new file mode 100644 index 00000000000..bb746ca7de8 --- /dev/null +++ b/.changelog/2732.bugfix.md @@ -0,0 +1,4 @@ +go/badger: enable truncate to recover from corrupted value log file + +Apparently badger is not at all resilient to crashes unless the truncate +option is enabled. diff --git a/go/common/persistent/persistent.go b/go/common/persistent/persistent.go index fbc613573c6..8299e151d26 100644 --- a/go/common/persistent/persistent.go +++ b/go/common/persistent/persistent.go @@ -49,6 +49,9 @@ func NewCommonStore(dataDir string) (*CommonStore, error) { opts := badger.DefaultOptions(filepath.Join(dataDir, dbName)) opts = opts.WithLogger(cmnBadger.NewLogAdapter(logger)) opts = opts.WithSyncWrites(true) + // Allow value log truncation if required (this is needed to recover the + // value log file which can get corrupted in crashes). + opts = opts.WithTruncate(true) opts = opts.WithCompression(options.None) // Reduce cache size to 128 KiB as the default is 1 GiB. opts = opts.WithMaxCacheSize(128 * 1024) diff --git a/go/consensus/tendermint/db/badger/badger.go b/go/consensus/tendermint/db/badger/badger.go index fdd40286553..e1a592e7d0d 100644 --- a/go/consensus/tendermint/db/badger/badger.go +++ b/go/consensus/tendermint/db/badger/badger.go @@ -66,6 +66,9 @@ func New(fn string, noSuffix bool) (dbm.DB, error) { opts := badger.DefaultOptions(fn) // This may benefit from LSMOnlyOptions. opts = opts.WithLogger(cmnBadger.NewLogAdapter(logger)) opts = opts.WithSyncWrites(false) + // Allow value log truncation if required (this is needed to recover the + // value log file which can get corrupted in crashes). + opts = opts.WithTruncate(true) opts = opts.WithCompression(options.Snappy) // Reduce cache size to 64 MiB as the default is 1 GiB. opts = opts.WithMaxCacheSize(64 * 1024 * 1024) diff --git a/go/runtime/history/db.go b/go/runtime/history/db.go index 2a1708e2dac..f30dcad20e5 100644 --- a/go/runtime/history/db.go +++ b/go/runtime/history/db.go @@ -53,6 +53,9 @@ func newDB(fn string, runtimeID common.Namespace) (*DB, error) { opts := badger.DefaultOptions(fn) opts = opts.WithLogger(cmnBadger.NewLogAdapter(logger)) opts = opts.WithSyncWrites(true) + // Allow value log truncation if required (this is needed to recover the + // value log file which can get corrupted in crashes). + opts = opts.WithTruncate(true) opts = opts.WithCompression(options.None) // Reduce cache size to 10 MiB as the default is 1 GiB. opts = opts.WithMaxCacheSize(10 * 1024 * 1024) diff --git a/go/runtime/localstorage/localstorage.go b/go/runtime/localstorage/localstorage.go index 056aef61a3b..36895ae7577 100644 --- a/go/runtime/localstorage/localstorage.go +++ b/go/runtime/localstorage/localstorage.go @@ -111,6 +111,9 @@ func New(dataDir, fn string, runtimeID common.Namespace) (LocalStorage, error) { opts := badger.DefaultOptions(filepath.Join(dataDir, fn)) opts = opts.WithLogger(cmnBadger.NewLogAdapter(s.logger)) opts = opts.WithSyncWrites(true) + // Allow value log truncation if required (this is needed to recover the + // value log file which can get corrupted in crashes). + opts = opts.WithTruncate(true) opts = opts.WithCompression(options.None) // Reduce cache size to 128 KiB as the default is 1 GiB. opts = opts.WithMaxCacheSize(128 * 1024) diff --git a/go/storage/mkvs/urkel/db/badger/badger.go b/go/storage/mkvs/urkel/db/badger/badger.go index a5a473563ee..46ee3dc7293 100644 --- a/go/storage/mkvs/urkel/db/badger/badger.go +++ b/go/storage/mkvs/urkel/db/badger/badger.go @@ -125,6 +125,9 @@ func New(cfg *api.Config) (api.NodeDB, error) { opts := badger.DefaultOptions(cfg.DB) opts = opts.WithLogger(cmnBadger.NewLogAdapter(db.logger)) opts = opts.WithSyncWrites(!cfg.DebugNoFsync) + // Allow value log truncation if required (this is needed to recover the + // value log file which can get corrupted in crashes). + opts = opts.WithTruncate(true) opts = opts.WithCompression(options.None) opts = opts.WithMaxCacheSize(cfg.MaxCacheSize)