From e990c7a54c4620bf4ebad8ae669281a2c3dfda5b Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Thu, 6 Aug 2020 13:12:32 -0700 Subject: [PATCH 1/3] Stop forcing RAM mode for the w directory. --- worker/server_state.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/worker/server_state.go b/worker/server_state.go index 898b5a99a88..fa972dcbe02 100644 --- a/worker/server_state.go +++ b/worker/server_state.go @@ -132,13 +132,6 @@ func (s *ServerState) initStorage() { opt.ValueLogMaxEntries = 10000 // Allow for easy space reclamation. opt.MaxCacheSize = 10 << 20 // 10 mb of cache size for WAL. - // We should always force load LSM tables to memory, disregarding user settings, because - // Raft.Advance hits the WAL many times. If the tables are not in memory, retrieval slows - // down way too much, causing cluster membership issues. Because of prefix compression and - // value separation provided by Badger, this is still better than using the memory based WAL - // storage provided by the Raft library. - opt.TableLoadingMode = options.LoadToRAM - // Print the options w/o exposing key. // TODO: Build a stringify interface in Badger options, which is used to print nicely here. key := opt.EncryptionKey From 4549e0fecebad8ea3bc37e1a450b36d5690464e3 Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Thu, 20 Aug 2020 12:45:15 -0700 Subject: [PATCH 2/3] Add flags for setting badger options for wal separately. --- dgraph/cmd/alpha/run.go | 13 +++++++++++-- worker/config.go | 9 +++++++-- worker/server_state.go | 22 +++++++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index 5e1db23da07..d439466f0fb 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -105,11 +105,18 @@ they form a Raft group and provide synchronous replication. // Options around how to set up Badger. flag.String("badger.tables", "mmap", - "[ram, mmap, disk] Specifies how Badger LSM tree is stored. "+ + "[ram, mmap, disk] Specifies how Badger LSM tree is stored for the postings directory. "+ "Option sequence consume most to least RAM while providing best to worst read "+ "performance respectively.") flag.String("badger.vlog", "mmap", - "[mmap, disk] Specifies how Badger Value log is stored."+ + "[mmap, disk] Specifies how Badger Value log is stored for the postings directory."+ + " mmap consumes more RAM, but provides better performance.") + flag.String("badger.wal_tables", "mmap", + "[ram, mmap, disk] Specifies how Badger LSM tree is stored for the write-ahead log. "+ + "Option sequence consume most to least RAM while providing best to worst read "+ + "performance respectively.") + flag.String("badger.wal_vlog", "mmap", + "[mmap, disk] Specifies how Badger Value log is stored for the write-ahead log."+ " mmap consumes more RAM, but provides better performance.") flag.Int("badger.compression_level", 3, "The compression level for Badger. A higher value uses more resources.") @@ -575,6 +582,8 @@ func run() { opts := worker.Options{ BadgerTables: Alpha.Conf.GetString("badger.tables"), BadgerVlog: Alpha.Conf.GetString("badger.vlog"), + BadgerWalTables: Alpha.Conf.GetString("badger.wal_tables"), + BadgerWalVlog: Alpha.Conf.GetString("badger.wal_vlog"), BadgerCompressionLevel: Alpha.Conf.GetInt("badger.compression_level"), PostingDir: Alpha.Conf.GetString("postings"), WALDir: Alpha.Conf.GetString("wal"), diff --git a/worker/config.go b/worker/config.go index 4a5a8fad244..33651f97ac2 100644 --- a/worker/config.go +++ b/worker/config.go @@ -39,10 +39,15 @@ const ( type Options struct { // PostingDir is the path to the directory storing the postings.. PostingDir string - // BadgerTables is the name of the mode used to load the badger tables. + // BadgerTables is the name of the mode used to load the badger tables for the p directory. BadgerTables string - // BadgerVlog is the name of the mode used to load the badger value log. + // BadgerVlog is the name of the mode used to load the badger value log for the p directory. BadgerVlog string + // BadgerWalTables is the name of the mode used to load the badger tables for the w directory. + BadgerWalTables string + // BadgerWalVlog is the name of the mode used to load the badger value log for the w directory. + BadgerWalVlog string + // BadgerCompressionLevel is the ZSTD compression level used by badger. A // higher value means more CPU intensive compression and better compression // ratio. diff --git a/worker/server_state.go b/worker/server_state.go index fa972dcbe02..b02041dba58 100644 --- a/worker/server_state.go +++ b/worker/server_state.go @@ -62,7 +62,7 @@ func InitServerState() { x.WorkerConfig.ProposedGroupId = groupId } -func setBadgerOptions(opt badger.Options) badger.Options { +func setBadgerOptions(opt badger.Options, wal bool) badger.Options { opt = opt.WithSyncWrites(false). WithTruncate(true). WithLogger(&x.ToGlog{}). @@ -86,8 +86,20 @@ func setBadgerOptions(opt badger.Options) badger.Options { opt.ZSTDCompressionLevel = Config.BadgerCompressionLevel } + var badgerTables string + var badgerVlog string + if wal { + // Settings for the write-ahead log. + badgerTables = Config.BadgerWalTables + badgerVlog = Config.BadgerWalVlog + } else { + // Settings for the data directory. + badgerTables = Config.BadgerTables + badgerVlog = Config.BadgerVlog + } + glog.Infof("Setting Badger table load option: %s", Config.BadgerTables) - switch Config.BadgerTables { + switch badgerTables { case "mmap": opt.TableLoadingMode = options.MemoryMap case "ram": @@ -99,7 +111,7 @@ func setBadgerOptions(opt badger.Options) badger.Options { } glog.Infof("Setting Badger value log load option: %s", Config.BadgerVlog) - switch Config.BadgerVlog { + switch badgerVlog { case "mmap": opt.ValueLogLoadingMode = options.MemoryMap case "disk": @@ -128,7 +140,7 @@ func (s *ServerState) initStorage() { // Write Ahead Log directory x.Checkf(os.MkdirAll(Config.WALDir, 0700), "Error while creating WAL dir.") opt := badger.LSMOnlyOptions(Config.WALDir) - opt = setBadgerOptions(opt) + opt = setBadgerOptions(opt, true) opt.ValueLogMaxEntries = 10000 // Allow for easy space reclamation. opt.MaxCacheSize = 10 << 20 // 10 mb of cache size for WAL. @@ -154,7 +166,7 @@ func (s *ServerState) initStorage() { WithKeepBlockIndicesInCache(true). WithKeepBlocksInCache(true). WithMaxBfCacheSize(500 << 20) // 500 MB of bloom filter cache. - opt = setBadgerOptions(opt) + opt = setBadgerOptions(opt, false) // Print the options w/o exposing key. // TODO: Build a stringify interface in Badger options, which is used to print nicely here. From 9e6e2da3d2be1f7766fa309d6fd81064ad3f40be Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Fri, 21 Aug 2020 12:17:01 -0700 Subject: [PATCH 3/3] Merge flags. --- dgraph/cmd/alpha/run.go | 55 +++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index d439466f0fb..b0afd4c6029 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -104,20 +104,17 @@ they form a Raft group and provide synchronous replication. flag.StringP("postings", "p", "p", "Directory to store posting lists.") // Options around how to set up Badger. - flag.String("badger.tables", "mmap", - "[ram, mmap, disk] Specifies how Badger LSM tree is stored for the postings directory. "+ - "Option sequence consume most to least RAM while providing best to worst read "+ - "performance respectively.") - flag.String("badger.vlog", "mmap", - "[mmap, disk] Specifies how Badger Value log is stored for the postings directory."+ - " mmap consumes more RAM, but provides better performance.") - flag.String("badger.wal_tables", "mmap", - "[ram, mmap, disk] Specifies how Badger LSM tree is stored for the write-ahead log. "+ - "Option sequence consume most to least RAM while providing best to worst read "+ - "performance respectively.") - flag.String("badger.wal_vlog", "mmap", - "[mmap, disk] Specifies how Badger Value log is stored for the write-ahead log."+ - " mmap consumes more RAM, but provides better performance.") + flag.String("badger.tables", "mmap,mmap", + "[ram, mmap, disk] Specifies how Badger LSM tree is stored for the postings and "+ + "write-ahead directory. Option sequence consume most to least RAM while providing "+ + "best to worst read performance respectively. If you pass two values separated by a "+ + "comma, the first value will be used for the postings directory and the second for "+ + "the write-ahead log directory.") + flag.String("badger.vlog", "mmap,mmap", + "[mmap, disk] Specifies how Badger Value log is stored for the postings and write-ahead "+ + "log directory. mmap consumes more RAM, but provides better performance. If you pass "+ + "two values separated by a comma the first value will be used for the postings "+ + "directory and the second for the w directory.") flag.Int("badger.compression_level", 3, "The compression level for Badger. A higher value uses more resources.") enc.RegisterFlags(flag) @@ -580,10 +577,6 @@ func run() { bindall = Alpha.Conf.GetBool("bindall") opts := worker.Options{ - BadgerTables: Alpha.Conf.GetString("badger.tables"), - BadgerVlog: Alpha.Conf.GetString("badger.vlog"), - BadgerWalTables: Alpha.Conf.GetString("badger.wal_tables"), - BadgerWalVlog: Alpha.Conf.GetString("badger.wal_vlog"), BadgerCompressionLevel: Alpha.Conf.GetInt("badger.compression_level"), PostingDir: Alpha.Conf.GetString("postings"), WALDir: Alpha.Conf.GetString("wal"), @@ -593,6 +586,32 @@ func run() { AllottedMemory: Alpha.Conf.GetFloat64("lru_mb"), } + badgerTables := strings.Split(Alpha.Conf.GetString("badger.tables"), ",") + if len(badgerTables) != 1 && len(badgerTables) != 2 { + glog.Fatalf("Unable to read badger.tables options. Expected single value or two "+ + "comma-separated values. Got %s", Alpha.Conf.GetString("badger.tables")) + } + if len(badgerTables) == 1 { + opts.BadgerTables = badgerTables[0] + opts.BadgerWalTables = badgerTables[0] + } else { + opts.BadgerTables = badgerTables[0] + opts.BadgerWalTables = badgerTables[1] + } + + badgerVlog := strings.Split(Alpha.Conf.GetString("badger.vlog"), ",") + if len(badgerVlog) != 1 && len(badgerVlog) != 2 { + glog.Fatalf("Unable to read badger.vlog options. Expected single value or two "+ + "comma-separated values. Got %s", Alpha.Conf.GetString("badger.vlog")) + } + if len(badgerVlog) == 1 { + opts.BadgerVlog = badgerVlog[0] + opts.BadgerWalVlog = badgerVlog[0] + } else { + opts.BadgerVlog = badgerVlog[0] + opts.BadgerWalVlog = badgerVlog[1] + } + secretFile := Alpha.Conf.GetString("acl_secret_file") if secretFile != "" { hmacSecret, err := ioutil.ReadFile(secretFile)