diff --git a/changelog/24010.txt b/changelog/24010.txt new file mode 100644 index 000000000000..aa72bc977912 --- /dev/null +++ b/changelog/24010.txt @@ -0,0 +1,3 @@ +```release-note:improvement +storage/raft: Upgrade to bbolt 1.3.8, along with an extra patch to reduce time scanning large freelist maps. +``` diff --git a/command/agent/cache/cacheboltdb/bolt.go b/command/agent/cache/cacheboltdb/bolt.go index a6740861a1b6..25c8fc3d8381 100644 --- a/command/agent/cache/cacheboltdb/bolt.go +++ b/command/agent/cache/cacheboltdb/bolt.go @@ -12,10 +12,10 @@ import ( "time" "github.com/golang/protobuf/proto" + bolt "github.com/hashicorp-forge/bbolt" "github.com/hashicorp/go-hclog" wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/go-multierror" - bolt "go.etcd.io/bbolt" ) const ( diff --git a/command/agent/cache/cacheboltdb/bolt_test.go b/command/agent/cache/cacheboltdb/bolt_test.go index 7099919cd667..a3ee327e5f1a 100644 --- a/command/agent/cache/cacheboltdb/bolt_test.go +++ b/command/agent/cache/cacheboltdb/bolt_test.go @@ -15,11 +15,11 @@ import ( "time" "github.com/golang/protobuf/proto" + bolt "github.com/hashicorp-forge/bbolt" "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/command/agent/cache/keymanager" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - bolt "go.etcd.io/bbolt" ) func getTestKeyManager(t *testing.T) keymanager.KeyManager { diff --git a/go.mod b/go.mod index d8a6be31d630..cc899d974b7f 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/google/go-github v17.0.0+incompatible github.com/google/go-metrics-stackdriver v0.2.0 github.com/google/tink/go v1.6.1 + github.com/hashicorp-forge/bbolt v1.3.8-hc3 github.com/hashicorp/cap v0.2.1-0.20230221194157-7894fed1633d github.com/hashicorp/consul-template v0.29.5 github.com/hashicorp/consul/api v1.17.0 diff --git a/go.sum b/go.sum index fda904b33f6e..d46a306f381e 100644 --- a/go.sum +++ b/go.sum @@ -1033,6 +1033,8 @@ github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hashicorp-forge/bbolt v1.3.8-hc3 h1:iTWR3RDPj0TGChAvJ8QjHFcNFWAUVgNQV73IE6gAX4E= +github.com/hashicorp-forge/bbolt v1.3.8-hc3/go.mod h1:sQBu5UIJ+rcUFU4Fo9rpTHNV935jwmGWS3dQ/MV8810= github.com/hashicorp/cap v0.2.1-0.20230221194157-7894fed1633d h1:29noMC2UssBX3F/BUmk0/j4PRUU4QvPTfyeOn3tmcOA= github.com/hashicorp/cap v0.2.1-0.20230221194157-7894fed1633d/go.mod h1:dHTmyMIVbzT981XxRoci5G//dfWmd/HhuNiCH6J5+IA= github.com/hashicorp/consul-template v0.29.5 h1:tzEo93RqODAX2cgOe/ke8xcpdPdxg5rxl6d22wE3f6c= diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index abc5e87a0fc8..3323f987df8b 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -20,6 +20,7 @@ import ( "github.com/armon/go-metrics" "github.com/golang/protobuf/proto" + bolt "github.com/hashicorp-forge/bbolt" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-raftchunking" @@ -28,7 +29,6 @@ import ( "github.com/hashicorp/vault/sdk/helper/jsonutil" "github.com/hashicorp/vault/sdk/physical" "github.com/hashicorp/vault/sdk/plugin/pb" - bolt "go.etcd.io/bbolt" ) const ( diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 9c8a26d2113f..bbbe3a442647 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -20,6 +20,7 @@ import ( "github.com/armon/go-metrics" "github.com/golang/protobuf/proto" + bolt "github.com/hashicorp-forge/bbolt" log "github.com/hashicorp/go-hclog" wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/go-raftchunking" @@ -38,7 +39,7 @@ import ( "github.com/hashicorp/vault/vault/cluster" "github.com/hashicorp/vault/vault/seal" "github.com/hashicorp/vault/version" - bolt "go.etcd.io/bbolt" + etcdbolt "go.etcd.io/bbolt" ) const ( @@ -401,7 +402,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend // Create the backend raft store for logs and stable storage. dbPath := filepath.Join(path, "raft.db") - opts := boltOptions(dbPath) + opts := etcdboltOptions(dbPath) raftOptions := raftboltdb.Options{ Path: dbPath, BoltOptions: opts, @@ -632,7 +633,7 @@ func (b *RaftBackend) CollectMetrics(sink *metricsutil.ClusterMetricSink) { stats = b.raft.Stats() } b.l.RUnlock() - b.collectMetricsWithStats(logstoreStats, sink, "logstore") + b.collectEtcdBoltMetricsWithStats(logstoreStats, sink, "logstore") b.collectMetricsWithStats(fsmStats, sink, "fsm") labels := []metrics.Label{ { @@ -673,6 +674,29 @@ func (b *RaftBackend) collectMetricsWithStats(stats bolt.Stats, sink *metricsuti sink.AddSampleWithLabels([]string{"raft_storage", "bolt", "write", "time"}, float32(txstats.WriteTime.Milliseconds()), labels) } +func (b *RaftBackend) collectEtcdBoltMetricsWithStats(stats etcdbolt.Stats, sink *metricsutil.ClusterMetricSink, database string) { + txstats := stats.TxStats + labels := []metricsutil.Label{{"database", database}} + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "freelist", "free_pages"}, float32(stats.FreePageN), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "freelist", "pending_pages"}, float32(stats.PendingPageN), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "freelist", "allocated_bytes"}, float32(stats.FreeAlloc), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "freelist", "used_bytes"}, float32(stats.FreelistInuse), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "transaction", "started_read_transactions"}, float32(stats.TxN), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "transaction", "currently_open_read_transactions"}, float32(stats.OpenTxN), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "page", "count"}, float32(txstats.GetPageCount()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "page", "bytes_allocated"}, float32(txstats.GetPageAlloc()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "cursor", "count"}, float32(txstats.GetCursorCount()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "node", "count"}, float32(txstats.GetNodeCount()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "node", "dereferences"}, float32(txstats.GetNodeDeref()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "rebalance", "count"}, float32(txstats.GetRebalance()), labels) + sink.AddSampleWithLabels([]string{"raft_storage", "bolt", "rebalance", "time"}, float32(txstats.GetRebalanceTime().Milliseconds()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "split", "count"}, float32(txstats.GetSplit()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "spill", "count"}, float32(txstats.GetSpill()), labels) + sink.AddSampleWithLabels([]string{"raft_storage", "bolt", "spill", "time"}, float32(txstats.GetSpillTime().Milliseconds()), labels) + sink.SetGaugeWithLabels([]string{"raft_storage", "bolt", "write", "count"}, float32(txstats.GetWrite()), labels) + sink.IncrCounterWithLabels([]string{"raft_storage", "bolt", "write", "time"}, float32(txstats.GetWriteTime().Milliseconds()), labels) +} + // RaftServer has information about a server in the Raft configuration type RaftServer struct { // NodeID is the name of the server @@ -1967,3 +1991,39 @@ func boltOptions(path string) *bolt.Options { return o } + +func etcdboltOptions(path string) *etcdbolt.Options { + o := &etcdbolt.Options{ + Timeout: 1 * time.Second, + FreelistType: etcdbolt.FreelistMapType, + NoFreelistSync: true, + MmapFlags: getMmapFlags(path), + } + + if os.Getenv("VAULT_RAFT_FREELIST_TYPE") == "array" { + o.FreelistType = etcdbolt.FreelistArrayType + } + + if os.Getenv("VAULT_RAFT_FREELIST_SYNC") != "" { + o.NoFreelistSync = false + } + + // By default, we want to set InitialMmapSize to 100GB, but only on 64bit platforms. + // Otherwise, we set it to whatever the value of VAULT_RAFT_INITIAL_MMAP_SIZE + // is, assuming it can be parsed as an int. Bolt itself sets this to 0 by default, + // so if users are wanting to turn this off, they can also set it to 0. Setting it + // to a negative value is the same as not setting it at all. + if os.Getenv("VAULT_RAFT_INITIAL_MMAP_SIZE") == "" { + o.InitialMmapSize = initialMmapSize + } else { + imms, err := strconv.Atoi(os.Getenv("VAULT_RAFT_INITIAL_MMAP_SIZE")) + + // If there's an error here, it means they passed something that's not convertible to + // a number. Rather than fail startup, just ignore it. + if err == nil && imms > 0 { + o.InitialMmapSize = imms + } + } + + return o +} diff --git a/physical/raft/raft_test.go b/physical/raft/raft_test.go index cc3594f0f12d..e7532b992a99 100644 --- a/physical/raft/raft_test.go +++ b/physical/raft/raft_test.go @@ -21,13 +21,13 @@ import ( "github.com/go-test/deep" "github.com/golang/protobuf/proto" + bolt "github.com/hashicorp-forge/bbolt" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-secure-stdlib/base62" "github.com/hashicorp/go-uuid" "github.com/hashicorp/raft" "github.com/hashicorp/vault/sdk/helper/jsonutil" "github.com/hashicorp/vault/sdk/physical" - bolt "go.etcd.io/bbolt" ) func connectPeers(nodes ...*RaftBackend) { diff --git a/physical/raft/snapshot.go b/physical/raft/snapshot.go index 3eb818574958..adcfac4e1c44 100644 --- a/physical/raft/snapshot.go +++ b/physical/raft/snapshot.go @@ -18,10 +18,10 @@ import ( "time" "github.com/golang/protobuf/proto" + bolt "github.com/hashicorp-forge/bbolt" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/sdk/plugin/pb" "github.com/rboyer/safeio" - bolt "go.etcd.io/bbolt" "go.uber.org/atomic" "github.com/hashicorp/raft"