metrics: expose fsync latency as prometheus histogram

Previously, the fsync latency was computed using an HDR histogram
however this histogram generated too many buckets of various widths. As
a result, a prometheus histogram was chosen instead. In addition, the
metrics were being accumulated and stored on each `LogWriter` and then
merged into the cumulative metrics on each WAL rotation. However, with
this new promethesus histogram the writes occur to the same histogram
and are separate from the lifetime of the WAL.

Additonally, remove all the callback listeners and configurations for
the callback based approach introduced in:
5fdb3ea

commit_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/record"
 	"github.com/cockroachdb/pebble/vfs"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/rand"
 )
@@ -184,7 +185,9 @@ func TestCommitPipelineWALClose(t *testing.T) {
 	}
 
 	// A basic commitEnv which writes to a WAL.
-	wal := record.NewLogWriter(sf, 0 /* logNum */, record.LogWriterConfig{})
+	wal := record.NewLogWriter(sf, 0 /* logNum */, record.LogWriterConfig{
+		WALFsyncLatency: prometheus.NewHistogram(prometheus.HistogramOpts{}),
+	})
 	var walDone sync.WaitGroup
 	testEnv := commitEnv{
 		logSeqNum:     new(uint64),
@@ -235,7 +238,9 @@ func BenchmarkCommitPipeline(b *testing.B) {
 		b.Run(fmt.Sprintf("parallel=%d", parallelism), func(b *testing.B) {
 			b.SetParallelism(parallelism)
 			mem := newMemTable(memTableOptions{})
-			wal := record.NewLogWriter(io.Discard, 0 /* logNum */, record.LogWriterConfig{})
+			wal := record.NewLogWriter(io.Discard, 0 /* logNum */, record.LogWriterConfig{
+				WALFsyncLatency: prometheus.NewHistogram(prometheus.HistogramOpts{}),
+			})
 
 			nullCommitEnv := commitEnv{
 				logSeqNum:     new(uint64),

db.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/vfs/atomicfs"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
@@ -341,7 +342,10 @@ type DB struct {
 			// (i.e. makeRoomForWrite).
 			*record.LogWriter
 			// Can be nil.
-			metrics record.LogWriterMetrics
+			metrics struct {
+				fsyncLatency prometheus.Histogram
+				record.LogWriterMetrics
+			}
 		}
 
 		mem struct {
@@ -1566,7 +1570,8 @@ func (d *DB) Metrics() *Metrics {
 	metrics.private.manifestFileSize = uint64(d.mu.versions.manifest.Size())
 	d.mu.versions.logUnlock()
 
-	if err := metrics.LogWriter.Merge(&d.mu.log.metrics); err != nil {
+	metrics.LogWriter.FsyncLatency = d.mu.log.metrics.fsyncLatency
+	if err := metrics.LogWriter.Merge(&d.mu.log.metrics.LogWriterMetrics); err != nil {
 		d.opts.Logger.Infof("metrics error: %s", err)
 	}
 	metrics.Flush.WriteThroughput = d.mu.compact.flushWriteThroughput
@@ -1950,7 +1955,7 @@ func (d *DB) makeRoomForWrite(b *Batch) error {
 		if !d.opts.DisableWAL {
 			d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: newLogSize})
 			d.mu.log.LogWriter = record.NewLogWriter(newLogFile, newLogNum, record.LogWriterConfig{
-				OnFsync:            d.opts.MetricEventListener.WALFsyncLatency,
+				WALFsyncLatency:    d.mu.log.metrics.fsyncLatency,
 				WALMinSyncInterval: d.opts.WALMinSyncInterval,
 			})
 		}

go.mod

@@ -3,31 +3,40 @@ module github.com/cockroachdb/pebble
 require (
 	github.com/DataDog/zstd v1.4.5
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2
-	github.com/cespare/xxhash/v2 v2.1.1
+	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cockroachdb/errors v1.8.1
 	github.com/cockroachdb/redact v1.0.8
 	github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9
-	github.com/golang/snappy v0.0.3
-	github.com/klauspost/compress v1.11.7
-	github.com/kr/pretty v0.1.0
+	github.com/golang/snappy v0.0.4
+	github.com/klauspost/compress v1.11.13
+	github.com/kr/pretty v0.2.1
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/spf13/cobra v0.0.5
+	github.com/prometheus/client_golang v1.12.0
+	github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a
+	github.com/spf13/cobra v1.0.0
 	github.com/stretchr/testify v1.7.0
 	golang.org/x/exp v0.0.0-20200513190911-00229845015e
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
 	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/spf13/pflag v1.0.3 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 
 go 1.17