changefeedccl: Add metrics scope library for changefeeds.

Add a utility library that allows creation of fixed number (8) of
custome "metric" scopes.  Each scope is intended to keep track of
a small set of SLI related metrics.
Scopes are named "tier0", "tier1", ... "tier7".

Follow on changes will integrate changefeed code with this library.

Release Notes: None

pkg/ccl/changefeedccl/cdcutils/BUILD.bazel

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "cdcutils",
-    srcs = ["throttle.go"],
+    srcs = [
+        "metrics_scope.go",
+        "throttle.go",
+    ],
     importpath = "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/cdcutils",
     visibility = ["//visibility:public"],
     deps = [
@@ -18,13 +21,17 @@ go_library(
 
 go_test(
     name = "cdcutils_test",
-    srcs = ["throttle_test.go"],
+    srcs = [
+        "metrics_scope_test.go",
+        "throttle_test.go",
+    ],
     embed = [":cdcutils"],
     deps = [
         "//pkg/ccl/changefeedccl/changefeedbase",
         "//pkg/settings/cluster",
         "//pkg/util/leaktest",
         "//pkg/util/log",
+        "//pkg/util/metric",
         "@com_github_stretchr_testify//require",
     ],
 )

pkg/ccl/changefeedccl/cdcutils/metrics_scope.go

@@ -0,0 +1,77 @@
+// Copyright 2021 The Cockroach Authors.
+//
+// Licensed as a CockroachDB Enterprise file under the Cockroach Community
+// License (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+//     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
+
+package cdcutils
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/cockroach/pkg/util/metric"
+)
+
+// maxSLIScopes is a static limit on the number of SLI scopes -- that is the number
+// of SLI metrics we will keep track of.
+// The limit is static due to metric.Registry limitations.
+const maxSLIScopes = 8
+
+// SLIMetrics is the list of SLI related metrics for changefeeds.
+type SLIMetrics struct {
+	ErrorRetries *metric.Counter
+	// TODO(yevgeniy): Add more SLI related metrics.
+}
+
+// MetricStruct implements metric.Struct interface
+func (*SLIMetrics) MetricStruct() {}
+
+func makeSLIMetrics(prefix string) *SLIMetrics {
+	withPrefix := func(meta metric.Metadata) metric.Metadata {
+		meta.Name = fmt.Sprintf("%s.%s", prefix, meta.Name)
+		return meta
+	}
+
+	return &SLIMetrics{
+		ErrorRetries: metric.NewCounter(withPrefix(metric.Metadata{
+			Name:        "error_retries",
+			Help:        "Total retryable errors encountered this SLI",
+			Measurement: "Errors",
+			Unit:        metric.Unit_COUNT,
+		})),
+	}
+}
+
+// SLIScopes represents a set of SLI related metrics for a particular "scope".
+type SLIScopes struct {
+	Scopes [maxSLIScopes]*SLIMetrics // Exported so that we can register w/ metrics registry.
+	names  map[string]*SLIMetrics
+}
+
+// MetricStruct implements metric.Struct interface
+func (*SLIScopes) MetricStruct() {}
+
+// CreateSLIScopes creates changefeed specific SLI scope: a metric.Struct containing
+// SLI specific metrics for each scope.
+// The scopes are statically named "tier<number>", and each metric name
+// contained in SLIMetrics will be prefixed by "changefeed.tier<number" prefix.
+func CreateSLIScopes() *SLIScopes {
+	scope := &SLIScopes{
+		names: make(map[string]*SLIMetrics, maxSLIScopes),
+	}
+
+	for i := 0; i < maxSLIScopes; i++ {
+		scopeName := fmt.Sprintf("tier%d", i)
+		scope.Scopes[i] = makeSLIMetrics(fmt.Sprintf("changefeed.%s", scopeName))
+		scope.names[scopeName] = scope.Scopes[i]
+	}
+	return scope
+}
+
+// GetSLIMetrics returns a metric.Struct associated with the specified scope, or nil
+// of no such scope exists.
+func (s *SLIScopes) GetSLIMetrics(scopeName string) *SLIMetrics {
+	return s.names[scopeName]
+}

pkg/ccl/changefeedccl/cdcutils/metrics_scope_test.go

@@ -0,0 +1,57 @@
+// Copyright 2021 The Cockroach Authors.
+//
+// Licensed as a CockroachDB Enterprise file under the Cockroach Community
+// License (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+//     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
+
+package cdcutils
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/metric"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetricScope(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	scope := CreateSLIScopes()
+	require.NotNil(t, scope)
+
+	sliMetricNames := []string{
+		"error_retries",
+	}
+
+	expectedMetrics := make(map[string]struct{})
+	expectScopedMetrics := func(scope string) {
+		for _, n := range sliMetricNames {
+			expectedMetrics[fmt.Sprintf("changefeed.%s.%s", scope, n)] = struct{}{}
+		}
+	}
+
+	for i := 0; i < maxSLIScopes; i++ {
+		scopeName := fmt.Sprintf("tier%d", i)
+		require.NotNil(t, scope.GetSLIMetrics(scopeName))
+		expectScopedMetrics(scopeName)
+	}
+
+	require.Nil(t, scope.GetSLIMetrics("does not exist"))
+	require.Nil(t, scope.GetSLIMetrics(fmt.Sprintf("tier%d", maxSLIScopes+1)))
+
+	registry := metric.NewRegistry()
+	registry.AddMetricStruct(scope)
+	registry.Each(func(name string, _ interface{}) {
+		_, exists := expectedMetrics[name]
+		require.True(t, exists)
+		delete(expectedMetrics, name)
+	})
+	require.Equal(t, 0, len(expectedMetrics),
+		"remaining metrics: %s", expectedMetrics)
+}