sql: created a scheduled logger to capture index usage stats

This change introduces a scheduled logger that runs background processes intended to emit logs on a time interval. Release note (sql change): Initial implementation of a scheduled logger, used to capture index usage statistics to the telemetry logging channel.
cockroachdb · Feb 25, 2022 · 0ee384d · 0ee384d
1 parent 5b3067f
commit 0ee384d
Show file tree

Hide file tree

Showing 15 changed files with 922 additions and 29 deletions.
diff --git a/docs/generated/eventlog.md b/docs/generated/eventlog.md
@@ -2305,6 +2305,32 @@ are automatically converted server-side.
 Events in this category are logged to the `TELEMETRY` channel.
 
 
+### `captured_index_usage_stats`
+
+An event of type `captured_index_usage_stats`
+
+
+| Field | Description | Sensitive |
+|--|--|--|
+| `TotalReadCount` | TotalReadCount is the number of times this index has been read from. | no |
+| `LastRead` | LastRead is the timestamp that this index was last being read from. | yes |
+| `TableID` | TableID is the ID of the table this index is created on. This is same as descpb.TableID and is unique within the cluster. | no |
+| `IndexID` | IndexID is the ID of the index within the scope of the given table. | no |
+| `DatabaseName` |  | yes |
+| `TableName` |  | yes |
+| `IndexName` |  | yes |
+| `IndexType` |  | yes |
+| `IsUnique` |  | no |
+| `IsInverted` |  | no |
+
+
+#### Common fields
+
+| Field | Description | Sensitive |
+|--|--|--|
+| `Timestamp` | The timestamp of the event. Expressed as nanoseconds since the Unix epoch. | no |
+| `EventType` | The type of the event. | no |
+
 ### `sampled_query`
 
 An event of type `sampled_query` is the SQL query event logged to the telemetry channel. It

diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
@@ -319,6 +319,7 @@ ALL_TESTS = [
     "//pkg/sql/rowexec:rowexec_test",
     "//pkg/sql/rowflow:rowflow_test",
     "//pkg/sql/scanner:scanner_test",
+    "//pkg/sql/scheduledlogging:scheduledlogging_test",
     "//pkg/sql/schemachange:schemachange_test",
     "//pkg/sql/schemachanger/rel:rel_test",
     "//pkg/sql/schemachanger/scbuild:scbuild_test",

diff --git a/pkg/base/testing_knobs.go b/pkg/base/testing_knobs.go
@@ -19,32 +19,33 @@ type ModuleTestingKnobs interface {
 // TestingKnobs contains facilities for controlling various parts of the
 // system for testing.
 type TestingKnobs struct {
-	Store                       ModuleTestingKnobs
-	KVClient                    ModuleTestingKnobs
-	RangeFeed                   ModuleTestingKnobs
-	SQLExecutor                 ModuleTestingKnobs
-	SQLLeaseManager             ModuleTestingKnobs
-	SQLSchemaChanger            ModuleTestingKnobs
-	SQLDeclarativeSchemaChanger ModuleTestingKnobs
-	SQLTypeSchemaChanger        ModuleTestingKnobs
-	GCJob                       ModuleTestingKnobs
-	PGWireTestingKnobs          ModuleTestingKnobs
-	StartupMigrationManager     ModuleTestingKnobs
-	DistSQL                     ModuleTestingKnobs
-	SQLEvalContext              ModuleTestingKnobs
-	NodeLiveness                ModuleTestingKnobs
-	Server                      ModuleTestingKnobs
-	TenantTestingKnobs          ModuleTestingKnobs
-	JobsTestingKnobs            ModuleTestingKnobs
-	BackupRestore               ModuleTestingKnobs
-	TTL                         ModuleTestingKnobs
-	Streaming                   ModuleTestingKnobs
-	MigrationManager            ModuleTestingKnobs
-	IndexUsageStatsKnobs        ModuleTestingKnobs
-	SQLStatsKnobs               ModuleTestingKnobs
-	SpanConfig                  ModuleTestingKnobs
-	SQLLivenessKnobs            ModuleTestingKnobs
-	TelemetryLoggingKnobs       ModuleTestingKnobs
-	DialerKnobs                 ModuleTestingKnobs
-	ProtectedTS                 ModuleTestingKnobs
+	Store                        ModuleTestingKnobs
+	KVClient                     ModuleTestingKnobs
+	RangeFeed                    ModuleTestingKnobs
+	SQLExecutor                  ModuleTestingKnobs
+	SQLLeaseManager              ModuleTestingKnobs
+	SQLSchemaChanger             ModuleTestingKnobs
+	SQLDeclarativeSchemaChanger  ModuleTestingKnobs
+	SQLTypeSchemaChanger         ModuleTestingKnobs
+	GCJob                        ModuleTestingKnobs
+	PGWireTestingKnobs           ModuleTestingKnobs
+	StartupMigrationManager      ModuleTestingKnobs
+	DistSQL                      ModuleTestingKnobs
+	SQLEvalContext               ModuleTestingKnobs
+	NodeLiveness                 ModuleTestingKnobs
+	Server                       ModuleTestingKnobs
+	TenantTestingKnobs           ModuleTestingKnobs
+	JobsTestingKnobs             ModuleTestingKnobs
+	BackupRestore                ModuleTestingKnobs
+	TTL                          ModuleTestingKnobs
+	Streaming                    ModuleTestingKnobs
+	MigrationManager             ModuleTestingKnobs
+	IndexUsageStatsKnobs         ModuleTestingKnobs
+	SQLStatsKnobs                ModuleTestingKnobs
+	SpanConfig                   ModuleTestingKnobs
+	SQLLivenessKnobs             ModuleTestingKnobs
+	TelemetryLoggingKnobs        ModuleTestingKnobs
+	DialerKnobs                  ModuleTestingKnobs
+	ProtectedTS                  ModuleTestingKnobs
+	CapturedIndexUsageStatsKnobs ModuleTestingKnobs
 }
diff --git a/pkg/server/BUILD.bazel b/pkg/server/BUILD.bazel
@@ -167,6 +167,7 @@ go_library(
         "//pkg/sql/physicalplan",
         "//pkg/sql/querycache",
         "//pkg/sql/roleoption",
+        "//pkg/sql/scheduledlogging",
         "//pkg/sql/schemachanger/scdeps",
         "//pkg/sql/schemachanger/scjob",
         "//pkg/sql/schemachanger/scrun",

diff --git a/pkg/server/server_sql.go b/pkg/server/server_sql.go
@@ -72,6 +72,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/optionalnodeliveness"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire"
 	"github.com/cockroachdb/cockroach/pkg/sql/querycache"
+	"github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging"
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scdeps"
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
@@ -787,6 +788,9 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
 	if spanConfigKnobs := cfg.TestingKnobs.SpanConfig; spanConfigKnobs != nil {
 		execCfg.SpanConfigTestingKnobs = spanConfigKnobs.(*spanconfig.TestingKnobs)
 	}
+	if capturedIndexUsageStatsKnobs := cfg.TestingKnobs.CapturedIndexUsageStatsKnobs; capturedIndexUsageStatsKnobs != nil {
+		execCfg.CaptureIndexUsageStatsKnobs = capturedIndexUsageStatsKnobs.(*scheduledlogging.CaptureIndexUsageStatsTestingKnobs)
+	}
 
 	statsRefresher := stats.MakeRefresher(
 		cfg.AmbientCtx,
@@ -1245,6 +1249,8 @@ func (s *SQLServer) preStart(
 		scheduledjobs.ProdJobSchedulerEnv,
 	)
 
+	scheduledlogging.Start(ctx, stopper, s.execCfg.DB, s.execCfg.Settings, s.internalExecutor, s.execCfg.CaptureIndexUsageStatsKnobs)
+
 	return nil
 }
 

diff --git a/pkg/sql/BUILD.bazel b/pkg/sql/BUILD.bazel
@@ -357,6 +357,7 @@ go_library(
         "//pkg/sql/rowenc",
         "//pkg/sql/rowexec",
         "//pkg/sql/rowinfra",
+        "//pkg/sql/scheduledlogging",
         "//pkg/sql/schemachange",
         "//pkg/sql/schemachanger/scbuild",
         "//pkg/sql/schemachanger/scdeps",

diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go
@@ -75,6 +75,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/querycache"
 	"github.com/cockroachdb/cockroach/pkg/sql/row"
 	"github.com/cockroachdb/cockroach/pkg/sql/rowenc"
+	"github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging"
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec"
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
@@ -1190,6 +1191,7 @@ type ExecutorConfig struct {
 	SQLStatsTestingKnobs                 *sqlstats.TestingKnobs
 	TelemetryLoggingTestingKnobs         *TelemetryLoggingTestingKnobs
 	SpanConfigTestingKnobs               *spanconfig.TestingKnobs
+	CaptureIndexUsageStatsKnobs          *scheduledlogging.CaptureIndexUsageStatsTestingKnobs
 	// HistogramWindowInterval is (server.Config).HistogramWindowInterval.
 	HistogramWindowInterval time.Duration
 

diff --git a/pkg/sql/scheduledlogging/BUILD.bazel b/pkg/sql/scheduledlogging/BUILD.bazel
@@ -0,0 +1,50 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "scheduledlogging",
+    srcs = ["captured_index_usage_stats.go"],
+    importpath = "github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/kv",
+        "//pkg/roachpb",
+        "//pkg/security",
+        "//pkg/settings",
+        "//pkg/settings/cluster",
+        "//pkg/sql/sem/tree",
+        "//pkg/sql/sessiondata",
+        "//pkg/sql/sqlutil",
+        "//pkg/util/log",
+        "//pkg/util/log/eventpb",
+        "//pkg/util/stop",
+        "//pkg/util/timeutil",
+        "@com_github_cockroachdb_errors//:errors",
+    ],
+)
+
+go_test(
+    name = "scheduledlogging_test",
+    srcs = [
+        "captured_index_usage_stats_test.go",
+        "main_test.go",
+    ],
+    embed = [":scheduledlogging"],
+    deps = [
+        "//pkg/base",
+        "//pkg/security",
+        "//pkg/security/securitytest",
+        "//pkg/server",
+        "//pkg/settings/cluster",
+        "//pkg/testutils",
+        "//pkg/testutils/serverutils",
+        "//pkg/testutils/sqlutils",
+        "//pkg/testutils/testcluster",
+        "//pkg/util/leaktest",
+        "//pkg/util/log",
+        "//pkg/util/log/channel",
+        "//pkg/util/log/logconfig",
+        "//pkg/util/randutil",
+        "//pkg/util/syncutil",
+        "@com_github_stretchr_testify//require",
+    ],
+)