release-21.2: sql: scheduled logger to capture index usage stats

docs/generated/eventlog.md

@@ -2205,6 +2205,32 @@ An event of type `drop_role` is recorded when a role is dropped.
 Events in this category are logged to the `TELEMETRY` channel.
 
 
+### `captured_index_usage_stats`
+
+An event of type `captured_index_usage_stats`
+
+
+| Field | Description | Sensitive |
+|--|--|--|
+| `TotalReadCount` | TotalReadCount is the number of times this index has been read from. | no |
+| `LastRead` | LastRead is the timestamp that this index was last being read from. | yes |
+| `TableID` | TableID is the ID of the table this index is created on. This is same as descpb.TableID and is unique within the cluster. | no |
+| `IndexID` | IndexID is the ID of the index within the scope of the given table. | no |
+| `DatabaseName` |  | yes |
+| `TableName` |  | yes |
+| `IndexName` |  | yes |
+| `IndexType` |  | yes |
+| `IsUnique` |  | no |
+| `IsInverted` |  | no |
+
+
+#### Common fields
+
+| Field | Description | Sensitive |
+|--|--|--|
+| `Timestamp` | The timestamp of the event. Expressed as nanoseconds since the Unix epoch. | no |
+| `EventType` | The type of the event. | no |
+
 ### `sampled_query`
 
 An event of type `sampled_query` is the SQL query event logged to the telemetry channel. It

docs/generated/settings/settings-for-tenants.txt

@@ -159,6 +159,7 @@ sql.stats.multi_column_collection.enabled	boolean	true	multi-column statistics c
 sql.stats.persisted_rows.max	integer	1000000	maximum number of rows of statement and transaction statistics that will be persisted in the system tables
 sql.stats.post_events.enabled	boolean	false	if set, an event is logged for every CREATE STATISTICS job
 sql.stats.response.max	integer	20000	the maximum number of statements and transaction stats returned in a CombinedStatements request
+sql.telemetry.capture_index_usage_stats.enabled	boolean	true	enable/disable capturing index usage statistics to the telemetry logging channel
 sql.telemetry.query_sampling.enabled	boolean	false	when set to true, executed queries will emit an event on the telemetry logging channel
 sql.temp_object_cleaner.cleanup_interval	duration	30m0s	how often to clean up orphaned temporary objects
 sql.temp_object_cleaner.wait_interval	duration	30m0s	how long after creation a temporary object will be cleaned up

docs/generated/settings/settings.html

@@ -164,6 +164,7 @@
 <tr><td><code>sql.stats.persisted_rows.max</code></td><td>integer</td><td><code>1000000</code></td><td>maximum number of rows of statement and transaction statistics that will be persisted in the system tables</td></tr>
 <tr><td><code>sql.stats.post_events.enabled</code></td><td>boolean</td><td><code>false</code></td><td>if set, an event is logged for every CREATE STATISTICS job</td></tr>
 <tr><td><code>sql.stats.response.max</code></td><td>integer</td><td><code>20000</code></td><td>the maximum number of statements and transaction stats returned in a CombinedStatements request</td></tr>
+<tr><td><code>sql.telemetry.capture_index_usage_stats.enabled</code></td><td>boolean</td><td><code>true</code></td><td>enable/disable capturing index usage statistics to the telemetry logging channel</td></tr>
 <tr><td><code>sql.telemetry.query_sampling.enabled</code></td><td>boolean</td><td><code>false</code></td><td>when set to true, executed queries will emit an event on the telemetry logging channel</td></tr>
 <tr><td><code>sql.temp_object_cleaner.cleanup_interval</code></td><td>duration</td><td><code>30m0s</code></td><td>how often to clean up orphaned temporary objects</td></tr>
 <tr><td><code>sql.temp_object_cleaner.wait_interval</code></td><td>duration</td><td><code>30m0s</code></td><td>how long after creation a temporary object will be cleaned up</td></tr>

pkg/BUILD.bazel

@@ -287,6 +287,7 @@ ALL_TESTS = [
     "//pkg/sql/rowexec:rowexec_test",
     "//pkg/sql/rowflow:rowflow_test",
     "//pkg/sql/scanner:scanner_test",
+    "//pkg/sql/scheduledlogging:scheduledlogging_test",
     "//pkg/sql/schemachange:schemachange_test",
     "//pkg/sql/schemachanger/scbuild:scbuild_test",
     "//pkg/sql/schemachanger/scexec:scexec_test",

pkg/base/testing_knobs.go

@@ -19,29 +19,30 @@ type ModuleTestingKnobs interface {
 // TestingKnobs contains facilities for controlling various parts of the
 // system for testing.
 type TestingKnobs struct {
-	Store                   ModuleTestingKnobs
-	KVClient                ModuleTestingKnobs
-	RangeFeed               ModuleTestingKnobs
-	SQLExecutor             ModuleTestingKnobs
-	SQLLeaseManager         ModuleTestingKnobs
-	SQLSchemaChanger        ModuleTestingKnobs
-	SQLNewSchemaChanger     ModuleTestingKnobs
-	SQLTypeSchemaChanger    ModuleTestingKnobs
-	GCJob                   ModuleTestingKnobs
-	PGWireTestingKnobs      ModuleTestingKnobs
-	StartupMigrationManager ModuleTestingKnobs
-	DistSQL                 ModuleTestingKnobs
-	SQLEvalContext          ModuleTestingKnobs
-	NodeLiveness            ModuleTestingKnobs
-	Server                  ModuleTestingKnobs
-	TenantTestingKnobs      ModuleTestingKnobs
-	JobsTestingKnobs        ModuleTestingKnobs
-	BackupRestore           ModuleTestingKnobs
-	Streaming               ModuleTestingKnobs
-	MigrationManager        ModuleTestingKnobs
-	IndexUsageStatsKnobs    ModuleTestingKnobs
-	SQLStatsKnobs           ModuleTestingKnobs
-	SpanConfig              ModuleTestingKnobs
-	SQLLivenessKnobs        ModuleTestingKnobs
-	TelemetryLoggingKnobs   ModuleTestingKnobs
+	Store                        ModuleTestingKnobs
+	KVClient                     ModuleTestingKnobs
+	RangeFeed                    ModuleTestingKnobs
+	SQLExecutor                  ModuleTestingKnobs
+	SQLLeaseManager              ModuleTestingKnobs
+	SQLSchemaChanger             ModuleTestingKnobs
+	SQLNewSchemaChanger          ModuleTestingKnobs
+	SQLTypeSchemaChanger         ModuleTestingKnobs
+	GCJob                        ModuleTestingKnobs
+	PGWireTestingKnobs           ModuleTestingKnobs
+	StartupMigrationManager      ModuleTestingKnobs
+	DistSQL                      ModuleTestingKnobs
+	SQLEvalContext               ModuleTestingKnobs
+	NodeLiveness                 ModuleTestingKnobs
+	Server                       ModuleTestingKnobs
+	TenantTestingKnobs           ModuleTestingKnobs
+	JobsTestingKnobs             ModuleTestingKnobs
+	BackupRestore                ModuleTestingKnobs
+	Streaming                    ModuleTestingKnobs
+	MigrationManager             ModuleTestingKnobs
+	IndexUsageStatsKnobs         ModuleTestingKnobs
+	SQLStatsKnobs                ModuleTestingKnobs
+	SpanConfig                   ModuleTestingKnobs
+	SQLLivenessKnobs             ModuleTestingKnobs
+	TelemetryLoggingKnobs        ModuleTestingKnobs
+	CapturedIndexUsageStatsKnobs ModuleTestingKnobs
 }

pkg/server/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "doc.go",
         "drain.go",
         "grpc_server.go",
+        "import_ts.go",
         "index_usage_stats.go",
         "init.go",
         "init_handshake.go",
@@ -145,6 +146,7 @@ go_library(
         "//pkg/sql/physicalplan",
         "//pkg/sql/querycache",
         "//pkg/sql/roleoption",
+        "//pkg/sql/scheduledlogging",
         "//pkg/sql/schemachanger/scexec",
         "//pkg/sql/schemachanger/scjob",
         "//pkg/sql/sem/tree",
@@ -404,6 +406,7 @@ go_test(
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_grpc//codes",
         "@org_golang_google_grpc//credentials",
+        "@org_golang_google_grpc//metadata",
         "@org_golang_google_grpc//status",
         "@org_golang_x_crypto//bcrypt",
     ],

pkg/server/server_sql.go

@@ -68,6 +68,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/optionalnodeliveness"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire"
 	"github.com/cockroachdb/cockroach/pkg/sql/querycache"
+	"github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging"
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
@@ -735,6 +736,9 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
 	if telemetryLoggingKnobs := cfg.TestingKnobs.TelemetryLoggingKnobs; telemetryLoggingKnobs != nil {
 		execCfg.TelemetryLoggingTestingKnobs = telemetryLoggingKnobs.(*sql.TelemetryLoggingTestingKnobs)
 	}
+	if capturedIndexUsageStatsKnobs := cfg.TestingKnobs.CapturedIndexUsageStatsKnobs; capturedIndexUsageStatsKnobs != nil {
+		execCfg.CaptureIndexUsageStatsKnobs = capturedIndexUsageStatsKnobs.(*scheduledlogging.CaptureIndexUsageStatsTestingKnobs)
+	}
 
 	statsRefresher := stats.MakeRefresher(
 		cfg.Settings,
@@ -1124,6 +1128,7 @@ func (s *SQLServer) preStart(
 		scheduledjobs.ProdJobSchedulerEnv,
 	)
 
+	scheduledlogging.Start(ctx, stopper, s.execCfg.DB, s.execCfg.Settings, s.internalExecutor, s.execCfg.CaptureIndexUsageStatsKnobs)
 	return nil
 }
 

pkg/sql/BUILD.bazel

@@ -343,6 +343,7 @@ go_library(
         "//pkg/sql/rowenc",
         "//pkg/sql/rowexec",
         "//pkg/sql/rowinfra",
+        "//pkg/sql/scheduledlogging",
         "//pkg/sql/schemachange",
         "//pkg/sql/schemachanger/scbuild",
         "//pkg/sql/schemachanger/scexec",

pkg/sql/exec_util.go

@@ -72,6 +72,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/querycache"
 	"github.com/cockroachdb/cockroach/pkg/sql/row"
 	"github.com/cockroachdb/cockroach/pkg/sql/rowenc"
+	"github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging"
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
@@ -1102,6 +1103,7 @@ type ExecutorConfig struct {
 	StreamingTestingKnobs         *StreamingTestingKnobs
 	SQLStatsTestingKnobs          *sqlstats.TestingKnobs
 	TelemetryLoggingTestingKnobs  *TelemetryLoggingTestingKnobs
+	CaptureIndexUsageStatsKnobs   *scheduledlogging.CaptureIndexUsageStatsTestingKnobs
 	// HistogramWindowInterval is (server.Config).HistogramWindowInterval.
 	HistogramWindowInterval time.Duration
 

pkg/sql/scheduledlogging/BUILD.bazel

@@ -0,0 +1,51 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "scheduledlogging",
+    srcs = ["captured_index_usage_stats.go"],
+    importpath = "github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/kv",
+        "//pkg/roachpb:with-mocks",
+        "//pkg/security",
+        "//pkg/settings",
+        "//pkg/settings/cluster",
+        "//pkg/sql/sem/tree",
+        "//pkg/sql/sessiondata",
+        "//pkg/sql/sqlutil",
+        "//pkg/util/log",
+        "//pkg/util/log/eventpb",
+        "//pkg/util/stop",
+        "//pkg/util/timeutil",
+        "@com_github_cockroachdb_errors//:errors",
+    ],
+)
+
+go_test(
+    name = "scheduledlogging_test",
+    srcs = [
+        "captured_index_usage_stats_test.go",
+        "main_test.go",
+    ],
+    embed = [":scheduledlogging"],
+    deps = [
+        "//pkg/base",
+        "//pkg/security",
+        "//pkg/security/securitytest",
+        "//pkg/server",
+        "//pkg/settings/cluster",
+        "//pkg/testutils",
+        "//pkg/testutils/serverutils",
+        "//pkg/testutils/sqlutils",
+        "//pkg/testutils/testcluster",
+        "//pkg/util/leaktest",
+        "//pkg/util/log",
+        "//pkg/util/log/channel",
+        "//pkg/util/log/logconfig",
+        "//pkg/util/randutil",
+        "//pkg/util/syncutil",
+        "@com_github_cockroachdb_errors//:errors",
+        "@com_github_stretchr_testify//require",
+    ],
+)