util/log,cli: new TELEMETRY channel

See discussion in cockroachdb#63815: we would like to leverage the logging subsystem, in particular network logging, as a new way to channel diagnostic reports. This work will operate in multiple phases. This commit is only about the first phase: creating a new logging channel TELEMETRY. Release note (cli change): CockroachDB now supports a new logging channel called TELEMETRY. This will be used in later versions to report diagnostic events useful to Cockroach Labs for product analytics. By default, this channel is connected to file output, with a maximum retention of 1MiB. To redirect the diagnostic output, one can define a new sink that captures this channel. For example, to see diagnostics reports on the standard error, one can use: `--log='sinks: {stderr: {channels: TELEMETRY, filter: INFO}}'` (At the time of this writing, no events are defined for the TELEMETRY channel yet.)
knz · Jun 14, 2021 · 1295502 · 1295502
1 parent 128e94f
commit 1295502
Show file tree

Hide file tree

Showing 13 changed files with 503 additions and 67 deletions.
diff --git a/docs/generated/logging.md b/docs/generated/logging.md
@@ -164,3 +164,9 @@ helping developers of CockroachDB itself. It exists as a separate
 channel so as to not pollute the `SQL_PERF` logging output with
 internal troubleshooting details.
 
+## TELEMETRY
+
+The TELEMETRY channel reports telemetry events. Telemetry events describe
+feature usage within CockroachDB and anonymizes any application-
+specific data.
+
diff --git a/pkg/cli/log_flags.go b/pkg/cli/log_flags.go
@@ -432,13 +432,24 @@ func addPredefinedLogFiles(c *logconfig.Config) {
 		if prefix == "sql-audit" && cliCtx.deprecatedLogOverrides.sqlAuditLogDir.isSet {
 			dir = &cliCtx.deprecatedLogOverrides.sqlAuditLogDir.s
 		}
-		m[prefix] = &logconfig.FileSinkConfig{
+
+		sinkConfig := &logconfig.FileSinkConfig{
 			Channels: logconfig.ChannelList{Channels: []logpb.Channel{ch}},
 			Dir:      dir,
 			CommonSinkConfig: logconfig.CommonSinkConfig{
 				Auditable: &b,
 			},
 		}
+
+		if ch == channel.TELEMETRY {
+			// Keep less data for telemetry.
+			sz := logconfig.ByteSize(100 * 1024)             // 100KiB
+			groupSize := logconfig.ByteSize(1 * 1024 * 1024) // 1MiB
+			sinkConfig.MaxFileSize = &sz
+			sinkConfig.MaxGroupSize = &groupSize
+		}
+
+		m[prefix] = sinkConfig
 	}
 }
 
@@ -451,6 +462,7 @@ var predefinedLogFiles = map[logpb.Channel]string{
 	channel.SQL_EXEC:          "sql-exec",
 	channel.SQL_PERF:          "sql-slow",
 	channel.SQL_INTERNAL_PERF: "sql-slow-internal-only",
+	channel.TELEMETRY:         "telemetry",
 }
 
 // predefinedAuditFiles indicate which channel-specific files are

diff --git a/pkg/cli/testdata/logflags b/pkg/cli/testdata/logflags
@@ -25,7 +25,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],<defaultLogDir>,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],<defaultLogDir>,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],<defaultLogDir>,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],<defaultLogDir>,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: <defaultLogDir>,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(<defaultLogDir>)>}
 
@@ -46,7 +54,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],<defaultLogDir>,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],<defaultLogDir>,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],<defaultLogDir>,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],<defaultLogDir>,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: <defaultLogDir>,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(<defaultLogDir>)>}
 
@@ -127,7 +143,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],/pathA/logs,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],/pathA/logs,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/pathA/logs,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/pathA/logs,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA/logs,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA/logs,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: /pathA/logs,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(/pathA/logs)>}
 
@@ -150,7 +174,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],/mypath,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
+telemetry: {channels: [ TELEMETRY],
+dir: /mypath,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(/mypath)>}
 
@@ -174,7 +206,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],/pathA/logs,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],/pathA/logs,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/pathA/logs,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/pathA/logs,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA/logs,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA/logs,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: /pathA/logs,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(/pathA/logs)>}
 
@@ -203,7 +243,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],/mypath,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
+telemetry: {channels: [ TELEMETRY],
+dir: /mypath,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(/mypath)>}
 
@@ -225,7 +273,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],<defaultLogDir>,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],<defaultLogDir>,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],<defaultLogDir>,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],<defaultLogDir>,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: <defaultLogDir>,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrCfg(ERROR,true)>},
 <stdCaptureFd2(<defaultLogDir>)>}
 
@@ -248,7 +304,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],<defaultLogDir>,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],<defaultLogDir>,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],<defaultLogDir>,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],<defaultLogDir>,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: <defaultLogDir>,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>}}
 
 # Logging to stderr without stderr capture causes an error in the default config.
@@ -302,7 +366,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],/mypath,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
+telemetry: {channels: [ TELEMETRY],
+dir: /mypath,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(/mypath)>}
 
@@ -326,7 +398,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],/pathA,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],/pathA,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/pathA,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/pathA,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA,true,crdb-v2)>,
+telemetry: {channels: [ TELEMETRY],
+dir: /pathA,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrDisabled>},
 <stdCaptureFd2(/pathA)>}
 
@@ -366,7 +446,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],<defaultLogDir>,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],<defaultLogDir>,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],<defaultLogDir>,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],<defaultLogDir>,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: <defaultLogDir>,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrCfg(INFO,true)>},
 <stdCaptureFd2(<defaultLogDir>)>}
 
@@ -388,7 +476,15 @@ sql-audit: <fileCfg([SENSITIVE_ACCESS],<defaultLogDir>,false,crdb-v2)>,
 sql-auth: <fileCfg([SESSIONS],<defaultLogDir>,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],<defaultLogDir>,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],<defaultLogDir>,true,crdb-v2)>,
-sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>},
+sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],<defaultLogDir>,true,crdb-v2)>,
+telemetry: {channels: [TELEMETRY],
+dir: <defaultLogDir>,
+max-file-size: 100KiB,
+max-group-size: 1.0MiB,
+buffered-writes: true,
+filter: INFO,
+format: crdb-v2,
+redactable: true}},
 <stderrCfg(INFO,true)>},
 <stdCaptureFd2(<defaultLogDir>)>}
 

diff --git a/pkg/util/log/channel/channel_generated.go b/pkg/util/log/channel/channel_generated.go
diff --git a/pkg/util/log/eventpb/BUILD.bazel b/pkg/util/log/eventpb/BUILD.bazel
@@ -44,6 +44,7 @@ proto_library(
         "role_events.proto",
         "session_events.proto",
         "sql_audit_events.proto",
+        "telemetry.proto",
         "zone_events.proto",
     ],
     strip_import_prefix = "/pkg",

diff --git a/pkg/util/log/eventpb/eventlog_channels_generated.go b/pkg/util/log/eventpb/eventlog_channels_generated.go
diff --git a/pkg/util/log/eventpb/json_encode_generated.go b/pkg/util/log/eventpb/json_encode_generated.go