util/log: add http log sink

Release note (cli change): Added a new HTTP sink to the logging system. This can be configured similarly to other log sinks with the new 'http-servers' and 'http-defaults' sections of the logging config passed via the "--log" or "--log-config-file" command line flags.
cockroachdb · Jul 29, 2021 · ad23285 · ad23285
1 parent d8ba460
commit ad23285
Show file tree

Hide file tree

Showing 14 changed files with 812 additions and 293 deletions.
diff --git a/docs/generated/logsinks.md b/docs/generated/logsinks.md
@@ -6,6 +6,8 @@ The supported log output sink types are documented below.
 
 - [Output to Fluentd-compatible log collectors](#output-to-fluentd-compatible-log-collectors)
 
+- [Output to HTTP servers.](#output-to-http-servers.)
+
 - [Standard error stream](#standard-error-stream)
 
 
@@ -91,7 +93,7 @@ Configuration options shared across all sink types:
 ## Sink type: Output to Fluentd-compatible log collectors
 
 
-This sink type causes logging data to be sent over the network, to
+This sink type causes logging data to be sent over the network to
 a log collector that can ingest log data in a
 [Fluentd](https://www.fluentd.org)-compatible protocol.
 
@@ -168,6 +170,70 @@ Configuration options shared across all sink types:
 
 
 
+<a name="output-to-http-servers.">
+
+## Sink type: Output to HTTP servers.
+
+
+This sink type causes logging data to be sent over the network
+as requests to an HTTP server.
+
+The configuration key under the `sinks` key in the YAML
+configuration is `http-servers`. Example configuration:
+
+     sinks:
+        http-servers:
+           health:
+              channels: HEALTH
+              address: http://127.0.0.1
+
+Every new server sink configured automatically inherits the configuration set in the `http-defaults` section.
+
+For example:
+
+     http-defaults:
+         redactable: false # default: disable redaction markers
+     sinks:
+       http-servers:
+         health:
+            channels: HEALTH
+            # This sink has redactable set to false,
+            # as the setting is inherited from fluent-defaults
+            # unless overridden here.
+
+The default output format for HTTP sinks is
+`json-compact`. [Other supported formats.](log-formats.html)
+
+{{site.data.alerts.callout_info}}
+Run `cockroach debug check-log-config` to verify the effect of defaults inheritance.
+{{site.data.alerts.end}}
+
+
+
+Type-specific configuration options:
+
+| Field | Description |
+|--|--|
+| `channels` | the list of logging channels that use this sink. See the [channel selection configuration](#channel-format) section for details.  |
+| `address` | the network address of the http server. The host/address and port parts are separated with a colon. IPv6 numeric addresses should be included within square brackets, e.g.: [::1]:1234. Inherited from `http-defaults.address` if not specified. |
+| `method` | the HTTP method to be used.  POST and GET are supported; defaults to POST. Inherited from `http-defaults.method` if not specified. |
+| `unsafe-tls` | enables certificate authentication to be bypassed. Defaults to false. Inherited from `http-defaults.unsafe-tls` if not specified. |
+| `timeout` | the HTTP timeout. Defaults to 0 for no timeout. Inherited from `http-defaults.timeout` if not specified. |
+
+
+Configuration options shared across all sink types:
+
+| Field | Description |
+|--|--|
+| `filter` | the minimum severity for log events to be emitted to this sink. This can be set to NONE to disable the sink. |
+| `format` | the entry format to use. |
+| `redact` | whether to strip sensitive information before log events are emitted to this sink. |
+| `redactable` | whether to keep redaction markers in the sink's output. The presence of redaction markers makes it possible to strip sensitive data reliably. |
+| `exit-on-error` | whether the logging system should terminate the process if an error is encountered while writing to this sink. |
+| `auditable` | translated to tweaks to the other settings for this sink during validation. For example, it enables `exit-on-error` and changes the format of files from `crdb-v1` to `crdb-v1-count`. |
+
+
+
 <a name="standard-error-stream">
 
 ## Sink type: Standard error stream

diff --git a/pkg/cli/log_flags_test.go b/pkg/cli/log_flags_test.go
@@ -31,7 +31,7 @@ func TestSetupLogging(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 
 	reWhitespace := regexp.MustCompile(`(?ms:((\s|\n)+))`)
-	reWhitespace2 := regexp.MustCompile(`{\s+`)
+	reBracketWhitespace := regexp.MustCompile(`(?P<bracket>[{[])\s+`)
 
 	reSimplify := regexp.MustCompile(`(?ms:^\s*(auditable: false|redact: false|exit-on-error: true|max-group-size: 100MiB)\n)`)
 
@@ -41,6 +41,14 @@ func TestSetupLogging(t *testing.T) {
 		`redactable: true, ` +
 		`exit-on-error: false` +
 		`}`
+	const defaultHTTPConfig = `http-defaults: {` +
+		`method: POST, ` +
+		`unsafe-tls: false, ` +
+		`timeout: 0s, ` +
+		`filter: INFO, ` +
+		`format: json-compact, ` +
+		`redactable: true, ` +
+		`exit-on-error: false}`
 	stdFileDefaultsRe := regexp.MustCompile(
 		`file-defaults: \{dir: (?P<path>[^,]+), max-file-size: 10MiB, buffered-writes: true, filter: INFO, format: crdb-v2, redactable: true\}`)
 	fileDefaultsNoMaxSizeRe := regexp.MustCompile(
@@ -103,10 +111,11 @@ func TestSetupLogging(t *testing.T) {
 			t.Fatal(err)
 		}
 		actual = reWhitespace.ReplaceAllString(h.String(), " ")
-		actual = reWhitespace2.ReplaceAllString(actual, "{")
+		actual = reBracketWhitespace.ReplaceAllString(actual, "$bracket")
 
 		// Shorten the configuration for legibility during reviews of test changes.
 		actual = strings.ReplaceAll(actual, defaultFluentConfig, "<fluentDefaults>")
+		actual = strings.ReplaceAll(actual, defaultHTTPConfig, "<httpDefaults>")
 		actual = stdFileDefaultsRe.ReplaceAllString(actual, "<stdFileDefaults($path)>")
 		actual = fileDefaultsNoMaxSizeRe.ReplaceAllString(actual, "<fileDefaultsNoMaxSize($path)>")
 		actual = strings.ReplaceAll(actual, fileDefaultsNoDir, "<fileDefaultsNoDir>")

diff --git a/pkg/cli/testdata/logflags b/pkg/cli/testdata/logflags
@@ -14,6 +14,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -43,6 +44,7 @@ start-single-node
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -76,13 +78,15 @@ sql
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledWarningNoRedaction>}}
 
 run
 init
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledWarningNoRedaction>}}
 
 
@@ -95,6 +99,7 @@ bank
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 
@@ -105,6 +110,7 @@ demo
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrCfg(NONE,false)>}}
 
 
@@ -120,6 +126,7 @@ start
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 
@@ -132,6 +139,7 @@ start
 ----
 config: {<stdFileDefaults(/pathA/logs)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -163,6 +171,7 @@ start
 ----
 config: {<stdFileDefaults(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -175,7 +184,7 @@ sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /mypath,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -195,6 +204,7 @@ start
 ----
 config: {<stdFileDefaults(/pathA/logs)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -232,6 +242,7 @@ start
 ----
 config: {<stdFileDefaults(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -244,7 +255,7 @@ sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /mypath,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -262,6 +273,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -293,6 +305,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -345,6 +358,7 @@ start
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 
@@ -355,6 +369,7 @@ start
 ----
 config: {<stdFileDefaults(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -367,7 +382,7 @@ sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /mypath,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -387,6 +402,7 @@ start
 ----
 config: {<stdFileDefaults(/pathA)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -399,7 +415,7 @@ sql-auth: <fileCfg([SESSIONS],/pathA,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/pathA,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/pathA,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /pathA,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -419,6 +435,7 @@ init
 ----
 config: {<fileDefaultsNoMaxSize(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: {channels: all,
 dir: /mypath,
 buffered-writes: true,
@@ -435,6 +452,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -465,6 +483,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -495,6 +514,7 @@ init
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 # Default when no severity is specified is WARNING.
@@ -504,6 +524,7 @@ init
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledWarningNoRedaction>}}
 
 

diff --git a/pkg/util/log/BUILD.bazel b/pkg/util/log/BUILD.bazel
@@ -22,6 +22,7 @@ go_library(
         "format_json.go",
         "formats.go",
         "get_stacks.go",
+        "http_sink.go",
         "intercept.go",
         "log.go",
         "log_bridge.go",
@@ -133,6 +134,7 @@ go_test(
         "format_crdb_v2_test.go",
         "format_json_test.go",
         "helpers_test.go",
+        "http_sink_test.go",
         "intercept_test.go",
         "main_test.go",
         "redact_test.go",
@@ -153,6 +155,7 @@ go_test(
         "//pkg/util/log/logconfig",
         "//pkg/util/log/logpb",
         "//pkg/util/log/severity",
+        "//pkg/util/netutil/addr",
         "//pkg/util/randutil",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",