Merge #66196 #68336

66196: util/log: add http log sink r=knz a=rauchenstein Release note (cli change): Added a new HTTP sink to the logging system. This can be configured similarly to other log sinks with the new 'http-servers' and 'http-defaults' sections of the logging config passed via the "--log" or "--log-config-file" command line flags. 68336: sql: fix literal reparse issue with timestamps r=rafiss a=e-mbrown Related to: #60722 There were reparse issues in multiple test when timestamps were replaced with underscores. Making the underscore characters strings solved the issue. Release note: None Co-authored-by: Jay Rauchenstein <[email protected]> Co-authored-by: e-mbrown <[email protected]>
cockroachdb · Aug 3, 2021 · 6115e79 · 6115e79
3 parents 57ec5f2 + e72da90 + 86701b2
commit 6115e79
Show file tree

Hide file tree

Showing 18 changed files with 874 additions and 360 deletions.
diff --git a/docs/generated/logsinks.md b/docs/generated/logsinks.md
@@ -6,6 +6,8 @@ The supported log output sink types are documented below.
 
 - [Output to Fluentd-compatible log collectors](#output-to-fluentd-compatible-log-collectors)
 
+- [Output to HTTP servers.](#output-to-http-servers.)
+
 - [Standard error stream](#standard-error-stream)
 
 
@@ -91,7 +93,7 @@ Configuration options shared across all sink types:
 ## Sink type: Output to Fluentd-compatible log collectors
 
 
-This sink type causes logging data to be sent over the network, to
+This sink type causes logging data to be sent over the network to
 a log collector that can ingest log data in a
 [Fluentd](https://www.fluentd.org)-compatible protocol.
 
@@ -168,6 +170,71 @@ Configuration options shared across all sink types:
 
 
 
+<a name="output-to-http-servers.">
+
+## Sink type: Output to HTTP servers.
+
+
+This sink type causes logging data to be sent over the network
+as requests to an HTTP server.
+
+The configuration key under the `sinks` key in the YAML
+configuration is `http-servers`. Example configuration:
+
+     sinks:
+        http-servers:
+           health:
+              channels: HEALTH
+              address: http://127.0.0.1
+
+Every new server sink configured automatically inherits the configuration set in the `http-defaults` section.
+
+For example:
+
+     http-defaults:
+         redactable: false # default: disable redaction markers
+     sinks:
+       http-servers:
+         health:
+            channels: HEALTH
+            # This sink has redactable set to false,
+            # as the setting is inherited from fluent-defaults
+            # unless overridden here.
+
+The default output format for HTTP sinks is
+`json-compact`. [Other supported formats.](log-formats.html)
+
+{{site.data.alerts.callout_info}}
+Run `cockroach debug check-log-config` to verify the effect of defaults inheritance.
+{{site.data.alerts.end}}
+
+
+
+Type-specific configuration options:
+
+| Field | Description |
+|--|--|
+| `channels` | the list of logging channels that use this sink. See the [channel selection configuration](#channel-format) section for details.  |
+| `address` | the network address of the http server. The host/address and port parts are separated with a colon. IPv6 numeric addresses should be included within square brackets, e.g.: [::1]:1234. Inherited from `http-defaults.address` if not specified. |
+| `method` | the HTTP method to be used.  POST and GET are supported; defaults to POST. Inherited from `http-defaults.method` if not specified. |
+| `unsafe-tls` | enables certificate authentication to be bypassed. Defaults to false. Inherited from `http-defaults.unsafe-tls` if not specified. |
+| `timeout` | the HTTP timeout. Defaults to 0 for no timeout. Inherited from `http-defaults.timeout` if not specified. |
+| `disable-keep-alives` | causes the logging sink to re-establish a new connection for every outgoing log message. This option is intended for testing only and can cause excessive network overhead in production systems. Inherited from `http-defaults.disable-keep-alives` if not specified. |
+
+
+Configuration options shared across all sink types:
+
+| Field | Description |
+|--|--|
+| `filter` | the minimum severity for log events to be emitted to this sink. This can be set to NONE to disable the sink. |
+| `format` | the entry format to use. |
+| `redact` | whether to strip sensitive information before log events are emitted to this sink. |
+| `redactable` | whether to keep redaction markers in the sink's output. The presence of redaction markers makes it possible to strip sensitive data reliably. |
+| `exit-on-error` | whether the logging system should terminate the process if an error is encountered while writing to this sink. |
+| `auditable` | translated to tweaks to the other settings for this sink during validation. For example, it enables `exit-on-error` and changes the format of files from `crdb-v1` to `crdb-v1-count`. |
+
+
+
 <a name="standard-error-stream">
 
 ## Sink type: Standard error stream

diff --git a/pkg/cli/log_flags_test.go b/pkg/cli/log_flags_test.go
@@ -31,7 +31,7 @@ func TestSetupLogging(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 
 	reWhitespace := regexp.MustCompile(`(?ms:((\s|\n)+))`)
-	reWhitespace2 := regexp.MustCompile(`{\s+`)
+	reBracketWhitespace := regexp.MustCompile(`(?P<bracket>[{[])\s+`)
 
 	reSimplify := regexp.MustCompile(`(?ms:^\s*(auditable: false|redact: false|exit-on-error: true|max-group-size: 100MiB)\n)`)
 
@@ -41,6 +41,15 @@ func TestSetupLogging(t *testing.T) {
 		`redactable: true, ` +
 		`exit-on-error: false` +
 		`}`
+	const defaultHTTPConfig = `http-defaults: {` +
+		`method: POST, ` +
+		`unsafe-tls: false, ` +
+		`timeout: 0s, ` +
+		`disable-keep-alives: false, ` +
+		`filter: INFO, ` +
+		`format: json-compact, ` +
+		`redactable: true, ` +
+		`exit-on-error: false}`
 	stdFileDefaultsRe := regexp.MustCompile(
 		`file-defaults: \{dir: (?P<path>[^,]+), max-file-size: 10MiB, buffered-writes: true, filter: INFO, format: crdb-v2, redactable: true\}`)
 	fileDefaultsNoMaxSizeRe := regexp.MustCompile(
@@ -103,10 +112,11 @@ func TestSetupLogging(t *testing.T) {
 			t.Fatal(err)
 		}
 		actual = reWhitespace.ReplaceAllString(h.String(), " ")
-		actual = reWhitespace2.ReplaceAllString(actual, "{")
+		actual = reBracketWhitespace.ReplaceAllString(actual, "$bracket")
 
 		// Shorten the configuration for legibility during reviews of test changes.
 		actual = strings.ReplaceAll(actual, defaultFluentConfig, "<fluentDefaults>")
+		actual = strings.ReplaceAll(actual, defaultHTTPConfig, "<httpDefaults>")
 		actual = stdFileDefaultsRe.ReplaceAllString(actual, "<stdFileDefaults($path)>")
 		actual = fileDefaultsNoMaxSizeRe.ReplaceAllString(actual, "<fileDefaultsNoMaxSize($path)>")
 		actual = strings.ReplaceAll(actual, fileDefaultsNoDir, "<fileDefaultsNoDir>")

diff --git a/pkg/cli/testdata/logflags b/pkg/cli/testdata/logflags
@@ -14,6 +14,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -43,6 +44,7 @@ start-single-node
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -76,13 +78,15 @@ sql
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledWarningNoRedaction>}}
 
 run
 init
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledWarningNoRedaction>}}
 
 
@@ -95,6 +99,7 @@ bank
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 
@@ -105,6 +110,7 @@ demo
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrCfg(NONE,false)>}}
 
 
@@ -120,6 +126,7 @@ start
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 
@@ -132,6 +139,7 @@ start
 ----
 config: {<stdFileDefaults(/pathA/logs)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -163,6 +171,7 @@ start
 ----
 config: {<stdFileDefaults(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -175,7 +184,7 @@ sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /mypath,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -195,6 +204,7 @@ start
 ----
 config: {<stdFileDefaults(/pathA/logs)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -232,6 +242,7 @@ start
 ----
 config: {<stdFileDefaults(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -244,7 +255,7 @@ sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /mypath,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -262,6 +273,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -293,6 +305,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -345,6 +358,7 @@ start
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 
@@ -355,6 +369,7 @@ start
 ----
 config: {<stdFileDefaults(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -367,7 +382,7 @@ sql-auth: <fileCfg([SESSIONS],/mypath,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/mypath,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/mypath,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/mypath,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /mypath,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -387,6 +402,7 @@ start
 ----
 config: {<stdFileDefaults(/pathA)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -399,7 +415,7 @@ sql-auth: <fileCfg([SESSIONS],/pathA,false,crdb-v2)>,
 sql-exec: <fileCfg([SQL_EXEC],/pathA,true,crdb-v2)>,
 sql-slow: <fileCfg([SQL_PERF],/pathA,true,crdb-v2)>,
 sql-slow-internal-only: <fileCfg([SQL_INTERNAL_PERF],/pathA,true,crdb-v2)>,
-telemetry: {channels: [ TELEMETRY],
+telemetry: {channels: [TELEMETRY],
 dir: /pathA,
 max-file-size: 100KiB,
 max-group-size: 1.0MiB,
@@ -419,6 +435,7 @@ init
 ----
 config: {<fileDefaultsNoMaxSize(/mypath)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: {channels: all,
 dir: /mypath,
 buffered-writes: true,
@@ -435,6 +452,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -465,6 +483,7 @@ start
 ----
 config: {<stdFileDefaults(<defaultLogDir>)>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {file-groups: {default: <fileCfg([DEV,
 OPS,
 HEALTH,
@@ -495,6 +514,7 @@ init
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledInfoNoRedaction>}}
 
 # Default when no severity is specified is WARNING.
@@ -504,6 +524,7 @@ init
 ----
 config: {<fileDefaultsNoDir>,
 <fluentDefaults>,
+<httpDefaults>,
 sinks: {<stderrEnabledWarningNoRedaction>}}
 
 

diff --git a/pkg/sql/parser/testdata/alter_table b/pkg/sql/parser/testdata/alter_table
@@ -630,8 +630,7 @@ ALTER TABLE a SPLIT AT VALUES (1) WITH EXPIRATION TIMESTAMP '2200-01-01 00:00:00
 ----
 ALTER TABLE a SPLIT AT VALUES (1) WITH EXPIRATION TIMESTAMP '2200-01-01 00:00:00.0'
 ALTER TABLE a SPLIT AT VALUES ((1)) WITH EXPIRATION (TIMESTAMP ('2200-01-01 00:00:00.0')) -- fully parenthesized
-ALTER TABLE a SPLIT AT VALUES (_) WITH EXPIRATION TIMESTAMP _ -- literals removed
-REPARSE WITHOUT LITERALS FAILS: at or near "_": syntax error
+ALTER TABLE a SPLIT AT VALUES (_) WITH EXPIRATION TIMESTAMP '_' -- literals removed
 ALTER TABLE _ SPLIT AT VALUES (1) WITH EXPIRATION TIMESTAMP '2200-01-01 00:00:00.0' -- identifiers removed
 
 parse
@@ -647,8 +646,7 @@ ALTER TABLE a SPLIT AT VALUES (1) WITH EXPIRATION TIMESTAMPTZ '2200-01-01 00:00:
 ----
 ALTER TABLE a SPLIT AT VALUES (1) WITH EXPIRATION TIMESTAMPTZ '2200-01-01 00:00:00.0'
 ALTER TABLE a SPLIT AT VALUES ((1)) WITH EXPIRATION (TIMESTAMPTZ ('2200-01-01 00:00:00.0')) -- fully parenthesized
-ALTER TABLE a SPLIT AT VALUES (_) WITH EXPIRATION TIMESTAMPTZ _ -- literals removed
-REPARSE WITHOUT LITERALS FAILS: at or near "_": syntax error
+ALTER TABLE a SPLIT AT VALUES (_) WITH EXPIRATION TIMESTAMPTZ '_' -- literals removed
 ALTER TABLE _ SPLIT AT VALUES (1) WITH EXPIRATION TIMESTAMPTZ '2200-01-01 00:00:00.0' -- identifiers removed
 
 parse