From eabe829fcb634e72b1155faced62b7a33592e8dd Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 3 Jul 2024 12:58:00 +0100 Subject: [PATCH] [exporter/elasticsearch] Add `telemetry.log_request_body` and `telemetry.log_response_body` config (#33854) **Description:** - Add `telemetry.log_request_body` and `telemetry.log_response_body` config for debugging. Debug log will contain field `request_body` and/or `response_body` in the same log line instead of separate lines to avoid interleaved log lines. - Change "Request failed" log level to debug. Output: ``` 2024-07-02T14:09:24.983+0100 debug elasticsearchexporter/elasticsearch_bulk.go:67 Request roundtrip completed. {"kind": "exporter", "data_type": "logs", "name": "elasticsearch", "response_body": "{\"version\":{\"number\":\"1.2.3\"}}\n", "path": "/", "method": "GET", "duration": 0.000865486, "status": "200 OK"} 2024-07-02T14:09:24.984+0100 debug elasticsearchexporter/elasticsearch_bulk.go:67 Request roundtrip completed. {"kind": "exporter", "data_type": "logs", "name": "elasticsearch", "request_body": "{\"create\":{\"_index\":\"logs-test-idx\"}}\n{\"@timestamp\":\"2024-07-02T13:09:24.970187592Z\",\"Attributes\":{\"a\":\"test\",\"b\":5,\"batch_index\":\"batch_1\",\"c\":3,\"d\":true,\"item_index\":\"item_1\"},\"Body\":\"Load Generator Counter #0\",\"Scope\":{\"name\":\"\",\"version\":\"\"},\"SeverityNumber\":11,\"SeverityText\":\"INFO3\",\"TraceFlags\":1}\n{\"create\":{\"_index\":\"logs-test-idx\"}}\n{\"@timestamp\":\"2024-07-02T13:09:24.970187592Z\",\"Attributes\":{\"a\":\"test\",\"b\":5,\"batch_index\":\"batch_1\",\"c\":3,\"d\":true,\"item_index\":\"item_2\"},\"Body\":\"Load Generator Counter #1\",\"Scope\":{\"name\":\"\",\"version\":\"\"},\"SeverityNumber\":11,\"SeverityText\":\"INFO3\",\"TraceFlags\":1}\n", "response_body": "{\"took\":0,\"errors\":false,\"items\":[{\"create\":{\"_index\":\"logs-test-idx\",\"_id\":\"\",\"_version\":0,\"result\":\"\",\"status\":201,\"_seq_no\":0,\"_primary_term\":0,\"_shards\":{\"total\":0,\"successful\":0,\"failed\":0},\"error\":{\"type\":\"\",\"reason\":\"\",\"caused_by\":{\"type\":\"\",\"reason\":\"\"}}}},{\"create\":{\"_index\":\"logs-test-idx\",\"_id\":\"\",\"_version\":0,\"result\":\"\",\"status\":201,\"_seq_no\":0,\"_primary_term\":0,\"_shards\":{\"total\":0,\"successful\":0,\"failed\":0},\"error\":{\"type\":\"\",\"reason\":\"\",\"caused_by\":{\"type\":\"\",\"reason\":\"\"}}}}]}\n", "path": "/_bulk", "method": "POST", "duration": 0.000539979, "status": "200 OK"} ``` Required config to log ``` exporters: elasticsearch: telemetry: log_request_body: true log_response_body: true service: telemetry: logs: level: debug ``` For easier analysis, limit the size of request body size. Use `num_workers`=1 and lower `flush.bytes` and/or `flush.interval`. **Link to tracking Issue:** **Testing:** Manually verified with a modified integration test. **Documentation:** --- ...sticsearchexporter_telemetry_settings.yaml | 27 +++++++++ exporter/elasticsearchexporter/README.md | 10 ++++ exporter/elasticsearchexporter/config.go | 9 +++ .../elasticsearch_bulk.go | 60 ++++++++++++++----- exporter/elasticsearchexporter/factory.go | 4 ++ 5 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 .chloggen/elasticsearchexporter_telemetry_settings.yaml diff --git a/.chloggen/elasticsearchexporter_telemetry_settings.yaml b/.chloggen/elasticsearchexporter_telemetry_settings.yaml new file mode 100644 index 000000000000..f4ae41675eca --- /dev/null +++ b/.chloggen/elasticsearchexporter_telemetry_settings.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Introduce experimental `telemetry.log_request_body` and `telemetry.log_response_body` config + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33854] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 6c2db78b52da..1c0597e76797 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -184,6 +184,16 @@ Settings related to node discovery are: Node discovery can be disabled by setting `discover.interval` to 0. +### Telemetry settings + +The Elasticsearch Exporter's own telemetry settings for testing and debugging purposes. + +⚠️ This is experimental and may change at any time. + +- `telemetry`: + - `log_request_body` (default=false): Logs Elasticsearch client request body as a field in a log line at DEBUG level. It requires `service::telemetry::logs::level` to be set to `debug`. WARNING: Enabling this config may expose sensitive data. + - `log_response_body` (default=false): Logs Elasticsearch client response body as a field in a log line at DEBUG level. It requires `service::telemetry::logs::level` to be set to `debug`. WARNING: Enabling this config may expose sensitive data. + ## Exporting metrics Metrics support is currently in development. diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 1bcabb99a325..ec745c77050a 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -72,6 +72,15 @@ type Config struct { Flush FlushSettings `mapstructure:"flush"` Mapping MappingsSettings `mapstructure:"mapping"` LogstashFormat LogstashFormatSettings `mapstructure:"logstash_format"` + + // TelemetrySettings contains settings useful for testing/debugging purposes + // This is experimental and may change at any time. + TelemetrySettings `mapstructure:"telemetry"` +} + +type TelemetrySettings struct { + LogRequestBody bool `mapstructure:"log_request_body"` + LogResponseBody bool `mapstructure:"log_response_body"` } type LogstashFormatSettings struct { diff --git a/exporter/elasticsearchexporter/elasticsearch_bulk.go b/exporter/elasticsearchexporter/elasticsearch_bulk.go index c44a66f3db43..5beb768bc582 100644 --- a/exporter/elasticsearchexporter/elasticsearch_bulk.go +++ b/exporter/elasticsearchexporter/elasticsearch_bulk.go @@ -32,37 +32,59 @@ type esBulkIndexerItem = docappender.BulkIndexerItem // clientLogger implements the estransport.Logger interface // that is required by the Elasticsearch client for logging. -type clientLogger zap.Logger +type clientLogger struct { + *zap.Logger + logRequestBody bool + logResponseBody bool +} // LogRoundTrip should not modify the request or response, except for consuming and closing the body. // Implementations have to check for nil values in request and response. -func (cl *clientLogger) LogRoundTrip(requ *http.Request, resp *http.Response, err error, _ time.Time, dur time.Duration) error { - zl := (*zap.Logger)(cl) +func (cl *clientLogger) LogRoundTrip(requ *http.Request, resp *http.Response, clientErr error, _ time.Time, dur time.Duration) error { + zl := cl.Logger + + var fields []zap.Field + if cl.logRequestBody && requ != nil && requ.Body != nil { + if b, err := io.ReadAll(requ.Body); err == nil { + fields = append(fields, zap.ByteString("request_body", b)) + } + } + if cl.logResponseBody && resp != nil && resp.Body != nil { + if b, err := io.ReadAll(resp.Body); err == nil { + fields = append(fields, zap.ByteString("response_body", b)) + } + } + switch { - case err == nil && resp != nil: - zl.Debug("Request roundtrip completed.", + case clientErr == nil && resp != nil: + fields = append( + fields, zap.String("path", sanitize.String(requ.URL.Path)), zap.String("method", requ.Method), zap.Duration("duration", dur), - zap.String("status", resp.Status)) - - case err != nil: - zl.Error("Request failed.", zap.NamedError("reason", err)) + zap.String("status", resp.Status), + ) + zl.Debug("Request roundtrip completed.", fields...) + + case clientErr != nil: + fields = append( + fields, + zap.NamedError("reason", clientErr), + ) + zl.Debug("Request failed.", fields...) } return nil } // RequestBodyEnabled makes the client pass a copy of request body to the logger. -func (*clientLogger) RequestBodyEnabled() bool { - // TODO: introduce setting log the bodies for more detailed debug logs - return false +func (cl *clientLogger) RequestBodyEnabled() bool { + return cl.logRequestBody } // ResponseBodyEnabled makes the client pass a copy of response body to the logger. -func (*clientLogger) ResponseBodyEnabled() bool { - // TODO: introduce setting log the bodies for more detailed debug logs - return false +func (cl *clientLogger) ResponseBodyEnabled() bool { + return cl.logResponseBody } func newElasticsearchClient( @@ -97,6 +119,12 @@ func newElasticsearchClient( return nil, err } + esLogger := clientLogger{ + Logger: telemetry.Logger, + logRequestBody: config.LogRequestBody, + logResponseBody: config.LogResponseBody, + } + return elasticsearch7.NewClient(esConfigCurrent{ Transport: httpClient.Transport, @@ -122,7 +150,7 @@ func newElasticsearchClient( // configure internal metrics reporting and logging EnableMetrics: false, // TODO EnableDebugLogger: false, // TODO - Logger: (*clientLogger)(telemetry.Logger), + Logger: &esLogger, }) } diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 7826fb59a47e..3fb8b295e352 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -84,6 +84,10 @@ func createDefaultConfig() component.Config { PrefixSeparator: "-", DateFormat: "%Y.%m.%d", }, + TelemetrySettings: TelemetrySettings{ + LogRequestBody: false, + LogResponseBody: false, + }, } }