From 53e628169a661e560018e811132bf7fb090092c6 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sat, 25 Jun 2022 23:02:48 -0700 Subject: [PATCH] [exporter/splunk_hec] Update limits for max_content_length settings Splunk Core/Cloud support request payloads up to 800 Mb (https://docs.splunk.com/Documentation/Splunk/8.2.2/Admin/Limitsconf#.5Bhttp_input.5D), but the exporter currently doesn't allow sending payloads more than 2 Mb due to a Splunk Observability ingest limitation. This change updates the limits to make it possible to send bigger requests. --- CHANGELOG.md | 1 + exporter/splunkhecexporter/README.md | 12 ++++++++++-- exporter/splunkhecexporter/config.go | 20 +++++++++++++------- exporter/splunkhecexporter/factory.go | 6 +++--- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 248054b412a0..c24b574fdc27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - `prometheusreceiver`: Add `target_info` labels to resource attributes. (#11034) - `saphanareceiver`: Fix component memory query, add better error handling (#11507) - `sapmexporter`: Add config option to log responses from Splunk APM. (#11425) +- `splunkhecexporter`: Update limits for max_content_length settings (#11550) ### 🧰 Bug fixes 🧰 diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md index 64b0426515af..5f31d45f3bf5 100644 --- a/exporter/splunkhecexporter/README.md +++ b/exporter/splunkhecexporter/README.md @@ -27,8 +27,16 @@ The following configuration options can also be configured: - `ca_file` (no default) Path to the CA cert to verify the server being connected to. - `cert_file` (no default) Path to the TLS cert to use for client connections when TLS client auth is required. - `key_file` (no default) Path to the TLS key to use for TLS required connections. -- `max_content_length_logs` (default: 2097152): Maximum log data size in bytes per HTTP post limited to 2097152 bytes (2 MiB). -- `max_content_length_metrics` (default: 2097152): Maximum metric data size in bytes per HTTP post limited to 2097152 bytes (2 MiB). +- `max_content_length_logs` (default: 2097152): Maximum log payload size in bytes. Log batches of bigger size will be + broken down into several requests. Default value is 2097152 bytes (2 MiB). Maximum allowed value is 838860800 + (~ 800 MB). Keep in mind that Splunk Observability backend doesn't accept requests bigger than 2 MiB. This + configuration value can be raised only if used with Splunk Core/Cloud. +- `max_content_length_metrics` (default: 2097152): Maximum metric payload size in bytes. Metric batches of bigger size + will be broken down into several requests. Default value is 2097152 bytes (2 MiB). Maximum allowed value is 838860800 + (~ 800 MB). +- `max_content_length_metrics` (default: 2097152): Maximum trace payload size in bytes. Trace batches of bigger size + will be broken down into several requests. Default value is 2097152 bytes (2 MiB). Maximum allowed value is 838860800 + (~ 800 MB). - `splunk_app_name` (default: "OpenTelemetry Collector Contrib") App name is used to track telemetry information for Splunk App's using HEC by App name. - `splunk_app_version` (default: Current OpenTelemetry Collector Contrib Build Version): App version is used to track telemetry information for Splunk App's using HEC by App version. - `log_data_enabled` (default: true): Specifies whether the log data is exported. Set it to `false` if you want the log diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go index 9776d7c2979b..3e1eb6f014de 100644 --- a/exporter/splunkhecexporter/config.go +++ b/exporter/splunkhecexporter/config.go @@ -29,10 +29,13 @@ import ( const ( // hecPath is the default HEC path on the Splunk instance. - hecPath = "services/collector" - maxContentLengthLogsLimit = 2 * 1024 * 1024 - maxContentLengthMetricsLimit = 2 * 1024 * 1024 - maxContentLengthTracesLimit = 2 * 1024 * 1024 + hecPath = "services/collector" + defaultContentLengthLogsLimit = 2 * 1024 * 1024 + defaultContentLengthMetricsLimit = 2 * 1024 * 1024 + defaultContentLengthTracesLimit = 2 * 1024 * 1024 + maxContentLengthLogsLimit = 800 * 1024 * 1024 + maxContentLengthMetricsLimit = 800 * 1024 * 1024 + maxContentLengthTracesLimit = 800 * 1024 * 1024 ) // OtelToHecFields defines the mapping of attributes to HEC fields @@ -80,13 +83,16 @@ type Config struct { // Disable GZip compression. Defaults to false. DisableCompression bool `mapstructure:"disable_compression"` - // Maximum log data size in bytes per HTTP post. Defaults to the backend limit of 2097152 bytes (2MiB). + // Maximum log payload size in bytes. Default value is 2097152 bytes (2MiB). + // Maximum allowed value is 838860800 (~ 800 MB). MaxContentLengthLogs uint `mapstructure:"max_content_length_logs"` - // Maximum metric data size in bytes per HTTP post. Defaults to the backend limit of 2097152 bytes (2MiB). + // Maximum metric payload size in bytes. Default value is 2097152 bytes (2MiB). + // Maximum allowed value is 838860800 (~ 800 MB). MaxContentLengthMetrics uint `mapstructure:"max_content_length_metrics"` - // Maximum trace data size in bytes per HTTP post. Defaults to the backend limit of 2097152 bytes (2MiB). + // Maximum trace payload size in bytes. Default value is 2097152 bytes (2MiB). + // Maximum allowed value is 838860800 (~ 800 MB). MaxContentLengthTraces uint `mapstructure:"max_content_length_traces"` // TLSSetting struct exposes TLS client configuration. diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go index 4d10bbb18deb..beaec256a7eb 100644 --- a/exporter/splunkhecexporter/factory.go +++ b/exporter/splunkhecexporter/factory.go @@ -70,9 +70,9 @@ func createDefaultConfig() config.Exporter { QueueSettings: exporterhelper.NewDefaultQueueSettings(), DisableCompression: false, MaxConnections: defaultMaxIdleCons, - MaxContentLengthLogs: maxContentLengthLogsLimit, - MaxContentLengthMetrics: maxContentLengthMetricsLimit, - MaxContentLengthTraces: maxContentLengthTracesLimit, + MaxContentLengthLogs: defaultContentLengthLogsLimit, + MaxContentLengthMetrics: defaultContentLengthMetricsLimit, + MaxContentLengthTraces: defaultContentLengthTracesLimit, HecToOtelAttrs: splunk.HecToOtelAttrs{ Source: splunk.DefaultSourceLabel, SourceType: splunk.DefaultSourceTypeLabel,