[exporter/elasticsearch] deprecate 'dedup' config

.chloggen/elasticsearch-deprecate-dedup.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: deprecation
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/elasticsearch
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Deprecate the "dedup" configuration.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33773]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: dedup has been deprecated, and will always be enabled in future.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/elasticsearchexporter/README.md

@@ -127,10 +127,10 @@ behaviours, which may be configured throug the following settings:
              field names for span events. 
   - `fields` (optional): Configure additional fields mappings.
   - `file` (optional): Read additional field mappings from the provided YAML file.
-  - `dedup` (default=true): Try to find and remove duplicate fields/attributes
-    from events before publishing to Elasticsearch. Some structured logging
-    libraries can produce duplicate fields (for example zap). Elasticsearch
-    will reject documents that have duplicate fields.
+  - `dedup` (default=true; DEPRECATED, in future deduplication will always be enabled):
+    Try to find and remove duplicate fields/attributes from events before publishing
+    to Elasticsearch. Some structured logging libraries can produce duplicate fields
+    (for example zap). Elasticsearch will reject documents that have duplicate fields.
   - `dedot` (default=true): When enabled attributes with `.` will be split into
     proper json objects.
 

exporter/elasticsearchexporter/config.go

@@ -15,6 +15,7 @@ import (
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/config/configopaque"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
+	"go.uber.org/zap"
 )
 
 // Config defines configuration for Elastic exporter.
@@ -157,6 +158,10 @@ type MappingsSettings struct {
 	File string `mapstructure:"file"`
 
 	// Try to find and remove duplicate fields
+	//
+	// Deprecated: [v0.104.0] deduplication will always be applied in future,
+	// with no option to disable. Disabling deduplication is not meaningful,
+	// as Elasticsearch will reject documents with duplicate JSON object keys.
 	Dedup bool `mapstructure:"dedup"`
 
 	Dedot bool `mapstructure:"dedot"`
@@ -309,3 +314,9 @@ func parseCloudID(input string) (*url.URL, error) {
 func (cfg *Config) MappingMode() MappingMode {
 	return mappingModes[cfg.Mapping.Mode]
 }
+
+func logConfigDeprecationWarnings(cfg *Config, logger *zap.Logger) {
+	if !cfg.Mapping.Dedup {
+		logger.Warn("dedup has been deprecated, and will always be enabled in future")
+	}
+}

exporter/elasticsearchexporter/factory.go

@@ -92,6 +92,7 @@ func createLogsExporter(
 		set.Logger.Warn("index option are deprecated and replaced with logs_index and traces_index.")
 		index = cf.Index
 	}
+	logConfigDeprecationWarnings(cf, set.Logger)
 
 	exporter, err := newExporter(cf, set, index, cf.LogsDynamicIndex.Enabled)
 	if err != nil {
@@ -115,6 +116,7 @@ func createMetricsExporter(
 	cfg component.Config,
 ) (exporter.Metrics, error) {
 	cf := cfg.(*Config)
+	logConfigDeprecationWarnings(cf, set.Logger)
 
 	exporter, err := newExporter(cf, set, cf.MetricsIndex, cf.MetricsDynamicIndex.Enabled)
 	if err != nil {
@@ -136,6 +138,7 @@ func createTracesExporter(ctx context.Context,
 	cfg component.Config) (exporter.Traces, error) {
 
 	cf := cfg.(*Config)
+	logConfigDeprecationWarnings(cf, set.Logger)
 
 	exporter, err := newExporter(cf, set, cf.TracesIndex, cf.TracesDynamicIndex.Enabled)
 	if err != nil {

exporter/elasticsearchexporter/factory_test.go

@@ -11,6 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component/componenttest"
 	"go.opentelemetry.io/collector/exporter/exportertest"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest/observer"
 )
 
 func TestCreateDefaultConfig(t *testing.T) {
@@ -94,3 +96,33 @@ func TestFactory_CreateLogsAndTracesExporterWithDeprecatedIndexOption(t *testing
 	require.NotNil(t, tracesExporter)
 	require.NoError(t, tracesExporter.Shutdown(context.Background()))
 }
+
+func TestFactory_DedupDeprecated(t *testing.T) {
+	factory := NewFactory()
+	cfg := withDefaultConfig(func(cfg *Config) {
+		cfg.Endpoint = "http://testing.invalid:9200"
+		cfg.Mapping.Dedup = false
+	})
+
+	loggerCore, logObserver := observer.New(zap.WarnLevel)
+	set := exportertest.NewNopSettings()
+	set.Logger = zap.New(loggerCore)
+
+	logsExporter, err := factory.CreateLogsExporter(context.Background(), set, cfg)
+	require.NoError(t, err)
+	require.NoError(t, logsExporter.Shutdown(context.Background()))
+
+	tracesExporter, err := factory.CreateTracesExporter(context.Background(), set, cfg)
+	require.NoError(t, err)
+	require.NoError(t, tracesExporter.Shutdown(context.Background()))
+
+	metricsExporter, err := factory.CreateMetricsExporter(context.Background(), set, cfg)
+	require.NoError(t, err)
+	require.NoError(t, metricsExporter.Shutdown(context.Background()))
+
+	records := logObserver.AllUntimed()
+	assert.Len(t, records, 3)
+	assert.Equal(t, "dedup has been deprecated, and will always be enabled in future", records[0].Message)
+	assert.Equal(t, "dedup has been deprecated, and will always be enabled in future", records[1].Message)
+	assert.Equal(t, "dedup has been deprecated, and will always be enabled in future", records[2].Message)
+}