[otlp] Add experimental feature flag for enabling retries (open-telem…

…etry#5435)

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md

@@ -39,6 +39,12 @@
   to `null` will now result in an `ArgumentNullException` being thrown.
   ([#5434](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5434))
 
+* Introduced experimental support for automatically retrying export to the otlp
+  endpoint when transient network errors occur. Users can enable this feature by
+  setting `OTEL_DOTNET_EXPERIMENTAL_OTLP_ENABLE_INMEMORY_RETRY` environment
+  variable to true.
+  ([#5435](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5435))
+
 ## 1.7.0
 
 Released 2023-Dec-08

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs

@@ -16,6 +16,8 @@ internal sealed class ExperimentalOptions
 
     public const string EmitLogEventEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES";
 
+    public const string EnableInMemoryRetryEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_ENABLE_INMEMORY_RETRY";
+
     public ExperimentalOptions()
         : this(new ConfigurationBuilder().AddEnvironmentVariables().Build())
     {
@@ -27,10 +29,24 @@ public ExperimentalOptions(IConfiguration configuration)
         {
             this.EmitLogEventAttributes = emitLogEventAttributes;
         }
+
+        if (configuration.TryGetBoolValue(EnableInMemoryRetryEnvVar, out var enableInMemoryRetry))
+        {
+            this.EnableInMemoryRetry = enableInMemoryRetry;
+        }
     }
 
     /// <summary>
-    /// Gets or sets a value indicating whether log event attributes should be exported.
+    /// Gets a value indicating whether log event attributes should be exported.
+    /// </summary>
+    public bool EmitLogEventAttributes { get; }
+
+    /// <summary>
+    /// Gets a value indicating whether or not in-memory retry should be enabled for transient errors.
     /// </summary>
-    public bool EmitLogEventAttributes { get; set; } = false;
+    /// <remarks>
+    /// Specification: <see
+    /// href="https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry"/>.
+    /// </remarks>
+    public bool EnableInMemoryRetry { get; }
 }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptions.cs

@@ -11,6 +11,7 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
+using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 using OpenTelemetry.Internal;
 using OpenTelemetry.Trace;
 
@@ -225,6 +226,7 @@ public Func<HttpClient> HttpClientFactory
     internal static void RegisterOtlpExporterOptionsFactory(IServiceCollection services)
     {
         services.RegisterOptionsFactory(CreateOtlpExporterOptions);
+        services.RegisterOptionsFactory(configuration => new ExperimentalOptions(configuration));
     }
 
     internal static OtlpExporterOptions CreateOtlpExporterOptions(

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs

@@ -6,6 +6,7 @@
 #endif
 using System.Reflection;
 using Grpc.Core;
+using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
 #if NETSTANDARD2_1 || NET6_0_OR_GREATER
 using Grpc.Net.Client;
@@ -88,7 +89,7 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
         return headers;
     }
 
-    public static OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportTransmissionHandler(this OtlpExporterOptions options)
+    public static OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
     {
         var exportClient = GetTraceExportClient(options);
 
@@ -99,10 +100,12 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
             ? httpTraceExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
+        return experimentalOptions.EnableInMemoryRetry
+            ? new OtlpExporterRetryTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds)
+            : new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
     }
 
-    public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options)
+    public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
     {
         var exportClient = GetMetricsExportClient(options);
 
@@ -113,21 +116,21 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
             ? httpMetricsExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
+        return experimentalOptions.EnableInMemoryRetry
+            ? new OtlpExporterRetryTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds)
+            : new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
     }
 
-    public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options)
+    public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
     {
         var exportClient = GetLogExportClient(options);
-
-        // `HttpClient.Timeout.TotalMilliseconds` would be populated with the correct timeout value for both the exporter configuration cases:
-        // 1. User provides their own HttpClient. This case is straightforward as the user wants to use their `HttpClient` and thereby the same client's timeout value.
-        // 2. If the user configures timeout via the exporter options, then the timeout set for the `HttpClient` initialized by the exporter will be set to user provided value.
         double timeoutMilliseconds = exportClient is OtlpHttpLogExportClient httpLogExportClient
             ? httpLogExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
+        return experimentalOptions.EnableInMemoryRetry
+            ? new OtlpExporterRetryTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds)
+            : new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
     }
 
     public static IExportClient<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportClient(this OtlpExporterOptions options) =>

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs

@@ -62,7 +62,7 @@ internal OtlpLogExporter(
             OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable(key, value);
         };
 
-        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetLogsExportTransmissionHandler();
+        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetLogsExportTransmissionHandler(experimentalOptions!);
 
         this.otlpLogRecordTransformer = new OtlpLogRecordTransformer(sdkLimitOptions!, experimentalOptions!);
     }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporterHelperExtensions.cs

@@ -404,7 +404,6 @@ private static void RegisterOptions(IServiceCollection services)
     {
         OtlpExporterOptions.RegisterOtlpExporterOptionsFactory(services);
         services.RegisterOptionsFactory(configuration => new SdkLimitOptions(configuration));
-        services.RegisterOptionsFactory(configuration => new ExperimentalOptions(configuration));
     }
 
     private static T GetOptions<T>(

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporter.cs

@@ -25,17 +25,19 @@ public class OtlpMetricExporter : BaseExporter<Metric>
     /// </summary>
     /// <param name="options">Configuration options for the exporter.</param>
     public OtlpMetricExporter(OtlpExporterOptions options)
-        : this(options, transmissionHandler: null)
+        : this(options, experimentalOptions: new(), transmissionHandler: null)
     {
     }
 
     /// <summary>
     /// Initializes a new instance of the <see cref="OtlpMetricExporter"/> class.
     /// </summary>
     /// <param name="options">Configuration options for the export.</param>
+    /// <param name="experimentalOptions"><see cref="ExperimentalOptions"/>.</param>
     /// <param name="transmissionHandler"><see cref="OtlpExporterTransmissionHandler{T}"/>.</param>
     internal OtlpMetricExporter(
         OtlpExporterOptions options,
+        ExperimentalOptions experimentalOptions,
         OtlpExporterTransmissionHandler<OtlpCollector.ExportMetricsServiceRequest> transmissionHandler = null)
     {
         // Each of the Otlp exporters: Traces, Metrics, and Logs set the same value for `OtlpKeyValueTransformer.LogUnsupportedAttributeType`
@@ -50,7 +52,7 @@ internal OtlpMetricExporter(
             OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable(key, value);
         };
 
-        this.transmissionHandler = transmissionHandler ?? options.GetMetricsExportTransmissionHandler();
+        this.transmissionHandler = transmissionHandler ?? options.GetMetricsExportTransmissionHandler(experimentalOptions);
     }
 
     internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource();

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporterExtensions.cs

@@ -7,6 +7,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using OpenTelemetry.Exporter;
+using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 using OpenTelemetry.Internal;
 
 namespace OpenTelemetry.Metrics;
@@ -98,6 +99,7 @@ public static MeterProviderBuilder AddOtlpExporter(
             return BuildOtlpExporterMetricReader(
                 exporterOptions,
                 sp.GetRequiredService<IOptionsMonitor<MetricReaderOptions>>().Get(finalOptionsName),
+                sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
                 sp);
         });
     }
@@ -169,19 +171,24 @@ public static MeterProviderBuilder AddOtlpExporter(
 
             configureExporterAndMetricReader?.Invoke(exporterOptions, metricReaderOptions);
 
-            return BuildOtlpExporterMetricReader(exporterOptions, metricReaderOptions, sp);
+            return BuildOtlpExporterMetricReader(
+                exporterOptions,
+                metricReaderOptions,
+                sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
+                sp);
         });
     }
 
     internal static MetricReader BuildOtlpExporterMetricReader(
         OtlpExporterOptions exporterOptions,
         MetricReaderOptions metricReaderOptions,
+        ExperimentalOptions experimentalOptions,
         IServiceProvider serviceProvider,
         Func<BaseExporter<Metric>, BaseExporter<Metric>>? configureExporterInstance = null)
     {
         exporterOptions.TryEnableIHttpClientFactoryIntegration(serviceProvider, "OtlpMetricExporter");
 
-        BaseExporter<Metric> metricExporter = new OtlpMetricExporter(exporterOptions);
+        BaseExporter<Metric> metricExporter = new OtlpMetricExporter(exporterOptions, experimentalOptions);
 
         if (configureExporterInstance != null)
         {

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporter.cs

@@ -26,7 +26,7 @@ public class OtlpTraceExporter : BaseExporter<Activity>
     /// </summary>
     /// <param name="options">Configuration options for the export.</param>
     public OtlpTraceExporter(OtlpExporterOptions options)
-        : this(options, sdkLimitOptions: new(), transmissionHandler: null)
+        : this(options, sdkLimitOptions: new(), experimentalOptions: new(), transmissionHandler: null)
     {
     }
 
@@ -35,10 +35,12 @@ public OtlpTraceExporter(OtlpExporterOptions options)
     /// </summary>
     /// <param name="exporterOptions"><see cref="OtlpExporterOptions"/>.</param>
     /// <param name="sdkLimitOptions"><see cref="SdkLimitOptions"/>.</param>
+    /// <param name="experimentalOptions"><see cref="ExperimentalOptions"/>.</param>
     /// <param name="transmissionHandler"><see cref="OtlpExporterTransmissionHandler{T}"/>.</param>
     internal OtlpTraceExporter(
     OtlpExporterOptions exporterOptions,
     SdkLimitOptions sdkLimitOptions,
+    ExperimentalOptions experimentalOptions,
     OtlpExporterTransmissionHandler<OtlpCollector.ExportTraceServiceRequest> transmissionHandler = null)
     {
         Debug.Assert(exporterOptions != null, "exporterOptions was null");
@@ -50,7 +52,7 @@ internal OtlpTraceExporter(
 
         ConfigurationExtensions.LogInvalidEnvironmentVariable = OpenTelemetryProtocolExporterEventSource.Log.InvalidEnvironmentVariable;
 
-        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetTraceExportTransmissionHandler();
+        this.transmissionHandler = transmissionHandler ?? exporterOptions.GetTraceExportTransmissionHandler(experimentalOptions);
     }
 
     internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource();

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporterHelperExtensions.cs

@@ -92,19 +92,24 @@ public static TracerProviderBuilder AddOtlpExporter(
             // instance.
             var sdkOptionsManager = sp.GetRequiredService<IOptionsMonitor<SdkLimitOptions>>().CurrentValue;
 
-            return BuildOtlpExporterProcessor(exporterOptions, sdkOptionsManager, sp);
+            return BuildOtlpExporterProcessor(
+                exporterOptions,
+                sdkOptionsManager,
+                sp.GetRequiredService<IOptionsMonitor<ExperimentalOptions>>().Get(finalOptionsName),
+                sp);
         });
     }
 
     internal static BaseProcessor<Activity> BuildOtlpExporterProcessor(
         OtlpExporterOptions exporterOptions,
         SdkLimitOptions sdkLimitOptions,
+        ExperimentalOptions experimentalOptions,
         IServiceProvider serviceProvider,
         Func<BaseExporter<Activity>, BaseExporter<Activity>>? configureExporterInstance = null)
     {
         exporterOptions.TryEnableIHttpClientFactoryIntegration(serviceProvider, "OtlpTraceExporter");
 
-        BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(exporterOptions, sdkLimitOptions);
+        BaseExporter<Activity> otlpExporter = new OtlpTraceExporter(exporterOptions, sdkLimitOptions, experimentalOptions);
 
         if (configureExporterInstance != null)
         {

test/Benchmarks/Exporter/OtlpGrpcExporterBenchmarks.cs

@@ -35,6 +35,7 @@ public void GlobalSetup()
         this.exporter = new OtlpTraceExporter(
             options,
             new SdkLimitOptions(),
+            new ExperimentalOptions(),
             new OtlpExporterTransmissionHandler<ExportTraceServiceRequest>(new OtlpGrpcTraceExportClient(options, new TestTraceServiceClient()), options.TimeoutMilliseconds));
 
         this.activity = ActivityHelper.CreateTestActivity();

test/Benchmarks/Exporter/OtlpHttpExporterBenchmarks.cs

@@ -63,6 +63,7 @@ public void GlobalSetup()
         this.exporter = new OtlpTraceExporter(
             options,
             new SdkLimitOptions(),
+            new ExperimentalOptions(),
             new OtlpExporterTransmissionHandler<ExportTraceServiceRequest>(new OtlpHttpTraceExportClient(options, options.HttpClientFactory()), options.TimeoutMilliseconds));
 
         this.activity = ActivityHelper.CreateTestActivity();

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/IntegrationTest/IntegrationTests.cs

@@ -72,6 +72,7 @@ public void TraceExportResultIsSuccess(OtlpExportProtocol protocol, string endpo
         builder.AddProcessor(OtlpTraceExporterHelperExtensions.BuildOtlpExporterProcessor(
             exporterOptions,
             DefaultSdkLimitOptions,
+            experimentalOptions: new(),
             serviceProvider: null,
             configureExporterInstance: otlpExporter =>
             {
@@ -153,6 +154,7 @@ public void MetricExportResultIsSuccess(OtlpExportProtocol protocol, string endp
         builder.AddReader(OtlpMetricExporterExtensions.BuildOtlpExporterMetricReader(
             exporterOptions,
             readerOptions,
+            experimentalOptions: new(),
             serviceProvider: null,
             configureExporterInstance: otlpExporter =>
             {

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/MockCollectorIntegrationTests.cs

@@ -8,11 +8,11 @@
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
-using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
-using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Transmission;
+using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Proto.Collector.Trace.V1;
 using OpenTelemetry.Tests;
@@ -172,23 +172,13 @@ public async Task GrpcRetryTests(bool useRetryTransmissionHandler, ExportResult
 
         var endpoint = new Uri($"http://localhost:{testGrpcPort}");
 
-        var exporterOptions = new OtlpExporterOptions() { Endpoint = endpoint, TimeoutMilliseconds = 20000 };
+        var exporterOptions = new OtlpExporterOptions() { Endpoint = endpoint, TimeoutMilliseconds = 20000, Protocol = OtlpExportProtocol.Grpc };
 
-        var exportClient = new OtlpGrpcTraceExportClient(exporterOptions);
+        var configuration = new ConfigurationBuilder()
+         .AddInMemoryCollection(new Dictionary<string, string> { [ExperimentalOptions.EnableInMemoryRetryEnvVar] = useRetryTransmissionHandler.ToString() })
+         .Build();
 
-        OtlpExporterTransmissionHandler<ExportTraceServiceRequest> transmissionHandler;
-
-        // TODO: update this to configure via experimental environment variable.
-        if (useRetryTransmissionHandler)
-        {
-            transmissionHandler = new OtlpExporterRetryTransmissionHandler<ExportTraceServiceRequest>(exportClient, exporterOptions.TimeoutMilliseconds);
-        }
-        else
-        {
-            transmissionHandler = new OtlpExporterTransmissionHandler<ExportTraceServiceRequest>(exportClient, exporterOptions.TimeoutMilliseconds);
-        }
-
-        var otlpExporter = new OtlpTraceExporter(exporterOptions, new(), transmissionHandler);
+        var otlpExporter = new OtlpTraceExporter(exporterOptions, new SdkLimitOptions(), new ExperimentalOptions(configuration));
 
         var activitySourceName = "otel.grpc.retry.test";
         using var source = new ActivitySource(activitySourceName);
@@ -266,23 +256,13 @@ public async Task HttpRetryTests(bool useRetryTransmissionHandler, ExportResult
 
         var endpoint = new Uri($"http://localhost:{testHttpPort}/v1/traces");
 
-        var exporterOptions = new OtlpExporterOptions() { Endpoint = endpoint, TimeoutMilliseconds = 20000 };
+        var exporterOptions = new OtlpExporterOptions() { Endpoint = endpoint, TimeoutMilliseconds = 20000, Protocol = OtlpExportProtocol.HttpProtobuf };
 
-        var exportClient = new OtlpHttpTraceExportClient(exporterOptions, new HttpClient());
-
-        OtlpExporterTransmissionHandler<ExportTraceServiceRequest> transmissionHandler;
-
-        // TODO: update this to configure via experimental environment variable.
-        if (useRetryTransmissionHandler)
-        {
-            transmissionHandler = new OtlpExporterRetryTransmissionHandler<ExportTraceServiceRequest>(exportClient, exporterOptions.TimeoutMilliseconds);
-        }
-        else
-        {
-            transmissionHandler = new OtlpExporterTransmissionHandler<ExportTraceServiceRequest>(exportClient, exporterOptions.TimeoutMilliseconds);
-        }
+        var configuration = new ConfigurationBuilder()
+         .AddInMemoryCollection(new Dictionary<string, string> { [ExperimentalOptions.EnableInMemoryRetryEnvVar] = useRetryTransmissionHandler.ToString() })
+         .Build();
 
-        var otlpExporter = new OtlpTraceExporter(exporterOptions, new(), transmissionHandler);
+        var otlpExporter = new OtlpTraceExporter(exporterOptions, new SdkLimitOptions(), new ExperimentalOptions(configuration));
 
         var activitySourceName = "otel.http.retry.test";
         using var source = new ActivitySource(activitySourceName);