[Otlp] Add disk retry enablement

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md

@@ -13,6 +13,14 @@
   `parent_is_remote` information.
   ([#5563](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5563))
 
+* Introduced experimental support for automatically retrying export to the otlp
+  endpoint by storing the telemetry offline during transient network errors.
+  Users can enable this feature by setting the
+  `OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY` environment variable to `disk` and
+  setting `OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH` to the path
+  on disk for storing the telemetry.
+  ([#5527](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5527))
+
 ## 1.8.1
 
 Released 2024-Apr-17

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs

@@ -17,6 +17,8 @@ internal sealed class ExperimentalOptions
 
     public const string OtlpRetryEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY";
 
+    public const string OtlpDiskRetryDirectoryPathEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH";
+
     public ExperimentalOptions()
         : this(new ConfigurationBuilder().AddEnvironmentVariables().Build())
     {
@@ -29,9 +31,28 @@ public ExperimentalOptions(IConfiguration configuration)
             this.EmitLogEventAttributes = emitLogEventAttributes;
         }
 
-        if (configuration.TryGetStringValue(OtlpRetryEnvVar, out var retryPolicy) && retryPolicy != null && retryPolicy.Equals("in_memory", StringComparison.OrdinalIgnoreCase))
+        if (configuration.TryGetStringValue(OtlpRetryEnvVar, out var retryPolicy) && retryPolicy != null)
         {
-            this.EnableInMemoryRetry = true;
+            if (retryPolicy.Equals("in_memory", StringComparison.OrdinalIgnoreCase))
+            {
+                this.EnableInMemoryRetry = true;
+            }
+            else if (retryPolicy.Equals("disk", StringComparison.OrdinalIgnoreCase))
+            {
+                if (configuration.TryGetStringValue(OtlpDiskRetryDirectoryPathEnvVar, out var path) && path != null)
+                {
+                    this.EnableDiskRetry = true;
+                    this.DiskRetryDirectoryPath = path;
+                }
+                else
+                {
+                    throw new ArgumentException($"{OtlpDiskRetryDirectoryPathEnvVar} is required when '{retryPolicy}' retry is used");
+                }
+            }
+            else
+            {
+                throw new NotSupportedException($"Retry Policy '{retryPolicy}' is not supported.");
+            }
         }
     }
 
@@ -48,4 +69,14 @@ public ExperimentalOptions(IConfiguration configuration)
     /// href="https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry"/>.
     /// </remarks>
     public bool EnableInMemoryRetry { get; }
+
+    /// <summary>
+    /// Gets a value indicating whether or not retry via disk should be enabled for transient errors.
+    /// </summary>
+    public bool EnableDiskRetry { get; }
+
+    /// <summary>
+    /// Gets the path on disk where the telemetry will be stored for retries at a later point.
+    /// </summary>
+    public string? DiskRetryDirectoryPath { get; }
 }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs

@@ -11,6 +11,7 @@
 #if NETSTANDARD2_1 || NET6_0_OR_GREATER
 using Grpc.Net.Client;
 #endif
+using Google.Protobuf;
 using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Transmission;
 using LogOtlpCollector = OpenTelemetry.Proto.Collector.Logs.V1;
 using MetricsOtlpCollector = OpenTelemetry.Proto.Collector.Metrics.V1;
@@ -100,9 +101,27 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
             ? httpTraceExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return experimentalOptions.EnableInMemoryRetry
-            ? new OtlpExporterRetryTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds)
-            : new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
+        if (experimentalOptions.EnableInMemoryRetry)
+        {
+            return new OtlpExporterRetryTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
+        }
+        else if (experimentalOptions.EnableDiskRetry)
+        {
+            return new OtlpExporterPersistentStorageTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(
+                exportClient,
+                timeoutMilliseconds,
+                (byte[] data) =>
+                {
+                    var request = new TraceOtlpCollector.ExportTraceServiceRequest();
+                    request.MergeFrom(data);
+                    return request;
+                },
+                Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "traces"));
+        }
+        else
+        {
+            return new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
+        }
     }
 
     public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
@@ -116,9 +135,27 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
             ? httpMetricsExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return experimentalOptions.EnableInMemoryRetry
-            ? new OtlpExporterRetryTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds)
-            : new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
+        if (experimentalOptions.EnableInMemoryRetry)
+        {
+            return new OtlpExporterRetryTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
+        }
+        else if (experimentalOptions.EnableDiskRetry)
+        {
+            return new OtlpExporterPersistentStorageTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(
+                exportClient,
+                timeoutMilliseconds,
+                (byte[] data) =>
+                {
+                    var request = new MetricsOtlpCollector.ExportMetricsServiceRequest();
+                    request.MergeFrom(data);
+                    return request;
+                },
+                Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "metrics"));
+        }
+        else
+        {
+            return new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
+        }
     }
 
     public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
@@ -128,9 +165,27 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
             ? httpLogExportClient.HttpClient.Timeout.TotalMilliseconds
             : options.TimeoutMilliseconds;
 
-        return experimentalOptions.EnableInMemoryRetry
-            ? new OtlpExporterRetryTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds)
-            : new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
+        if (experimentalOptions.EnableInMemoryRetry)
+        {
+            return new OtlpExporterRetryTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
+        }
+        else if (experimentalOptions.EnableDiskRetry)
+        {
+            return new OtlpExporterPersistentStorageTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(
+                exportClient,
+                timeoutMilliseconds,
+                (byte[] data) =>
+                {
+                    var request = new LogOtlpCollector.ExportLogsServiceRequest();
+                    request.MergeFrom(data);
+                    return request;
+                },
+                Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "logs"));
+        }
+        else
+        {
+            return new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
+        }
     }
 
     public static IExportClient<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportClient(this OtlpExporterOptions options) =>

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md

@@ -632,6 +632,11 @@ want to solicit feedback from the community.
 
     Added in `1.8.0`.
 
+    When set to `disk` along with setting
+    `OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH` to the path on
+    disk, it enables retries by storing telemetry on disk during transient
+    errors. TODO: Add package version.
+
 * Logs
 
   * `OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES`