[metrics branch] Add configuration of persistent storage to OTLP Exporter

OpenTelemetry.sln

@@ -249,6 +249,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "getting-started-console", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "getting-started-jaeger", "docs\trace\getting-started-jaeger\getting-started-jaeger.csproj", "{A0C0B77C-6C7B-4EC2-AC61-EA1F489811B9}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage", "src\OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage\OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.csproj", "{2DA349A7-814F-43A6-AA22-28D03421EAF7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.Tests", "test\OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.Tests\OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.Tests.csproj", "{448837FD-6DEA-4B37-A1F5-42BC7AD8E9BE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -523,6 +527,14 @@ Global
 		{A0C0B77C-6C7B-4EC2-AC61-EA1F489811B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A0C0B77C-6C7B-4EC2-AC61-EA1F489811B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A0C0B77C-6C7B-4EC2-AC61-EA1F489811B9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2DA349A7-814F-43A6-AA22-28D03421EAF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2DA349A7-814F-43A6-AA22-28D03421EAF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2DA349A7-814F-43A6-AA22-28D03421EAF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2DA349A7-814F-43A6-AA22-28D03421EAF7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{448837FD-6DEA-4B37-A1F5-42BC7AD8E9BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{448837FD-6DEA-4B37-A1F5-42BC7AD8E9BE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{448837FD-6DEA-4B37-A1F5-42BC7AD8E9BE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{448837FD-6DEA-4B37-A1F5-42BC7AD8E9BE}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

build/Common.props

@@ -45,6 +45,7 @@
     <MicrosoftSourceLinkGitHubPkgVer>[1.1.1,2.0)</MicrosoftSourceLinkGitHubPkgVer>
     <OpenTracingPkgVer>[0.12.1,0.13)</OpenTracingPkgVer>
     <OTelPreviousStableVer>1.3.2</OTelPreviousStableVer>
+    <PersistentStoragePkgVer>1.0.0-beta.1</PersistentStoragePkgVer>
     <SerilogPkgVer>[2.8.0,3.0)</SerilogPkgVer>
     <StyleCopAnalyzersPkgVer>[1.2.0-beta.435,2.0)</StyleCopAnalyzersPkgVer>
     <SystemCollectionsImmutablePkgVer>1.4.0</SystemCollectionsImmutablePkgVer>

src/OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage/OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <!-- OmniSharp/VS Code requires TargetFrameworks to be in descending order for IntelliSense and analysis. -->
+    <TargetFrameworks>net6.0;netstandard2.1;netstandard2.0;net462</TargetFrameworks>
+    <Description>OpenTelemetry protocol exporter persistent storage for OpenTelemetry .NET</Description>
+    <PackageTags>$(PackageTags);OTLP</PackageTags>
+    <MinVerTagPrefix>core-</MinVerTagPrefix>
+  </PropertyGroup>
+
+  <!--Do not run ApiCompat as this is newly added. Remove this property once we have released a stable version.-->
+  <PropertyGroup>
+    <RunApiCompat>false</RunApiCompat>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.OpenTelemetryProtocol\OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj" />
+  </ItemGroup>
+
+</Project>

src/OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage/OtlpTraceExporterPersistentStorageExtensions.cs

@@ -0,0 +1,95 @@
+// <copyright file="OtlpTraceExporterPersistentStorageExtensions.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using Microsoft.Extensions.DependencyInjection;
+using OpenTelemetry.Exporter;
+using OpenTelemetry.Extensions.PersistentStorage.Abstractions;
+using OpenTelemetry.Internal;
+
+namespace OpenTelemetry.Trace;
+
+/// <summary>
+/// Extension methods to simplify registering of the OpenTelemetry Protocol (OTLP) exporter
+/// with persistent storage.
+/// </summary>
+public static class OtlpTraceExporterPersistentStorageExtensions
+{
+    /// <summary>
+    /// Adds OpenTelemetry Protocol (OTLP) exporter to the TracerProvider
+    /// with access to persistent storage.
+    /// </summary>
+    /// <param name="builder"><see cref="TracerProviderBuilder"/> builder to use.</param>
+    /// <param name="name">Name which is used when retrieving options.</param>
+    /// <param name="configure">Callback action for configuring <see cref="OtlpExporterOptions"/>.</param>
+    /// <param name="persistentStorageFactory">Factory function to create a <see cref="PersistentBlobProvider"/>.</param>
+    /// <returns>The instance of <see cref="TracerProviderBuilder"/> to chain the calls.</returns>
+    public static TracerProviderBuilder AddOtlpExporterWithPersistentStorage(
+        this TracerProviderBuilder builder,
+        string? name,
+        Action<OtlpExporterOptions> configure,
+        Func<IServiceProvider, PersistentBlobProvider> persistentStorageFactory)
+    {
+        Guard.ThrowIfNull(persistentStorageFactory);
+
+        Action<OtlpExporterOptions, IServiceProvider>? inlineConfigurationAction;
+        if (name is not null)
+        {
+            // Note: If we are using named options we can safely use Options API
+            // to act on the instance that will be created. See:
+            // https://github.com/open-telemetry/opentelemetry-dotnet/issues/4043
+            builder.ConfigureServices(services =>
+            {
+                services
+                      .AddOptions<OtlpExporterOptions>(name)
+                      .Configure<IServiceProvider>(ConfigureOptionsAction);
+            });
+
+            inlineConfigurationAction = null;
+        }
+        else
+        {
+            // Note: If we are NOT using named options we need to execute inline
+            // to prevent potentially impacting options for other signals.
+            inlineConfigurationAction = ConfigureOptionsAction;
+        }
+
+        return builder.AddOtlpExporter(
+            name,
+            configure,
+            inlineConfigurationAction);
+
+        void ConfigureOptionsAction(OtlpExporterOptions options, IServiceProvider serviceProvider)
+        {
+            options.PersistentBlobProvider = persistentStorageFactory(serviceProvider);
+        }
+    }
+
+    /// <summary>
+    /// Adds OpenTelemetry Protocol (OTLP) exporter to the TracerProvider
+    /// with access to persistent storage.
+    /// </summary>
+    /// <param name="builder"><see cref="TracerProviderBuilder"/> builder to use.</param>
+    /// <param name="configure">Callback action for configuring <see cref="OtlpExporterOptions"/>.</param>
+    /// <param name="persistentStorageFactory">Factory function to create a <see cref="PersistentBlobProvider"/>.</param>
+    /// <returns>The instance of <see cref="TracerProviderBuilder"/> to chain the calls.</returns>
+    public static TracerProviderBuilder AddOtlpExporterWithPersistentStorage(
+        this TracerProviderBuilder builder,
+        Action<OtlpExporterOptions> configure,
+        Func<IServiceProvider, PersistentBlobProvider> persistentStorageFactory)
+    {
+        return builder.AddOtlpExporterWithPersistentStorage(name: null, configure, persistentStorageFactory);
+    }
+}

src/OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage/README.md

@@ -0,0 +1,14 @@
+# OTLP Exporter Persistent Storage for OpenTelemetry .NET
+
+## Usage
+
+```csharp
+var storagePath = Path.GetTempPath();
+
+using var tracerProvider = Sdk.CreateTracerProviderBuilder()
+    .AddSource("ActivitySourceName")
+    .AddOtlpExporterWithPersistentStorage(
+        opt => {},
+        serviceProvider => new FileBlobProvider(storagePath))
+    .Build();
+```

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/AssemblyInfo.cs

@@ -21,13 +21,17 @@
 [assembly: InternalsVisibleTo("Benchmarks, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")]
 [assembly: InternalsVisibleTo("MockOpenTelemetryCollector, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")]
 [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")]
+[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")]
+[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")]
 
 // Used by Moq.
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
 #else
 [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests")]
 [assembly: InternalsVisibleTo("Benchmarks")]
 [assembly: InternalsVisibleTo("MockOpenTelemetryCollector")]
+[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage")]
+[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.Tests")]
 
 // Used by Moq.
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpGrpcExportClient.cs

@@ -15,6 +15,7 @@
 // </copyright>
 
 using Grpc.Core;
+using OpenTelemetry.Extensions.PersistentStorage.Abstractions;
 using OpenTelemetry.Internal;
 #if NETSTANDARD2_1 || NET6_0_OR_GREATER
 using Grpc.Net.Client;
@@ -36,6 +37,7 @@ protected BaseOtlpGrpcExportClient(OtlpExporterOptions options)
             this.Endpoint = new UriBuilder(options.Endpoint).Uri;
             this.Headers = options.GetMetadataFromHeaders();
             this.TimeoutMilliseconds = options.TimeoutMilliseconds;
+            this.PersistentBlobProvider = options.PersistentBlobProvider;
         }
 
 #if NETSTANDARD2_1 || NET6_0_OR_GREATER
@@ -50,6 +52,8 @@ protected BaseOtlpGrpcExportClient(OtlpExporterOptions options)
 
         internal int TimeoutMilliseconds { get; }
 
+        internal PersistentBlobProvider PersistentBlobProvider { get; }
+
         /// <inheritdoc/>
         public abstract bool SendExportRequest(TRequest request, CancellationToken cancellationToken = default);
 

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcTraceExportClient.cs

@@ -14,6 +14,7 @@
 // limitations under the License.
 // </copyright>
 
+using Google.Protobuf;
 using Grpc.Core;
 using OtlpCollector = OpenTelemetry.Proto.Collector.Trace.V1;
 
@@ -49,6 +50,11 @@ public override bool SendExportRequest(OtlpCollector.ExportTraceServiceRequest r
             }
             catch (RpcException ex)
             {
+                if (this.PersistentBlobProvider != null)
+                {
+                    this.PersistentBlobProvider.TryCreateBlob(request.ToByteArray(), out _);
+                }
+
                 OpenTelemetryProtocolExporterEventSource.Log.FailedToReachCollector(this.Endpoint, ex);
 
                 return false;

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj

@@ -19,6 +19,7 @@
     <PackageReference Include="Grpc" Version="$(GrpcPkgVer)" Condition="'$(TargetFramework)' == 'netstandard2.0' OR '$(TargetFramework)' == 'net462'" />
     <PackageReference Include="Google.Protobuf" Version="$(GoogleProtobufPkgVer)" />
     <PackageReference Include="Grpc.Tools" Version="$(GrpcToolsPkgVer)" PrivateAssets="all" />
+    <PackageReference Include="OpenTelemetry.Extensions.PersistentStorage.Abstractions" Version="$(PersistentStoragePkgVer)" />
     <PackageReference Include="System.Reflection.Emit.Lightweight" Version="$(SystemReflectionEmitLightweightPkgVer)" Condition="'$(TargetFramework)' != 'net6.0'" />
   </ItemGroup>
 

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptions.cs

@@ -22,6 +22,7 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
+using OpenTelemetry.Extensions.PersistentStorage.Abstractions;
 using OpenTelemetry.Internal;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Trace;
@@ -195,6 +196,12 @@ public Uri Endpoint
         /// </summary>
         internal bool ProgrammaticallyModifiedEndpoint { get; private set; }
 
+        /// <summary>
+        /// Gets or sets the <see cref="PersistentBlobProvider"/> used at
+        /// runtime to save data from retryable errors to offline storage.
+        /// </summary>
+        internal PersistentBlobProvider PersistentBlobProvider { get; set; }
+
         internal static void RegisterOtlpExporterOptionsFactory(IServiceCollection services)
         {
             services.RegisterOptionsFactory(

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpTraceExporterHelperExtensions.cs

@@ -34,7 +34,7 @@ public static class OtlpTraceExporterHelperExtensions
         /// <param name="builder"><see cref="TracerProviderBuilder"/> builder to use.</param>
         /// <returns>The instance of <see cref="TracerProviderBuilder"/> to chain the calls.</returns>
         public static TracerProviderBuilder AddOtlpExporter(this TracerProviderBuilder builder)
-            => AddOtlpExporter(builder, name: null, configure: null);
+            => AddOtlpExporter(builder, name: null, configure: null, inlineConfigurationAction: null);
 
         /// <summary>
         /// Adds OpenTelemetry Protocol (OTLP) exporter to the TracerProvider.
@@ -43,7 +43,7 @@ public static TracerProviderBuilder AddOtlpExporter(this TracerProviderBuilder b
         /// <param name="configure">Callback action for configuring <see cref="OtlpExporterOptions"/>.</param>
         /// <returns>The instance of <see cref="TracerProviderBuilder"/> to chain the calls.</returns>
         public static TracerProviderBuilder AddOtlpExporter(this TracerProviderBuilder builder, Action<OtlpExporterOptions> configure)
-            => AddOtlpExporter(builder, name: null, configure);
+            => AddOtlpExporter(builder, name: null, configure, inlineConfigurationAction: null);
 
         /// <summary>
         /// Adds OpenTelemetry Protocol (OTLP) exporter to the TracerProvider.
@@ -56,6 +56,13 @@ public static TracerProviderBuilder AddOtlpExporter(
             this TracerProviderBuilder builder,
             string name,
             Action<OtlpExporterOptions> configure)
+            => AddOtlpExporter(builder, name, configure, inlineConfigurationAction: null);
+
+        internal static TracerProviderBuilder AddOtlpExporter(
+            this TracerProviderBuilder builder,
+            string name,
+            Action<OtlpExporterOptions> configure,
+            Action<OtlpExporterOptions, IServiceProvider> inlineConfigurationAction)
         {
             Guard.ThrowIfNull(builder);
 
@@ -97,6 +104,8 @@ public static TracerProviderBuilder AddOtlpExporter(
                     exporterOptions = sp.GetRequiredService<IOptionsMonitor<OtlpExporterOptions>>().Get(finalOptionsName);
                 }
 
+                inlineConfigurationAction?.Invoke(exporterOptions, sp);
+
                 // Note: Not using finalOptionsName here for SdkLimitOptions.
                 // There should only be one provider for a given service
                 // collection so SdkLimitOptions is treated as a single default

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.Tests/OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.Tests.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <Nullable>disable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="xunit" Version="$(XUnitPkgVer)" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="$(XUnitRunnerVisualStudioPkgVer)" />
+    <PackageReference Include="Grpc.AspNetCore.Server" Version="$(GrpcAspNetCoreServerPkgVer)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage\OpenTelemetry.Exporter.OpenTelemetryProtocol.PersistentStorage.csproj" />
+  </ItemGroup>
+
+</Project>