Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Retries to gRPC version of OTLP Exporter #3883

Closed
wants to merge 15 commits into from
Closed

Add Retries to gRPC version of OTLP Exporter #3883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
11c692a
Add retries to gRPC OTLP Exporter
mic-max Nov 9, 2022
c68b911
formatting
mic-max Nov 9, 2022
7930415
Add publicApi for net6.0
mic-max Nov 9, 2022
70e5f2a
Update CHANGELOG.md
mic-max Nov 9, 2022
b27938a
Update OtlpExporterOptionsExtensions.cs
mic-max Nov 9, 2022
1b3e3ed
Remove final OK status
mic-max Nov 9, 2022
6dc5fd8
Update CHANGELOG.md
mic-max Nov 9, 2022
cef6d45
Update PublicAPI.Unshipped.txt
mic-max Nov 9, 2022
35ffea6
Hide options
mic-max Nov 16, 2022
611e3ec
Remove publicApi changes
mic-max Nov 16, 2022
9a487ab
ci rerun
mic-max Nov 17, 2022
86aa870
Merge branch 'main' into otlp-persist
vishweshbankwar Nov 17, 2022
daff9b0
Fix tests
mic-max Nov 17, 2022
7456397
Merge branch 'otlp-persist' of https://github.com/mic-max/opentelemet…
mic-max Nov 17, 2022
a5fac98
ci rerun
mic-max Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ Released 2022-Nov-07
scopes.
([3843](https://github.com/open-telemetry/opentelemetry-dotnet/pull/3843))

* Add ability to retry failed exports to the gRPC version of the exporter.
([#3883](https://github.com/open-telemetry/opentelemetry-dotnet/pull/3883))

## 1.4.0-beta.2

Released 2022-Oct-17
Expand Down
39 changes: 38 additions & 1 deletion src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
#if NETSTANDARD2_1 || NET6_0_OR_GREATER
using Grpc.Net.Client;
using Grpc.Net.Client.Configuration;
#endif
using LogOtlpCollector = OpenTelemetry.Proto.Collector.Logs.V1;
using MetricsOtlpCollector = OpenTelemetry.Proto.Collector.Metrics.V1;
Expand All @@ -30,6 +31,11 @@ namespace OpenTelemetry.Exporter
{
internal static class OtlpExporterOptionsExtensions
{
private const int RetryMaxAttempts = 5;
private const double RetryBackoffMultiplier = 1.5;
private static readonly TimeSpan RetryInitialBackoff = TimeSpan.FromSeconds(1);
private static readonly TimeSpan RetryMaxBackoff = TimeSpan.FromSeconds(5);

#if NETSTANDARD2_1 || NET6_0_OR_GREATER
public static GrpcChannel CreateChannel(this OtlpExporterOptions options)
#else
Expand All @@ -42,7 +48,38 @@ public static Channel CreateChannel(this OtlpExporterOptions options)
}

#if NETSTANDARD2_1 || NET6_0_OR_GREATER
return GrpcChannel.ForAddress(options.Endpoint);
var retryPolicy = new RetryPolicy
{
MaxAttempts = RetryMaxAttempts,
InitialBackoff = RetryInitialBackoff,
MaxBackoff = RetryMaxBackoff,
BackoffMultiplier = RetryBackoffMultiplier,

// See table showing gRPC codes and if they are retryable:
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#failures
RetryableStatusCodes =
{
StatusCode.Cancelled,
StatusCode.DeadlineExceeded,
StatusCode.ResourceExhausted, // TODO: Investigate this one.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks like we need to follow https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc-throttling for this one.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My initial assumption is that the gRPC client handles this. Can probably validate the behavior conforms with the specs requirements with the mock server.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks like it does https://github.com/grpc/grpc-dotnet/blob/23047e0d498501dc95bb58b010ed3c11f8cb3cf8/src/Grpc.Net.Client/Internal/Retry/RetryCall.cs#L190
if I am looking at right place 😄

StatusCode.Aborted,
StatusCode.OutOfRange,
StatusCode.Unavailable,
StatusCode.DataLoss,
},
};

var methodConfig = new MethodConfig
{
Names = { MethodName.Default },
RetryPolicy = retryPolicy,
};

var serviceConfig = new ServiceConfig { MethodConfigs = { methodConfig } };

var channelOptions = new GrpcChannelOptions { ServiceConfig = serviceConfig };

return GrpcChannel.ForAddress(options.Endpoint, channelOptions);
#else
ChannelCredentials channelCredentials;
if (options.Endpoint.Scheme == Uri.UriSchemeHttps)
Expand Down
92 changes: 91 additions & 1 deletion test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/MockCollectorIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public async Task TestRecoveryAfterFailedExport()

var httpClient = new HttpClient() { BaseAddress = new System.Uri("http://localhost:5050") };

var codes = new[] { Grpc.Core.StatusCode.Unimplemented, Grpc.Core.StatusCode.OK };
var codes = new[] { Grpc.Core.StatusCode.Unimplemented };
await httpClient.GetAsync($"/MockCollector/SetResponseCodes/{string.Join(",", codes.Select(x => (int)x))}");

var exportResults = new List<ExportResult>();
Expand Down Expand Up @@ -108,6 +108,96 @@ public async Task TestRecoveryAfterFailedExport()
await host.StopAsync().ConfigureAwait(false);
}

[Theory]
[InlineData(ExportResult.Success, new Grpc.Core.StatusCode[] { })]
[InlineData(ExportResult.Success, new[] { Grpc.Core.StatusCode.Cancelled })]
[InlineData(ExportResult.Success, new[] { Grpc.Core.StatusCode.DeadlineExceeded })]
[InlineData(ExportResult.Success, new[] { Grpc.Core.StatusCode.ResourceExhausted })]
[InlineData(ExportResult.Success, new[] { Grpc.Core.StatusCode.Aborted })]
[InlineData(ExportResult.Success, new[] { Grpc.Core.StatusCode.OutOfRange })]
[InlineData(ExportResult.Success, new[] { Grpc.Core.StatusCode.Unavailable })]
[InlineData(ExportResult.Success, new[] { Grpc.Core.StatusCode.DataLoss })]

[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.Cancelled, Grpc.Core.StatusCode.Cancelled, Grpc.Core.StatusCode.Cancelled, Grpc.Core.StatusCode.Cancelled, Grpc.Core.StatusCode.Cancelled })]

[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.Unknown })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.InvalidArgument })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.NotFound })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.AlreadyExists })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.PermissionDenied })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.FailedPrecondition })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.Unimplemented })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.Internal })]
[InlineData(ExportResult.Failure, new[] { Grpc.Core.StatusCode.Unauthenticated })]

public async Task TestOtlpExporterRetry(ExportResult result, Grpc.Core.StatusCode[] codes)
{
using var host = await new HostBuilder()
.ConfigureWebHostDefaults(webBuilder => webBuilder
.ConfigureKestrel(options =>
{
options.ListenLocalhost(5050, listenOptions => listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http1);
options.ListenLocalhost(4317, listenOptions => listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http2);
})
.ConfigureServices(services =>
{
services.AddSingleton(new MockCollectorState());
services.AddGrpc();
})
.Configure(app =>
{
app.UseRouting();

app.UseEndpoints(endpoints =>
{
endpoints.MapGet(
"/MockCollector/SetResponseCodes/{responseCodesCsv}",
(MockCollectorState collectorState, string responseCodesCsv) =>
{
var codes = responseCodesCsv.Split(",").Select(x => int.Parse(x)).ToArray();
collectorState.SetStatusCodes(codes);
});

endpoints.MapGrpcService<MockTraceService>();
});
}))
.StartAsync();

var httpClient = new HttpClient() { BaseAddress = new System.Uri("http://localhost:5050") };
await httpClient.GetAsync($"/MockCollector/SetResponseCodes/{string.Join(",", codes.Select(x => (int)x))}");

var exportResults = new List<ExportResult>();
var otlpExporter = new OtlpTraceExporter(new OtlpExporterOptions()
{
Endpoint = new System.Uri("http://localhost:4317"),
});
var delegatingExporter = new DelegatingExporter<Activity>
{
OnExportFunc = (batch) =>
{
var result = otlpExporter.Export(batch);
exportResults.Add(result);
return result;
},
};

var activitySourceName = "otel.mock.collector.test";

using var tracerProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(activitySourceName)
.AddProcessor(new SimpleActivityExportProcessor(delegatingExporter))
.Build();

using var source = new ActivitySource(activitySourceName);

source.StartActivity().Stop();

Assert.Single(exportResults);
Assert.Equal(result, exportResults[0]);

await host.StopAsync().ConfigureAwait(false);
}

private class MockCollectorState
{
private Grpc.Core.StatusCode[] statusCodes = { };
Expand Down