Add TelemetrySource to ExecutionRejectedException (#2346)

Add `TelemetrySource` for use with `ExecutionRejectedException`.

src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs

@@ -95,7 +95,9 @@ public ValueTask IsolateCircuitAsync(ResilienceContext context)
 
         lock (_lock)
         {
-            SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(new IsolatedCircuitException()));
+            var exception = new IsolatedCircuitException();
+            _telemetry.SetTelemetrySource(exception);
+            SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(exception));
             OpenCircuitFor_NeedsLock(Outcome.FromResult<T>(default), TimeSpan.MaxValue, manual: true, context, out task);
             _circuitState = CircuitState.Isolated;
         }
@@ -123,7 +125,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)
     {
         EnsureNotDisposed();
 
-        Exception? exception = null;
+        BrokenCircuitException? exception = null;
         bool isHalfOpen = false;
 
         Task? task = null;
@@ -157,6 +159,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)
 
         if (exception is not null)
         {
+            _telemetry.SetTelemetrySource(exception);
             return Outcome.FromException<T>(exception);
         }
 
@@ -308,11 +311,13 @@ private void SetLastHandledOutcome_NeedsLock(Outcome<T> outcome)
     private BrokenCircuitException CreateBrokenCircuitException()
     {
         TimeSpan retryAfter = _blockedUntil - _timeProvider.GetUtcNow();
-        return _breakingException switch
+        var exception = _breakingException switch
         {
-            Exception exception => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, exception),
+            Exception ex => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, ex),
             _ => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter)
         };
+        _telemetry.SetTelemetrySource(exception);
+        return exception;
     }
 
     private void OpenCircuit_NeedsLock(Outcome<T> outcome, bool manual, ResilienceContext context, out Task? scheduledTask)

src/Polly.Core/ExecutionRejectedException.cs

@@ -2,6 +2,8 @@
 using System.Runtime.Serialization;
 #endif
 
+using Polly.Telemetry;
+
 namespace Polly;
 
 /// <summary>
@@ -49,4 +51,9 @@ protected ExecutionRejectedException(SerializationInfo info, StreamingContext co
     }
 #endif
 #pragma warning restore RS0016 // Add public types and members to the declared API
+
+    /// <summary>
+    /// Gets the source of the strategy which has thrown the exception, if known.
+    /// </summary>
+    public virtual ResilienceTelemetrySource? TelemetrySource { get; internal set; }
 }

src/Polly.Core/PublicAPI.Unshipped.txt

@@ -3,3 +3,5 @@ Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! messa
 Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! message, System.TimeSpan retryAfter, System.Exception! inner) -> void
 Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(System.TimeSpan retryAfter) -> void
 Polly.CircuitBreaker.BrokenCircuitException.RetryAfter.get -> System.TimeSpan?
+virtual Polly.ExecutionRejectedException.TelemetrySource.get -> Polly.Telemetry.ResilienceTelemetrySource?
+Polly.Telemetry.ResilienceStrategyTelemetry.SetTelemetrySource(Polly.ExecutionRejectedException! exception) -> void

src/Polly.Core/Telemetry/ResilienceStrategyTelemetry.cs

@@ -1,3 +1,5 @@
+using System.ComponentModel;
+
 namespace Polly.Telemetry;
 
 /// <summary>
@@ -21,6 +23,18 @@ internal ResilienceStrategyTelemetry(ResilienceTelemetrySource source, Telemetry
 
     internal ResilienceTelemetrySource TelemetrySource { get; }
 
+    /// <summary>
+    ///  Sets the source of the telemetry on the provided exception.
+    /// </summary>
+    /// <param name="exception">The to-be-set exception.</param>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public void SetTelemetrySource(ExecutionRejectedException exception)
+    {
+        Guard.NotNull(exception);
+
+        exception.TelemetrySource = TelemetrySource;
+    }
+
     /// <summary>
     /// Reports an event that occurred in a resilience strategy.
     /// </summary>

src/Polly.Core/Timeout/TimeoutResilienceStrategy.cs

@@ -74,6 +74,7 @@ protected internal override async ValueTask<Outcome<TResult>> ExecuteCore<TResul
                 timeout,
                 e);
 
+            _telemetry.SetTelemetrySource(timeoutException);
             return Outcome.FromException<TResult>(timeoutException.TrySetStackTrace());
         }
 

src/Polly.RateLimiting/RateLimiterRejectedException.cs

@@ -44,7 +44,8 @@ public RateLimiterRejectedException(string message)
     /// <param name="message">The message that describes the error.</param>
     /// <param name="retryAfter">The retry after value.</param>
     public RateLimiterRejectedException(string message, TimeSpan retryAfter)
-        : base(message) => RetryAfter = retryAfter;
+        : base(message)
+        => RetryAfter = retryAfter;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="RateLimiterRejectedException"/> class.
@@ -63,7 +64,8 @@ public RateLimiterRejectedException(string message, Exception inner)
     /// <param name="retryAfter">The retry after value.</param>
     /// <param name="inner">The inner exception.</param>
     public RateLimiterRejectedException(string message, TimeSpan retryAfter, Exception inner)
-        : base(message, inner) => RetryAfter = retryAfter;
+        : base(message, inner)
+        => RetryAfter = retryAfter;
 
     /// <summary>
     /// Gets the amount of time to wait before retrying again.
@@ -84,10 +86,10 @@ public RateLimiterRejectedException(string message, TimeSpan retryAfter, Excepti
     private RateLimiterRejectedException(SerializationInfo info, StreamingContext context)
         : base(info, context)
     {
-        var value = info.GetDouble("RetryAfter");
-        if (value >= 0.0)
+        var retryAfter = info.GetDouble(nameof(RetryAfter));
+        if (retryAfter >= 0.0)
         {
-            RetryAfter = TimeSpan.FromSeconds(value);
+            RetryAfter = TimeSpan.FromSeconds(retryAfter);
         }
     }
 
@@ -96,14 +98,7 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont
     {
         Guard.NotNull(info);
 
-        if (RetryAfter.HasValue)
-        {
-            info.AddValue("RetryAfter", RetryAfter.Value.TotalSeconds);
-        }
-        else
-        {
-            info.AddValue("RetryAfter", -1.0);
-        }
+        info.AddValue(nameof(RetryAfter), RetryAfter.HasValue ? RetryAfter.Value.TotalSeconds : -1.0);
 
         base.GetObjectData(info, context);
     }

src/Polly.RateLimiting/RateLimiterResilienceStrategy.cs

@@ -65,7 +65,11 @@ protected override async ValueTask<Outcome<TResult>> ExecuteCore<TResult, TState
             await OnLeaseRejected(new OnRateLimiterRejectedArguments(context, lease)).ConfigureAwait(context.ContinueOnCapturedContext);
         }
 
-        var exception = retryAfter.HasValue ? new RateLimiterRejectedException(retryAfter.Value) : new RateLimiterRejectedException();
+        var exception = retryAfter is not null
+            ? new RateLimiterRejectedException(retryAfterValue)
+            : new RateLimiterRejectedException();
+
+        _telemetry.SetTelemetrySource(exception);
 
         return Outcome.FromException<TResult>(exception.TrySetStackTrace());
     }

test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs

@@ -52,8 +52,9 @@ public async Task IsolateAsync_Ok()
         called.Should().BeTrue();
 
         var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
-        outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>()
-            .And.Subject.As<IsolatedCircuitException>().RetryAfter.Should().BeNull();
+        var exception = outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>().Subject;
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().NotBeNull();
 
         // now close it
         await controller.CloseCircuitAsync(ResilienceContextPool.Shared.Get());
@@ -119,8 +120,9 @@ public async Task OnActionPreExecute_CircuitOpenedByValue()
         using var controller = CreateController();
 
         await OpenCircuit(controller, Outcome.FromResult(99));
-        var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
-        error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
+        var exception = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
 
         GetBlockedTill(controller).Should().Be(_timeProvider.GetUtcNow() + _options.BreakDuration);
     }
@@ -149,6 +151,7 @@ await OpenCircuit(
                 stacks.Add(e.StackTrace!);
                 e.Message.Should().Be("The circuit is now open and is not allowing calls.");
                 e.RetryAfter.Should().NotBeNull();
+                e.TelemetrySource.Should().NotBeNull();
 
                 if (innerException)
                 {
@@ -206,9 +209,10 @@ public async Task OnActionPreExecute_CircuitOpenedByException()
         using var controller = CreateController();
 
         await OpenCircuit(controller, Outcome.FromException<int>(new InvalidOperationException()));
-        var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
-        error.InnerException.Should().BeOfType<InvalidOperationException>();
-        error.RetryAfter.Should().NotBeNull();
+        var exception = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
+        exception.InnerException.Should().BeOfType<InvalidOperationException>();
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
     }
 
     [Fact]
@@ -261,9 +265,11 @@ public async Task OnActionPreExecute_HalfOpen()
         // act
         await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
         var error = (await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception;
-        error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
 
         // assert
+        var exception = error.Should().BeOfType<BrokenCircuitException>().Subject;
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
         controller.CircuitState.Should().Be(CircuitState.HalfOpen);
         called.Should().BeTrue();
     }
@@ -465,7 +471,9 @@ public async Task OnActionFailureAsync_VoidResult_EnsureBreakingExceptionNotSet(
         // assert
         controller.LastException.Should().BeNull();
         var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
-        outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
+        var exception = outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().Subject;
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
     }
 
     [Fact]
@@ -501,8 +509,9 @@ public async Task Flow_Closed_HalfOpen_Open_HalfOpen_Closed()
         TimeSpan advanceTimeRejected = TimeSpan.FromMilliseconds(1);
         AdvanceTime(advanceTimeRejected);
         var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
-        outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>()
-            .And.Subject.As<BrokenCircuitException>().RetryAfter.Should().Be(_options.BreakDuration - advanceTimeRejected);
+        var exception = outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().Subject;
+        exception.RetryAfter.Should().Be(_options.BreakDuration - advanceTimeRejected);
+        exception.TelemetrySource.Should().NotBeNull();
 
         // wait and try, transition to half open
         AdvanceTime(_options.BreakDuration + _options.BreakDuration);

test/Polly.Core.Tests/Telemetry/ResilienceStrategyTelemetryTests.cs

@@ -1,4 +1,5 @@
 using Polly.Telemetry;
+using Polly.Timeout;
 
 namespace Polly.Core.Tests.Telemetry;
 
@@ -94,4 +95,25 @@ public void Report_NoListener_ShouldNotThrow()
            .Should()
            .NotThrow();
     }
+
+    [Fact]
+    public void SetTelemetrySource_Ok()
+    {
+        var sut = new ResilienceStrategyTelemetry(_source, null);
+        var exception = new TimeoutRejectedException();
+
+        sut.SetTelemetrySource(exception);
+
+        exception.TelemetrySource.Should().Be(_source);
+    }
+
+    [Fact]
+    public void SetTelemetrySource_ShouldThrow()
+    {
+        ExecutionRejectedException? exception = null;
+
+        _sut.Invoking(s => s.SetTelemetrySource(exception!))
+            .Should()
+            .Throw<ArgumentNullException>();
+    }
 }

test/Polly.Core.Tests/Timeout/TimeoutResilienceStrategyTests.cs

@@ -173,6 +173,27 @@ public async Task Execute_Timeout_EnsureStackTrace()
         }
     }
 
+    [Fact]
+    public async Task Execute_Timeout_EnsureTelemetrySource()
+    {
+        SetTimeout(TimeSpan.FromSeconds(2));
+        var sut = CreateSut();
+
+        var outcome = await sut.ExecuteOutcomeAsync(
+            async (c, _) =>
+            {
+                var delay = _timeProvider.Delay(TimeSpan.FromSeconds(4), c.CancellationToken);
+                _timeProvider.Advance(TimeSpan.FromSeconds(2));
+                await delay;
+
+                return Outcome.FromResult("dummy");
+            },
+            ResilienceContextPool.Shared.Get(),
+            "state");
+
+        outcome.Exception.Should().BeOfType<TimeoutRejectedException>().Subject.TelemetrySource.Should().NotBeNull();
+    }
+
     [Fact]
     public async Task Execute_Cancelled_EnsureNoTimeout()
     {

test/Polly.RateLimiting.Tests/RateLimiterRejectedExceptionTests.cs

@@ -4,19 +4,67 @@ namespace Polly.Core.Tests.Timeout;
 
 public class RateLimiterRejectedExceptionTests
 {
+    private readonly string _message = "dummy";
+    private readonly TimeSpan _retryAfter = TimeSpan.FromSeconds(4);
+
     [Fact]
     public void Ctor_Ok()
     {
-        var retryAfter = TimeSpan.FromSeconds(4);
+        var exception = new RateLimiterRejectedException();
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be("The operation could not be executed because it was rejected by the rate limiter.");
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_RetryAfter_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_retryAfter);
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be($"The operation could not be executed because it was rejected by the rate limiter. It can be retried after '00:00:04'.");
+        exception.RetryAfter.Should().Be(_retryAfter);
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_Message_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message);
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_Message_RetryAfter_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message, _retryAfter);
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().Be(_retryAfter);
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_Message_InnerException_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message, new InvalidOperationException());
+        exception.InnerException.Should().BeOfType<InvalidOperationException>();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().BeNull();
+    }
 
-        new RateLimiterRejectedException().Message.Should().Be("The operation could not be executed because it was rejected by the rate limiter.");
-        new RateLimiterRejectedException().RetryAfter.Should().BeNull();
-        new RateLimiterRejectedException("dummy").Message.Should().Be("dummy");
-        new RateLimiterRejectedException("dummy", new InvalidOperationException()).Message.Should().Be("dummy");
-        new RateLimiterRejectedException(retryAfter).RetryAfter.Should().Be(retryAfter);
-        new RateLimiterRejectedException(retryAfter).Message.Should().Be($"The operation could not be executed because it was rejected by the rate limiter. It can be retried after '{retryAfter}'.");
-        new RateLimiterRejectedException("dummy", retryAfter).RetryAfter.Should().Be(retryAfter);
-        new RateLimiterRejectedException("dummy", retryAfter, new InvalidOperationException()).RetryAfter.Should().Be(retryAfter);
+    [Fact]
+    public void Ctor_Message_RetryAfter_InnerException_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message, _retryAfter, new InvalidOperationException());
+        exception.InnerException.Should().BeOfType<InvalidOperationException>();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().Be(_retryAfter);
+        exception.TelemetrySource.Should().BeNull();
     }
 
 #if !NETCOREAPP