From ea789aa489176dcbbd511a8461e9aabe6226431f Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sat, 30 Mar 2024 12:06:23 -0500 Subject: [PATCH 01/47] Switching DataFlows to OpenTelemetry vs. IMetricsProvider. --- RabbitMQ.Dataflows.sln | 7 + src/HouseofCat.Dataflows/BaseDataflow.cs | 65 ++++----- .../Extensions/WorkStateExtensions.cs | 137 ++++++++++++++++++ src/HouseofCat.Dataflows/IWorkState.cs | 7 +- .../HouseofCat.Metrics.csproj | 1 + src/HouseofCat.Metrics/IMetricsProvider.cs | 57 +++++++- src/HouseofCat.Metrics/NullMetricsProvider.cs | 31 +++- .../OpenTelemetryMetricsProvider.cs | 90 +++++++++++- .../Prometheus/PrometheusMetricsProvider.cs | 29 ++++ .../Dataflows/ConsumerDataflow.cs | 120 ++++++++------- .../Dataflows/RabbitWorkState.cs | 3 + .../Extensions/MessageExtensions.cs | 2 +- src/HouseofCat.RabbitMQ/Messages/Letter.cs | 28 ++-- .../OpenTelemetry.Console.Tests.csproj | 19 +++ tests/OpenTelemetry.Console.Tests/Program.cs | 26 ++++ .../Tests/SpanTests.cs | 94 ++++++++++++ version.props | 6 +- 17 files changed, 596 insertions(+), 126 deletions(-) create mode 100644 src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs create mode 100644 tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj create mode 100644 tests/OpenTelemetry.Console.Tests/Program.cs create mode 100644 tests/OpenTelemetry.Console.Tests/Tests/SpanTests.cs diff --git a/RabbitMQ.Dataflows.sln b/RabbitMQ.Dataflows.sln index 6a292ccf..8f19ad99 100644 --- a/RabbitMQ.Dataflows.sln +++ b/RabbitMQ.Dataflows.sln @@ -86,6 +86,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "rabbitmq", "rabbitmq", "{5C guides\rabbitmq\Serialization.md = guides\rabbitmq\Serialization.md EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Console.Tests", "tests\OpenTelemetry.Console.Tests\OpenTelemetry.Console.Tests.csproj", "{077E07C3-9A35-42A0-8228-E9778F02DFCE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -136,6 +138,10 @@ Global {528C015E-58FC-44B4-BAC5-05BCFCD506C9}.Debug|Any CPU.Build.0 = Debug|Any CPU {528C015E-58FC-44B4-BAC5-05BCFCD506C9}.Release|Any CPU.ActiveCfg = Release|Any CPU {528C015E-58FC-44B4-BAC5-05BCFCD506C9}.Release|Any CPU.Build.0 = Release|Any CPU + {077E07C3-9A35-42A0-8228-E9778F02DFCE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {077E07C3-9A35-42A0-8228-E9778F02DFCE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {077E07C3-9A35-42A0-8228-E9778F02DFCE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {077E07C3-9A35-42A0-8228-E9778F02DFCE}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -160,6 +166,7 @@ Global {0930ACF5-372E-49AF-B53F-A06C3A5FA997} = {D01D59E9-75CC-4C74-A887-22B0A55C994B} {528C015E-58FC-44B4-BAC5-05BCFCD506C9} = {D01D59E9-75CC-4C74-A887-22B0A55C994B} {5C94F432-8229-4FFA-8DBF-AC1DEDE66265} = {1AB5E832-AD36-40AF-A7E9-A105A778116C} + {077E07C3-9A35-42A0-8228-E9778F02DFCE} = {D01D59E9-75CC-4C74-A887-22B0A55C994B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D7099845-267C-4232-8764-5DF25D6B2C79} diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index e34e1bbb..c4d62c55 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -1,7 +1,8 @@ using HouseofCat.Compression; +using HouseofCat.Dataflows.Extensions; using HouseofCat.Encryption; -using HouseofCat.Metrics; using HouseofCat.Serialization; +using OpenTelemetry.Trace; using System; using System.Runtime.ExceptionServices; using System.Threading.Tasks; @@ -17,7 +18,6 @@ namespace HouseofCat.Dataflows; protected ISerializationProvider _serializationProvider; protected IEncryptionProvider _encryptionProvider; protected ICompressionProvider _compressProvider; - protected IMetricsProvider _metricsProvider; protected ISourceBlock _currentBlock; public Task Completion { get; protected set; } @@ -60,20 +60,20 @@ public TransformBlock GetTransformBlock( public TransformBlock GetWrappedTransformBlock( Func action, ExecutionDataflowBlockOptions options, - string metricIdentifier, - bool metricMicroScale = false, - string metricUnit = null, - string metricDescription = null) + string metricIdentifier) { TState WrapAction(TState state) { + using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(metricIdentifier, metricMicroScale, metricUnit, metricDescription, state.MetricTags); + return action(state); } catch (Exception ex) { + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); + childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); return state; @@ -86,20 +86,19 @@ TState WrapAction(TState state) public TransformBlock GetWrappedTransformBlock( Func> action, ExecutionDataflowBlockOptions options, - string metricIdentifier, - bool metricMicroScale = false, - string metricUnit = null, - string metricDescription = null) + string metricIdentifier) { async Task WrapActionAsync(TState state) { + using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(metricIdentifier, metricMicroScale, metricUnit, metricDescription, state.MetricTags); return await action(state).ConfigureAwait(false); } catch (Exception ex) { + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); + childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); return state; @@ -112,20 +111,20 @@ async Task WrapActionAsync(TState state) public ActionBlock GetWrappedActionBlock( Action action, ExecutionDataflowBlockOptions options, - string metricIdentifier, - bool metricMicroScale = false, - string metricUnit = null, - string metricDescription = null) + string spanName) { void WrapAction(TState state) { + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(metricIdentifier, metricMicroScale, metricUnit, metricDescription, state.MetricTags); action(state); } - catch - { /* Actions are terminating block, so swallow (maybe log) */ } + catch (Exception ex) + { + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); + childSpan?.RecordException(ex); + } } return new ActionBlock(WrapAction, options); @@ -134,20 +133,20 @@ void WrapAction(TState state) public ActionBlock GetWrappedActionBlock( Func action, ExecutionDataflowBlockOptions options, - string metricIdentifier, - bool metricMicroScale = false, - string metricUnit = null, - string metricDescription = null) + string spanName) { void WrapAction(TState state) { + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(metricIdentifier, metricMicroScale, metricUnit, metricDescription, state.MetricTags); action(state); } - catch - { /* Actions are terminating block, so swallow (maybe log) */ } + catch (Exception ex) + { + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); + childSpan?.RecordException(ex); + } } return new ActionBlock(WrapAction, options); @@ -156,20 +155,20 @@ void WrapAction(TState state) public ActionBlock GetWrappedActionBlock( Func action, ExecutionDataflowBlockOptions options, - string metricIdentifier, - bool metricMicroScale = false, - string metricUnit = null, - string metricDescription = null) + string spanName) { async Task WrapActionAsync(TState state) { + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(metricIdentifier, metricMicroScale, metricUnit, metricDescription, state.MetricTags); await action(state).ConfigureAwait(false); } - catch - { /* Actions are terminating block, so swallow (maybe log) */ } + catch (Exception ex) + { + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); + childSpan?.RecordException(ex); + } } return new ActionBlock(WrapActionAsync, options); diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs new file mode 100644 index 00000000..47ee4e8a --- /dev/null +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -0,0 +1,137 @@ +using OpenTelemetry.Trace; +using System.Diagnostics; + +namespace HouseofCat.Dataflows.Extensions; + +public static class WorkStateExtensions +{ + public static void SetOpenTelemetryError(this IWorkState state, string message = null) + { + if (state is null) return; + state.SetCurrentActivityAsError(message); + + if (state.RootSpan is null) return; + state.SetCurrentSpanAsError(message); + } + + public static void SetCurrentActivityAsError(this IWorkState state, string message = null) + { + var activity = Activity.Current; + if (activity is null) return; + + activity.SetStatus(ActivityStatusCode.Error, message ?? state.EDI.SourceException?.Message); + if (state?.EDI is not null) + { + if (state.EDI.SourceException is not null) + { + activity.RecordException(state.EDI.SourceException); + } + } + else + { + activity.SetStatus(ActivityStatusCode.Error, message); + } + } + + public static void SetCurrentSpanAsError(this IWorkState state, string message = null) + { + var span = Tracer.CurrentSpan; + if (span is null) return; + + span.SetStatus(Status.Error); + span.SetAttribute("Error", message); + if (state?.EDI is not null) + { + if (state.EDI.SourceException is not null) + { + span.RecordException(state.EDI.SourceException); + } + } + } + + public static string OpenTelemetryDefaultProviderTraceSourceNameKey { get; set; } = "OpenTelemetryTraceSourceName"; + public static string OpenTelemetryDefaultProviderTracerServiceVersionKey { get; set; } = "OpenTelemetryTraceSourceVersion"; + + private static readonly string _defaultProviderTracerSourceName = "HouseofCat.Dataflows"; + + public static void SetWorkflowNameAsOpenTelemetrySourceName( + this IWorkState state, + string workflowName, + string version = null) + { + state.Data[OpenTelemetryDefaultProviderTraceSourceNameKey] = workflowName ?? _defaultProviderTracerSourceName; + state.Data[OpenTelemetryDefaultProviderTracerServiceVersionKey] = version; + } + + public static void StartRootSpan( + this IWorkState state, + string spanName, + SpanKind spanKind = SpanKind.Internal, + SpanAttributes spanAttributes = null) + { + if (state.Data.TryGetValue(OpenTelemetryDefaultProviderTraceSourceNameKey, out var workflowName)) + { + state.Data.TryGetValue(OpenTelemetryDefaultProviderTracerServiceVersionKey, out var version); + + state.RootSpan = TracerProvider + .Default + ?.GetTracer( + workflowName?.ToString() ?? _defaultProviderTracerSourceName, + version?.ToString()) + ?.StartRootSpan(spanName, spanKind, initialAttributes: spanAttributes); + } + else + { + state.RootSpan = TracerProvider + .Default + ?.GetTracer(_defaultProviderTracerSourceName, null) + ?.StartRootSpan(spanName, spanKind, initialAttributes: spanAttributes); + } + } + + public static TelemetrySpan CreateActiveSpan( + this IWorkState state, + string spanName, + SpanKind spanKind = SpanKind.Internal, + SpanAttributes spanAttributes = null) + { + if (state.Data.TryGetValue(OpenTelemetryDefaultProviderTraceSourceNameKey, out var workflowName)) + { + state.Data.TryGetValue(OpenTelemetryDefaultProviderTracerServiceVersionKey, out var version); + + return TracerProvider + .Default + ?.GetTracer( + workflowName?.ToString() ?? _defaultProviderTracerSourceName, + version?.ToString()) + ?.StartActiveSpan( + spanName, + spanKind, + parentContext: state.RootSpan?.Context ?? default, + initialAttributes: spanAttributes); + } + else + { + return TracerProvider + .Default + ?.GetTracer(_defaultProviderTracerSourceName, null) + ?.StartActiveSpan( + spanName, + spanKind, + parentContext: state.RootSpan?.Context ?? default, + initialAttributes: spanAttributes); + } + } + + public static void EndRootSpan( + this IWorkState state, + bool includeErrorWhenFaulted = false) + { + if (includeErrorWhenFaulted && state.IsFaulted) + { + state.SetOpenTelemetryError(); + } + state?.RootSpan?.End(); + state?.RootSpan?.Dispose(); + } +} diff --git a/src/HouseofCat.Dataflows/IWorkState.cs b/src/HouseofCat.Dataflows/IWorkState.cs index 23e7450d..adfdba54 100644 --- a/src/HouseofCat.Dataflows/IWorkState.cs +++ b/src/HouseofCat.Dataflows/IWorkState.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using OpenTelemetry.Trace; +using System.Collections.Generic; using System.Runtime.ExceptionServices; namespace HouseofCat.Dataflows; @@ -18,6 +19,6 @@ public interface IWorkState // Outbound byte[] SendData { get; set; } - // Metrics - IDictionary MetricTags { get; set; } + // RootSpan + TelemetrySpan RootSpan { get; set; } } diff --git a/src/HouseofCat.Metrics/HouseofCat.Metrics.csproj b/src/HouseofCat.Metrics/HouseofCat.Metrics.csproj index 2482b37d..b3a00626 100644 --- a/src/HouseofCat.Metrics/HouseofCat.Metrics.csproj +++ b/src/HouseofCat.Metrics/HouseofCat.Metrics.csproj @@ -1,6 +1,7 @@  + diff --git a/src/HouseofCat.Metrics/IMetricsProvider.cs b/src/HouseofCat.Metrics/IMetricsProvider.cs index 3f9dd436..1bfa668e 100644 --- a/src/HouseofCat.Metrics/IMetricsProvider.cs +++ b/src/HouseofCat.Metrics/IMetricsProvider.cs @@ -1,5 +1,7 @@ -using System; +using OpenTelemetry.Trace; +using System; using System.Collections.Generic; +using System.Diagnostics; namespace HouseofCat.Metrics; @@ -11,8 +13,57 @@ public interface IMetricsProvider void IncrementGauge(string name, string unit = null, string description = null); void ObserveValue(string name, double value, string unit = null, string description = null); void ObserveValueFluctuation(string name, double value, string unit = null, string description = null); + IDisposable Duration(string name, bool microScale = false, string unit = null, string description = null); + IDisposable Track(string name, string unit = null, string description = null); - IDisposable TrackAndDuration(string name, bool microScale = false, string unit = null, string description = null, IDictionary metricTags = null); - IDisposable Trace(string name, IDictionary metricTags = null); + + IDisposable TrackAndDuration( + string name, + bool microScale = false, + string unit = null, + string description = null, + ActivityKind activityKind = ActivityKind.Internal, + IDictionary metricTags = null); + + IDisposable Trace( + string name, + ActivityKind activityKind = ActivityKind.Internal, + IDictionary metricTags = null); + + /// + /// Creates a new active span. + /// + /// + /// + /// + /// + IDisposable GetSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null); + + /// + /// Attempts to create a new active child span from CURRENT active span or just a new active span if no current span. + /// + /// + /// + /// + /// + IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null); + + /// + /// Creates a new active child span from provided span context. + /// + /// + /// + /// + /// + IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + SpanContext parentSpanContext = default); } diff --git a/src/HouseofCat.Metrics/NullMetricsProvider.cs b/src/HouseofCat.Metrics/NullMetricsProvider.cs index 69764826..30addafb 100644 --- a/src/HouseofCat.Metrics/NullMetricsProvider.cs +++ b/src/HouseofCat.Metrics/NullMetricsProvider.cs @@ -1,5 +1,7 @@ -using System; +using OpenTelemetry.Trace; +using System; using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.CompilerServices; namespace HouseofCat.Metrics; @@ -48,6 +50,7 @@ public IDisposable TrackAndDuration( bool microScale = false, string unit = null, string description = null, + ActivityKind activityKind = ActivityKind.Internal, IDictionary metricTags = null) { return null; @@ -56,8 +59,34 @@ public IDisposable TrackAndDuration( [MethodImpl(MethodImplOptions.AggressiveInlining)] public IDisposable Trace( string name, + ActivityKind activityKind = ActivityKind.Internal, IDictionary metricTags = null) { return null; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public IDisposable GetSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null) + { + return null; + } + + public IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null) + { + return null; + } + + public IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + SpanContext parentSpanContext = default) + { + return null; + } } diff --git a/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs b/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs index 72670dfd..bdccb679 100644 --- a/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs +++ b/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs @@ -1,4 +1,5 @@ using HouseofCat.Utilities.Errors; +using OpenTelemetry.Trace; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -14,6 +15,7 @@ public class OpenTelemetryMetricsProvider : IMetricsProvider, IDisposable private readonly Meter _meter; private readonly string _meterName; private readonly ActivitySource _activitySource; + private readonly Tracer _tracer; public ConcurrentDictionary Counters { get; } = new ConcurrentDictionary(); public ConcurrentDictionary Gauges { get; } = new ConcurrentDictionary(); @@ -30,6 +32,7 @@ public OpenTelemetryMetricsProvider(IMeterFactory meterFactory, string meterName _meterName = meterName; _meter = _factory.Create(_meterName); + _tracer = TracerProvider.Default.GetTracer(activitySourceName); _activitySource = new ActivitySource(activitySourceName ?? "HouseofCat.Metrics", activityVersion); } @@ -143,25 +146,37 @@ public IDisposable TrackAndDuration( bool microScale = false, string unit = null, string description = null, + ActivityKind activityKind = ActivityKind.Internal, IDictionary tags = null) { - return GetActivity(name, tags); + return GetActivity(name, activityKind, tags); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public IDisposable Trace( string name, + ActivityKind activityKind = ActivityKind.Internal, IDictionary metricTags = null) { - return GetActivity(name, metricTags); + return GetActivity(name, activityKind, metricTags); } - private Activity GetActivity(string name, IDictionary metricTags) + private Activity GetActivity( + string name, + ActivityKind activityKind = ActivityKind.Internal, + IDictionary metricTags = null) { - var activity = _activitySource.StartActivity(name); - if (activity is not null - && metricTags is not null - && activity.IsAllDataRequested) + var parentActivity = Activity.Current; + + Activity activity; + if (parentActivity is not null) + { activity = _activitySource?.StartActivity(name, activityKind, parentActivity.Context); } + else + { activity = _activitySource?.StartActivity(name, activityKind); } + + if (activity is null) return null; + + if (metricTags is not null && activity.IsAllDataRequested) { foreach (var tag in metricTags) { @@ -172,5 +187,66 @@ private Activity GetActivity(string name, IDictionary metricTags return activity; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public IDisposable GetSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null) + { + return GetDefaultTelemetrySpan(name, false, spanKind, metricTags); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null) + { + return GetDefaultTelemetrySpan(name, true, spanKind, metricTags); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + SpanContext parentSpanContext = default) + { + return GetChildTelemetrySpan(name, spanKind, parentSpanContext); + } + + private TelemetrySpan GetDefaultTelemetrySpan( + string name, + bool childSpanOfCurrent = false, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null) + { + if (childSpanOfCurrent) + { + var parentSpan = Tracer.CurrentSpan; + if (parentSpan is not null) return _tracer?.StartActiveSpan(name, spanKind, parentSpan.Context); + } + + var span = _tracer?.StartActiveSpan(name); + if (span is null) return null; + + if (metricTags is not null) + { + foreach (var tag in metricTags) + { + span.SetAttribute(tag.Key, tag.Value); + } + } + + return span; + } + + private TelemetrySpan GetChildTelemetrySpan( + string name, + SpanKind spanKind = SpanKind.Internal, + SpanContext parentSpanContext = default) + { + return _tracer?.StartActiveSpan(name, spanKind, parentSpanContext); + } + #endregion } diff --git a/src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs b/src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs index 3aad3c95..75465dbd 100644 --- a/src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs +++ b/src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs @@ -1,11 +1,13 @@ using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; +using OpenTelemetry.Trace; using Prometheus; using Prometheus.DotNetRuntime; using Prometheus.DotNetRuntime.Metrics.Producers; using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.CompilerServices; namespace HouseofCat.Metrics; @@ -191,6 +193,7 @@ public IDisposable TrackAndDuration( bool microScale = false, string unit = null, string description = null, + ActivityKind activityKind = ActivityKind.Internal, IDictionary metricTags = null) { var duration = Duration(name, microScale, description: description); @@ -201,10 +204,36 @@ public IDisposable TrackAndDuration( [MethodImpl(MethodImplOptions.AggressiveInlining)] public IDisposable Trace( string name, + ActivityKind activityKind = ActivityKind.Internal, IDictionary metricTags = null) { return null; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public IDisposable GetSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null) + { + return null; + } + + public IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + IDictionary metricTags = null) + { + return null; + } + + public IDisposable GetChildSpan( + string name, + SpanKind spanKind = SpanKind.Internal, + SpanContext parentSpanContext = default) + { + return null; + } + #endregion } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 0e43ecf0..fc961d7d 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -1,16 +1,18 @@ using HouseofCat.Compression; using HouseofCat.Dataflows; +using HouseofCat.Dataflows.Extensions; using HouseofCat.Encryption; -using HouseofCat.Metrics; using HouseofCat.RabbitMQ.Services; using HouseofCat.Serialization; using HouseofCat.Utilities.Errors; +using OpenTelemetry.Trace; using System; using System.Collections.Generic; using System.Linq; using System.Runtime.ExceptionServices; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using static HouseofCat.Dataflows.Extensions.WorkStateExtensions; namespace HouseofCat.RabbitMQ.Dataflows; @@ -208,12 +210,6 @@ public ConsumerDataflow SetEncryptionProvider(IEncryptionProvider provid return this; } - public ConsumerDataflow SetMetricsProvider(IMetricsProvider provider) - { - _metricsProvider = provider ?? new NullMetricsProvider(); - return this; - } - #region Step Adders protected virtual ITargetBlock CreateTargetBlock( @@ -237,7 +233,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler", false); + _errorAction = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); } return this; } @@ -254,7 +250,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler", false); + _errorAction = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); } return this; } @@ -267,35 +263,31 @@ public ConsumerDataflow WithReadyToProcessBuffer(int boundedCapacity, Ta public ConsumerDataflow AddStep( Func suppliedStep, - string metricIdentifier, - bool metricMicroScale = false, - string metricDescription = null, + string spanName, int? maxDoP = null, bool? ensureOrdered = null, int? boundedCapacity = null, TaskScheduler taskScheduler = null) { Guard.AgainstNull(suppliedStep, nameof(suppliedStep)); - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps", metricDescription); + var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, metricIdentifier, metricMicroScale)); + _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, spanName)); return this; } public ConsumerDataflow AddStep( Func> suppliedStep, - string metricIdentifier, - bool metricMicroScale = false, - string metricDescription = null, + string spanName, int? maxDoP = null, bool? ensureOrdered = null, int? boundedCapacity = null, TaskScheduler taskScheduler = null) { Guard.AgainstNull(suppliedStep, nameof(suppliedStep)); - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps", metricDescription); + var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, metricIdentifier, metricMicroScale)); + _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, spanName)); return this; } @@ -316,9 +308,8 @@ public ConsumerDataflow WithFinalization( Guard.AgainstNull(action, nameof(action)); if (_finalization == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization", true); + _finalization = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); } return this; } @@ -333,9 +324,8 @@ public ConsumerDataflow WithFinalization( Guard.AgainstNull(action, nameof(action)); if (_finalization == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization", true); + _finalization = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); } return this; } @@ -347,11 +337,12 @@ public ConsumerDataflow WithBuildState( int? boundedCapacity = null, TaskScheduler taskScheduler = null) { + Guard.AgainstNullOrEmpty(stateKey, nameof(stateKey)); + if (_buildStateBlock == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _buildStateBlock = GetBuildStateBlock(_serializationProvider, stateKey, executionOptions); + _buildStateBlock = GetBuildStateBlock(stateKey, executionOptions); } return this; } @@ -365,7 +356,6 @@ public ConsumerDataflow WithDecryptionStep( Guard.AgainstNull(_encryptionProvider, nameof(_encryptionProvider)); if (_decryptBlock == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); _decryptBlock = GetByteManipulationTransformBlock( @@ -373,8 +363,7 @@ public ConsumerDataflow WithDecryptionStep( executionOptions, false, x => x.ReceivedData.Encrypted, - $"{WorkflowName}_Decrypt", - true); + $"{WorkflowName}_Decrypt"); } return this; } @@ -388,7 +377,6 @@ public ConsumerDataflow WithDecompressionStep( Guard.AgainstNull(_compressProvider, nameof(_compressProvider)); if (_decompressBlock == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); _decompressBlock = GetByteManipulationTransformBlock( @@ -396,8 +384,7 @@ public ConsumerDataflow WithDecompressionStep( executionOptions, false, x => x.ReceivedData.Compressed, - $"{WorkflowName}_Decompress", - true); + $"{WorkflowName}_Decompress"); } return this; @@ -412,7 +399,6 @@ public ConsumerDataflow WithCreateSendLetter( { if (_createSendLetter == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); _createSendLetter = GetWrappedTransformBlock(createLetter, executionOptions, $"{WorkflowName}_CreateSendLetter"); } @@ -428,7 +414,6 @@ public ConsumerDataflow WithCompression( Guard.AgainstNull(_compressProvider, nameof(_compressProvider)); if (_compressBlock == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); _compressBlock = GetByteManipulationTransformBlock( @@ -436,8 +421,7 @@ public ConsumerDataflow WithCompression( executionOptions, true, x => !x.ReceivedData.Compressed, - $"{WorkflowName}_Compress", - true); + $"{WorkflowName}_Compress"); } return this; } @@ -451,7 +435,6 @@ public ConsumerDataflow WithEncryption( Guard.AgainstNull(_encryptionProvider, nameof(_encryptionProvider)); if (_encryptBlock == null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); _encryptBlock = GetByteManipulationTransformBlock( @@ -459,8 +442,7 @@ public ConsumerDataflow WithEncryption( executionOptions, true, x => !x.ReceivedData.Encrypted, - $"{WorkflowName}_Encrypt", - true); + $"{WorkflowName}_Encrypt"); } return this; } @@ -471,7 +453,6 @@ public ConsumerDataflow WithSendStep( int? boundedCapacity = null, TaskScheduler taskScheduler = null) { - _metricsProvider.IncrementCounter($"{WorkflowName}_Steps"); if (_sendLetterBlock == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); @@ -601,8 +582,9 @@ private void LinkWithFaultRoute(ISourceBlock source, IPropagatorBlock $"{WorkflowName}_StateBuild"; - public virtual TState BuildState(ISerializationProvider provider, string key, ReceivedData data) + private static string RootSpanIdentifier => "{0}_RootSpan_{1}"; + + public virtual TState BuildState(string key, ReceivedData data) { var state = new TState { @@ -611,26 +593,44 @@ public virtual TState BuildState(ISerializationProvider provider, string k }; // If the SerializationProvider was assigned, use it, else it's raw bytes. - if (provider != null) - { state.Data[key] = provider.Deserialize(data.Data); } + if (_serializationProvider != null) + { state.Data[key] = _serializationProvider.Deserialize(data.Data); } else { state.Data[key] = data.Data; } + state.SetWorkflowNameAsOpenTelemetrySourceName(WorkflowName); + + var spanAttributes = new SpanAttributes(); + spanAttributes.Add(nameof(WorkflowName), WorkflowName); + spanAttributes.Add(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName); + if (state.ReceivedData?.Letter?.MessageId is not null) + { + spanAttributes.Add(nameof(state.ReceivedData.Letter.MessageId), state.ReceivedData.Letter.MessageId); + } + if (state.ReceivedData?.Letter?.Metadata?.Id is not null) + { + spanAttributes.Add(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id); + } + + state.StartRootSpan( + string.Format( + RootSpanIdentifier, + WorkflowName, + state.ReceivedData?.Letter?.MessageId ?? Guid.NewGuid().ToString()), + SpanKind.Consumer, + spanAttributes); + return state; } public TransformBlock GetBuildStateBlock( - ISerializationProvider provider, string key, ExecutionDataflowBlockOptions options) { TState BuildStateWrap(ReceivedData data) { try - { - using var multiDispose = _metricsProvider.TrackAndDuration(StateIdentifier, true); - return BuildState(provider, key, data); - } + { return BuildState(key, data); } catch { return null; } } @@ -643,17 +643,13 @@ public TransformBlock GetByteManipulationTransformBlock( ExecutionDataflowBlockOptions options, bool outbound, Predicate predicate, - string metricIdentifier, - bool metricMicroScale = false, - string metricUnit = null, - string metricDescription = null) + string metricIdentifier) { TState WrapAction(TState state) { + using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(metricIdentifier, metricMicroScale, metricUnit, metricDescription, state.MetricTags); - if (outbound) { if (state.SendData?.Length > 0) @@ -678,6 +674,8 @@ TState WrapAction(TState state) } catch (Exception ex) { + childSpan?.SetStatus(Status.Error); + childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); return state; @@ -692,16 +690,13 @@ public TransformBlock GetByteManipulationTransformBlock( ExecutionDataflowBlockOptions options, bool outbound, Predicate predicate, - string metricIdentifier, - bool metricMicroScale = false, - string metricUnit = null, - string metricDescription = null) + string metricIdentifier) { async Task WrapActionAsync(TState state) { + using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(metricIdentifier, metricMicroScale, metricUnit, metricDescription, state.MetricTags); if (outbound) { @@ -726,6 +721,8 @@ async Task WrapActionAsync(TState state) } catch (Exception ex) { + childSpan?.SetStatus(Status.Error); + childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); return state; @@ -742,10 +739,9 @@ public TransformBlock GetWrappedPublishTransformBlock( { async Task WrapPublishAsync(TState state) { + using var childSpan = state.CreateActiveSpan(PublishStepIdentifier, SpanKind.Producer); try { - using var multiDispose = _metricsProvider.TrackAndDuration(PublishStepIdentifier, true, metricTags: state.MetricTags); - await service.Publisher.PublishAsync(state.SendMessage, true, true).ConfigureAwait(false); state.SendMessageSent = true; @@ -753,6 +749,8 @@ async Task WrapPublishAsync(TState state) } catch (Exception ex) { + childSpan?.SetStatus(Status.Error); + childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); return state; diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index 1c256409..52299935 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -1,4 +1,5 @@ using HouseofCat.Dataflows; +using OpenTelemetry.Trace; using System.Collections.Generic; using System.Runtime.ExceptionServices; using System.Runtime.Serialization; @@ -30,4 +31,6 @@ public abstract class RabbitWorkState : IRabbitWorkState public ExceptionDispatchInfo EDI { get; set; } public IDictionary MetricTags { get; set; } + + public TelemetrySpan RootSpan { get; set; } } diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 9fcb04f9..eb83783f 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -83,7 +83,7 @@ public static IMessage CreateSimpleRandomLetter(string queueName, int bodySize = return new Letter { MessageId = Guid.NewGuid().ToString(), - LetterMetadata = new LetterMetadata(), + Metadata = new LetterMetadata(), Envelope = new Envelope { Exchange = string.Empty, diff --git a/src/HouseofCat.RabbitMQ/Messages/Letter.cs b/src/HouseofCat.RabbitMQ/Messages/Letter.cs index 16d25931..fde51ca1 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Letter.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Letter.cs @@ -32,14 +32,14 @@ public class Letter : IMessage public Envelope Envelope { get; set; } public string MessageId { get; set; } - public LetterMetadata LetterMetadata { get; set; } + public IMetadata Metadata { get; set; } public ReadOnlyMemory Body { get; set; } public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders) { MessageId ??= Guid.NewGuid().ToString(); - var props = this.CreateBasicProperties(channelHost, withOptionalHeaders, LetterMetadata); + var props = this.CreateBasicProperties(channelHost, withOptionalHeaders, Metadata); props.MessageId = MessageId; // Non-optional Header. @@ -59,7 +59,7 @@ public Letter(string exchange, string routingKey, ReadOnlyMemory data, Let RoutingOptions = routingOptions ?? RoutingOptions.CreateDefaultRoutingOptions() }; Body = data; - LetterMetadata = metadata ?? new LetterMetadata(); + Metadata = metadata ?? new LetterMetadata(); } public Letter(string exchange, string routingKey, ReadOnlyMemory data, string id, RoutingOptions routingOptions = null) @@ -72,9 +72,9 @@ public Letter(string exchange, string routingKey, ReadOnlyMemory data, str }; Body = data; if (!string.IsNullOrWhiteSpace(id)) - { LetterMetadata = new LetterMetadata { Id = id }; } + { Metadata = new LetterMetadata { Id = id }; } else - { LetterMetadata = new LetterMetadata(); } + { Metadata = new LetterMetadata(); } } public Letter(string exchange, string routingKey, byte[] data, string id, byte priority) @@ -87,29 +87,29 @@ public Letter(string exchange, string routingKey, byte[] data, string id, byte p }; Body = data; if (!string.IsNullOrWhiteSpace(id)) - { LetterMetadata = new LetterMetadata { Id = id }; } + { Metadata = new LetterMetadata { Id = id }; } else - { LetterMetadata = new LetterMetadata(); } + { Metadata = new LetterMetadata(); } } public Letter Clone() { var clone = this.Clone(); - clone.LetterMetadata = LetterMetadata.Clone(); + clone.Metadata = Metadata.Clone(); return clone; } - public IMetadata GetMetadata() => LetterMetadata; + public IMetadata GetMetadata() => Metadata; public IMetadata CreateMetadataIfMissing() { - LetterMetadata ??= new LetterMetadata(); - return LetterMetadata; + Metadata ??= new LetterMetadata(); + return Metadata; } - public T GetHeader(string key) => LetterMetadata.GetHeader(key); - public bool RemoveHeader(string key) => LetterMetadata.RemoveHeader(key); - public IDictionary GetHeadersOutOfMetadata() => LetterMetadata.GetHeadersOutOfMetadata(); + public T GetHeader(string key) => Metadata.GetHeader(key); + public bool RemoveHeader(string key) => Metadata.RemoveHeader(key); + public IDictionary GetHeadersOutOfMetadata() => Metadata.GetHeadersOutOfMetadata(); public ReadOnlyMemory GetBodyToPublish(ISerializationProvider serializationProvider) => serializationProvider.Serialize(this).ToArray(); diff --git a/tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj b/tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj new file mode 100644 index 00000000..f9523c37 --- /dev/null +++ b/tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + enable + + + + + + + + + + + + + diff --git a/tests/OpenTelemetry.Console.Tests/Program.cs b/tests/OpenTelemetry.Console.Tests/Program.cs new file mode 100644 index 00000000..e31acda3 --- /dev/null +++ b/tests/OpenTelemetry.Console.Tests/Program.cs @@ -0,0 +1,26 @@ +using HouseofCat.Utilities; +using Microsoft.Extensions.Logging; +using OpenTelemetry; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using OpenTelmetry.Tests; + +var loggerFactory = LogHelper.CreateConsoleLoggerFactory(LogLevel.Information); +LogHelper.LoggerFactory = loggerFactory; +var logger = loggerFactory.CreateLogger(); + +var applicationName = "OpenTelemetry.ConsoleTests"; +var workflowName = "MyWorkflowName"; + +using var traceProvider = Sdk + .CreateTracerProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(applicationName)) + .AddSource(workflowName) + .AddConsoleExporter() + .Build(); + +SpanTests.RunRootSpanWithChildSpanErrorTest(logger, workflowName); + +logger.LogInformation("Tests complete! Press return to exit...."); + +Console.ReadLine(); diff --git a/tests/OpenTelemetry.Console.Tests/Tests/SpanTests.cs b/tests/OpenTelemetry.Console.Tests/Tests/SpanTests.cs new file mode 100644 index 00000000..8e9aee6b --- /dev/null +++ b/tests/OpenTelemetry.Console.Tests/Tests/SpanTests.cs @@ -0,0 +1,94 @@ +using HouseofCat.RabbitMQ.Dataflows; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Trace; +using static HouseofCat.Dataflows.Extensions.WorkStateExtensions; + +namespace OpenTelmetry.Tests; + +public static class SpanTests +{ + public class CustomWorkState : RabbitWorkState + { + public CustomWorkState() + { + Data = new Dictionary(); + } + } + + public static void RunRootSpanTest(ILogger logger, string sourceName) + { + logger.LogInformation($"Starting {nameof(RunRootSpanTest)}..."); + + var workstate = new CustomWorkState(); + + workstate.SetWorkflowNameAsOpenTelemetrySourceName(sourceName, "v1.0.0"); + workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); + workstate.EndRootSpan(); + + logger.LogInformation($"Finished {nameof(RunRootSpanTest)}."); + } + + public static void RunRootSpanWithChildSpanTest(ILogger logger, string workflowName) + { + logger.LogInformation($"Starting {RunRootSpanWithChildSpanTest}..."); + + var workstate = new CustomWorkState(); + + workstate.SetWorkflowNameAsOpenTelemetrySourceName(workflowName, "v1.0.0"); + workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); + using (var span = workstate.CreateActiveSpan("RunBasicSpanTest.Child", SpanKind.Internal)) + { + span.SetStatus(Status.Ok); + } + workstate.EndRootSpan(); + + logger.LogInformation($"Finished {nameof(RunRootSpanWithChildSpanTest)}."); + } + + public static void RunRootSpanWithChildSpanErrorTest(ILogger logger, string workflowName) + { + logger.LogInformation($"Starting {RunRootSpanWithChildSpanTest}..."); + + var workstate = new CustomWorkState(); + + workstate.SetWorkflowNameAsOpenTelemetrySourceName(workflowName, "v1.0.0"); + workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); + + using (var span = workstate.CreateActiveSpan("RunBasicSpanTest.Child", SpanKind.Internal)) + { + workstate.SetCurrentSpanAsError("Span had an error!"); + } + + workstate.EndRootSpan(); + + logger.LogInformation($"Finished {nameof(RunRootSpanWithChildSpanTest)}."); + } + + public static void RunRootSpanWithManyChildSpanFlatLevelTest(ILogger logger, string workflowName) + { + logger.LogInformation($"Starting {RunRootSpanWithChildSpanTest}..."); + + var workstate = new CustomWorkState(); + + workstate.SetWorkflowNameAsOpenTelemetrySourceName(workflowName, "v1.0.0"); + workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); + + for (var i = 0; i < 10; i++) + { + using var span = workstate.CreateActiveSpan($"RunBasicSpanTest.Child.{i}", SpanKind.Internal); + + if (i == 9) + { + workstate.SetCurrentSpanAsError("Span 9 had an error!"); + } + else + { + span.SetStatus(Status.Ok); + } + } + + workstate.EndRootSpan(); + + logger.LogInformation($"Finished {nameof(RunRootSpanWithChildSpanTest)}."); + } +} diff --git a/version.props b/version.props index 9cb3e1fe..42b37052 100644 --- a/version.props +++ b/version.props @@ -1,7 +1,7 @@ - 3.2.2 - 3.2.2 - 3.2.2 + 3.3.0 + 3.3.0 + 3.3.0 From 41508b676186cfd68d1095772671fd5f0a11629e Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sat, 30 Mar 2024 13:34:24 -0500 Subject: [PATCH 02/47] WorkState Extension enhancement and cleanup. --- .../Extensions/WorkStateExtensions.cs | 188 ++++++++++++------ .../OpenTelemetryMetricsProvider.cs | 6 +- .../Dataflows/ConsumerDataflow.cs | 26 +-- .../Extensions/AssemblyExtensions.cs | 43 ++++ .../Extensions/StringExtensions.cs | 72 +++++++ tests/OpenTelemetry.Console.Tests/Program.cs | 2 +- .../Tests/{SpanTests.cs => WorkStateTests.cs} | 28 ++- 7 files changed, 273 insertions(+), 92 deletions(-) create mode 100644 src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs create mode 100644 src/HouseofCat.Utilities/Extensions/StringExtensions.cs rename tests/OpenTelemetry.Console.Tests/Tests/{SpanTests.cs => WorkStateTests.cs} (59%) diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 47ee4e8a..d5cf8c86 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -1,5 +1,8 @@ -using OpenTelemetry.Trace; +using HouseofCat.Utilities.Extensions; +using OpenTelemetry.Trace; +using System.Collections.Generic; using System.Diagnostics; +using System.Reflection; namespace HouseofCat.Dataflows.Extensions; @@ -36,6 +39,11 @@ public static void SetCurrentActivityAsError(this IWorkState state, string messa public static void SetCurrentSpanAsError(this IWorkState state, string message = null) { var span = Tracer.CurrentSpan; + state.SetSpanAsError(span, message); + } + + public static void SetSpanAsError(this IWorkState state, TelemetrySpan span, string message = null) + { if (span is null) return; span.SetStatus(Status.Error); @@ -49,89 +57,155 @@ public static void SetCurrentSpanAsError(this IWorkState state, string message = } } - public static string OpenTelemetryDefaultProviderTraceSourceNameKey { get; set; } = "OpenTelemetryTraceSourceName"; - public static string OpenTelemetryDefaultProviderTracerServiceVersionKey { get; set; } = "OpenTelemetryTraceSourceVersion"; + public static string DefaultRootSpanNameFormat { get; set; } = "{0}.root"; + public static string DefaultChildSpanNameFormat { get; set; } = "{0}.{1}.child"; + public static string DefaultWorkflowNameKey { get; set; } = "hoc.workflow.name"; + public static string DefaultWorkflowVersionKey { get; set; } = "hoc.workflow.version"; - private static readonly string _defaultProviderTracerSourceName = "HouseofCat.Dataflows"; - - public static void SetWorkflowNameAsOpenTelemetrySourceName( + /// + /// Start a new RootSpan with respect to State. + /// + /// + /// + /// + /// + /// + /// + public static void StartRootSpan( this IWorkState state, string workflowName, - string version = null) + string workflowVersion = null, + SpanKind spanKind = SpanKind.Internal, + IEnumerable> suppliedAttributes = null) { - state.Data[OpenTelemetryDefaultProviderTraceSourceNameKey] = workflowName ?? _defaultProviderTracerSourceName; - state.Data[OpenTelemetryDefaultProviderTracerServiceVersionKey] = version; + if (state is null) return; + + if (string.IsNullOrWhiteSpace(workflowVersion)) + { + var assembly = Assembly.GetExecutingAssembly(); + workflowVersion = assembly.GetExecutingSemanticVersion(); + } + + var traceProvider = TracerProvider + .Default + .GetTracer( + workflowName, + workflowVersion); + + if (traceProvider is null) return; + + state.Data[DefaultWorkflowNameKey] = workflowName; + state.Data[DefaultWorkflowVersionKey] = workflowVersion; + + var rootSpanName = string.Format(DefaultRootSpanNameFormat, workflowName); + + var attributes = new SpanAttributes(); + attributes.Add(DefaultWorkflowNameKey, workflowName); + attributes.Add(DefaultWorkflowVersionKey, workflowVersion); + + if (suppliedAttributes is not null) + { + foreach (var kvp in suppliedAttributes) + { + attributes.Add(kvp.Key, kvp.Value); + } + } + + state.RootSpan = traceProvider + .StartRootSpan( + rootSpanName, + spanKind, + initialAttributes: attributes); } - public static void StartRootSpan( + /// + /// Create a new active span with respect to State and potentially a parent RootSpan. + /// + /// + /// + /// + /// + /// + public static TelemetrySpan CreateActiveSpan( this IWorkState state, string spanName, SpanKind spanKind = SpanKind.Internal, SpanAttributes spanAttributes = null) { - if (state.Data.TryGetValue(OpenTelemetryDefaultProviderTraceSourceNameKey, out var workflowName)) - { - state.Data.TryGetValue(OpenTelemetryDefaultProviderTracerServiceVersionKey, out var version); - - state.RootSpan = TracerProvider - .Default - ?.GetTracer( - workflowName?.ToString() ?? _defaultProviderTracerSourceName, - version?.ToString()) - ?.StartRootSpan(spanName, spanKind, initialAttributes: spanAttributes); - } - else - { - state.RootSpan = TracerProvider - .Default - ?.GetTracer(_defaultProviderTracerSourceName, null) - ?.StartRootSpan(spanName, spanKind, initialAttributes: spanAttributes); - } + if (state?.Data is null) return null; + + state.Data.TryGetValue(DefaultWorkflowNameKey, out var workflowName); + state.Data.TryGetValue(DefaultWorkflowVersionKey, out var workflowVersion); + + if (workflowName is null) return null; + + var traceProvider = TracerProvider + .Default + .GetTracer( + workflowName.ToString(), + workflowVersion?.ToString()); + + if (traceProvider is null) return null; + + var childSpanName = string.Format(DefaultChildSpanNameFormat, workflowName, spanName); + + return traceProvider + .StartActiveSpan( + childSpanName, + spanKind, + parentContext: state.RootSpan?.Context ?? default, + initialAttributes: spanAttributes); } - public static TelemetrySpan CreateActiveSpan( + /// + /// Create a new active span with respect to state and span context provided. + /// + /// + /// + /// + /// + /// + public static TelemetrySpan CreateActiveChildSpan( this IWorkState state, string spanName, + SpanContext spanContext, SpanKind spanKind = SpanKind.Internal, SpanAttributes spanAttributes = null) { - if (state.Data.TryGetValue(OpenTelemetryDefaultProviderTraceSourceNameKey, out var workflowName)) - { - state.Data.TryGetValue(OpenTelemetryDefaultProviderTracerServiceVersionKey, out var version); - - return TracerProvider - .Default - ?.GetTracer( - workflowName?.ToString() ?? _defaultProviderTracerSourceName, - version?.ToString()) - ?.StartActiveSpan( - spanName, - spanKind, - parentContext: state.RootSpan?.Context ?? default, - initialAttributes: spanAttributes); - } - else - { - return TracerProvider - .Default - ?.GetTracer(_defaultProviderTracerSourceName, null) - ?.StartActiveSpan( - spanName, - spanKind, - parentContext: state.RootSpan?.Context ?? default, - initialAttributes: spanAttributes); - } + if (state?.Data is null) return null; + + state.Data.TryGetValue(DefaultWorkflowNameKey, out var workflowName); + state.Data.TryGetValue(DefaultWorkflowVersionKey, out var workflowVersion); + + var traceProvider = TracerProvider + .Default + .GetTracer( + workflowName.ToString(), + workflowVersion?.ToString()); + + if (traceProvider is null) return null; + + var childSpanName = string.Format(DefaultChildSpanNameFormat, workflowName, spanName); + + return traceProvider + .StartActiveSpan( + childSpanName, + spanKind, + parentContext: spanContext, + initialAttributes: spanAttributes); } public static void EndRootSpan( this IWorkState state, bool includeErrorWhenFaulted = false) { + if (state is null) return; + if (includeErrorWhenFaulted && state.IsFaulted) { state.SetOpenTelemetryError(); } - state?.RootSpan?.End(); - state?.RootSpan?.Dispose(); + state.RootSpan?.End(); + state.RootSpan?.Dispose(); } } diff --git a/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs b/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs index bdccb679..e728bc1c 100644 --- a/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs +++ b/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs @@ -23,7 +23,11 @@ public class OpenTelemetryMetricsProvider : IMetricsProvider, IDisposable private bool _disposedValue; - public OpenTelemetryMetricsProvider(IMeterFactory meterFactory, string meterName, string activitySourceName = null, string activityVersion = null) + public OpenTelemetryMetricsProvider( + IMeterFactory meterFactory, + string meterName, + string activitySourceName = null, + string activityVersion = null) { Guard.AgainstNull(meterFactory, nameof(meterFactory)); Guard.AgainstNullOrEmpty(meterName, nameof(meterName)); diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index fc961d7d..22fd4185 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -534,7 +534,7 @@ private void LinkSuppliedSteps(DataflowLinkOptions overrideOptions = null) // Link all user steps. if (_suppliedTransforms?.Count > 0) { - for (int i = 0; i < _suppliedTransforms.Count; i++) + for (var i = 0; i < _suppliedTransforms.Count; i++) { if (i == 0) { LinkWithFaultRoute(_currentBlock, _suppliedTransforms[i], x => x.IsFaulted, overrideOptions ?? _linkStepOptions); } @@ -582,8 +582,6 @@ private void LinkWithFaultRoute(ISourceBlock source, IPropagatorBlock "{0}_RootSpan_{1}"; - public virtual TState BuildState(string key, ReceivedData data) { var state = new TState @@ -592,33 +590,27 @@ public virtual TState BuildState(string key, ReceivedData data) Data = new Dictionary() }; - // If the SerializationProvider was assigned, use it, else it's raw bytes. if (_serializationProvider != null) { state.Data[key] = _serializationProvider.Deserialize(data.Data); } else { state.Data[key] = data.Data; } - state.SetWorkflowNameAsOpenTelemetrySourceName(WorkflowName); - var spanAttributes = new SpanAttributes(); - spanAttributes.Add(nameof(WorkflowName), WorkflowName); - spanAttributes.Add(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName); + var attributes = new List>() + { + KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName) + }; + if (state.ReceivedData?.Letter?.MessageId is not null) { - spanAttributes.Add(nameof(state.ReceivedData.Letter.MessageId), state.ReceivedData.Letter.MessageId); + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.MessageId), state.ReceivedData.Letter.MessageId)); } if (state.ReceivedData?.Letter?.Metadata?.Id is not null) { - spanAttributes.Add(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id); + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id)); } - state.StartRootSpan( - string.Format( - RootSpanIdentifier, - WorkflowName, - state.ReceivedData?.Letter?.MessageId ?? Guid.NewGuid().ToString()), - SpanKind.Consumer, - spanAttributes); + state.StartRootSpan(WorkflowName, spanKind: SpanKind.Consumer, suppliedAttributes: attributes); return state; } diff --git a/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs b/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs new file mode 100644 index 00000000..bc535525 --- /dev/null +++ b/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Concurrent; +using System.Reflection; + +namespace HouseofCat.Utilities.Extensions; + +public static class AssemblyExtensions +{ + private static readonly ConcurrentDictionary _assemblyVersion = new ConcurrentDictionary(); + + public static string GetExecutingSemanticVersion(this Assembly assembly) + { + if (_assemblyVersion.TryGetValue(assembly, out var cachedVersion)) + { + return cachedVersion; + } + + var version = Assembly.GetExecutingAssembly().GetName().GetFlexibleSemVersion(); + _assemblyVersion.TryAdd(assembly, version); + return version; + } + + public static string GetFlexibleSemVersion(this AssemblyName assemblyName) + { + if (assemblyName == null) + { + return null; + } + + var versionParts = assemblyName + .Version + .ToString() + .Split('.', StringSplitOptions.RemoveEmptyEntries); + + return (versionParts?.Length ?? 0) switch + { + 0 => null, + 1 => versionParts[0], + 2 => $"{versionParts[0]}.{versionParts[1]}", + _ => $"{versionParts[0]}.{versionParts[1]}.{versionParts[2]}" + }; + } +} diff --git a/src/HouseofCat.Utilities/Extensions/StringExtensions.cs b/src/HouseofCat.Utilities/Extensions/StringExtensions.cs new file mode 100644 index 00000000..2b35b32b --- /dev/null +++ b/src/HouseofCat.Utilities/Extensions/StringExtensions.cs @@ -0,0 +1,72 @@ +using System; + +namespace HouseofCat.Utilities.Extensions; + +public static class StringExtensions +{ + private static readonly string _charSet = "abcdefghijklmnopqrstuvwxyz0123456789"; + + public static string RandomAlphaNumericChars(int numChars) + { + Span buffer = numChars <= 1024 + ? stackalloc char[numChars] + : new char[numChars]; + + for (var i = 0; i < numChars; i++) + { + buffer[i] = _charSet[System.Random.Shared.Next(_charSet.Length)]; + } + + return new string(buffer); + } + + public static int CountWordsInText(this ReadOnlySpan text) + { + if (text.Length == 0) return 0; + + var wordCount = 0; + var index = 0; + + while (index < text.Length && char.IsWhiteSpace(text[index])) + { index++; } + + while (index < text.Length) + { + while (index < text.Length && !char.IsWhiteSpace(text[index])) + { index++; } + + wordCount++; + + while (index < text.Length && char.IsWhiteSpace(text[index])) + { index++; } + } + + return wordCount; + } + + public static string ReplaceAt(this string input, int index, int length, string replacement) + { + if (length == 0) return input; + + return string.Create( + input.Length - length + replacement.Length, + (input, index, length, replacement), + (span, state) => + { + state + .input + .AsSpan()[..state.index]. + CopyTo(span); + + state + .replacement + .AsSpan() + .CopyTo(span[state.index..]); + + state + .input + .AsSpan()[(state.index + state.length)..] + .CopyTo(span[(state.index + state.replacement.Length)..]); + }); + } +} \ No newline at end of file diff --git a/tests/OpenTelemetry.Console.Tests/Program.cs b/tests/OpenTelemetry.Console.Tests/Program.cs index e31acda3..415e7446 100644 --- a/tests/OpenTelemetry.Console.Tests/Program.cs +++ b/tests/OpenTelemetry.Console.Tests/Program.cs @@ -19,7 +19,7 @@ .AddConsoleExporter() .Build(); -SpanTests.RunRootSpanWithChildSpanErrorTest(logger, workflowName); +WorkStateTests.RunRootSpanWithChildSpanErrorTest(logger, workflowName); logger.LogInformation("Tests complete! Press return to exit...."); diff --git a/tests/OpenTelemetry.Console.Tests/Tests/SpanTests.cs b/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs similarity index 59% rename from tests/OpenTelemetry.Console.Tests/Tests/SpanTests.cs rename to tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs index 8e9aee6b..918bb0e0 100644 --- a/tests/OpenTelemetry.Console.Tests/Tests/SpanTests.cs +++ b/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs @@ -5,7 +5,7 @@ namespace OpenTelmetry.Tests; -public static class SpanTests +public static class WorkStateTests { public class CustomWorkState : RabbitWorkState { @@ -15,14 +15,13 @@ public CustomWorkState() } } - public static void RunRootSpanTest(ILogger logger, string sourceName) + public static void RunRootSpanTest(ILogger logger, string workflowName) { logger.LogInformation($"Starting {nameof(RunRootSpanTest)}..."); var workstate = new CustomWorkState(); - workstate.SetWorkflowNameAsOpenTelemetrySourceName(sourceName, "v1.0.0"); - workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); + workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); workstate.EndRootSpan(); logger.LogInformation($"Finished {nameof(RunRootSpanTest)}."); @@ -30,13 +29,12 @@ public static void RunRootSpanTest(ILogger logger, string sourceName) public static void RunRootSpanWithChildSpanTest(ILogger logger, string workflowName) { - logger.LogInformation($"Starting {RunRootSpanWithChildSpanTest}..."); + logger.LogInformation($"Starting {nameof(RunRootSpanWithChildSpanTest)}..."); var workstate = new CustomWorkState(); - workstate.SetWorkflowNameAsOpenTelemetrySourceName(workflowName, "v1.0.0"); - workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); - using (var span = workstate.CreateActiveSpan("RunBasicSpanTest.Child", SpanKind.Internal)) + workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); + using (var span = workstate.CreateActiveSpan("ChildStep", spanKind: SpanKind.Internal)) { span.SetStatus(Status.Ok); } @@ -47,14 +45,13 @@ public static void RunRootSpanWithChildSpanTest(ILogger logger, string workflowN public static void RunRootSpanWithChildSpanErrorTest(ILogger logger, string workflowName) { - logger.LogInformation($"Starting {RunRootSpanWithChildSpanTest}..."); + logger.LogInformation($"Starting {nameof(RunRootSpanWithChildSpanTest)}..."); var workstate = new CustomWorkState(); - workstate.SetWorkflowNameAsOpenTelemetrySourceName(workflowName, "v1.0.0"); - workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); + workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); - using (var span = workstate.CreateActiveSpan("RunBasicSpanTest.Child", SpanKind.Internal)) + using (var span = workstate.CreateActiveSpan("ChildStep", spanKind: SpanKind.Internal)) { workstate.SetCurrentSpanAsError("Span had an error!"); } @@ -66,16 +63,15 @@ public static void RunRootSpanWithChildSpanErrorTest(ILogger logger, string work public static void RunRootSpanWithManyChildSpanFlatLevelTest(ILogger logger, string workflowName) { - logger.LogInformation($"Starting {RunRootSpanWithChildSpanTest}..."); + logger.LogInformation($"Starting {nameof(RunRootSpanWithChildSpanTest)}..."); var workstate = new CustomWorkState(); - workstate.SetWorkflowNameAsOpenTelemetrySourceName(workflowName, "v1.0.0"); - workstate.StartRootSpan("RunBasicSpanTest.Root", SpanKind.Internal); + workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); for (var i = 0; i < 10; i++) { - using var span = workstate.CreateActiveSpan($"RunBasicSpanTest.Child.{i}", SpanKind.Internal); + using var span = workstate.CreateActiveSpan($"ChildStep.{i}", SpanKind.Internal); if (i == 9) { From ac8c235857ee720e96918c73d4d73de107740a74 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sat, 30 Mar 2024 15:28:48 -0500 Subject: [PATCH 03/47] Test ConsumerDataflow with OpenTelemetry project. --- RabbitMQ.Dataflows.sln | 7 ++ src/HouseofCat.Dataflows/BaseDataflow.cs | 31 +++++-- .../Dataflows/ConsumerDataflow.cs | 29 ++++-- .../Dataflows/RabbitWorkState.cs | 3 +- .../Messages/ReceivedData.cs | 1 - tests/OpenTelemetry.Console.Tests/Program.cs | 2 +- .../Tests/WorkStateTests.cs | 2 +- tests/RabbitMQ.Console.Tests/Program.cs | 2 +- .../Tests/BasicGetTests.cs | 2 +- .../Tests/ConsumerTests.cs | 2 +- .../Tests/PubSubTests.cs | 2 +- .../Tests/PublisherTests.cs | 2 +- .../Tests/RabbitServiceTests.cs | 2 +- tests/RabbitMQ.Console.Tests/Tests/Shared.cs | 7 +- .../Program.cs | 65 +++++++++++++ .../RabbitMQ.ConsumerDataflows.Tests.csproj | 26 ++++++ .../RabbitMQ.ConsumerDataflows.json | 55 +++++++++++ .../Tests/ConsumerDataflowOptions.cs | 27 ++++++ .../Tests/ConsumerDataflowService.cs | 92 +++++++++++++++++++ .../Tests/CustomWorkState.cs | 8 ++ .../Tests/Shared.cs | 81 ++++++++++++++++ 21 files changed, 412 insertions(+), 36 deletions(-) create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs diff --git a/RabbitMQ.Dataflows.sln b/RabbitMQ.Dataflows.sln index 8f19ad99..1767474f 100644 --- a/RabbitMQ.Dataflows.sln +++ b/RabbitMQ.Dataflows.sln @@ -88,6 +88,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "rabbitmq", "rabbitmq", "{5C EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Console.Tests", "tests\OpenTelemetry.Console.Tests\OpenTelemetry.Console.Tests.csproj", "{077E07C3-9A35-42A0-8228-E9778F02DFCE}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RabbitMQ.ConsumerDataflows.Tests", "tests\RabbitMQ.ConsumerDataflows.Tests\RabbitMQ.ConsumerDataflows.Tests.csproj", "{F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -142,6 +144,10 @@ Global {077E07C3-9A35-42A0-8228-E9778F02DFCE}.Debug|Any CPU.Build.0 = Debug|Any CPU {077E07C3-9A35-42A0-8228-E9778F02DFCE}.Release|Any CPU.ActiveCfg = Release|Any CPU {077E07C3-9A35-42A0-8228-E9778F02DFCE}.Release|Any CPU.Build.0 = Release|Any CPU + {F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -167,6 +173,7 @@ Global {528C015E-58FC-44B4-BAC5-05BCFCD506C9} = {D01D59E9-75CC-4C74-A887-22B0A55C994B} {5C94F432-8229-4FFA-8DBF-AC1DEDE66265} = {1AB5E832-AD36-40AF-A7E9-A105A778116C} {077E07C3-9A35-42A0-8228-E9778F02DFCE} = {D01D59E9-75CC-4C74-A887-22B0A55C994B} + {F6C0B657-E70B-4DE9-96AE-E7612C89AF5F} = {D01D59E9-75CC-4C74-A887-22B0A55C994B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D7099845-267C-4232-8764-5DF25D6B2C79} diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index c4d62c55..d5f2399c 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -60,11 +60,11 @@ public TransformBlock GetTransformBlock( public TransformBlock GetWrappedTransformBlock( Func action, ExecutionDataflowBlockOptions options, - string metricIdentifier) + string spanName) { TState WrapAction(TState state) { - using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { @@ -86,11 +86,11 @@ TState WrapAction(TState state) public TransformBlock GetWrappedTransformBlock( Func> action, ExecutionDataflowBlockOptions options, - string metricIdentifier) + string spanName) { async Task WrapActionAsync(TState state) { - using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { return await action(state).ConfigureAwait(false); @@ -108,14 +108,14 @@ async Task WrapActionAsync(TState state) return new TransformBlock(WrapActionAsync, options); } - public ActionBlock GetWrappedActionBlock( + public ActionBlock GetLastWrappedActionBlock( Action action, ExecutionDataflowBlockOptions options, string spanName) { void WrapAction(TState state) { - using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { action(state); @@ -124,51 +124,62 @@ void WrapAction(TState state) { childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.Dispose(); } + + state.EndRootSpan(); } return new ActionBlock(WrapAction, options); } - public ActionBlock GetWrappedActionBlock( + public ActionBlock GetLastWrappedActionBlock( Func action, ExecutionDataflowBlockOptions options, string spanName) { void WrapAction(TState state) { - using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { action(state); + childSpan.Dispose(); } catch (Exception ex) { childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.Dispose(); } + + state.EndRootSpan(); } return new ActionBlock(WrapAction, options); } - public ActionBlock GetWrappedActionBlock( + public ActionBlock GetLastWrappedActionBlock( Func action, ExecutionDataflowBlockOptions options, string spanName) { async Task WrapActionAsync(TState state) { - using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); try { await action(state).ConfigureAwait(false); + childSpan.Dispose(); } catch (Exception ex) { childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.Dispose(); } + + state.EndRootSpan(); } return new ActionBlock(WrapActionAsync, options); diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 22fd4185..3440a58e 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -233,7 +233,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); + _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); } return this; } @@ -250,7 +250,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); + _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); } return this; } @@ -309,7 +309,7 @@ public ConsumerDataflow WithFinalization( if (_finalization == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); + _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); } return this; } @@ -325,7 +325,7 @@ public ConsumerDataflow WithFinalization( if (_finalization == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); + _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); } return this; } @@ -590,12 +590,6 @@ public virtual TState BuildState(string key, ReceivedData data) Data = new Dictionary() }; - if (_serializationProvider != null) - { state.Data[key] = _serializationProvider.Deserialize(data.Data); } - else - { state.Data[key] = data.Data; } - - var attributes = new List>() { KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName) @@ -612,6 +606,21 @@ public virtual TState BuildState(string key, ReceivedData data) state.StartRootSpan(WorkflowName, spanKind: SpanKind.Consumer, suppliedAttributes: attributes); + if (_serializationProvider != null + && data.ContentType != Constants.HeaderValueForUnknown) + { + try + { state.Data[key] = _serializationProvider.Deserialize(data.Data); } + catch (Exception ex) + { + state.IsFaulted = true; + state.EDI = ExceptionDispatchInfo.Capture(ex); + return state; + } + } + else + { state.Data[key] = data.Data; } + return state; } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index 52299935..ca60e6e3 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -8,7 +8,6 @@ namespace HouseofCat.RabbitMQ.Dataflows; public interface IRabbitWorkState : IWorkState { - // Inbound Data IReceivedData ReceivedData { get; set; } IMessage SendMessage { get; set; } bool SendMessageSent { get; set; } @@ -18,8 +17,10 @@ public abstract class RabbitWorkState : IRabbitWorkState { [IgnoreDataMember] public virtual IReceivedData ReceivedData { get; set; } + public virtual byte[] SendData { get; set; } public virtual IMessage SendMessage { get; set; } + public virtual bool SendMessageSent { get; set; } public virtual IDictionary Data { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs index f6f34317..521a52bf 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs @@ -191,7 +191,6 @@ public bool RejectMessage(bool requeue) /// public void Complete() => _completionSource.SetResult(true); - protected virtual void Dispose(bool disposing) { if (!_disposedValue) diff --git a/tests/OpenTelemetry.Console.Tests/Program.cs b/tests/OpenTelemetry.Console.Tests/Program.cs index 415e7446..0c2f3389 100644 --- a/tests/OpenTelemetry.Console.Tests/Program.cs +++ b/tests/OpenTelemetry.Console.Tests/Program.cs @@ -3,7 +3,7 @@ using OpenTelemetry; using OpenTelemetry.Resources; using OpenTelemetry.Trace; -using OpenTelmetry.Tests; +using OpenTelemetry.Tests; var loggerFactory = LogHelper.CreateConsoleLoggerFactory(LogLevel.Information); LogHelper.LoggerFactory = loggerFactory; diff --git a/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs b/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs index 918bb0e0..d111171e 100644 --- a/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs +++ b/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs @@ -3,7 +3,7 @@ using OpenTelemetry.Trace; using static HouseofCat.Dataflows.Extensions.WorkStateExtensions; -namespace OpenTelmetry.Tests; +namespace OpenTelemetry.Tests; public static class WorkStateTests { diff --git a/tests/RabbitMQ.Console.Tests/Program.cs b/tests/RabbitMQ.Console.Tests/Program.cs index b844af03..3219bacf 100644 --- a/tests/RabbitMQ.Console.Tests/Program.cs +++ b/tests/RabbitMQ.Console.Tests/Program.cs @@ -1,4 +1,4 @@ -using ConnectivityTests.Tests; +using RabbitMQ.Console.Tests; using HouseofCat.Utilities; using Microsoft.Extensions.Logging; diff --git a/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs b/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs index d9c63071..6cc2fac1 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs @@ -2,7 +2,7 @@ using RabbitMQ.Client; using System.Text; -namespace ConnectivityTests.Tests; +namespace RabbitMQ.Console.Tests; public static class BasicGetTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs index 60e75657..213ee795 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs @@ -2,7 +2,7 @@ using Microsoft.Extensions.Logging; using System.Text; -namespace ConnectivityTests.Tests; +namespace RabbitMQ.Console.Tests; public static class ConsumerTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs index a8cbadab..ec9b3717 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs @@ -5,7 +5,7 @@ using System.Text; using System.Text.Json; -namespace ConnectivityTests.Tests; +namespace RabbitMQ.Console.Tests; public static class PubSubTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index 56570b12..cd3e59b1 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -5,7 +5,7 @@ using System.Text; using System.Threading.Channels; -namespace ConnectivityTests.Tests; +namespace RabbitMQ.Console.Tests; public static class PublisherTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index 96329c38..454f0eb2 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -1,7 +1,7 @@ using HouseofCat.RabbitMQ; using Microsoft.Extensions.Logging; -namespace ConnectivityTests.Tests; +namespace RabbitMQ.Console.Tests; public static class RabbitServiceTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/Shared.cs b/tests/RabbitMQ.Console.Tests/Tests/Shared.cs index d02ee3eb..e1553f39 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/Shared.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/Shared.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using RabbitMQ.Client; -namespace ConnectivityTests.Tests; +namespace RabbitMQ.Console.Tests; public static class Shared { @@ -40,11 +40,6 @@ public static async Task SetupTestsAsync(ILogger logger, string co channel.QueueBind(QueueName, ExchangeName, RoutingKey); - logger.LogInformation( - "Publishing message to Exchange [{exchangeName}] with RoutingKey [{routingKey}]", - ExchangeName, - RoutingKey); - channelHost.Close(); return channelPool; diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs new file mode 100644 index 00000000..b375bc18 --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -0,0 +1,65 @@ +using HouseofCat.Utilities; +using Microsoft.Extensions.Logging; +using OpenTelemetry; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using RabbitMQ.ConsumerDataflows.Tests; +using System.Text; + +var loggerFactory = LogHelper.CreateConsoleLoggerFactory(LogLevel.Information); +LogHelper.LoggerFactory = loggerFactory; +var logger = loggerFactory.CreateLogger(); + +var applicationName = "RabbitMQ.ConsumerDataflow.Tests"; + +using var traceProvider = Sdk + .CreateTracerProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(applicationName)) + .AddSource(Shared.ConsumerWorkflowName) + .AddConsoleExporter() + .Build(); + +var rabbitService = await Shared.SetupRabbitServiceAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); +var dataflowService = new ConsumerDataflowService(rabbitService); + +dataflowService.AddStep( + "WriteToRabbitMessageToConsole", + (state) => + { + Console.WriteLine(Encoding.UTF8.GetString(state.ReceivedData.Data.Span)); + return state; + }); + +dataflowService.AddFinalization( + (state) => + { + logger.LogInformation("Finalization Step!"); + state.ReceivedData.AckMessage(); + state.ReceivedData.Complete(); + }); + +dataflowService.AddErrorHandling( + (state) => + { + logger.LogError(state?.EDI?.SourceException, "Error Step!"); + state.ReceivedData.NackMessage(requeue: true); + state.ReceivedData.Complete(); + }); + +await dataflowService.StartAsync(); + +logger.LogInformation("Listening for Messages! Press Return to stop consumer..."); + +Console.ReadLine(); + +logger.LogInformation("ConsumerService stopping..."); + +await dataflowService.StopAsync(); + +logger.LogInformation("RabbitMQ AutoPublish stopping..."); + +await rabbitService.Publisher.StopAutoPublishAsync(); + +logger.LogInformation("All stopped! Press return to exit..."); + +Console.ReadLine(); diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj new file mode 100644 index 00000000..50a4a09e --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj @@ -0,0 +1,26 @@ + + + + Exe + net8.0 + enable + + + + + + + + + + + + + + + + Always + + + + diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json new file mode 100644 index 00000000..d0b9b8e2 --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json @@ -0,0 +1,55 @@ +{ + "FactoryOptions": { + "Uri": "amqp://guest:guest@localhost:5672/", + "MaxChannelsPerConnection": 2000, + "HeartbeatInterval": 6, + "AutoRecovery": true, + "TopologyRecovery": true, + "NetRecoveryTimeout": 5, + "ContinuationTimeout": 10, + "EnableDispatchConsumersAsync": true + }, + "PoolOptions": { + "ServiceName": "HoC.RabbitMQ", + "MaxConnections": 2, + "MaxChannels": 10, + "MaxAckableChannels": 0, + "SleepOnErrorInterval": 5000, + "TansientChannelStartRange": 10000, + "UseTransientChannels": false + }, + "PublisherOptions": { + "LetterQueueBufferSize": 100, + "PriorityLetterQueueBufferSize": 100, + "BehaviorWhenFull": 0, + "AutoPublisherSleepInterval": 1, + "CreatePublishReceipts": true, + "Compress": true, + "Encrypt": true + }, + "GlobalConsumerOptions": { + "ModerateSettings": { + "ErrorSuffix": "Error", + "BatchSize": 12, + "BehaviorWhenFull": 0, + "SleepOnIdleInterval": 0, + "UseTransientChannels": true, + "AutoAck": false, + "NoLocal": false, + "Exclusive": false, + "GlobalConsumerPipelineOptions": { + "WaitForCompletion": true, + "MaxDegreesOfParallelism": 4, + "EnsureOrdered": false + } + } + }, + "ConsumerOptions": { + "TestConsumer": { + "Enabled": true, + "GlobalSettings": "ModerateSettings", + "ConsumerName": "TestConsumer", + "QueueName": "TestQueue" + } + } +} \ No newline at end of file diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs new file mode 100644 index 00000000..ff628b03 --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs @@ -0,0 +1,27 @@ +using HouseofCat.RabbitMQ; +using HouseofCat.RabbitMQ.Dataflows; +using HouseofCat.RabbitMQ.Services; + +namespace RabbitMQ.ConsumerDataflows.Tests; + +public sealed class ConsumerDataflowOptions +{ + public string WorkflowName { get; set; } + public string WorkStateKey { get; set; } = "State"; + public string ConsumerName { get; set; } + public int ConsumerCount { get; set; } = 1; + public int MaxDegreeOfParallelism { get; set; } = 1; + public bool EnsureOrdered { get; set; } = true; + public int Capacity { get; set; } = 1000; + public string ErrorQueueName { get; set; } + + public ConsumerDataflow BuildConsumerDataflow(IRabbitService rabbitService) + { + return new ConsumerDataflow( + rabbitService, + WorkflowName, + ConsumerName, + ConsumerCount) + .WithBuildState(WorkStateKey); + } +} diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs new file mode 100644 index 00000000..1f4fd230 --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs @@ -0,0 +1,92 @@ +using HouseofCat.RabbitMQ.Dataflows; +using HouseofCat.RabbitMQ.Services; + +namespace RabbitMQ.ConsumerDataflows.Tests; + +public class ConsumerDataflowService +{ + private readonly ConsumerDataflowOptions _consumerWorkflowOptions; + private readonly ConsumerDataflow _workflow; + + public ConsumerDataflowService(IRabbitService rabbitService) + { + _consumerWorkflowOptions = new ConsumerDataflowOptions + { + WorkflowName = Shared.ConsumerWorkflowName, + ConsumerName = Shared.ConsumerName, + ConsumerCount = 1, + MaxDegreeOfParallelism = 1, + EnsureOrdered = true, + Capacity = 1000, + ErrorQueueName = Shared.ErrorQueue + }; + + _workflow = _consumerWorkflowOptions.BuildConsumerDataflow(rabbitService); + } + + public void AddStep(string stepName, Func step) + { + _workflow.AddStep( + step, + stepName, + _consumerWorkflowOptions.MaxDegreeOfParallelism, + _consumerWorkflowOptions.EnsureOrdered, + _consumerWorkflowOptions.Capacity); + } + + public void AddStep(string stepName, Func> step) + { + _workflow.AddStep( + step, + stepName, + _consumerWorkflowOptions.MaxDegreeOfParallelism, + _consumerWorkflowOptions.EnsureOrdered, + _consumerWorkflowOptions.Capacity); + } + + public void AddFinalization(Action step) + { + _workflow.WithFinalization( + step, + _consumerWorkflowOptions.MaxDegreeOfParallelism, + _consumerWorkflowOptions.EnsureOrdered, + _consumerWorkflowOptions.Capacity); + } + + public void AddFinalization(Func step) + { + _workflow.WithFinalization( + step, + _consumerWorkflowOptions.MaxDegreeOfParallelism, + _consumerWorkflowOptions.EnsureOrdered, + _consumerWorkflowOptions.Capacity); + } + + public void AddErrorHandling(Action step) + { + _workflow.WithErrorHandling( + step, + _consumerWorkflowOptions.Capacity, + _consumerWorkflowOptions.MaxDegreeOfParallelism, + _consumerWorkflowOptions.EnsureOrdered); + } + + public void AddErrorHandling(Func step) + { + _workflow.WithErrorHandling( + step, + _consumerWorkflowOptions.Capacity, + _consumerWorkflowOptions.MaxDegreeOfParallelism, + _consumerWorkflowOptions.EnsureOrdered); + } + + public async Task StartAsync() + { + await _workflow.StartAsync(); + } + + public async Task StopAsync() + { + await _workflow.StopAsync(); + } +} diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs new file mode 100644 index 00000000..3f2436f7 --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs @@ -0,0 +1,8 @@ +using HouseofCat.RabbitMQ.Dataflows; + +namespace RabbitMQ.ConsumerDataflows.Tests; + +public sealed class CustomWorkState : RabbitWorkState +{ + +} diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs new file mode 100644 index 00000000..5016fab4 --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs @@ -0,0 +1,81 @@ +using HouseofCat.Compression.Recyclable; +using HouseofCat.Encryption; +using HouseofCat.Hashing; +using HouseofCat.RabbitMQ; +using HouseofCat.RabbitMQ.Pools; +using HouseofCat.RabbitMQ.Services; +using HouseofCat.RabbitMQ.Services.Extensions; +using HouseofCat.Serialization; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; + +namespace RabbitMQ.ConsumerDataflows.Tests; + +public static class Shared +{ + public static readonly string ExchangeName = "TestExchange"; + public static readonly string QueueName = "TestQueue"; + public static readonly string RoutingKey = "TestRoutingKey"; + + public static readonly string ConsumerWorkflowName = "TestConsumerWorkflow"; + public static readonly string ConsumerName = "TestConsumer"; + public static readonly string ErrorQueue = "TestQueue.Error"; + + public static async Task SetupTestsAsync(ILogger logger, string configFileNamePath) + { + var rabbitOptions = await RabbitExtensions.GetRabbitOptionsFromJsonFileAsync(configFileNamePath); + var channelPool = new ChannelPool(rabbitOptions); + + var channelHost = await channelPool.GetTransientChannelAsync(true); + var channel = channelHost.GetChannel(); + + logger.LogInformation("Declaring Exchange: [{ExchangeName}]", ExchangeName); + channel.ExchangeDeclare(ExchangeName, ExchangeType.Direct, true, false, null); + + logger.LogInformation("Declaring Queue: [{QueueName}]", QueueName); + channel.QueueDeclare(QueueName, true, false, false, null); + + logger.LogInformation( + "Binding Queue [{queueName}] To Exchange: [{exchangeName}]. RoutingKey: [{routingKey}]", + QueueName, + ExchangeName, + RoutingKey); + + channel.QueueBind(QueueName, ExchangeName, RoutingKey); + + logger.LogInformation( + "Publishing message to Exchange [{exchangeName}] with RoutingKey [{routingKey}]", + ExchangeName, + RoutingKey); + + channelHost.Close(); + + return channelPool; + } + + public static readonly string EncryptionPassword = "PasswordyPasswordPassword"; + public static readonly string EncryptionSalt = "SaltySaltSalt"; + public static readonly int KeySize = 32; + + public static async Task SetupRabbitServiceAsync(ILoggerFactory loggerFactory, string configFileNamePath) + { + var rabbitOptions = await RabbitExtensions.GetRabbitOptionsFromJsonFileAsync(configFileNamePath); + var jsonProvider = new JsonProvider(); + + var hashProvider = new ArgonHashingProvider(); + var aes256Key = hashProvider.GetHashKey(EncryptionPassword, EncryptionSalt, KeySize); + var aes256Provider = new AesGcmEncryptionProvider(aes256Key); + + var gzipProvider = new RecyclableGzipProvider(); + + var rabbitService = await rabbitOptions.BuildRabbitServiceAsync( + jsonProvider, + aes256Provider, + gzipProvider, + loggerFactory); + + await rabbitService.Publisher.StartAutoPublishAsync(); + + return rabbitService; + } +} From 1b2f61a4e9057250a0905eb40962d11f9252a9ac Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sat, 30 Mar 2024 15:34:31 -0500 Subject: [PATCH 04/47] Build fix. --- src/HouseofCat.Dataflows/BaseDataflow.cs | 8 +++----- tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs | 2 +- tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs | 2 +- tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs | 2 +- tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs | 3 ++- tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs | 2 +- tests/RabbitMQ.Console.Tests/Tests/Shared.cs | 2 +- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index d5f2399c..ccfbbbb3 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -124,9 +124,9 @@ void WrapAction(TState state) { childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); - childSpan?.Dispose(); } + childSpan?.Dispose(); state.EndRootSpan(); } @@ -144,15 +144,14 @@ void WrapAction(TState state) try { action(state); - childSpan.Dispose(); } catch (Exception ex) { childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); - childSpan?.Dispose(); } + childSpan?.Dispose(); state.EndRootSpan(); } @@ -170,15 +169,14 @@ async Task WrapActionAsync(TState state) try { await action(state).ConfigureAwait(false); - childSpan.Dispose(); } catch (Exception ex) { childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); - childSpan?.Dispose(); } + childSpan?.Dispose(); state.EndRootSpan(); } diff --git a/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs b/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs index 6cc2fac1..15939843 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs @@ -2,7 +2,7 @@ using RabbitMQ.Client; using System.Text; -namespace RabbitMQ.Console.Tests; +namespace RabbitMQ.ConsoleTests; public static class BasicGetTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs index 213ee795..28847bc0 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs @@ -2,7 +2,7 @@ using Microsoft.Extensions.Logging; using System.Text; -namespace RabbitMQ.Console.Tests; +namespace RabbitMQ.ConsoleTests; public static class ConsumerTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs index ec9b3717..ecc134a1 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs @@ -5,7 +5,7 @@ using System.Text; using System.Text.Json; -namespace RabbitMQ.Console.Tests; +namespace RabbitMQ.ConsoleTests; public static class PubSubTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index cd3e59b1..1e2bac4f 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -2,10 +2,11 @@ using HouseofCat.RabbitMQ.Pools; using HouseofCat.Serialization; using Microsoft.Extensions.Logging; +using System; using System.Text; using System.Threading.Channels; -namespace RabbitMQ.Console.Tests; +namespace RabbitMQ.ConsoleTests; public static class PublisherTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index 454f0eb2..7f8901e8 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -1,7 +1,7 @@ using HouseofCat.RabbitMQ; using Microsoft.Extensions.Logging; -namespace RabbitMQ.Console.Tests; +namespace RabbitMQ.ConsoleTests; public static class RabbitServiceTests { diff --git a/tests/RabbitMQ.Console.Tests/Tests/Shared.cs b/tests/RabbitMQ.Console.Tests/Tests/Shared.cs index e1553f39..f6506ff2 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/Shared.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/Shared.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using RabbitMQ.Client; -namespace RabbitMQ.Console.Tests; +namespace RabbitMQ.ConsoleTests; public static class Shared { From 17d72971e167df73271f9a508c60531deb4628d6 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sat, 30 Mar 2024 17:08:05 -0500 Subject: [PATCH 05/47] Cleanup. --- .../DataflowExtensions.cs | 1 - .../Extensions/WorkStateExtensions.cs | 1 - src/HouseofCat.RabbitMQ/Constants.cs | 6 +- .../Dataflows/ConsumerDataflow.cs | 88 ++++++-- .../Dataflows/RabbitWorkState.cs | 2 - src/HouseofCat.RabbitMQ/Enums.cs | 192 ------------------ .../Extensions/MessageExtensions.cs | 2 +- src/HouseofCat.RabbitMQ/Messages/Letter.cs | 2 +- .../Messages/ReceivedData.cs | 23 ++- .../Options/RoutingOptions.cs | 4 +- .../Publisher/Publisher.cs | 8 +- .../Tests/PublisherTests.cs | 1 - .../Tests/ConsumerDataflowOptions.cs | 2 +- 13 files changed, 100 insertions(+), 232 deletions(-) delete mode 100644 src/HouseofCat.RabbitMQ/Enums.cs diff --git a/src/HouseofCat.Dataflows/DataflowExtensions.cs b/src/HouseofCat.Dataflows/DataflowExtensions.cs index 7047777c..7e2e0c43 100644 --- a/src/HouseofCat.Dataflows/DataflowExtensions.cs +++ b/src/HouseofCat.Dataflows/DataflowExtensions.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index d5cf8c86..d7525676 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -205,7 +205,6 @@ public static void EndRootSpan( { state.SetOpenTelemetryError(); } - state.RootSpan?.End(); state.RootSpan?.Dispose(); } } diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index 20d0897c..339f440a 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -9,10 +9,12 @@ public static class Constants public static string HeaderPrefix { get; set; } = "X-"; // Consumer + public static string HeaderForContentType { get; set; } = "ContentType"; + public static string HeaderValueForContentTypeApplicationJson { get; set; } = "application/json;"; public static string HeaderForObjectType { get; set; } = "X-RD-OBJECTTYPE"; - public static string HeaderValueForMessage { get; set; } = "IMESSAGE"; + public static string HeaderValueForMessageObjectType { get; set; } = "IMESSAGE"; - public static string HeaderValueForUnknown { get; set; } = "UNKNOWN"; + public static string HeaderValueForUnknownObjectType { get; set; } = "UNK"; public static string HeaderForEncrypted { get; set; } = "X-RD-ENCRYPTED"; public static string HeaderForEncryption { get; set; } = "X-RD-ENCRYPTION"; public static string HeaderForEncryptDate { get; set; } = "X-RD-ENCRYPTDATE"; diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 3440a58e..f16a8173 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -233,7 +233,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); + _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.ErrorHandler"); } return this; } @@ -250,7 +250,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_ErrorHandler"); + _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.ErrorHandler"); } return this; } @@ -309,7 +309,7 @@ public ConsumerDataflow WithFinalization( if (_finalization == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); + _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.Finalization"); } return this; } @@ -325,12 +325,26 @@ public ConsumerDataflow WithFinalization( if (_finalization == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}_Finalization"); + _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.Finalization"); } return this; } - public ConsumerDataflow WithBuildState( + public ConsumerDataflow WithBuildState( + int? maxDoP = null, + bool? ensureOrdered = null, + int? boundedCapacity = null, + TaskScheduler taskScheduler = null) + { + if (_buildStateBlock == null) + { + var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); + _buildStateBlock = GetBuildStateBlock(executionOptions); + } + return this; + } + + public ConsumerDataflow WithBuildStateAndPayload( string stateKey, int? maxDoP = null, bool? ensureOrdered = null, @@ -342,7 +356,7 @@ public ConsumerDataflow WithBuildState( if (_buildStateBlock == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _buildStateBlock = GetBuildStateBlock(stateKey, executionOptions); + _buildStateBlock = GetBuildStateWithPayloadBlock(stateKey, executionOptions); } return this; } @@ -582,7 +596,34 @@ private void LinkWithFaultRoute(ISourceBlock source, IPropagatorBlock(string key, ReceivedData data) + public virtual TState BuildState(ReceivedData data) + { + var state = new TState + { + ReceivedData = data, + Data = new Dictionary() + }; + + var attributes = new List>() + { + KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName) + }; + + if (state.ReceivedData?.Letter?.MessageId is not null) + { + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.MessageId), state.ReceivedData.Letter.MessageId)); + } + if (state.ReceivedData?.Letter?.Metadata?.Id is not null) + { + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id)); + } + + state.StartRootSpan(WorkflowName, spanKind: SpanKind.Consumer, suppliedAttributes: attributes); + + return state; + } + + public virtual TState BuildStateAndPayload(string key, ReceivedData data) { var state = new TState { @@ -607,31 +648,38 @@ public virtual TState BuildState(string key, ReceivedData data) state.StartRootSpan(WorkflowName, spanKind: SpanKind.Consumer, suppliedAttributes: attributes); if (_serializationProvider != null - && data.ContentType != Constants.HeaderValueForUnknown) + && data.ObjectType != Constants.HeaderValueForUnknownObjectType) { try { state.Data[key] = _serializationProvider.Deserialize(data.Data); } - catch (Exception ex) - { - state.IsFaulted = true; - state.EDI = ExceptionDispatchInfo.Capture(ex); - return state; - } + catch { } } - else - { state.Data[key] = data.Data; } return state; } - public TransformBlock GetBuildStateBlock( + public TransformBlock GetBuildStateWithPayloadBlock( string key, ExecutionDataflowBlockOptions options) { TState BuildStateWrap(ReceivedData data) { try - { return BuildState(key, data); } + { return BuildStateAndPayload(key, data); } + catch + { return null; } + } + + return new TransformBlock(BuildStateWrap, options); + } + + public TransformBlock GetBuildStateBlock( + ExecutionDataflowBlockOptions options) + { + TState BuildStateWrap(ReceivedData data) + { + try + { return BuildState(data); } catch { return null; } } @@ -660,7 +708,7 @@ TState WrapAction(TState state) } else if (predicate.Invoke(state)) { - if (state.ReceivedData.ContentType == Constants.HeaderValueForMessage) + if (state.ReceivedData.ObjectType == Constants.HeaderValueForMessageObjectType) { if (state.ReceivedData.Letter == null) { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } @@ -708,7 +756,7 @@ async Task WrapActionAsync(TState state) } else if (predicate.Invoke(state)) { - if (state.ReceivedData.ContentType == Constants.HeaderValueForMessage) + if (state.ReceivedData.ObjectType == Constants.HeaderValueForMessageObjectType) { if (state.ReceivedData.Letter == null) { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index ca60e6e3..efbee26b 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -31,7 +31,5 @@ public abstract class RabbitWorkState : IRabbitWorkState public bool IsFaulted { get; set; } public ExceptionDispatchInfo EDI { get; set; } - public IDictionary MetricTags { get; set; } - public TelemetrySpan RootSpan { get; set; } } diff --git a/src/HouseofCat.RabbitMQ/Enums.cs b/src/HouseofCat.RabbitMQ/Enums.cs deleted file mode 100644 index cb0ef0e4..00000000 --- a/src/HouseofCat.RabbitMQ/Enums.cs +++ /dev/null @@ -1,192 +0,0 @@ -using System.ComponentModel; - -namespace HouseofCat.RabbitMQ; - -public static class Enums -{ - /// - /// Allows for quickling setting ContentType for RabbitMQ IBasicProperties. - /// - public enum ContentType - { - /// - /// ContentType.Javascript - /// - [Description("application/javascript;")] - Javascript, - - /// - /// ContentType.Json - /// - [Description("application/json;")] - Json, - - /// - /// ContentType.Urlencoded - /// - [Description("application/x-www-form-urlencoded;")] - Urlencoded, - - /// - /// ContentType.Xml - /// - [Description("application/xml;")] - Xml, - - /// - /// ContentType.Zip - /// - [Description("application/zip;")] - Zip, - - /// - /// ContentType.Pdf - /// - [Description("application/pdf;")] - Pdf, - - /// - /// ContentType.Sql - /// - [Description("application/sql;")] - Sql, - - /// - /// ContentType.Graphql - /// - [Description("application/graphql;")] - Graphql, - - /// - /// ContentType.Ldjson - /// - [Description("application/ld+json;")] - Ldjson, - - /// - /// ContentType.Msword - /// - [Description("application/msword(.doc);")] - Msword, - - /// - /// ContentType.Openword - /// - [Description("application/vnd.openxmlformats-officedocument.wordprocessingml.document(.docx);")] - Openword, - - /// - /// ContentType.Excel - /// - [Description("application/vnd.ms-excel(.xls);")] - Excel, - - /// - /// ContentType.Openexcel - /// - [Description("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet(.xlsx);")] - Openexcel, - - /// - /// ContentType.Powerpoint - /// - [Description("application/vnd.ms-powerpoint(.ppt);")] - Powerpoint, - - /// - /// ContentType.Openpowerpoint - /// - [Description("application/vnd.openxmlformats-officedocument.presentationml.presentation(.pptx);")] - Openpowerpoint, - - /// - /// ContentType.Opendocument - /// - [Description("application/vnd.oasis.opendocument.text(.odt);")] - Opendocument, - - /// - /// ContentType.Audiompeg - /// - [Description("audio/mpeg;")] - Audiompeg, - - /// - /// ContentType.Audiovorbis - /// - [Description("audio/vorbis;")] - Audiovorbis, - - /// - /// ContentType.Multiformdata - /// - [Description("multipart/form-data;")] - Multiformdata, - - /// - /// ContentType.Textcss - /// - [Description("text/css;")] - Textcss, - - /// - /// ContentType.Texthtml - /// - [Description("text/html;")] - Texthtml, - - /// - /// ContentType.Textcsv - /// - [Description("text/csv;")] - Textcsv, - - /// - /// ContentType.Textplain - /// - [Description("text/plain;")] - Textplain, - - /// - /// ContentType.Png - /// - [Description("image/png;")] - Png, - - /// - /// ContentType.Jpeg - /// - [Description("image/jpeg;")] - Jpeg, - - /// - /// ContentType.Gif - /// - [Description("image/gif;")] - Gif - } - - /// - /// Allows for quickling combining Charset with ContentType for RabbitMQ IBasicProperties. - /// - public enum Charset - { - /// - /// Charset.Utf8 - /// - [Description("charset=utf-8")] - Utf8, - - /// - /// Charset.Utf16 - /// - [Description("charset=utf-16")] - Utf16, - - /// - /// Charset.Utf32 - /// - [Description("charset=utf-32")] - Utf32, - } -} diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index eb83783f..1c471939 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -50,7 +50,7 @@ public static IBasicProperties CreateBasicProperties( var props = channelHost.GetChannel().CreateBasicProperties(); props.DeliveryMode = message.Envelope.RoutingOptions.DeliveryMode; - props.ContentType = new string(message.Envelope.RoutingOptions.MessageType); + props.ContentType = new string(message.Envelope.RoutingOptions.ContentType); props.Priority = message.Envelope.RoutingOptions.PriorityLevel; props.MessageId = message.MessageId == null ? new string(message.MessageId) diff --git a/src/HouseofCat.RabbitMQ/Messages/Letter.cs b/src/HouseofCat.RabbitMQ/Messages/Letter.cs index fde51ca1..77dc849c 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Letter.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Letter.cs @@ -43,7 +43,7 @@ public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptio props.MessageId = MessageId; // Non-optional Header. - props.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessage; + props.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; return props; } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs index 521a52bf..078d3bc4 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs @@ -13,6 +13,7 @@ public interface IReceivedData IModel Channel { get; set; } string ContentType { get; } + string ObjectType { get; } bool Encrypted { get; } string EncryptionType { get; } DateTime EncryptedDateTime { get; } @@ -50,6 +51,7 @@ public class ReceivedData : IReceivedData, IDisposable // Headers public string ContentType { get; private set; } + public string ObjectType { get; private set; } public bool Encrypted { get; private set; } public string EncryptionType { get; private set; } public DateTime EncryptedDateTime { get; private set; } @@ -92,12 +94,19 @@ public ReceivedData( private void ReadHeaders() { - if (Properties?.Headers != null && Properties.Headers.TryGetValue(Constants.HeaderForObjectType, out object objectType)) + if (Properties?.Headers is null) return; + + if (Properties.Headers.TryGetValue(Constants.HeaderForObjectType, out object objectType)) { - ContentType = Encoding.UTF8.GetString((byte[])objectType); + ObjectType = Encoding.UTF8.GetString((byte[])objectType); + + if (Properties.Headers.TryGetValue(Constants.HeaderForObjectType, out object contentType)) + { + ContentType = Encoding.UTF8.GetString((byte[])contentType); + } - // ADD SERIALIZER TO HEADER AND && JSON THIS ONE - if (ContentType == Constants.HeaderValueForMessage && Data.Length > 0) + if (ObjectType == Constants.HeaderValueForMessageObjectType + && Data.Length > 0) { try { Letter = JsonSerializer.Deserialize(Data.Span); } @@ -122,7 +131,11 @@ private void ReadHeaders() } else { - ContentType = Constants.HeaderValueForUnknown; + ObjectType = Constants.HeaderValueForUnknownObjectType; + if (Properties.Headers.TryGetValue(Constants.HeaderForContentType, out object contentType)) + { + ContentType = Encoding.UTF8.GetString((byte[])contentType); + } } } diff --git a/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs b/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs index ffc0ceb7..8182f1f9 100644 --- a/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs @@ -14,7 +14,9 @@ public class RoutingOptions [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] public byte PriorityLevel { get; set; } - public string MessageType { get; set; } = $"{Enums.ContentType.Json.Description()} {Enums.Charset.Utf8.Description()}"; + private static readonly string _jsonContentType = "application/json;"; + + public string ContentType { get; set; } = _jsonContentType; public static RoutingOptions CreateDefaultRoutingOptions(byte priority = 0) { diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index ff9f6b28..7627a3ef 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -1,8 +1,8 @@ using HouseofCat.Compression; using HouseofCat.Encryption; -using HouseofCat.Utilities; using HouseofCat.RabbitMQ.Pools; using HouseofCat.Serialization; +using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; using HouseofCat.Utilities.Time; using Microsoft.Extensions.Logging; @@ -359,7 +359,7 @@ public async Task PublishAsync( } // Non-optional Header. - messageProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessage; + messageProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; try { @@ -456,7 +456,7 @@ public async Task PublishBatchAsync( } // Non-optional Header. - messageProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessage; + messageProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; try { @@ -781,7 +781,7 @@ private static IBasicProperties BuildProperties( } // Non-optional Header. - props.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessage; + props.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; return props; } diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index 1e2bac4f..ceef3ca1 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -2,7 +2,6 @@ using HouseofCat.RabbitMQ.Pools; using HouseofCat.Serialization; using Microsoft.Extensions.Logging; -using System; using System.Text; using System.Threading.Channels; diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs index ff628b03..2451c5c7 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs @@ -22,6 +22,6 @@ public ConsumerDataflow BuildConsumerDataflow(IRabbitService ra WorkflowName, ConsumerName, ConsumerCount) - .WithBuildState(WorkStateKey); + .WithBuildState(); } } From c5235b61adf627f3f977b720a659311ede364549 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 09:58:58 -0500 Subject: [PATCH 06/47] Refactoring for Publish Telemetry and create and use the traceparent header on Publish and Consumers. --- .../ChannelReaderBlock.cs | 4 +- .../ChannelReaderBlockEngine.cs | 4 +- src/HouseofCat.Dataflows/DataflowEngine.cs | 4 +- .../Extensions/WorkStateExtensions.cs | 105 ++++---- src/HouseofCat.Dataflows/IWorkState.cs | 4 +- src/HouseofCat.Dataflows/Pipeline.cs | 4 +- src/HouseofCat.RabbitMQ/Constants.cs | 2 + src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 4 +- .../Dataflows/ConsumerBlock.cs | 6 +- .../Dataflows/ConsumerDataflow.cs | 23 +- .../Dataflows/RabbitWorkState.cs | 2 +- src/HouseofCat.RabbitMQ/Messages/Letter.cs | 11 +- .../Messages/ReceivedData.cs | 6 + .../Options/RoutingOptions.cs | 6 +- .../Pipelines/ConsumerPipeline.cs | 4 +- src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs | 4 +- src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs | 4 +- .../Pools/ConnectionHost.cs | 4 +- .../Pools/ConnectionPool.cs | 6 +- .../Publisher/Publisher.cs | 54 ++-- .../Services/RabbitService.cs | 8 +- .../Extensions/AssemblyExtensions.cs | 23 +- .../Helpers/AppHelpers.cs | 43 ++++ .../LogHelper.cs => Helpers/LogHelpers.cs} | 4 +- .../Helpers/OpenTelemetryHelpers.cs | 238 ++++++++++++++++++ .../{Time/Time.cs => Helpers/TimeHelpers.cs} | 4 +- .../HouseofCat.Utilities.csproj | 4 + src/HouseofCat.Utilities/Runtime/App.cs | 19 -- tests/OpenTelemetry.Console.Tests/Program.cs | 8 +- .../Tests/WorkStateTests.cs | 8 +- tests/RabbitMQ.Console.Tests/Program.cs | 6 +- .../Program.cs | 10 +- .../Serialization/SerializationTests.cs | 2 +- .../UnitTests/Transform/DataTransformTests.cs | 2 +- .../Transform/RecyclableTransformTests.cs | 2 +- 35 files changed, 461 insertions(+), 181 deletions(-) create mode 100644 src/HouseofCat.Utilities/Helpers/AppHelpers.cs rename src/HouseofCat.Utilities/{Logging/LogHelper.cs => Helpers/LogHelpers.cs} (94%) create mode 100644 src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs rename src/HouseofCat.Utilities/{Time/Time.cs => Helpers/TimeHelpers.cs} (91%) delete mode 100644 src/HouseofCat.Utilities/Runtime/App.cs diff --git a/src/HouseofCat.Dataflows/ChannelReaderBlock.cs b/src/HouseofCat.Dataflows/ChannelReaderBlock.cs index 7afbc6ab..da197403 100644 --- a/src/HouseofCat.Dataflows/ChannelReaderBlock.cs +++ b/src/HouseofCat.Dataflows/ChannelReaderBlock.cs @@ -3,8 +3,8 @@ using System.Threading.Channels; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; -using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; namespace HouseofCat.Dataflows; @@ -27,7 +27,7 @@ public ChannelReaderBlock(ChannelReader channelReader, ExecutionDataflowBl protected ChannelReaderBlock(ChannelReader channelReader, ITargetBlock targetBlock) { Guard.AgainstNull(channelReader, nameof(channelReader)); - _logger = LogHelper.LoggerFactory.CreateLogger>(); + _logger = LogHelpers.LoggerFactory.CreateLogger>(); _channelReader = channelReader; if (targetBlock is ISourceBlock sourceBlock) diff --git a/src/HouseofCat.Dataflows/ChannelReaderBlockEngine.cs b/src/HouseofCat.Dataflows/ChannelReaderBlockEngine.cs index 27a7cd78..5fdf4314 100644 --- a/src/HouseofCat.Dataflows/ChannelReaderBlockEngine.cs +++ b/src/HouseofCat.Dataflows/ChannelReaderBlockEngine.cs @@ -3,7 +3,7 @@ using System.Threading.Channels; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; -using HouseofCat.Utilities; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; namespace HouseofCat.Dataflows; @@ -38,7 +38,7 @@ protected ChannelReaderBlockEngine( TaskScheduler taskScheduler, Func postWorkBodyAsync = null) { - _logger = LogHelper.GetLogger>(); + _logger = LogHelpers.GetLogger>(); _workBodyAsync = workBodyAsync ?? throw new ArgumentNullException(nameof(workBodyAsync)); _executeOptions = new ExecutionDataflowBlockOptions diff --git a/src/HouseofCat.Dataflows/DataflowEngine.cs b/src/HouseofCat.Dataflows/DataflowEngine.cs index 2e45d3ee..08a9fc30 100644 --- a/src/HouseofCat.Dataflows/DataflowEngine.cs +++ b/src/HouseofCat.Dataflows/DataflowEngine.cs @@ -1,4 +1,4 @@ -using HouseofCat.Utilities; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using System; using System.Threading.Tasks; @@ -35,7 +35,7 @@ public DataflowEngine( int boundedCapacity = 1000, TaskScheduler taskScheduler = null) { - _logger = LogHelper.GetLogger>(); + _logger = LogHelpers.GetLogger>(); _workBodyAsync = workBodyAsync ?? throw new ArgumentNullException(nameof(workBodyAsync)); _bufferBlock = new BufferBlock( diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index d7525676..12afd13f 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -1,4 +1,5 @@ using HouseofCat.Utilities.Extensions; +using HouseofCat.Utilities.Helpers; using OpenTelemetry.Trace; using System.Collections.Generic; using System.Diagnostics; @@ -13,7 +14,7 @@ public static void SetOpenTelemetryError(this IWorkState state, string message = if (state is null) return; state.SetCurrentActivityAsError(message); - if (state.RootSpan is null) return; + if (state.WorkflowSpan is null) return; state.SetCurrentSpanAsError(message); } @@ -46,24 +47,27 @@ public static void SetSpanAsError(this IWorkState state, TelemetrySpan span, str { if (span is null) return; - span.SetStatus(Status.Error); - span.SetAttribute("Error", message); if (state?.EDI is not null) { + span.SetStatus(Status.Error.WithDescription(message ?? state.EDI.SourceException?.Message)); if (state.EDI.SourceException is not null) { span.RecordException(state.EDI.SourceException); } } + else + { + span.SetStatus(Status.Error.WithDescription(message)); + } } - public static string DefaultRootSpanNameFormat { get; set; } = "{0}.root"; - public static string DefaultChildSpanNameFormat { get; set; } = "{0}.{1}.child"; + public static string DefaultSpanNameFormat { get; set; } = "{0}.workflow"; + public static string DefaultChildSpanNameFormat { get; set; } = "{0}.{1}.workflow.step"; public static string DefaultWorkflowNameKey { get; set; } = "hoc.workflow.name"; public static string DefaultWorkflowVersionKey { get; set; } = "hoc.workflow.version"; /// - /// Start a new RootSpan with respect to State. + /// Start a new RootSpan/ChildSpan with respect to WorkState. /// /// /// @@ -71,12 +75,13 @@ public static void SetSpanAsError(this IWorkState state, TelemetrySpan span, str /// /// /// - public static void StartRootSpan( + public static void StartWorkflowSpan( this IWorkState state, string workflowName, string workflowVersion = null, SpanKind spanKind = SpanKind.Internal, - IEnumerable> suppliedAttributes = null) + IEnumerable> suppliedAttributes = null, + string traceHeader = null) { if (state is null) return; @@ -86,18 +91,10 @@ public static void StartRootSpan( workflowVersion = assembly.GetExecutingSemanticVersion(); } - var traceProvider = TracerProvider - .Default - .GetTracer( - workflowName, - workflowVersion); - - if (traceProvider is null) return; - state.Data[DefaultWorkflowNameKey] = workflowName; state.Data[DefaultWorkflowVersionKey] = workflowVersion; - var rootSpanName = string.Format(DefaultRootSpanNameFormat, workflowName); + var spanName = string.Format(DefaultSpanNameFormat, workflowName); var attributes = new SpanAttributes(); attributes.Add(DefaultWorkflowNameKey, workflowName); @@ -111,11 +108,29 @@ public static void StartRootSpan( } } - state.RootSpan = traceProvider + if (traceHeader is not null) + { + var spanContext = OpenTelemetryHelpers.ExtractSpanContext(traceHeader); + if (spanContext.HasValue) + { + state.WorkflowSpan = OpenTelemetryHelpers + .StartActiveChildSpan( + workflowName?.ToString(), + spanName, + spanContext.Value, + workflowVersion?.ToString(), + spanKind: spanKind, + attributes: attributes); + } + } + + state.WorkflowSpan = OpenTelemetryHelpers .StartRootSpan( - rootSpanName, + workflowName?.ToString(), + spanName, + workflowVersion?.ToString(), spanKind, - initialAttributes: attributes); + attributes: attributes); } /// @@ -130,7 +145,7 @@ public static TelemetrySpan CreateActiveSpan( this IWorkState state, string spanName, SpanKind spanKind = SpanKind.Internal, - SpanAttributes spanAttributes = null) + IEnumerable> suppliedAttributes = null) { if (state?.Data is null) return null; @@ -139,22 +154,24 @@ public static TelemetrySpan CreateActiveSpan( if (workflowName is null) return null; - var traceProvider = TracerProvider - .Default - .GetTracer( - workflowName.ToString(), - workflowVersion?.ToString()); - - if (traceProvider is null) return null; + var attributes = new SpanAttributes(); - var childSpanName = string.Format(DefaultChildSpanNameFormat, workflowName, spanName); + if (suppliedAttributes is not null) + { + foreach (var kvp in suppliedAttributes) + { + attributes.Add(kvp.Key, kvp.Value); + } + } - return traceProvider + return OpenTelemetryHelpers .StartActiveSpan( - childSpanName, + workflowName?.ToString(), + spanName, + workflowVersion?.ToString(), spanKind, - parentContext: state.RootSpan?.Context ?? default, - initialAttributes: spanAttributes); + parentContext: state.WorkflowSpan?.Context ?? default, + attributes: attributes); } /// @@ -170,29 +187,23 @@ public static TelemetrySpan CreateActiveChildSpan( string spanName, SpanContext spanContext, SpanKind spanKind = SpanKind.Internal, - SpanAttributes spanAttributes = null) + SpanAttributes attributes = null) { if (state?.Data is null) return null; state.Data.TryGetValue(DefaultWorkflowNameKey, out var workflowName); state.Data.TryGetValue(DefaultWorkflowVersionKey, out var workflowVersion); - var traceProvider = TracerProvider - .Default - .GetTracer( - workflowName.ToString(), - workflowVersion?.ToString()); - - if (traceProvider is null) return null; - var childSpanName = string.Format(DefaultChildSpanNameFormat, workflowName, spanName); - return traceProvider - .StartActiveSpan( + return OpenTelemetryHelpers + .StartActiveChildSpan( + workflowName?.ToString(), childSpanName, + spanContext, + workflowVersion?.ToString(), spanKind, - parentContext: spanContext, - initialAttributes: spanAttributes); + attributes: attributes); } public static void EndRootSpan( @@ -205,6 +216,6 @@ public static void EndRootSpan( { state.SetOpenTelemetryError(); } - state.RootSpan?.Dispose(); + state.WorkflowSpan?.Dispose(); } } diff --git a/src/HouseofCat.Dataflows/IWorkState.cs b/src/HouseofCat.Dataflows/IWorkState.cs index adfdba54..aef1bc59 100644 --- a/src/HouseofCat.Dataflows/IWorkState.cs +++ b/src/HouseofCat.Dataflows/IWorkState.cs @@ -19,6 +19,6 @@ public interface IWorkState // Outbound byte[] SendData { get; set; } - // RootSpan - TelemetrySpan RootSpan { get; set; } + // RootSpan or ChildSpan derived from TraceParentHeader + TelemetrySpan WorkflowSpan { get; set; } } diff --git a/src/HouseofCat.Dataflows/Pipeline.cs b/src/HouseofCat.Dataflows/Pipeline.cs index bb386c6b..bccb081a 100644 --- a/src/HouseofCat.Dataflows/Pipeline.cs +++ b/src/HouseofCat.Dataflows/Pipeline.cs @@ -1,4 +1,4 @@ -using HouseofCat.Utilities; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; @@ -51,7 +51,7 @@ public Pipeline( TaskScheduler taskScheduler = null) { _cts = new CancellationTokenSource(); - _logger = LogHelper.GetLogger>(); + _logger = LogHelpers.GetLogger>(); _linkStepOptions = new DataflowLinkOptions { PropagateCompletion = true }; _executeStepOptions = new ExecutionDataflowBlockOptions diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index 339f440a..ea015537 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -21,6 +21,8 @@ public static class Constants public static string HeaderForCompressed { get; set; } = "X-RD-COMPRESSED"; public static string HeaderForCompression { get; set; } = "X-RD-COMPRESSION"; + public static string HeaderForTraceParent { get; set; } = "traceparent"; + public const string RangeErrorMessage = "Value for {0} must be between {1} and {2}."; // Pipeline diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 41ec376b..581d4177 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -1,5 +1,4 @@ using HouseofCat.Dataflows; -using HouseofCat.Utilities; using HouseofCat.RabbitMQ.Pools; using HouseofCat.Utilities.Errors; using Microsoft.Extensions.Logging; @@ -11,6 +10,7 @@ using System.Threading.Channels; using System.Threading.Tasks; using static HouseofCat.RabbitMQ.LogMessages; +using HouseofCat.Utilities.Helpers; namespace HouseofCat.RabbitMQ; @@ -100,7 +100,7 @@ public Consumer(IChannelPool channelPool, ConsumerOptions consumerOptions) Guard.AgainstNull(channelPool, nameof(channelPool)); Guard.AgainstNull(consumerOptions, nameof(consumerOptions)); - _logger = LogHelper.GetLogger(); + _logger = LogHelpers.GetLogger(); Options = channelPool.Options; ChannelPool = channelPool; ConsumerOptions = consumerOptions; diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs index da16cd99..f003fc00 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs @@ -1,5 +1,5 @@ -using HouseofCat.Utilities; -using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using System; using System.Threading; @@ -35,7 +35,7 @@ protected ConsumerBlock(ITargetBlock bufferBlock) : this(bufferBlock, (ISo protected ConsumerBlock(ITargetBlock bufferBlock, ISourceBlock sourceBufferBlock) { - _logger = LogHelper.LoggerFactory.CreateLogger>(); + _logger = LogHelpers.LoggerFactory.CreateLogger>(); _bufferBlock = bufferBlock; _sourceBufferBlock = sourceBufferBlock; Completion = _bufferBlock.Completion; diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index f16a8173..3c50614e 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -5,6 +5,7 @@ using HouseofCat.RabbitMQ.Services; using HouseofCat.Serialization; using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Helpers; using OpenTelemetry.Trace; using System; using System.Collections.Generic; @@ -618,7 +619,11 @@ public virtual TState BuildState(ReceivedData data) attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id)); } - state.StartRootSpan(WorkflowName, spanKind: SpanKind.Consumer, suppliedAttributes: attributes); + state.StartWorkflowSpan( + WorkflowName, + spanKind: SpanKind.Consumer, + suppliedAttributes: attributes, + traceHeader: data.TraceParentHeader); return state; } @@ -645,7 +650,11 @@ public virtual TState BuildStateAndPayload(string key, ReceivedData data) attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id)); } - state.StartRootSpan(WorkflowName, spanKind: SpanKind.Consumer, suppliedAttributes: attributes); + state.StartWorkflowSpan( + WorkflowName, + spanKind: SpanKind.Consumer, + suppliedAttributes: attributes, + traceHeader: data.TraceParentHeader); if (_serializationProvider != null && data.ObjectType != Constants.HeaderValueForUnknownObjectType) @@ -692,11 +701,11 @@ public TransformBlock GetByteManipulationTransformBlock( ExecutionDataflowBlockOptions options, bool outbound, Predicate predicate, - string metricIdentifier) + string spanName) { TState WrapAction(TState state) { - using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); + using var childSpan = state.CreateActiveChildSpan(spanName, state.WorkflowSpan.Context, SpanKind.Consumer); try { if (outbound) @@ -739,11 +748,11 @@ public TransformBlock GetByteManipulationTransformBlock( ExecutionDataflowBlockOptions options, bool outbound, Predicate predicate, - string metricIdentifier) + string spanName) { async Task WrapActionAsync(TState state) { - using var childSpan = state.CreateActiveSpan(metricIdentifier, SpanKind.Consumer); + using var childSpan = state.CreateActiveChildSpan(spanName, state.WorkflowSpan.Context, SpanKind.Consumer); try { @@ -788,7 +797,7 @@ public TransformBlock GetWrappedPublishTransformBlock( { async Task WrapPublishAsync(TState state) { - using var childSpan = state.CreateActiveSpan(PublishStepIdentifier, SpanKind.Producer); + using var childSpan = state.CreateActiveChildSpan(PublishStepIdentifier, state.WorkflowSpan.Context, SpanKind.Producer); try { await service.Publisher.PublishAsync(state.SendMessage, true, true).ConfigureAwait(false); diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index efbee26b..edf0d9ef 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -31,5 +31,5 @@ public abstract class RabbitWorkState : IRabbitWorkState public bool IsFaulted { get; set; } public ExceptionDispatchInfo EDI { get; set; } - public TelemetrySpan RootSpan { get; set; } + public TelemetrySpan WorkflowSpan { get; set; } } diff --git a/src/HouseofCat.RabbitMQ/Messages/Letter.cs b/src/HouseofCat.RabbitMQ/Messages/Letter.cs index 77dc849c..3b2a214e 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Letter.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Letter.cs @@ -1,5 +1,6 @@ using HouseofCat.RabbitMQ.Pools; using HouseofCat.Serialization; +using HouseofCat.Utilities.Helpers; using RabbitMQ.Client; using System; using System.Collections.Generic; @@ -39,13 +40,15 @@ public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptio { MessageId ??= Guid.NewGuid().ToString(); - var props = this.CreateBasicProperties(channelHost, withOptionalHeaders, Metadata); - props.MessageId = MessageId; + var basicProperties = this.CreateBasicProperties(channelHost, withOptionalHeaders, Metadata); + basicProperties.MessageId = MessageId; // Non-optional Header. - props.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + var openTelHeader = OpenTelemetryHelpers.CreateOpenTelemetryHeaderFromCurrentActivityOrDefault(); + basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; - return props; + return basicProperties; } public Letter() { } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs index 078d3bc4..0474c5c4 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs @@ -57,6 +57,7 @@ public class ReceivedData : IReceivedData, IDisposable public DateTime EncryptedDateTime { get; private set; } public bool Compressed { get; private set; } public string CompressionType { get; private set; } + public string TraceParentHeader { get; private set; } private readonly TaskCompletionSource _completionSource = new TaskCompletionSource(); public Task Completion => _completionSource.Task; @@ -137,6 +138,11 @@ private void ReadHeaders() ContentType = Encoding.UTF8.GetString((byte[])contentType); } } + + if (Properties.Headers.TryGetValue(Constants.HeaderForTraceParent, out object traceParentHeader)) + { + TraceParentHeader = Encoding.UTF8.GetString((byte[])traceParentHeader); + } } /// diff --git a/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs b/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs index 8182f1f9..8029047a 100644 --- a/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs @@ -3,7 +3,7 @@ namespace HouseofCat.RabbitMQ; -public class RoutingOptions +public record RoutingOptions { [Range(1, 2, ErrorMessage = Constants.RangeErrorMessage)] public byte DeliveryMode { get; set; } = 2; @@ -14,9 +14,7 @@ public class RoutingOptions [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] public byte PriorityLevel { get; set; } - private static readonly string _jsonContentType = "application/json;"; - - public string ContentType { get; set; } = _jsonContentType; + public string ContentType { get; set; } = Constants.HeaderValueForContentTypeApplicationJson; public static RoutingOptions CreateDefaultRoutingOptions(byte priority = 0) { diff --git a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs index eff39847..370dd95f 100644 --- a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs +++ b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs @@ -1,6 +1,6 @@ using HouseofCat.Dataflows.Pipelines; using HouseofCat.RabbitMQ.Dataflows; -using HouseofCat.Utilities; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using System; using System.Threading; @@ -40,7 +40,7 @@ public ConsumerPipeline( IConsumer consumer, IPipeline pipeline) { - _logger = LogHelper.GetLogger>(); + _logger = LogHelpers.GetLogger>(); Pipeline = pipeline ?? throw new ArgumentNullException(nameof(pipeline)); Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer)); ConsumerOptions = consumer.ConsumerOptions ?? throw new ArgumentNullException(nameof(consumer.Options)); diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs index d688aeb1..bbfd58af 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs @@ -1,5 +1,5 @@ -using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.Events; @@ -50,7 +50,7 @@ public class ChannelHost : IChannelHost, IDisposable public ChannelHost(ulong channelId, IConnectionHost connHost, bool ackable) { - _logger = LogHelper.GetLogger(); + _logger = LogHelpers.GetLogger(); ChannelId = channelId; _connHost = connHost; diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs index cdeca25d..414c122b 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs @@ -1,5 +1,5 @@ -using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using System; using System.Collections.Concurrent; @@ -77,7 +77,7 @@ public ChannelPool(IConnectionPool connPool) ? 10000 : Options.PoolOptions.TansientChannelStartRange; - _logger = LogHelper.GetLogger(); + _logger = LogHelpers.GetLogger(); _connectionPool = connPool; _flaggedChannels = new ConcurrentDictionary(); diff --git a/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs b/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs index 49db835a..4ce1a479 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs @@ -1,4 +1,4 @@ -using HouseofCat.Utilities; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.Events; @@ -37,7 +37,7 @@ public class ConnectionHost : IConnectionHost, IDisposable public ConnectionHost(ulong connectionId, IConnection connection) { - _logger = LogHelper.GetLogger(); + _logger = LogHelpers.GetLogger(); ConnectionId = connectionId; AssignConnection(connection); diff --git a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs index f614b555..03acdb88 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs @@ -1,5 +1,5 @@ -using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.OAuth2; @@ -40,7 +40,7 @@ public ConnectionPool(RabbitOptions options, HttpClientHandler oauth2ClientHandl Guard.AgainstNull(options, nameof(options)); Options = options; - _logger = LogHelper.GetLogger(); + _logger = LogHelpers.GetLogger(); _connections = Channel.CreateBounded(Options.PoolOptions.MaxConnections); @@ -61,7 +61,7 @@ public ConnectionPool(RabbitOptions options) Guard.AgainstNull(options, nameof(options)); Options = options; - _logger = LogHelper.GetLogger(); + _logger = LogHelpers.GetLogger(); _connections = Channel.CreateBounded(Options.PoolOptions.MaxConnections); _connectionFactory = BuildConnectionFactory(); diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index 7627a3ef..d2ac36a2 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -2,9 +2,8 @@ using HouseofCat.Encryption; using HouseofCat.RabbitMQ.Pools; using HouseofCat.Serialization; -using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; -using HouseofCat.Utilities.Time; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using System; @@ -94,7 +93,7 @@ public class Publisher : IPublisher, IDisposable private Task _processReceiptsAsync; private bool _disposedValue; - public string TimeFormat { get; set; } = Time.Formats.CatsAltFormat; + public string TimeFormat { get; set; } = TimeHelpers.Formats.CatsAltFormat; public Publisher( RabbitOptions options, @@ -118,7 +117,7 @@ public Publisher( Guard.AgainstNull(serializationProvider, nameof(serializationProvider)); Options = channelPool.Options; - _logger = LogHelper.GetLogger(); + _logger = LogHelpers.GetLogger(); _serializationProvider = serializationProvider; if (Options.PublisherOptions.Encrypt && encryptionProvider == null) @@ -287,7 +286,7 @@ private async Task ProcessMessagesAsync(ChannelReader channelReader) metadata.Encrypted = _encrypt; metadata.CustomFields[Constants.HeaderForEncrypted] = _encrypt; metadata.CustomFields[Constants.HeaderForEncryption] = _encryptionProvider.Type; - metadata.CustomFields[Constants.HeaderForEncryptDate] = Time.GetDateTimeNow(Time.Formats.RFC3339Long); + metadata.CustomFields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.RFC3339Long); } _logger.LogDebug(LogMessages.AutoPublishers.MessagePublished, message.MessageId, metadata?.Id); @@ -339,27 +338,26 @@ public async Task PublishAsync( string routingKey, ReadOnlyMemory payload, bool mandatory = false, - IBasicProperties messageProperties = null, + IBasicProperties basicProperties = null, string messageId = null) { Guard.AgainstBothNullOrEmpty(exchangeName, nameof(exchangeName), routingKey, nameof(routingKey)); var error = false; var channelHost = await _channelPool.GetChannelAsync().ConfigureAwait(false); - if (messageProperties == null) + if (basicProperties == null) { - messageProperties = channelHost.GetChannel().CreateBasicProperties(); - messageProperties.DeliveryMode = 2; - messageProperties.MessageId = messageId ?? Guid.NewGuid().ToString(); + basicProperties = channelHost.GetChannel().CreateBasicProperties(); + basicProperties.DeliveryMode = 2; + basicProperties.MessageId = messageId ?? Guid.NewGuid().ToString(); - if (!messageProperties.IsHeadersPresent()) + if (!basicProperties.IsHeadersPresent()) { - messageProperties.Headers = new Dictionary(); + basicProperties.Headers = new Dictionary(); } } - // Non-optional Header. - messageProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + SetMandatoryHeaders(basicProperties); try { @@ -369,7 +367,7 @@ public async Task PublishAsync( exchange: exchangeName ?? string.Empty, routingKey: routingKey, mandatory: mandatory, - basicProperties: messageProperties, + basicProperties: basicProperties, body: payload); } catch (Exception ex) @@ -743,6 +741,13 @@ await _receiptBuffer .ConfigureAwait(false); } + private static void SetMandatoryHeaders(IBasicProperties basicProperties) + { + basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + var openTelHeader = OpenTelemetryHelpers.CreateOpenTelemetryHeaderFromCurrentActivityOrDefault(); + basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; + } + /// /// Intended to bring feature parity and include properties when publishing byte[], which you get for free when publishing with IMessage objects. /// @@ -759,14 +764,14 @@ private static IBasicProperties BuildProperties( byte? priority = 0, byte? deliveryMode = 2) { - var props = channelHost.GetChannel().CreateBasicProperties(); - props.DeliveryMode = deliveryMode ?? 2; // Default Persisted - props.Priority = priority ?? 0; // Default Priority - props.MessageId = messageId ?? Guid.NewGuid().ToString(); + var basicProperties = channelHost.GetChannel().CreateBasicProperties(); + basicProperties.DeliveryMode = deliveryMode ?? 2; // Default Persisted + basicProperties.Priority = priority ?? 0; // Default Priority + basicProperties.MessageId = messageId ?? Guid.NewGuid().ToString(); - if (!props.IsHeadersPresent()) + if (!basicProperties.IsHeadersPresent()) { - props.Headers = new Dictionary(); + basicProperties.Headers = new Dictionary(); } if (headers?.Count > 0) @@ -775,15 +780,14 @@ private static IBasicProperties BuildProperties( { if (kvp.Key.StartsWith(Constants.HeaderPrefix, StringComparison.OrdinalIgnoreCase)) { - props.Headers[kvp.Key] = kvp.Value; + basicProperties.Headers[kvp.Key] = kvp.Value; } } } - // Non-optional Header. - props.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + SetMandatoryHeaders(basicProperties); - return props; + return basicProperties; } protected virtual void Dispose(bool disposing) diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index f75c71fa..d79a56ee 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -7,7 +7,7 @@ using HouseofCat.Serialization; using HouseofCat.Utilities; using HouseofCat.Utilities.Errors; -using HouseofCat.Utilities.Time; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using System; @@ -68,7 +68,7 @@ public class RabbitService : IRabbitService, IDisposable public ConcurrentDictionary> Consumers { get; private set; } = new ConcurrentDictionary>(); private ConcurrentDictionary ConsumerPipelineNameToConsumerOptions { get; set; } = new ConcurrentDictionary(); - public string TimeFormat { get; set; } = Time.Formats.CatsAltFormat; + public string TimeFormat { get; set; } = TimeHelpers.Formats.CatsAltFormat; public RabbitService( string fileNamePath, @@ -113,7 +113,7 @@ public RabbitService( { Guard.AgainstNull(chanPool, nameof(chanPool)); Guard.AgainstNull(serializationProvider, nameof(serializationProvider)); - LogHelper.LoggerFactory = loggerFactory; + LogHelpers.LoggerFactory = loggerFactory; Options = chanPool.Options; ChannelPool = chanPool; @@ -362,7 +362,7 @@ public bool Encrypt(IMessage message) metadata.Encrypted = true; metadata.CustomFields[Constants.HeaderForEncrypted] = true; metadata.CustomFields[Constants.HeaderForEncryption] = EncryptionProvider.Type; - metadata.CustomFields[Constants.HeaderForEncryptDate] = Time.GetDateTimeNow(TimeFormat); + metadata.CustomFields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeFormat); return true; } diff --git a/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs b/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs index bc535525..9d80b48a 100644 --- a/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs +++ b/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs @@ -1,4 +1,4 @@ -using System; +using HouseofCat.Utilities.Helpers; using System.Collections.Concurrent; using System.Reflection; @@ -15,29 +15,10 @@ public static string GetExecutingSemanticVersion(this Assembly assembly) return cachedVersion; } - var version = Assembly.GetExecutingAssembly().GetName().GetFlexibleSemVersion(); + var version = AppHelpers.GetFlexibleSemVersion(Assembly.GetExecutingAssembly().GetName()); _assemblyVersion.TryAdd(assembly, version); return version; } - public static string GetFlexibleSemVersion(this AssemblyName assemblyName) - { - if (assemblyName == null) - { - return null; - } - var versionParts = assemblyName - .Version - .ToString() - .Split('.', StringSplitOptions.RemoveEmptyEntries); - - return (versionParts?.Length ?? 0) switch - { - 0 => null, - 1 => versionParts[0], - 2 => $"{versionParts[0]}.{versionParts[1]}", - _ => $"{versionParts[0]}.{versionParts[1]}.{versionParts[2]}" - }; - } } diff --git a/src/HouseofCat.Utilities/Helpers/AppHelpers.cs b/src/HouseofCat.Utilities/Helpers/AppHelpers.cs new file mode 100644 index 00000000..dc052230 --- /dev/null +++ b/src/HouseofCat.Utilities/Helpers/AppHelpers.cs @@ -0,0 +1,43 @@ +using System; +using System.Reflection; + +namespace HouseofCat.Utilities.Helpers; + +public static class AppHelpers +{ + public static bool IsDebug + { + get + { +#if DEBUG + return true; +#else + return false; +#endif + } + } + + public static string Env => Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Debug"; + public static bool InDocker => Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER") == "true"; + + public static string GetFlexibleSemVersion(AssemblyName assemblyName) + { + if (assemblyName == null) + { + return null; + } + + var versionParts = assemblyName + .Version + .ToString() + .Split('.', StringSplitOptions.RemoveEmptyEntries); + + return (versionParts?.Length ?? 0) switch + { + 0 => null, + 1 => versionParts[0], + 2 => $"{versionParts[0]}.{versionParts[1]}", + _ => $"{versionParts[0]}.{versionParts[1]}.{versionParts[2]}" + }; + } +} diff --git a/src/HouseofCat.Utilities/Logging/LogHelper.cs b/src/HouseofCat.Utilities/Helpers/LogHelpers.cs similarity index 94% rename from src/HouseofCat.Utilities/Logging/LogHelper.cs rename to src/HouseofCat.Utilities/Helpers/LogHelpers.cs index f0b3b750..d564174a 100644 --- a/src/HouseofCat.Utilities/Logging/LogHelper.cs +++ b/src/HouseofCat.Utilities/Helpers/LogHelpers.cs @@ -1,9 +1,9 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace HouseofCat.Utilities; +namespace HouseofCat.Utilities.Helpers; -public static class LogHelper +public static class LogHelpers { private readonly static object _syncObj = new object(); private static ILoggerFactory _factory; diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs new file mode 100644 index 00000000..7d66f190 --- /dev/null +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -0,0 +1,238 @@ +using OpenTelemetry.Trace; +using System; +using System.Collections.Generic; +using System.Diagnostics; + +namespace HouseofCat.Utilities.Helpers; + +public static class OpenTelemetryHelpers +{ + public static string CreateOpenTelemetryHeader( + string traceId, + string spanId, + string version = "00", + ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) + { + return $"{version}-{traceId}-{spanId}-{(int)activityTraceFlags:00}"; + } + + public static string CreateOpenTelemetryHeaderFromCurrentActivityOrDefault( + ActivityTraceFlags flags = ActivityTraceFlags.None) + { + var activity = Activity.Current; + if (activity is null) + { + return CreateOpenTelemetryHeader( + ActivityTraceId.CreateRandom().ToHexString(), + ActivitySpanId.CreateRandom().ToHexString(), + "00", + flags); + } + + return CreateOpenTelemetryHeader( + activity.TraceId.ToHexString(), + activity.SpanId.ToHexString(), + "00", + activity.ActivityTraceFlags); + } + + public static string GetCurrentActivityId() + { + return Activity.Current?.Id; + } + + public static ActivitySpanId? GetCurrentActivitySpanId() + { + return Activity.Current?.SpanId; + } + + public static void SetCurrentActivityParentId(string parentId) + { + Activity.Current?.SetParentId(parentId); + } + + public static SpanContext? ExtractSpanContext(string traceHeader) + { + if (string.IsNullOrEmpty(traceHeader)) return default; + + var split = traceHeader.Split('-'); + + try + { + // With Version + var activityTraceFlags = ActivityTraceFlags.None; + ActivityTraceId activityTraceId; + ActivitySpanId activitySpanId; + if (split.Length == 4 + && split[0].Length == 2 + && split[1].Length == 32 + && split[2].Length == 16 + && split[3].Length == 2) + { + activityTraceId = ActivityTraceId.CreateFromString(split[1].AsSpan()); + activitySpanId = ActivitySpanId.CreateFromString(split[2].AsSpan()); + if (int.TryParse(split[3], out var flagsAsInt)) + { + activityTraceFlags = (ActivityTraceFlags)flagsAsInt; + } + } + // Without Version Pre-Appended + else if (split.Length == 3 + && split[0].Length == 32 + && split[1].Length == 16 + && split[2].Length == 2) + { + activityTraceId = ActivityTraceId.CreateFromString(split[0].AsSpan()); + activitySpanId = ActivitySpanId.CreateFromString(split[1].AsSpan()); + if (int.TryParse(split[2], out var flagsAsInt)) + { + activityTraceFlags = (ActivityTraceFlags)flagsAsInt; + } + } + else + { + return default; + } + + return new SpanContext(activityTraceId, activitySpanId, activityTraceFlags, isRemote: true); + } + catch + { + return default; + } + } + + public static void SetCurrentActivityParentIdFromTraceHeader(string traceheader) + { + if (string.IsNullOrEmpty(traceheader)) return; + + var activityTraceFlags = ActivityTraceFlags.None; + var split = traceheader.Split('-'); + + // With Version + ActivityTraceId activityTraceId; + ActivitySpanId activitySpanId; + if (split.Length == 4 && split[3].Length == 2) + { + activityTraceId = ActivityTraceId.CreateFromString(split[1].AsSpan()); + activitySpanId = ActivitySpanId.CreateFromString(split[2].AsSpan()); + if (int.TryParse(split[3], out var flagsAsInt)) + { + activityTraceFlags = (ActivityTraceFlags)flagsAsInt; + } + } + // Without Version Pre-Appended + else if (split.Length == 3 && split[0].Length > 2) + { + activityTraceId = ActivityTraceId.CreateFromString(split[0].AsSpan()); + activitySpanId = ActivitySpanId.CreateFromString(split[1].AsSpan()); + if (int.TryParse(split[2], out var flagsAsInt)) + { + activityTraceFlags = (ActivityTraceFlags)flagsAsInt; + } + } + else + { + return; + } + + Activity.Current?.SetParentId( + activityTraceId, + activitySpanId, + activityTraceFlags); + } + + public static void SetCurrentActivityParentId( + string traceId, + string spanId, + ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) + { + Activity.Current?.SetParentId( + ActivityTraceId.CreateFromString(traceId), + ActivitySpanId.CreateFromString(spanId), + activityTraceFlags); + } + + public static void SetCurrentActivityParentId( + ActivityTraceId traceId, + ActivitySpanId spanId, + ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) + { + Activity.Current?.SetParentId(traceId, spanId, activityTraceFlags); + } + + public static TelemetrySpan StartRootSpan( + string sourceName, + string spanName, + string sourceVersion = null, + SpanKind spanKind = SpanKind.Internal, + SpanAttributes attributes = null, + IEnumerable links = null, + DateTimeOffset startTime = default) + { + var provider = TracerProvider + .Default + .GetTracer( + sourceName, + sourceVersion); + + if (provider is null) return null; + + return provider.StartRootSpan( + spanName, + spanKind, + initialAttributes: attributes, + links: links, + startTime: startTime); + } + + public static TelemetrySpan StartActiveSpan( + string sourceName, + string spanName, + string sourceVersion = null, + SpanKind spanKind = SpanKind.Internal, + SpanContext parentContext = default, + SpanAttributes attributes = null, + IEnumerable links = null, + DateTimeOffset startTime = default) + { + var provider = TracerProvider + .Default + .GetTracer( + sourceName, + sourceVersion); + + if (provider is null) return null; + + return provider.StartActiveSpan( + spanName, + spanKind, + initialAttributes: attributes, + parentContext: parentContext, + links: links, + startTime: startTime); + } + + public static TelemetrySpan StartActiveChildSpan( + string sourceName, + string spanName, + SpanContext parentContext, + string sourceVersion = null, + SpanKind spanKind = SpanKind.Internal, + SpanAttributes attributes = null) + { + var provider = TracerProvider + .Default + .GetTracer( + sourceName, + sourceVersion); + + if (provider is null) return null; + + return provider.StartActiveSpan( + spanName, + spanKind, + initialAttributes: attributes, + parentContext: parentContext); + } +} diff --git a/src/HouseofCat.Utilities/Time/Time.cs b/src/HouseofCat.Utilities/Helpers/TimeHelpers.cs similarity index 91% rename from src/HouseofCat.Utilities/Time/Time.cs rename to src/HouseofCat.Utilities/Helpers/TimeHelpers.cs index 00ea0baa..43bed52d 100644 --- a/src/HouseofCat.Utilities/Time/Time.cs +++ b/src/HouseofCat.Utilities/Helpers/TimeHelpers.cs @@ -1,9 +1,9 @@ using System; using System.Globalization; -namespace HouseofCat.Utilities.Time; +namespace HouseofCat.Utilities.Helpers; -public static class Time +public static class TimeHelpers { public static string GetDateTimeNow(string format) => DateTime.Now.ToString(format, DateTimeFormatInfo.InvariantInfo); public static string GetDateTimeUtcNow(string format) => DateTime.UtcNow.ToString(format, DateTimeFormatInfo.InvariantInfo); diff --git a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj index 3d2cc8e6..164178b9 100644 --- a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj +++ b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj @@ -5,6 +5,10 @@ + + + + true diff --git a/src/HouseofCat.Utilities/Runtime/App.cs b/src/HouseofCat.Utilities/Runtime/App.cs deleted file mode 100644 index 587bffd5..00000000 --- a/src/HouseofCat.Utilities/Runtime/App.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; - -namespace HouseofCat.Utilities.Runtime; - -public static class App -{ - public static bool IsDebug - { - get - { -#if DEBUG - return true; -#else - return false; -#endif - } - } - public static string Env => Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Debug"; -} diff --git a/tests/OpenTelemetry.Console.Tests/Program.cs b/tests/OpenTelemetry.Console.Tests/Program.cs index 0c2f3389..cc70d86a 100644 --- a/tests/OpenTelemetry.Console.Tests/Program.cs +++ b/tests/OpenTelemetry.Console.Tests/Program.cs @@ -1,12 +1,12 @@ -using HouseofCat.Utilities; -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using OpenTelemetry; using OpenTelemetry.Resources; using OpenTelemetry.Trace; using OpenTelemetry.Tests; +using HouseofCat.Utilities.Helpers; -var loggerFactory = LogHelper.CreateConsoleLoggerFactory(LogLevel.Information); -LogHelper.LoggerFactory = loggerFactory; +var loggerFactory = LogHelpers.CreateConsoleLoggerFactory(LogLevel.Information); +LogHelpers.LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger(); var applicationName = "OpenTelemetry.ConsoleTests"; diff --git a/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs b/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs index d111171e..cc3fbcdd 100644 --- a/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs +++ b/tests/OpenTelemetry.Console.Tests/Tests/WorkStateTests.cs @@ -21,7 +21,7 @@ public static void RunRootSpanTest(ILogger logger, string workflowName) var workstate = new CustomWorkState(); - workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); + workstate.StartWorkflowSpan(workflowName, spanKind: SpanKind.Internal); workstate.EndRootSpan(); logger.LogInformation($"Finished {nameof(RunRootSpanTest)}."); @@ -33,7 +33,7 @@ public static void RunRootSpanWithChildSpanTest(ILogger logger, string workflowN var workstate = new CustomWorkState(); - workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); + workstate.StartWorkflowSpan(workflowName, spanKind: SpanKind.Internal); using (var span = workstate.CreateActiveSpan("ChildStep", spanKind: SpanKind.Internal)) { span.SetStatus(Status.Ok); @@ -49,7 +49,7 @@ public static void RunRootSpanWithChildSpanErrorTest(ILogger logger, string work var workstate = new CustomWorkState(); - workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); + workstate.StartWorkflowSpan(workflowName, spanKind: SpanKind.Internal); using (var span = workstate.CreateActiveSpan("ChildStep", spanKind: SpanKind.Internal)) { @@ -67,7 +67,7 @@ public static void RunRootSpanWithManyChildSpanFlatLevelTest(ILogger logger, str var workstate = new CustomWorkState(); - workstate.StartRootSpan(workflowName, spanKind: SpanKind.Internal); + workstate.StartWorkflowSpan(workflowName, spanKind: SpanKind.Internal); for (var i = 0; i < 10; i++) { diff --git a/tests/RabbitMQ.Console.Tests/Program.cs b/tests/RabbitMQ.Console.Tests/Program.cs index 3219bacf..5485f604 100644 --- a/tests/RabbitMQ.Console.Tests/Program.cs +++ b/tests/RabbitMQ.Console.Tests/Program.cs @@ -1,9 +1,9 @@ using RabbitMQ.Console.Tests; -using HouseofCat.Utilities; using Microsoft.Extensions.Logging; +using HouseofCat.Utilities.Helpers; -var loggerFactory = LogHelper.CreateConsoleLoggerFactory(LogLevel.Information); -LogHelper.LoggerFactory = loggerFactory; +var loggerFactory = LogHelpers.CreateConsoleLoggerFactory(LogLevel.Information); +LogHelpers.LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger(); // Basic Tests diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index b375bc18..a33c050f 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -1,4 +1,4 @@ -using HouseofCat.Utilities; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using OpenTelemetry; using OpenTelemetry.Resources; @@ -6,8 +6,8 @@ using RabbitMQ.ConsumerDataflows.Tests; using System.Text; -var loggerFactory = LogHelper.CreateConsoleLoggerFactory(LogLevel.Information); -LogHelper.LoggerFactory = loggerFactory; +var loggerFactory = LogHelpers.CreateConsoleLoggerFactory(LogLevel.Information); +LogHelpers.LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger(); var applicationName = "RabbitMQ.ConsumerDataflow.Tests"; @@ -42,8 +42,8 @@ (state) => { logger.LogError(state?.EDI?.SourceException, "Error Step!"); - state.ReceivedData.NackMessage(requeue: true); - state.ReceivedData.Complete(); + state?.ReceivedData?.NackMessage(requeue: true); + state?.ReceivedData?.Complete(); }); await dataflowService.StartAsync(); diff --git a/tests/UnitTests/Serialization/SerializationTests.cs b/tests/UnitTests/Serialization/SerializationTests.cs index f9703a47..1f82480b 100644 --- a/tests/UnitTests/Serialization/SerializationTests.cs +++ b/tests/UnitTests/Serialization/SerializationTests.cs @@ -1,5 +1,5 @@ using HouseofCat.Serialization; -using HouseofCat.Utilities.Time; +using HouseofCat.Utilities.Helpers; using MessagePack; namespace Serialization; diff --git a/tests/UnitTests/Transform/DataTransformTests.cs b/tests/UnitTests/Transform/DataTransformTests.cs index 9de9d43d..8fb173e2 100644 --- a/tests/UnitTests/Transform/DataTransformTests.cs +++ b/tests/UnitTests/Transform/DataTransformTests.cs @@ -3,7 +3,7 @@ using HouseofCat.Encryption; using HouseofCat.Hashing; using HouseofCat.Serialization; -using HouseofCat.Utilities.Time; +using HouseofCat.Utilities.Helpers; namespace Transform; diff --git a/tests/UnitTests/Transform/RecyclableTransformTests.cs b/tests/UnitTests/Transform/RecyclableTransformTests.cs index e6135625..b7b3a2fa 100644 --- a/tests/UnitTests/Transform/RecyclableTransformTests.cs +++ b/tests/UnitTests/Transform/RecyclableTransformTests.cs @@ -3,7 +3,7 @@ using HouseofCat.Encryption; using HouseofCat.Hashing; using HouseofCat.Serialization; -using HouseofCat.Utilities.Time; +using HouseofCat.Utilities.Helpers; namespace Transform; From f7425287888aed33236cd352f9a83dd667fe6a8b Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 10:19:02 -0500 Subject: [PATCH 07/47] Removing Envelope, RoutingOptions, and renaming Letter to Message. --- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 46 +++++----- .../Consumer/Extensions/ConsumerExtensions.cs | 6 +- .../Dataflows/ConsumerDataflow.cs | 46 +++++----- .../Dataflows/RabbitWorkState.cs | 4 +- .../Extensions/MessageExtensions.cs | 40 +++------ .../Extensions/MetadataExtensions.cs | 12 +-- src/HouseofCat.RabbitMQ/Messages/Envelope.cs | 8 -- .../Messages/{Letter.cs => Message.cs} | 85 +++++++++++-------- .../{LetterMetadata.cs => Metadata.cs} | 2 +- .../{ReceivedData.cs => ReceivedMessage.cs} | 14 +-- .../Options/RoutingOptions.cs | 28 ------ .../Pipelines/ConsumerPipeline.cs | 12 +-- .../Publisher/Publisher.cs | 30 +++---- .../Services/RabbitService.cs | 26 +++--- tests/RabbitMQ.Console.Tests/Program.cs | 2 +- .../Tests/PubSubTests.cs | 8 +- .../Tests/PublisherTests.cs | 4 +- .../Tests/RabbitServiceTests.cs | 4 +- .../Serialization/SerializationTests.cs | 8 +- .../UnitTests/Transform/DataTransformTests.cs | 4 +- .../Transform/RecyclableTransformTests.cs | 4 +- 21 files changed, 181 insertions(+), 212 deletions(-) delete mode 100644 src/HouseofCat.RabbitMQ/Messages/Envelope.cs rename src/HouseofCat.RabbitMQ/Messages/{Letter.cs => Message.cs} (58%) rename src/HouseofCat.RabbitMQ/Messages/{LetterMetadata.cs => Metadata.cs} (93%) rename src/HouseofCat.RabbitMQ/Messages/{ReceivedData.cs => ReceivedMessage.cs} (95%) delete mode 100644 src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 581d4177..906fc3ee 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -40,7 +40,7 @@ Task DataflowExecutionEngineAsync( CancellationToken token = default); Task ChannelExecutionEngineAsync( - Func> workBodyAsync, + Func> workBodyAsync, int maxDoP = 4, bool ensureOrdered = true, Func postWorkBodyAsync = null, @@ -49,7 +49,7 @@ Task ChannelExecutionEngineAsync( CancellationToken token = default); Task DirectChannelExecutionEngineAsync( - Func> workBodyAsync, + Func> workBodyAsync, int maxDoP = 4, bool ensureOrdered = true, TaskScheduler taskScheduler = null, @@ -64,14 +64,14 @@ Task DirectChannelExecutionEngineAsync( IAsyncEnumerable StreamUntilQueueEmptyAsync(); } -public class Consumer : IConsumer, IDisposable +public class Consumer : IConsumer, IDisposable { private readonly ILogger _logger; private readonly SemaphoreSlim _conLock = new SemaphoreSlim(1, 1); private readonly SemaphoreSlim _executionLock = new SemaphoreSlim(1, 1); private IChannelHost _chanHost; private bool _disposedValue; - private Channel _consumerChannel; + private Channel _consumerChannel; public string ConsumerTag { get; private set; } private bool _shutdown; @@ -116,7 +116,7 @@ public async Task StartConsumerAsync() { await SetChannelHostAsync().ConfigureAwait(false); _shutdown = false; - _consumerChannel = Channel.CreateBounded( + _consumerChannel = Channel.CreateBounded( new BoundedChannelOptions(ConsumerOptions.BatchSize!.Value) { FullMode = ConsumerOptions.BehaviorWhenFull!.Value @@ -334,7 +334,7 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs { await _consumerChannel .Writer - .WriteAsync(new ReceivedData(_chanHost.GetChannel(), bdea, !(ConsumerOptions.AutoAck ?? false))) + .WriteAsync(new ReceivedMessage(_chanHost.GetChannel(), bdea, !(ConsumerOptions.AutoAck ?? false))) .ConfigureAwait(false); return true; } @@ -406,9 +406,9 @@ await _chanHost _chanHost.ChannelId); } - public ChannelReader GetConsumerBuffer() => _consumerChannel.Reader; + public ChannelReader GetConsumerBuffer() => _consumerChannel.Reader; - public async ValueTask ReadAsync() + public async ValueTask ReadAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); @@ -418,11 +418,11 @@ public async ValueTask ReadAsync() .ConfigureAwait(false); } - public async Task> ReadUntilEmptyAsync() + public async Task> ReadUntilEmptyAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); - var list = new List(); + var list = new List(); await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false); while (_consumerChannel.Reader.TryRead(out var message)) { @@ -433,7 +433,7 @@ public async Task> ReadUntilEmptyAsync() return list; } - public async IAsyncEnumerable StreamUntilQueueEmptyAsync() + public async IAsyncEnumerable StreamUntilQueueEmptyAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); @@ -445,7 +445,7 @@ public async IAsyncEnumerable StreamUntilQueueEmptyAsync() } } - public async IAsyncEnumerable StreamUntilConsumerStopAsync() + public async IAsyncEnumerable StreamUntilConsumerStopAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); @@ -456,29 +456,29 @@ public async IAsyncEnumerable StreamUntilConsumerStopAsync() } public async Task DataflowExecutionEngineAsync( - Func> workBodyAsync, + Func> workBodyAsync, int maxDoP = 4, bool ensureOrdered = true, int boundedCapacity = 1000, TaskScheduler taskScheduler = null, CancellationToken token = default) { - var dataflowEngine = new DataflowEngine(workBodyAsync, maxDoP, ensureOrdered, null, null, boundedCapacity, taskScheduler); + var dataflowEngine = new DataflowEngine(workBodyAsync, maxDoP, ensureOrdered, null, null, boundedCapacity, taskScheduler); await TransferDataToDataflowEngine(dataflowEngine, token); } public async Task DataflowExecutionEngineAsync( - Func> workBodyAsync, + Func> workBodyAsync, int maxDoP = 4, bool ensureOrdered = true, - Func> preWorkBodyAsync = null, + Func> preWorkBodyAsync = null, Func postWorkBodyAsync = null, int boundedCapacity = 1000, TaskScheduler taskScheduler = null, CancellationToken token = default) { - var dataflowEngine = new DataflowEngine( + var dataflowEngine = new DataflowEngine( workBodyAsync, maxDoP, ensureOrdered, @@ -491,7 +491,7 @@ public async Task DataflowExecutionEngineAsync( } private async Task TransferDataToDataflowEngine( - DataflowEngine dataflowEngine, + DataflowEngine dataflowEngine, CancellationToken token = default) { await _executionLock.WaitAsync(2000, token).ConfigureAwait(false); @@ -533,7 +533,7 @@ await dataflowEngine } public async Task ChannelExecutionEngineAsync( - Func> workBodyAsync, + Func> workBodyAsync, int maxDoP = 4, bool ensureOrdered = true, Func postWorkBodyAsync = null, @@ -541,7 +541,7 @@ public async Task ChannelExecutionEngineAsync( TaskScheduler taskScheduler = null, CancellationToken token = default) { - var channelBlockEngine = new ChannelBlockEngine( + var channelBlockEngine = new ChannelBlockEngine( workBodyAsync, maxDoP, ensureOrdered, @@ -554,13 +554,13 @@ public async Task ChannelExecutionEngineAsync( } public async Task DirectChannelExecutionEngineAsync( - Func> workBodyAsync, + Func> workBodyAsync, int maxDoP = 4, bool ensureOrdered = true, TaskScheduler taskScheduler = null, CancellationToken token = default) { - _ = new ChannelBlockEngine( + _ = new ChannelBlockEngine( _consumerChannel, workBodyAsync, maxDoP, ensureOrdered, taskScheduler, token); await _executionLock.WaitAsync(2000, token).ConfigureAwait(false); @@ -589,7 +589,7 @@ public async Task DirectChannelExecutionEngineAsync( } private async Task TransferDataToChannelBlockEngine( - ChannelBlockEngine channelBlockEngine, + ChannelBlockEngine channelBlockEngine, CancellationToken token = default) { await _executionLock.WaitAsync(2000, token).ConfigureAwait(false); diff --git a/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs b/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs index 3d6070d9..0863eaa9 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs @@ -9,13 +9,13 @@ namespace HouseofCat.RabbitMQ; public static class ConsumerExtensions { public static ValueTask DirectChannelExecutionEngineAsync( - this IConsumer consumer, - Func> workBodyAsync, + this IConsumer consumer, + Func> workBodyAsync, Func postWorkBodyAsync = null, TaskScheduler taskScheduler = null, CancellationToken cancellationToken = default) { - var channelReaderBlockEngine = new ChannelReaderBlockEngine( + var channelReaderBlockEngine = new ChannelReaderBlockEngine( consumer.GetConsumerBuffer(), workBodyAsync, consumer.ConsumerOptions.ConsumerPipelineOptions.MaxDegreesOfParallelism, diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 3c50614e..78ae0f67 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -22,16 +22,16 @@ namespace HouseofCat.RabbitMQ.Dataflows; public string WorkflowName { get; } private readonly IRabbitService _rabbitService; - private readonly ICollection> _consumers; + private readonly ICollection> _consumers; private readonly ConsumerOptions _consumerOptions; private readonly TaskScheduler _taskScheduler; private readonly string _consumerName; private readonly int _consumerCount; // Main Flow - Ingestion - private readonly List> _consumerBlocks; - protected ITargetBlock _inputBuffer; - private TransformBlock _buildStateBlock; + private readonly List> _consumerBlocks; + protected ITargetBlock _inputBuffer; + private TransformBlock _buildStateBlock; private TransformBlock _createSendLetter; protected TransformBlock _decryptBlock; protected TransformBlock _decompressBlock; @@ -80,7 +80,7 @@ public ConsumerDataflow( TaskScheduler = _taskScheduler, }; - _consumerBlocks = new List>(); + _consumerBlocks = new List>(); } /// @@ -95,7 +95,7 @@ public ConsumerDataflow( public ConsumerDataflow( IRabbitService rabbitService, string workflowName, - ICollection> consumers, + ICollection> consumers, GlobalConsumerPipelineOptions globalConsumerPipelineOptions, TaskScheduler taskScheduler = null) : this( rabbitService, @@ -121,7 +121,7 @@ public ConsumerDataflow( public ConsumerDataflow( IRabbitService rabbitService, string workflowName, - ICollection> consumers, + ICollection> consumers, int maxDoP, bool ensureOrdered, TaskScheduler taskScheduler = null) @@ -146,12 +146,12 @@ public ConsumerDataflow( TaskScheduler = _taskScheduler, }; - _consumerBlocks = new List>(); + _consumerBlocks = new List>(); } - public virtual Task StartAsync() => StartAsync>(); + public virtual Task StartAsync() => StartAsync>(); - protected async Task StartAsync() where TConsumerBlock : ConsumerBlock, new() + protected async Task StartAsync() where TConsumerBlock : ConsumerBlock, new() { BuildLinkages(); @@ -481,13 +481,13 @@ public ConsumerDataflow WithSendStep( #region Step Linking protected virtual void BuildLinkages(DataflowLinkOptions overrideOptions = null) - where TConsumerBlock : ConsumerBlock, new() + where TConsumerBlock : ConsumerBlock, new() { Guard.AgainstNull(_buildStateBlock, nameof(_buildStateBlock)); // Create State Is Mandatory Guard.AgainstNull(_finalization, nameof(_finalization)); // Leaving The Workflow Is Mandatory Guard.AgainstNull(_errorAction, nameof(_errorAction)); // Processing Errors Is Mandatory - _inputBuffer ??= new BufferBlock(); + _inputBuffer ??= new BufferBlock(); _readyBuffer ??= new BufferBlock(); @@ -519,7 +519,7 @@ protected virtual void BuildLinkages(DataflowLinkOptions overrid } } - ((ISourceBlock)_inputBuffer).LinkTo(_buildStateBlock, overrideOptions ?? _linkStepOptions); + ((ISourceBlock)_inputBuffer).LinkTo(_buildStateBlock, overrideOptions ?? _linkStepOptions); _buildStateBlock.LinkTo(_errorBuffer, overrideOptions ?? _linkStepOptions, x => x == null); SetCurrentSourceBlock(_buildStateBlock); @@ -597,7 +597,7 @@ private void LinkWithFaultRoute(ISourceBlock source, IPropagatorBlock(string key, ReceivedData data) + public virtual TState BuildStateAndPayload(string key, ReceivedMessage data) { var state = new TState { @@ -667,11 +667,11 @@ public virtual TState BuildStateAndPayload(string key, ReceivedData data) return state; } - public TransformBlock GetBuildStateWithPayloadBlock( + public TransformBlock GetBuildStateWithPayloadBlock( string key, ExecutionDataflowBlockOptions options) { - TState BuildStateWrap(ReceivedData data) + TState BuildStateWrap(ReceivedMessage data) { try { return BuildStateAndPayload(key, data); } @@ -679,13 +679,13 @@ TState BuildStateWrap(ReceivedData data) { return null; } } - return new TransformBlock(BuildStateWrap, options); + return new TransformBlock(BuildStateWrap, options); } - public TransformBlock GetBuildStateBlock( + public TransformBlock GetBuildStateBlock( ExecutionDataflowBlockOptions options) { - TState BuildStateWrap(ReceivedData data) + TState BuildStateWrap(ReceivedMessage data) { try { return BuildState(data); } @@ -693,7 +693,7 @@ TState BuildStateWrap(ReceivedData data) { return null; } } - return new TransformBlock(BuildStateWrap, options); + return new TransformBlock(BuildStateWrap, options); } public TransformBlock GetByteManipulationTransformBlock( @@ -720,7 +720,7 @@ TState WrapAction(TState state) if (state.ReceivedData.ObjectType == Constants.HeaderValueForMessageObjectType) { if (state.ReceivedData.Letter == null) - { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } + { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } state.ReceivedData.Letter.Body = action(state.ReceivedData.Letter.Body).ToArray(); } @@ -768,7 +768,7 @@ async Task WrapActionAsync(TState state) if (state.ReceivedData.ObjectType == Constants.HeaderValueForMessageObjectType) { if (state.ReceivedData.Letter == null) - { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } + { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } state.ReceivedData.Letter.Body = await action(state.ReceivedData.Letter.Body).ConfigureAwait(false); } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index edf0d9ef..64ed717a 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -8,7 +8,7 @@ namespace HouseofCat.RabbitMQ.Dataflows; public interface IRabbitWorkState : IWorkState { - IReceivedData ReceivedData { get; set; } + IReceivedMessage ReceivedData { get; set; } IMessage SendMessage { get; set; } bool SendMessageSent { get; set; } } @@ -16,7 +16,7 @@ public interface IRabbitWorkState : IWorkState public abstract class RabbitWorkState : IRabbitWorkState { [IgnoreDataMember] - public virtual IReceivedData ReceivedData { get; set; } + public virtual IReceivedMessage ReceivedData { get; set; } public virtual byte[] SendData { get; set; } public virtual IMessage SendMessage { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 1c471939..9f6e7abb 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -15,17 +15,11 @@ public static TMessage Clone(this IMessage message) { return new TMessage { - Envelope = new Envelope - { - Exchange = new string(message.Envelope.Exchange), - RoutingKey = new string(message.Envelope.RoutingKey), - RoutingOptions = new RoutingOptions - { - DeliveryMode = message.Envelope.RoutingOptions?.DeliveryMode ?? 2, - Mandatory = message.Envelope.RoutingOptions?.Mandatory ?? false, - PriorityLevel = message.Envelope.RoutingOptions?.PriorityLevel ?? 0, - } - } + Exchange = new string(message.Exchange), + RoutingKey = new string(message.RoutingKey), + DeliveryMode = message.DeliveryMode, + Mandatory = message.Mandatory, + PriorityLevel = message.PriorityLevel }; } @@ -49,9 +43,9 @@ public static IBasicProperties CreateBasicProperties( { var props = channelHost.GetChannel().CreateBasicProperties(); - props.DeliveryMode = message.Envelope.RoutingOptions.DeliveryMode; - props.ContentType = new string(message.Envelope.RoutingOptions.ContentType); - props.Priority = message.Envelope.RoutingOptions.PriorityLevel; + props.DeliveryMode = message.DeliveryMode; + props.ContentType = new string(message.ContentType); + props.Priority = message.PriorityLevel; props.MessageId = message.MessageId == null ? new string(message.MessageId) : Guid.NewGuid().ToString(); @@ -80,20 +74,14 @@ public static IMessage CreateSimpleRandomLetter(string queueName, int bodySize = var payload = new byte[bodySize]; XorShift.FillBuffer(payload, 0, bodySize); - return new Letter + return new Message { MessageId = Guid.NewGuid().ToString(), - Metadata = new LetterMetadata(), - Envelope = new Envelope - { - Exchange = string.Empty, - RoutingKey = new string(queueName), - RoutingOptions = new RoutingOptions - { - DeliveryMode = 1, - PriorityLevel = 0 - } - }, + Metadata = new Metadata(), + Exchange = string.Empty, + RoutingKey = new string(queueName), + DeliveryMode = 1, + PriorityLevel = 0, Body = payload }; } diff --git a/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs index 7fb00a61..a2c2fdbe 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs @@ -25,8 +25,8 @@ public static T Clone(this IMetadata metadata) public static T GetHeader(this IMetadata metadata, string key) { - Guard.AgainstNull(metadata, nameof(LetterMetadata)); - Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(LetterMetadata.CustomFields)); + Guard.AgainstNull(metadata, nameof(Metadata)); + Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(Metadata.CustomFields)); if (metadata.CustomFields.TryGetValue(key, out object value)) { @@ -47,8 +47,8 @@ public static void UpsertHeader(this IMetadata metadata, string key, object valu public static bool RemoveHeader(this IMetadata metadata, string key) { - Guard.AgainstNull(metadata, nameof(LetterMetadata)); - Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(LetterMetadata.CustomFields)); + Guard.AgainstNull(metadata, nameof(Metadata)); + Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(Metadata.CustomFields)); return metadata .CustomFields @@ -57,8 +57,8 @@ public static bool RemoveHeader(this IMetadata metadata, string key) public static IDictionary GetHeadersOutOfMetadata(this IMetadata metadata) { - Guard.AgainstNull(metadata, nameof(LetterMetadata)); - Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(LetterMetadata.CustomFields)); + Guard.AgainstNull(metadata, nameof(Metadata)); + Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(Metadata.CustomFields)); var dict = new Dictionary(); diff --git a/src/HouseofCat.RabbitMQ/Messages/Envelope.cs b/src/HouseofCat.RabbitMQ/Messages/Envelope.cs deleted file mode 100644 index 212cc8e1..00000000 --- a/src/HouseofCat.RabbitMQ/Messages/Envelope.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace HouseofCat.RabbitMQ; - -public class Envelope -{ - public string Exchange { get; set; } - public string RoutingKey { get; set; } - public RoutingOptions RoutingOptions { get; set; } -} diff --git a/src/HouseofCat.RabbitMQ/Messages/Letter.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs similarity index 58% rename from src/HouseofCat.RabbitMQ/Messages/Letter.cs rename to src/HouseofCat.RabbitMQ/Messages/Message.cs index 3b2a214e..58ec2a62 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Letter.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -4,13 +4,24 @@ using RabbitMQ.Client; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; namespace HouseofCat.RabbitMQ; public interface IMessage { string MessageId { get; set; } - Envelope Envelope { get; set; } + string Exchange { get; set; } + string RoutingKey { get; set; } + + byte DeliveryMode { get; set; } + + bool Mandatory { get; set; } + + byte PriorityLevel { get; set; } + + string ContentType { get; set; } + ReadOnlyMemory Body { get; set; } IMetadata GetMetadata(); @@ -28,11 +39,24 @@ public interface IMessage IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders); } -public class Letter : IMessage +public class Message : IMessage { - public Envelope Envelope { get; set; } public string MessageId { get; set; } + public string Exchange { get; set; } + public string RoutingKey { get; set; } + + [Range(1, 2, ErrorMessage = Constants.RangeErrorMessage)] + public byte DeliveryMode { get; set; } = 2; + + public bool Mandatory { get; set; } + + // Max Priority letter level is 255, however, the max-queue priority though is 10, so > 10 is treated as 10. + [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] + public byte PriorityLevel { get; set; } + + public string ContentType { get; set; } = Constants.HeaderValueForContentTypeApplicationJson; + public IMetadata Metadata { get; set; } public ReadOnlyMemory Body { get; set; } @@ -51,54 +75,47 @@ public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptio return basicProperties; } - public Letter() { } + public Message() { } - public Letter(string exchange, string routingKey, ReadOnlyMemory data, LetterMetadata metadata = null, RoutingOptions routingOptions = null) + public Message( + string exchange, + string routingKey, + ReadOnlyMemory data, + Metadata metadata = null) { - Envelope = new Envelope - { - Exchange = exchange, - RoutingKey = routingKey, - RoutingOptions = routingOptions ?? RoutingOptions.CreateDefaultRoutingOptions() - }; + Exchange = exchange; + RoutingKey = routingKey; Body = data; - Metadata = metadata ?? new LetterMetadata(); + Metadata = metadata ?? new Metadata(); } - public Letter(string exchange, string routingKey, ReadOnlyMemory data, string id, RoutingOptions routingOptions = null) + public Message(string exchange, string routingKey, ReadOnlyMemory data, string id) { - Envelope = new Envelope - { - Exchange = exchange, - RoutingKey = routingKey, - RoutingOptions = routingOptions ?? RoutingOptions.CreateDefaultRoutingOptions() - }; + Exchange = exchange; + RoutingKey = routingKey; Body = data; + if (!string.IsNullOrWhiteSpace(id)) - { Metadata = new LetterMetadata { Id = id }; } + { Metadata = new Metadata { Id = id }; } else - { Metadata = new LetterMetadata(); } + { Metadata = new Metadata(); } } - public Letter(string exchange, string routingKey, byte[] data, string id, byte priority) + public Message(string exchange, string routingKey, byte[] data, string id, byte priority) { - Envelope = new Envelope - { - Exchange = exchange, - RoutingKey = routingKey, - RoutingOptions = RoutingOptions.CreateDefaultRoutingOptions(priority) - }; + Exchange = exchange; + RoutingKey = routingKey; Body = data; if (!string.IsNullOrWhiteSpace(id)) - { Metadata = new LetterMetadata { Id = id }; } + { Metadata = new Metadata { Id = id }; } else - { Metadata = new LetterMetadata(); } + { Metadata = new Metadata(); } } - public Letter Clone() + public Message Clone() { - var clone = this.Clone(); - clone.Metadata = Metadata.Clone(); + var clone = this.Clone(); + clone.Metadata = Metadata.Clone(); return clone; } @@ -106,7 +123,7 @@ public Letter Clone() public IMetadata CreateMetadataIfMissing() { - Metadata ??= new LetterMetadata(); + Metadata ??= new Metadata(); return Metadata; } diff --git a/src/HouseofCat.RabbitMQ/Messages/LetterMetadata.cs b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs similarity index 93% rename from src/HouseofCat.RabbitMQ/Messages/LetterMetadata.cs rename to src/HouseofCat.RabbitMQ/Messages/Metadata.cs index 01f7a222..c3eaf8b5 100644 --- a/src/HouseofCat.RabbitMQ/Messages/LetterMetadata.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs @@ -11,7 +11,7 @@ public interface IMetadata Dictionary CustomFields { get; set; } } -public class LetterMetadata : IMetadata +public class Metadata : IMetadata { /// /// An alternative Id field, intended to be user-supplied. diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs similarity index 95% rename from src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs rename to src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index 0474c5c4..26320a3e 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedData.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -7,7 +7,7 @@ namespace HouseofCat.RabbitMQ; -public interface IReceivedData +public interface IReceivedMessage { bool Ackable { get; } IModel Channel { get; set; } @@ -23,7 +23,7 @@ public interface IReceivedData ReadOnlyMemory Data { get; set; } string ConsumerTag { get; } ulong DeliveryTag { get; } - Letter Letter { get; set; } + Message Letter { get; set; } IBasicProperties Properties { get; } @@ -35,7 +35,7 @@ public interface IReceivedData bool RejectMessage(bool requeue); } -public class ReceivedData : IReceivedData, IDisposable +public class ReceivedMessage : IReceivedMessage, IDisposable { /// /// Indicates that the content was not deserializeable based on the provided headers. @@ -47,7 +47,7 @@ public class ReceivedData : IReceivedData, IDisposable public string ConsumerTag { get; } public ulong DeliveryTag { get; } public ReadOnlyMemory Data { get; set; } - public Letter Letter { get; set; } + public Message Letter { get; set; } // Headers public string ContentType { get; private set; } @@ -64,7 +64,7 @@ public class ReceivedData : IReceivedData, IDisposable private bool _disposedValue; - public ReceivedData( + public ReceivedMessage( IModel channel, BasicGetResult result, bool ackable) @@ -78,7 +78,7 @@ public ReceivedData( ReadHeaders(); } - public ReceivedData( + public ReceivedMessage( IModel channel, BasicDeliverEventArgs args, bool ackable) @@ -110,7 +110,7 @@ private void ReadHeaders() && Data.Length > 0) { try - { Letter = JsonSerializer.Deserialize(Data.Span); } + { Letter = JsonSerializer.Deserialize(Data.Span); } catch { FailedToDeserialize = true; } } diff --git a/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs b/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs deleted file mode 100644 index 8029047a..00000000 --- a/src/HouseofCat.RabbitMQ/Options/RoutingOptions.cs +++ /dev/null @@ -1,28 +0,0 @@ -using HouseofCat.Utilities.Extensions; -using System.ComponentModel.DataAnnotations; - -namespace HouseofCat.RabbitMQ; - -public record RoutingOptions -{ - [Range(1, 2, ErrorMessage = Constants.RangeErrorMessage)] - public byte DeliveryMode { get; set; } = 2; - - public bool Mandatory { get; set; } - - // Max Priority letter level is 255, however, the max-queue priority though is 10, so > 10 is treated as 10. - [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] - public byte PriorityLevel { get; set; } - - public string ContentType { get; set; } = Constants.HeaderValueForContentTypeApplicationJson; - - public static RoutingOptions CreateDefaultRoutingOptions(byte priority = 0) - { - return new RoutingOptions - { - DeliveryMode = 2, - Mandatory = false, - PriorityLevel = priority - }; - } -} diff --git a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs index 370dd95f..c0499b75 100644 --- a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs +++ b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs @@ -27,8 +27,8 @@ public class ConsumerPipeline : IConsumerPipeline, IDisposable where public bool Started { get; private set; } private readonly ILogger> _logger; - private IConsumer Consumer { get; } - private IPipeline Pipeline { get; } + private IConsumer Consumer { get; } + private IPipeline Pipeline { get; } private Task FeedPipelineWithDataTasks { get; set; } private TaskCompletionSource _completionSource; private CancellationTokenSource _cancellationTokenSource; @@ -37,8 +37,8 @@ public class ConsumerPipeline : IConsumerPipeline, IDisposable where private readonly SemaphoreSlim _pipeExecLock = new SemaphoreSlim(1, 1); public ConsumerPipeline( - IConsumer consumer, - IPipeline pipeline) + IConsumer consumer, + IPipeline pipeline) { _logger = LogHelpers.GetLogger>(); Pipeline = pipeline ?? throw new ArgumentNullException(nameof(pipeline)); @@ -130,7 +130,7 @@ await Consumer } public async Task PipelineStreamEngineAsync( - IPipeline pipeline, + IPipeline pipeline, bool waitForCompletion, CancellationToken token = default) { @@ -190,7 +190,7 @@ await receivedData finally { _pipeExecLock.Release(); } } - public async Task PipelineExecutionEngineAsync(IPipeline pipeline, bool waitForCompletion, CancellationToken token = default) + public async Task PipelineExecutionEngineAsync(IPipeline pipeline, bool waitForCompletion, CancellationToken token = default) { await _pipeExecLock .WaitAsync(2000, token) diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index d2ac36a2..22d8723b 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -549,9 +549,9 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp chanHost .GetChannel() .BasicPublish( - message.Envelope.Exchange, - message.Envelope.RoutingKey, - message.Envelope.RoutingOptions?.Mandatory ?? false, + message.Exchange, + message.RoutingKey, + message.Mandatory, message.BuildProperties(chanHost, withOptionalHeaders), body); } @@ -559,7 +559,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp { _logger.LogDebug( LogMessages.Publishers.PublishMessageFailed, - $"{message.Envelope.Exchange}->{message.Envelope.RoutingKey}", + $"{message.Exchange}->{message.RoutingKey}", message.MessageId, ex.Message); @@ -600,9 +600,9 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece chanHost .GetChannel() .BasicPublish( - message.Envelope.Exchange, - message.Envelope.RoutingKey, - message.Envelope.RoutingOptions?.Mandatory ?? false, + message.Exchange, + message.RoutingKey, + message.Mandatory, message.BuildProperties(chanHost, withOptionalHeaders), message.GetBodyToPublish(_serializationProvider)); @@ -612,7 +612,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece { _logger.LogDebug( LogMessages.Publishers.PublishMessageFailed, - $"{message.Envelope.Exchange}->{message.Envelope.RoutingKey}", + $"{message.Exchange}->{message.RoutingKey}", message.MessageId, ex.Message); @@ -649,9 +649,9 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, try { chanHost.GetChannel().BasicPublish( - messages[i].Envelope.Exchange, - messages[i].Envelope.RoutingKey, - messages[i].Envelope.RoutingOptions.Mandatory, + messages[i].Exchange, + messages[i].RoutingKey, + messages[i].Mandatory, messages[i].BuildProperties(chanHost, withOptionalHeaders), messages[i].GetBodyToPublish(_serializationProvider)); } @@ -659,7 +659,7 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, { _logger.LogDebug( LogMessages.Publishers.PublishMessageFailed, - $"{messages[i].Envelope.Exchange}->{messages[i].Envelope.RoutingKey}", + $"{messages[i].Exchange}->{messages[i].RoutingKey}", messages[i].MessageId, ex.Message); @@ -697,9 +697,9 @@ public async Task PublishManyAsBatchAsync(IList messages, bool createR for (int i = 0; i < messages.Count; i++) { publishBatch.Add( - messages[i].Envelope.Exchange, - messages[i].Envelope.RoutingKey, - messages[i].Envelope.RoutingOptions.Mandatory, + messages[i].Exchange, + messages[i].RoutingKey, + messages[i].Mandatory, messages[i].BuildProperties(chanHost, withOptionalHeaders), messages[i].GetBodyToPublish(_serializationProvider)); diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index d79a56ee..29824c64 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -30,14 +30,14 @@ public interface IRabbitService IEncryptionProvider EncryptionProvider { get; } ICompressionProvider CompressionProvider { get; } - ConcurrentDictionary> Consumers { get; } + ConcurrentDictionary> Consumers { get; } Task ComcryptAsync(IMessage message); Task CompressAsync(IMessage message); - IConsumerPipeline CreateConsumerPipeline(string consumerName, int batchSize, bool? ensureOrdered, Func> pipelineBuilder) where TOut : RabbitWorkState; - IConsumerPipeline CreateConsumerPipeline(string consumerName, IPipeline pipeline) where TOut : RabbitWorkState; + IConsumerPipeline CreateConsumerPipeline(string consumerName, int batchSize, bool? ensureOrdered, Func> pipelineBuilder) where TOut : RabbitWorkState; + IConsumerPipeline CreateConsumerPipeline(string consumerName, IPipeline pipeline) where TOut : RabbitWorkState; - IConsumerPipeline CreateConsumerPipeline(string consumerName, Func> pipelineBuilder) where TOut : RabbitWorkState; + IConsumerPipeline CreateConsumerPipeline(string consumerName, Func> pipelineBuilder) where TOut : RabbitWorkState; Task DecomcryptAsync(IMessage message); Task DecompressAsync(IMessage message); @@ -45,8 +45,8 @@ public interface IRabbitService bool Encrypt(IMessage message); Task?> GetAsync(string queueName); Task GetAsync(string queueName); - IConsumer GetConsumer(string consumerName); - IConsumer GetConsumerByPipelineName(string consumerPipelineName); + IConsumer GetConsumer(string consumerName); + IConsumer GetConsumerByPipelineName(string consumerPipelineName); ValueTask ShutdownAsync(bool immediately); } @@ -65,7 +65,7 @@ public class RabbitService : IRabbitService, IDisposable public IEncryptionProvider EncryptionProvider { get; } public ICompressionProvider CompressionProvider { get; } - public ConcurrentDictionary> Consumers { get; private set; } = new ConcurrentDictionary>(); + public ConcurrentDictionary> Consumers { get; private set; } = new ConcurrentDictionary>(); private ConcurrentDictionary ConsumerPipelineNameToConsumerOptions { get; set; } = new ConcurrentDictionary(); public string TimeFormat { get; set; } = TimeHelpers.Formats.CatsAltFormat; @@ -283,7 +283,7 @@ public IConsumerPipeline CreateConsumerPipeline( string consumerName, int batchSize, bool? ensureOrdered, - Func> pipelineBuilder) + Func> pipelineBuilder) where TOut : RabbitWorkState { var consumer = GetConsumer(consumerName); @@ -294,7 +294,7 @@ public IConsumerPipeline CreateConsumerPipeline( public IConsumerPipeline CreateConsumerPipeline( string consumerName, - Func> pipelineBuilder) + Func> pipelineBuilder) where TOut : RabbitWorkState { var consumer = GetConsumer(consumerName); @@ -307,7 +307,7 @@ public IConsumerPipeline CreateConsumerPipeline( public IConsumerPipeline CreateConsumerPipeline( string consumerName, - IPipeline pipeline) + IPipeline pipeline) where TOut : RabbitWorkState { var consumer = GetConsumer(consumerName); @@ -315,14 +315,14 @@ public IConsumerPipeline CreateConsumerPipeline( return new ConsumerPipeline(consumer, pipeline); } - public IConsumer GetConsumer(string consumerName) + public IConsumer GetConsumer(string consumerName) { - if (!Consumers.TryGetValue(consumerName, out IConsumer value)) + if (!Consumers.TryGetValue(consumerName, out IConsumer value)) { throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerOptionsMessage, consumerName)); } return value; } - public IConsumer GetConsumerByPipelineName(string consumerPipelineName) + public IConsumer GetConsumerByPipelineName(string consumerPipelineName) { if (!ConsumerPipelineNameToConsumerOptions.TryGetValue(consumerPipelineName, out ConsumerOptions value)) { throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerPipelineOptionsMessage, consumerPipelineName)); } diff --git a/tests/RabbitMQ.Console.Tests/Program.cs b/tests/RabbitMQ.Console.Tests/Program.cs index 5485f604..e7e06fbc 100644 --- a/tests/RabbitMQ.Console.Tests/Program.cs +++ b/tests/RabbitMQ.Console.Tests/Program.cs @@ -1,4 +1,4 @@ -using RabbitMQ.Console.Tests; +using RabbitMQ.ConsoleTests; using Microsoft.Extensions.Logging; using HouseofCat.Utilities.Helpers; diff --git a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs index ecc134a1..defd13ad 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs @@ -28,7 +28,7 @@ private static async Task StartPublisherAsync(ILogger logger, IChannelPool chann try { await publisher.StartAutoPublishAsync(); - var letterTemplate = new Letter("", Shared.QueueName, null, new LetterMetadata()); + var letterTemplate = new Message("", Shared.QueueName, null, new Metadata()); for (var i = 0; i < testCount; i++) { @@ -70,7 +70,7 @@ private static async Task StartConsumerAsync(ILogger logger, IChannelPool channe { try { - var letter = JsonSerializer.Deserialize(receivedData.Data.Span); + var letter = JsonSerializer.Deserialize(receivedData.Data.Span); var dataAsString = Encoding.UTF8.GetString(letter.Body.Span); if (dataAsString.StartsWith("exit")) @@ -123,7 +123,7 @@ private static async Task SendCountersToQueueSlowlyAsync(ILogger logger, IChanne try { await publisher.StartAutoPublishAsync(); - var letterTemplate = new Letter("", Shared.QueueName, null, new LetterMetadata()); + var letterTemplate = new Message("", Shared.QueueName, null, new Metadata()); for (var i = 0; i < testCount; i++) { @@ -156,7 +156,7 @@ private static async Task VerifyNoDuplicatesInQueueAsync(ILogger logger, IChanne await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) { - var letter = JsonSerializer.Deserialize(receivedData.Data.Span); + var letter = JsonSerializer.Deserialize(receivedData.Data.Span); var number = Encoding.UTF8.GetString(letter.Body.Span); if (!hashSet.Add(number)) diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index ceef3ca1..a748bcb4 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -26,7 +26,7 @@ private static async Task StartPublisherAsync(ILogger logger, IChannelPool chann try { await publisher.StartAutoPublishAsync(); - var letterTemplate = new Letter("", Shared.QueueName, null, new LetterMetadata()); + var letterTemplate = new Message("", Shared.QueueName, null, new Metadata()); for (var i = 0; i < testCount; i++) { @@ -91,7 +91,7 @@ await publisher.StartAutoPublishAsync( // Step 4: Create Message var data = Encoding.UTF8.GetBytes("Hello, RabbitMQ!"); - var message = new Letter(Shared.ExchangeName, Shared.RoutingKey, data, Guid.NewGuid().ToString()) + var message = new Message(Shared.ExchangeName, Shared.RoutingKey, data, Guid.NewGuid().ToString()) { // DeliveryId for tracking/routing through Publisher/Consumer. MessageId = Guid.NewGuid().ToString(), diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index 7f8901e8..ef14f326 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -15,7 +15,7 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger await consumer.StartConsumerAsync(); var dataAsBytes = rabbitService.SerializationProvider.Serialize(new { Name = "TestName", Age = 42 }); - var letter = new Letter( + var letter = new Message( exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, data: dataAsBytes, @@ -46,7 +46,7 @@ public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory log await consumer.StartConsumerAsync(); var dataAsBytes = rabbitService.SerializationProvider.Serialize(new { Name = "TestName", Age = 42 }); - var letter = new Letter( + var letter = new Message( exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, data: dataAsBytes, diff --git a/tests/UnitTests/Serialization/SerializationTests.cs b/tests/UnitTests/Serialization/SerializationTests.cs index 1f82480b..e20a7d80 100644 --- a/tests/UnitTests/Serialization/SerializationTests.cs +++ b/tests/UnitTests/Serialization/SerializationTests.cs @@ -29,7 +29,7 @@ public class MyCustomClass { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; public IDictionary AbstractData { get; set; } = new Dictionary @@ -37,7 +37,7 @@ public class MyCustomClass { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; public MyCustomSubClass SubClass { get; set; } = new MyCustomSubClass(); @@ -70,7 +70,7 @@ public class MyCustomClass2 { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; [Key(3)] @@ -79,7 +79,7 @@ public class MyCustomClass2 { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; [Key(4)] diff --git a/tests/UnitTests/Transform/DataTransformTests.cs b/tests/UnitTests/Transform/DataTransformTests.cs index 8fb173e2..8becfb23 100644 --- a/tests/UnitTests/Transform/DataTransformTests.cs +++ b/tests/UnitTests/Transform/DataTransformTests.cs @@ -56,7 +56,7 @@ public class MyCustomClass { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; public IDictionary AbstractData { get; set; } = new Dictionary @@ -64,7 +64,7 @@ public class MyCustomClass { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; public MyCustomSubClass SubClass { get; set; } = new MyCustomSubClass(); diff --git a/tests/UnitTests/Transform/RecyclableTransformTests.cs b/tests/UnitTests/Transform/RecyclableTransformTests.cs index b7b3a2fa..d8d73587 100644 --- a/tests/UnitTests/Transform/RecyclableTransformTests.cs +++ b/tests/UnitTests/Transform/RecyclableTransformTests.cs @@ -53,7 +53,7 @@ public class MyCustomClass { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; public IDictionary AbstractData { get; set; } = new Dictionary @@ -61,7 +61,7 @@ public class MyCustomClass { "I like to eat", "Apples and Bananas" }, { "TestKey", 12 }, { "TestKey2", 12.0 }, - { "Date", Time.GetDateTimeNow(Time.Formats.CatRFC3339) } + { "Date", TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.CatRFC3339) } }; public MyCustomSubClass SubClass { get; set; } = new MyCustomSubClass(); From 1bfc7ba7b23d00bdc3b22b0d72d02bf82845b399 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 10:25:58 -0500 Subject: [PATCH 08/47] More Letter renaming. --- guides/csharp/TopLevel.md | 4 +- guides/rabbitmq/AutoPublisher.md | 4 +- .../Dataflows/ConsumerDataflow.cs | 76 +++++++++---------- .../Dataflows/RabbitWorkState.cs | 4 +- .../Extensions/MessageExtensions.cs | 22 +++--- src/HouseofCat.RabbitMQ/Messages/Message.cs | 17 +++-- src/HouseofCat.RabbitMQ/Messages/Metadata.cs | 6 +- .../Messages/PublishReceipt.cs | 4 +- .../Messages/ReceivedMessage.cs | 8 +- .../Publisher/Publisher.cs | 6 +- .../Tests/PubSubTests.cs | 2 +- .../Tests/RabbitServiceTests.cs | 16 ++-- .../Program.cs | 10 +-- 13 files changed, 90 insertions(+), 89 deletions(-) diff --git a/guides/csharp/TopLevel.md b/guides/csharp/TopLevel.md index b45f7442..b0ea23dc 100644 --- a/guides/csharp/TopLevel.md +++ b/guides/csharp/TopLevel.md @@ -141,8 +141,8 @@ Let me copy in a basic HoC config with our consumer settings in it. This file ne "SleepOnErrorInterval": 1000 }, "PublisherOptions": { - "LetterQueueBufferSize": 100, - "PriorityLetterQueueBufferSize": 100, + "MessageQueueBufferSize": 100, + "PriorityMessageQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 0, "CreatePublishReceipts": true, diff --git a/guides/rabbitmq/AutoPublisher.md b/guides/rabbitmq/AutoPublisher.md index 12817b32..d0b02bd7 100644 --- a/guides/rabbitmq/AutoPublisher.md +++ b/guides/rabbitmq/AutoPublisher.md @@ -34,7 +34,7 @@ var rabbitOptions = new RabbitOptions PublisherOptions = new PublisherOptions { CreatePublishReceipts = true, - LetterQueueBufferSize = 10_000, + MessageQueueBufferSize = 10_000, BehaviorWhenFull = BoundedChannelFullMode.Wait, Compress = false, Encrypt = false, @@ -61,7 +61,7 @@ try // Step 4: Create IMessage var data = Encoding.UTF8.GetBytes("Hello, RabbitMQ!"); - var message = new Letter(Shared.ExchangeName, Shared.RoutingKey, data, Guid.NewGuid().ToString()) + var message = new Message(Shared.ExchangeName, Shared.RoutingKey, data, Guid.NewGuid().ToString()) { // DeliveryId for tracking/routing through Publisher/Consumer. MessageId = Guid.NewGuid().ToString(), diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 78ae0f67..6a18475b 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -32,7 +32,7 @@ namespace HouseofCat.RabbitMQ.Dataflows; private readonly List> _consumerBlocks; protected ITargetBlock _inputBuffer; private TransformBlock _buildStateBlock; - private TransformBlock _createSendLetter; + private TransformBlock _createSendMessage; protected TransformBlock _decryptBlock; protected TransformBlock _decompressBlock; @@ -44,7 +44,7 @@ namespace HouseofCat.RabbitMQ.Dataflows; protected ITargetBlock _postProcessingBuffer; protected TransformBlock _compressBlock; protected TransformBlock _encryptBlock; - protected TransformBlock _sendLetterBlock; + protected TransformBlock _sendMessageBlock; protected ActionBlock _finalization; // Error/Fault Flow @@ -377,7 +377,7 @@ public ConsumerDataflow WithDecryptionStep( _encryptionProvider.Decrypt, executionOptions, false, - x => x.ReceivedData.Encrypted, + x => x.ReceivedMessage.Encrypted, $"{WorkflowName}_Decrypt"); } return this; @@ -398,24 +398,24 @@ public ConsumerDataflow WithDecompressionStep( _compressProvider.Decompress, executionOptions, false, - x => x.ReceivedData.Compressed, + x => x.ReceivedMessage.Compressed, $"{WorkflowName}_Decompress"); } return this; } - public ConsumerDataflow WithCreateSendLetter( - Func> createLetter, + public ConsumerDataflow WithCreateSendMessage( + Func> createMessage, int? maxDoP = null, bool? ensureOrdered = null, int? boundedCapacity = null, TaskScheduler taskScheduler = null) { - if (_createSendLetter == null) + if (_createSendMessage == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _createSendLetter = GetWrappedTransformBlock(createLetter, executionOptions, $"{WorkflowName}_CreateSendLetter"); + _createSendMessage = GetWrappedTransformBlock(createMessage, executionOptions, $"{WorkflowName}_CreateSendMessage"); } return this; } @@ -435,7 +435,7 @@ public ConsumerDataflow WithCompression( _compressProvider.Compress, executionOptions, true, - x => !x.ReceivedData.Compressed, + x => !x.ReceivedMessage.Compressed, $"{WorkflowName}_Compress"); } return this; @@ -456,7 +456,7 @@ public ConsumerDataflow WithEncryption( _encryptionProvider.Encrypt, executionOptions, true, - x => !x.ReceivedData.Encrypted, + x => !x.ReceivedMessage.Encrypted, $"{WorkflowName}_Encrypt"); } return this; @@ -468,10 +468,10 @@ public ConsumerDataflow WithSendStep( int? boundedCapacity = null, TaskScheduler taskScheduler = null) { - if (_sendLetterBlock == null) + if (_sendMessageBlock == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _sendLetterBlock = GetWrappedPublishTransformBlock(_rabbitService, executionOptions); + _sendMessageBlock = GetWrappedPublishTransformBlock(_rabbitService, executionOptions); } return this; } @@ -557,11 +557,11 @@ private void LinkSuppliedSteps(DataflowLinkOptions overrideOptions = null) { LinkWithFaultRoute(_suppliedTransforms[i - 1], _suppliedTransforms[i], x => x.IsFaulted, overrideOptions ?? _linkStepOptions); } } - // Link the last user step to PostProcessingBuffer/CreateSendLetter. - if (_createSendLetter != null) + // Link the last user step to PostProcessingBuffer/CreateSendMessage. + if (_createSendMessage != null) { - LinkWithFaultRoute(_suppliedTransforms[^1], _createSendLetter, x => x.IsFaulted, overrideOptions ?? _linkStepOptions); - _createSendLetter.LinkTo(_postProcessingBuffer, overrideOptions ?? _linkStepOptions); + LinkWithFaultRoute(_suppliedTransforms[^1], _createSendMessage, x => x.IsFaulted, overrideOptions ?? _linkStepOptions); + _createSendMessage.LinkTo(_postProcessingBuffer, overrideOptions ?? _linkStepOptions); SetCurrentSourceBlock(_postProcessingBuffer); } else @@ -580,8 +580,8 @@ private void LinkPostProcessing(DataflowLinkOptions overrideOptions = null) if (_encryptBlock != null) { LinkWithFaultRoute(_currentBlock, _encryptBlock, x => x.IsFaulted, overrideOptions); } - if (_sendLetterBlock != null) - { LinkWithFaultRoute(_currentBlock, _sendLetterBlock, x => x.IsFaulted, overrideOptions); } + if (_sendMessageBlock != null) + { LinkWithFaultRoute(_currentBlock, _sendMessageBlock, x => x.IsFaulted, overrideOptions); } _currentBlock.LinkTo(_finalization, overrideOptions ?? _linkStepOptions); // Last Action } @@ -601,7 +601,7 @@ public virtual TState BuildState(ReceivedMessage data) { var state = new TState { - ReceivedData = data, + ReceivedMessage = data, Data = new Dictionary() }; @@ -610,13 +610,13 @@ public virtual TState BuildState(ReceivedMessage data) KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName) }; - if (state.ReceivedData?.Letter?.MessageId is not null) + if (state.ReceivedMessage?.Message?.MessageId is not null) { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.MessageId), state.ReceivedData.Letter.MessageId)); + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.MessageId), state.ReceivedMessage.Message.MessageId)); } - if (state.ReceivedData?.Letter?.Metadata?.Id is not null) + if (state.ReceivedMessage?.Message?.Metadata?.PayloadId is not null) { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id)); + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.Metadata.PayloadId), state.ReceivedMessage.Message.Metadata.PayloadId)); } state.StartWorkflowSpan( @@ -632,7 +632,7 @@ public virtual TState BuildStateAndPayload(string key, ReceivedMessage dat { var state = new TState { - ReceivedData = data, + ReceivedMessage = data, Data = new Dictionary() }; @@ -641,13 +641,13 @@ public virtual TState BuildStateAndPayload(string key, ReceivedMessage dat KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName) }; - if (state.ReceivedData?.Letter?.MessageId is not null) + if (state.ReceivedMessage?.Message?.MessageId is not null) { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.MessageId), state.ReceivedData.Letter.MessageId)); + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.MessageId), state.ReceivedMessage.Message.MessageId)); } - if (state.ReceivedData?.Letter?.Metadata?.Id is not null) + if (state.ReceivedMessage?.Message?.Metadata?.PayloadId is not null) { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedData.Letter.Metadata.Id), state.ReceivedData.Letter.Metadata.Id)); + attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.Metadata.PayloadId), state.ReceivedMessage.Message.Metadata.PayloadId)); } state.StartWorkflowSpan( @@ -717,15 +717,15 @@ TState WrapAction(TState state) } else if (predicate.Invoke(state)) { - if (state.ReceivedData.ObjectType == Constants.HeaderValueForMessageObjectType) + if (state.ReceivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType) { - if (state.ReceivedData.Letter == null) - { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } + if (state.ReceivedMessage.Message == null) + { state.ReceivedMessage.Message = _serializationProvider.Deserialize(state.ReceivedMessage.Data); } - state.ReceivedData.Letter.Body = action(state.ReceivedData.Letter.Body).ToArray(); + state.ReceivedMessage.Message.Body = action(state.ReceivedMessage.Message.Body).ToArray(); } else - { state.ReceivedData.Data = action(state.ReceivedData.Data).ToArray(); } + { state.ReceivedMessage.Data = action(state.ReceivedMessage.Data).ToArray(); } } return state; @@ -765,15 +765,15 @@ async Task WrapActionAsync(TState state) } else if (predicate.Invoke(state)) { - if (state.ReceivedData.ObjectType == Constants.HeaderValueForMessageObjectType) + if (state.ReceivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType) { - if (state.ReceivedData.Letter == null) - { state.ReceivedData.Letter = _serializationProvider.Deserialize(state.ReceivedData.Data); } + if (state.ReceivedMessage.Message == null) + { state.ReceivedMessage.Message = _serializationProvider.Deserialize(state.ReceivedMessage.Data); } - state.ReceivedData.Letter.Body = await action(state.ReceivedData.Letter.Body).ConfigureAwait(false); + state.ReceivedMessage.Message.Body = await action(state.ReceivedMessage.Message.Body).ConfigureAwait(false); } else - { state.ReceivedData.Data = await action(state.ReceivedData.Data).ConfigureAwait(false); } + { state.ReceivedMessage.Data = await action(state.ReceivedMessage.Data).ConfigureAwait(false); } } return state; } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index 64ed717a..1cfb23e2 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -8,7 +8,7 @@ namespace HouseofCat.RabbitMQ.Dataflows; public interface IRabbitWorkState : IWorkState { - IReceivedMessage ReceivedData { get; set; } + IReceivedMessage ReceivedMessage { get; set; } IMessage SendMessage { get; set; } bool SendMessageSent { get; set; } } @@ -16,7 +16,7 @@ public interface IRabbitWorkState : IWorkState public abstract class RabbitWorkState : IRabbitWorkState { [IgnoreDataMember] - public virtual IReceivedMessage ReceivedData { get; set; } + public virtual IReceivedMessage ReceivedMessage { get; set; } public virtual byte[] SendData { get; set; } public virtual IMessage SendMessage { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 9f6e7abb..8420958b 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -69,7 +69,7 @@ public static IBasicProperties CreateBasicProperties( return props; } - public static IMessage CreateSimpleRandomLetter(string queueName, int bodySize = 1000) + public static IMessage CreateSimpleRandomMessage(string queueName, int bodySize = 1000) { var payload = new byte[bodySize]; XorShift.FillBuffer(payload, 0, bodySize); @@ -86,29 +86,29 @@ public static IMessage CreateSimpleRandomLetter(string queueName, int bodySize = }; } - public static IList CreateManySimpleRandomLetters(List queueNames, int letterCount, int bodySize = 1000) + public static IList CreateManySimpleRandomMessages(List queueNames, int messageCount, int bodySize = 1000) { var random = new Random(); - var letters = new List(); + var messages = new List(); var queueCount = queueNames.Count; - for (var i = 0; i < letterCount; i++) + for (var i = 0; i < messageCount; i++) { - letters.Add(CreateSimpleRandomLetter(queueNames[random.Next(0, queueCount)], bodySize)); + messages.Add(CreateSimpleRandomMessage(queueNames[random.Next(0, queueCount)], bodySize)); } - return letters; + return messages; } - public static IList CreateManySimpleRandomLetters(string queueName, int letterCount, int bodySize = 1000) + public static IList CreateManySimpleRandomMessages(string queueName, int messageCount, int bodySize = 1000) { - var letters = new List(); + var messages = new List(); - for (var i = 0; i < letterCount; i++) + for (var i = 0; i < messageCount; i++) { - letters.Add(CreateSimpleRandomLetter(queueName, bodySize)); + messages.Add(CreateSimpleRandomMessage(queueName, bodySize)); } - return letters; + return messages; } } diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index 58ec2a62..0f326017 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -51,7 +51,7 @@ public class Message : IMessage public bool Mandatory { get; set; } - // Max Priority letter level is 255, however, the max-queue priority though is 10, so > 10 is treated as 10. + // Max Priority Message level is 255, however, the max-queue priority though is 10, so > 10 is treated as 10. [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] public byte PriorityLevel { get; set; } @@ -89,25 +89,26 @@ public Message( Metadata = metadata ?? new Metadata(); } - public Message(string exchange, string routingKey, ReadOnlyMemory data, string id) + public Message(string exchange, string routingKey, ReadOnlyMemory data, string payloadId) { Exchange = exchange; RoutingKey = routingKey; Body = data; - if (!string.IsNullOrWhiteSpace(id)) - { Metadata = new Metadata { Id = id }; } + if (!string.IsNullOrWhiteSpace(payloadId)) + { Metadata = new Metadata { PayloadId = payloadId }; } else { Metadata = new Metadata(); } } - public Message(string exchange, string routingKey, byte[] data, string id, byte priority) + public Message(string exchange, string routingKey, byte[] data, string payloadId, byte priority) { Exchange = exchange; RoutingKey = routingKey; Body = data; - if (!string.IsNullOrWhiteSpace(id)) - { Metadata = new Metadata { Id = id }; } + PriorityLevel = priority; + if (!string.IsNullOrWhiteSpace(payloadId)) + { Metadata = new Metadata { PayloadId = payloadId }; } else { Metadata = new Metadata(); } } @@ -135,5 +136,5 @@ public ReadOnlyMemory GetBodyToPublish(ISerializationProvider serializatio serializationProvider.Serialize(this).ToArray(); public IPublishReceipt GetPublishReceipt(bool error) => - new PublishReceipt { MessageId = MessageId, IsError = error, OriginalLetter = error ? this : null }; + new PublishReceipt { MessageId = MessageId, IsError = error, OriginalMessage = error ? this : null }; } diff --git a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs index c3eaf8b5..74cfe10f 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs @@ -4,7 +4,7 @@ namespace HouseofCat.RabbitMQ; public interface IMetadata { - string Id { get; } + string PayloadId { get; } bool Encrypted { get; set; } bool Compressed { get; set; } @@ -14,9 +14,9 @@ public interface IMetadata public class Metadata : IMetadata { /// - /// An alternative Id field, intended to be user-supplied. + /// User supplied Id for tracking purposes. Allows identifying the inner payload without deserializing it first. /// - public string Id { get; set; } + public string PayloadId { get; set; } public bool Encrypted { get; set; } public bool Compressed { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs b/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs index 6c7d3dac..f6d166ae 100644 --- a/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs +++ b/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs @@ -12,7 +12,7 @@ public struct PublishReceipt : IPublishReceipt { public bool IsError { get; set; } public string MessageId { get; set; } - public IMessage OriginalLetter { get; set; } + public IMessage OriginalMessage { get; set; } - public readonly IMessage GetOriginalMessage() => OriginalLetter; + public readonly IMessage GetOriginalMessage() => OriginalMessage; } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index 26320a3e..314f703c 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -23,7 +23,7 @@ public interface IReceivedMessage ReadOnlyMemory Data { get; set; } string ConsumerTag { get; } ulong DeliveryTag { get; } - Message Letter { get; set; } + Message Message { get; set; } IBasicProperties Properties { get; } @@ -47,7 +47,7 @@ public class ReceivedMessage : IReceivedMessage, IDisposable public string ConsumerTag { get; } public ulong DeliveryTag { get; } public ReadOnlyMemory Data { get; set; } - public Message Letter { get; set; } + public Message Message { get; set; } // Headers public string ContentType { get; private set; } @@ -110,7 +110,7 @@ private void ReadHeaders() && Data.Length > 0) { try - { Letter = JsonSerializer.Deserialize(Data.Span); } + { Message = JsonSerializer.Deserialize(Data.Span); } catch { FailedToDeserialize = true; } } @@ -220,7 +220,7 @@ protected virtual void Dispose(bool disposing) } if (Channel != null) { Channel = null; } - if (Letter != null) { Letter = null; } + if (Message != null) { Message = null; } _disposedValue = true; } diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index 22d8723b..d7151014 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -233,7 +233,7 @@ public void QueueMessage(IMessage message) Guard.AgainstNull(message, nameof(message)); var metadata = message.GetMetadata(); - _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, metadata?.Id); + _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, metadata?.PayloadId); _messageQueue.Writer.TryWrite(message); } @@ -252,7 +252,7 @@ public async ValueTask QueueMessageAsync(IMessage message) } var metadata = message.GetMetadata(); - _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, metadata?.Id); + _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, metadata?.PayloadId); await _messageQueue .Writer @@ -289,7 +289,7 @@ private async Task ProcessMessagesAsync(ChannelReader channelReader) metadata.CustomFields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.RFC3339Long); } - _logger.LogDebug(LogMessages.AutoPublishers.MessagePublished, message.MessageId, metadata?.Id); + _logger.LogDebug(LogMessages.AutoPublishers.MessagePublished, message.MessageId, metadata?.PayloadId); await PublishAsync(message, _createPublishReceipts, _withHeaders) .ConfigureAwait(false); diff --git a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs index defd13ad..aa49c6d5 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs @@ -83,7 +83,7 @@ private static async Task StartConsumerAsync(ILogger logger, IChannelPool channe { logger.LogInformation( "Received message [Id: {MessageId}]: [{data}]", - receivedData.Letter.MessageId, + receivedData.Message.MessageId, dataAsString); receivedData.AckMessage(); diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index ef14f326..5130004e 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -19,21 +19,21 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, data: dataAsBytes, - id: Guid.NewGuid().ToString()); + payloadId: Guid.NewGuid().ToString()); await rabbitService.Publisher.QueueMessageAsync(letter); // Ping pong the same message. await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) { - if (receivedData?.Letter is null) + if (receivedData?.Message is null) { receivedData?.AckMessage(); continue; } - await rabbitService.DecomcryptAsync(receivedData.Letter); - await rabbitService.Publisher.QueueMessageAsync(receivedData.Letter); + await rabbitService.DecomcryptAsync(receivedData.Message); + await rabbitService.Publisher.QueueMessageAsync(receivedData.Message); receivedData.AckMessage(); } } @@ -50,21 +50,21 @@ public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory log exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, data: dataAsBytes, - id: Guid.NewGuid().ToString()); + payloadId: Guid.NewGuid().ToString()); await rabbitService.Publisher.QueueMessageAsync(letter); // Ping pong the same message. await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) { - if (receivedData?.Letter is null) + if (receivedData?.Message is null) { receivedData?.AckMessage(); continue; } - await rabbitService.DecomcryptAsync(receivedData.Letter); - rabbitService.Publisher.QueueMessage(receivedData.Letter); + await rabbitService.DecomcryptAsync(receivedData.Message); + rabbitService.Publisher.QueueMessage(receivedData.Message); receivedData.AckMessage(); } } diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index a33c050f..6ca8ca5c 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -26,7 +26,7 @@ "WriteToRabbitMessageToConsole", (state) => { - Console.WriteLine(Encoding.UTF8.GetString(state.ReceivedData.Data.Span)); + Console.WriteLine(Encoding.UTF8.GetString(state.ReceivedMessage.Data.Span)); return state; }); @@ -34,16 +34,16 @@ (state) => { logger.LogInformation("Finalization Step!"); - state.ReceivedData.AckMessage(); - state.ReceivedData.Complete(); + state.ReceivedMessage.AckMessage(); + state.ReceivedMessage.Complete(); }); dataflowService.AddErrorHandling( (state) => { logger.LogError(state?.EDI?.SourceException, "Error Step!"); - state?.ReceivedData?.NackMessage(requeue: true); - state?.ReceivedData?.Complete(); + state?.ReceivedMessage?.NackMessage(requeue: true); + state?.ReceivedMessage?.Complete(); }); await dataflowService.StartAsync(); From 9247aeb198fbb75c0ead8bf8ffa46539957b386f Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 10:27:54 -0500 Subject: [PATCH 09/47] More letter renaming. --- src/HouseofCat.RabbitMQ/Options/PublisherOptions.cs | 2 +- src/HouseofCat.RabbitMQ/Publisher/Publisher.cs | 2 +- src/HouseofCat.RabbitMQ/README.md | 5 ++--- src/HouseofCat.RabbitMQ/SampleRabbitOptions.json | 3 +-- tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json | 3 +-- tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json | 1 - .../RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json | 3 +-- tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs | 2 +- .../RabbitMQ.ConsumerDataflows.json | 3 +-- 9 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Options/PublisherOptions.cs b/src/HouseofCat.RabbitMQ/Options/PublisherOptions.cs index d5fa0ac8..a4286670 100644 --- a/src/HouseofCat.RabbitMQ/Options/PublisherOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/PublisherOptions.cs @@ -5,7 +5,7 @@ namespace HouseofCat.RabbitMQ; public class PublisherOptions { public bool CreatePublishReceipts { get; set; } - public int LetterQueueBufferSize { get; set; } = 10_000; + public int MessageQueueBufferSize { get; set; } = 10_000; public BoundedChannelFullMode BehaviorWhenFull { get; set; } = BoundedChannelFullMode.Wait; public bool Compress { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index d7151014..1ff0f9ab 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -175,7 +175,7 @@ public async Task StartAutoPublishAsync(Func process private void SetupPublisher(Func processReceiptAsync = null) { _messageQueue = Channel.CreateBounded( - new BoundedChannelOptions(Options.PublisherOptions.LetterQueueBufferSize) + new BoundedChannelOptions(Options.PublisherOptions.MessageQueueBufferSize) { FullMode = Options.PublisherOptions.BehaviorWhenFull }); diff --git a/src/HouseofCat.RabbitMQ/README.md b/src/HouseofCat.RabbitMQ/README.md index 4ce48dc7..7d22f9c1 100644 --- a/src/HouseofCat.RabbitMQ/README.md +++ b/src/HouseofCat.RabbitMQ/README.md @@ -5,7 +5,7 @@ Review tutorials and documentation under `./guides/rabbitmq`. Sample Config File -```javascript +```json { "FactoryOptions": { "Uri": "amqp://guest:guest@localhost:5672/", @@ -27,8 +27,7 @@ Sample Config File "UseTransientChannels": false }, "PublisherOptions": { - "LetterQueueBufferSize": 100, - "PriorityLetterQueueBufferSize": 100, + "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, diff --git a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json index bacd0ea2..19642cef 100644 --- a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json +++ b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json @@ -30,8 +30,7 @@ "UseTransientChannels": false }, "PublisherOptions": { - "LetterQueueBufferSize": 100, - "PriorityLetterQueueBufferSize": 100, + "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 0, "CreatePublishReceipts": true, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json index 799ac87b..eecbca91 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json @@ -19,8 +19,7 @@ "UseTransientChannels": false }, "PublisherOptions": { - "LetterQueueBufferSize": 100, - "PriorityLetterQueueBufferSize": 100, + "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json index f2aa3300..90ac5ee0 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json @@ -20,7 +20,6 @@ }, "PublisherOptions": { "LetterQueueBufferSize": 100, - "PriorityLetterQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json index d0b9b8e2..ce2fbbf9 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json @@ -19,8 +19,7 @@ "UseTransientChannels": false }, "PublisherOptions": { - "LetterQueueBufferSize": 100, - "PriorityLetterQueueBufferSize": 100, + "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index a748bcb4..594dfaa0 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -68,7 +68,7 @@ public static async Task RunAutoPublisherStandaloneAsync() PublisherOptions = new PublisherOptions { CreatePublishReceipts = true, - LetterQueueBufferSize = 10_000, + MessageQueueBufferSize = 10_000, BehaviorWhenFull = BoundedChannelFullMode.Wait, Compress = false, Encrypt = false, diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json index d0b9b8e2..ce2fbbf9 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json @@ -19,8 +19,7 @@ "UseTransientChannels": false }, "PublisherOptions": { - "LetterQueueBufferSize": 100, - "PriorityLetterQueueBufferSize": 100, + "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, From 42335d57ad9cd5490515d2672e5f2e6317ebb963 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 10:37:32 -0500 Subject: [PATCH 10/47] Rename ReceivedData to ReceivedMessage. --- guides/csharp/TopLevel.md | 34 ++++++------- src/HouseofCat.RabbitMQ/Constants.cs | 2 +- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 20 ++++---- .../Pipelines/Constants.cs | 6 +-- .../Pipelines/ConsumerPipeline.cs | 28 +++++------ .../Publisher/Publisher.cs | 6 +-- .../RabbitMQ.PublisherTests.json | 2 +- .../Tests/ConsumerTests.cs | 6 +-- .../Tests/PubSubTests.cs | 50 +++++++++---------- .../Tests/PublisherTests.cs | 12 ++--- .../Tests/RabbitServiceTests.cs | 32 ++++++------ version.props | 6 +-- 12 files changed, 102 insertions(+), 102 deletions(-) diff --git a/guides/csharp/TopLevel.md b/guides/csharp/TopLevel.md index b0ea23dc..8216128b 100644 --- a/guides/csharp/TopLevel.md +++ b/guides/csharp/TopLevel.md @@ -217,32 +217,32 @@ await consumer.StartConsumerAsync(); ``` Messages at this point should be sitting in the `ConsumerBuffer`. I am going to use `IAsyncEnumerable` to stream those out of the local buffer for -further processing. `ForEach ReceivedData` we will read the inner body and then Ack/Nack the message as a processing step (do work step). +further processing. `ForEach ReceivedMessage` we will read the inner body and then Ack/Nack the message as a processing step (do work step). Rather than an ugly/bulky `foreach` let us create a `local function` called `ProcessMessage` to keep things nice and clean. We are not using auto-ack so we have to ack our messages for them be marked as finished (or nack/unfinished) with server-side. ```csharp -await foreach (var receivedData in consumer.StreamOutUntilClosedAsync()) // this will exit only when the internal buffer closes/exception +await foreach (var receivedMessage in consumer.StreamOutUntilClosedAsync()) // this will exit only when the internal buffer closes/exception { - ProcessMessage(receivedData); + ProcessMessage(receivedMessage); } -void ProcessMessage(IReceivedData receivedData) +void ProcessMessage(IReceivedMessage receivedMessage) { try { - var body = Encoding.UTF8.GetString(receivedData.Data); + var body = Encoding.UTF8.GetString(receivedMessage.Data); logger.LogInformation($"{DateTime.Now:yyyy/MM/dd hh:mm:ss.ffffff} - [Message Received]: {body}"); - if (receivedData.Ackable) - { receivedData.AckMessage(); } + if (receivedMessage.Ackable) + { receivedMessage.AckMessage(); } } catch (Exception ex) { logger.LogError(ex, "An error occurred processing messages from the consumer buffer."); - if (receivedData.Ackable) - { receivedData.NackMessage(requeue: true); } + if (receivedMessage.Ackable) + { receivedMessage.NackMessage(requeue: true); } } } ``` @@ -272,29 +272,29 @@ var rabbitService = new RabbitService( var consumer = rabbitService.GetConsumer("HoC-Consumer"); await consumer.StartConsumerAsync(); -await foreach (var receivedData in consumer.StreamOutUntilClosedAsync()) +await foreach (var receivedMessage in consumer.StreamOutUntilClosedAsync()) { - ProcessMessage(receivedData); + ProcessMessage(receivedMessage); } await rabbitService.ShutdownAsync(immediately: false); -void ProcessMessage(IReceivedData receivedData) +void ProcessMessage(IreceivedMessage receivedMessage) { try { - var body = Encoding.UTF8.GetString(receivedData.Data); + var body = Encoding.UTF8.GetString(receivedMessage.Data); logger.LogInformation($"{DateTime.Now:yyyy/MM/dd hh:mm:ss.ffffff} - [Message Received]: {body}"); - if (receivedData.Ackable) - { receivedData.AckMessage(); } + if (receivedMessage.Ackable) + { receivedMessage.AckMessage(); } } catch (Exception ex) { logger.LogError(ex, "An error occurred streaming out messages from the consumer."); - if (receivedData.Ackable) - { receivedData.NackMessage(requeue: true); } + if (receivedMessage.Ackable) + { receivedMessage.NackMessage(requeue: true); } } } ``` diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index ea015537..ab0b8134 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -139,6 +139,6 @@ public static class Consumers public const string ConsumerDataflowActionCancelled = "Consumer [{0}] dataflow engine actions were cancelled."; public const string ConsumerDataflowError = "Consumer [{0}] dataflow engine encountered an error. Error: {1}"; - public const string ConsumerDataflowQueueing = "Consumer [{0}] dataflow engine queueing unit of work [ReceivedData:DT:{1}]."; + public const string ConsumerDataflowQueueing = "Consumer [{0}] dataflow engine queueing unit of work [receivedMessage:DT:{1}]."; } } diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 906fc3ee..8f512c07 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -449,9 +449,9 @@ public async IAsyncEnumerable StreamUntilConsumerStopAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); - await foreach (var receivedData in _consumerChannel.Reader.ReadAllAsync()) + await foreach (var receivedMessage in _consumerChannel.Reader.ReadAllAsync()) { - yield return receivedData; + yield return receivedMessage; } } @@ -500,17 +500,17 @@ private async Task TransferDataToDataflowEngine( { while (await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false)) { - while (_consumerChannel.Reader.TryRead(out var receivedData)) + while (_consumerChannel.Reader.TryRead(out var receivedMessage)) { - if (receivedData != null) + if (receivedMessage != null) { _logger.LogDebug( Consumers.ConsumerDataflowQueueing, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); await dataflowEngine - .EnqueueWorkAsync(receivedData) + .EnqueueWorkAsync(receivedMessage) .ConfigureAwait(false); } } @@ -598,16 +598,16 @@ private async Task TransferDataToChannelBlockEngine( { while (await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false)) { - var receivedData = await _consumerChannel.Reader.ReadAsync(token); - if (receivedData != null) + var receivedMessage = await _consumerChannel.Reader.ReadAsync(token); + if (receivedMessage != null) { _logger.LogDebug( Consumers.ConsumerDataflowQueueing, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); await channelBlockEngine - .EnqueueWorkAsync(receivedData) + .EnqueueWorkAsync(receivedMessage) .ConfigureAwait(false); } } diff --git a/src/HouseofCat.RabbitMQ/Pipelines/Constants.cs b/src/HouseofCat.RabbitMQ/Pipelines/Constants.cs index 1a75dbd8..85735709 100644 --- a/src/HouseofCat.RabbitMQ/Pipelines/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Pipelines/Constants.cs @@ -4,9 +4,9 @@ public static class Constants { public static class ConsumerPipelines { - public readonly static string ConsumerPipelineQueueing = "Consumer ({0}) pipeline engine queueing unit of work (ReceivedData:DT:{1})."; - public readonly static string ConsumerPipelineWaiting = "Consumer ({0}) pipeline engine waiting on completion of unit of work (ReceivedData:DT:{1})..."; - public readonly static string ConsumerPipelineWaitingDone = "Consumer ({0}) pipeline engine waiting on completed unit of work (ReceivedData:DT:{1})."; + public readonly static string ConsumerPipelineQueueing = "Consumer ({0}) pipeline engine queueing unit of work (receivedMessage:DT:{1})."; + public readonly static string ConsumerPipelineWaiting = "Consumer ({0}) pipeline engine waiting on completion of unit of work (receivedMessage:DT:{1})..."; + public readonly static string ConsumerPipelineWaitingDone = "Consumer ({0}) pipeline engine waiting on completed unit of work (receivedMessage:DT:{1})."; public readonly static string ConsumerPipelineActionCancelled = "Consumer ({0}) pipeline engine actions were cancelled."; public readonly static string ConsumerPipelineError = "Consumer ({0}) pipeline engine encountered an error. Error: {1}"; } diff --git a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs index c0499b75..dec62575 100644 --- a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs +++ b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs @@ -140,17 +140,17 @@ await _pipeExecLock try { - await foreach (var receivedData in Consumer.GetConsumerBuffer().ReadAllAsync(token)) + await foreach (var receivedMessage in Consumer.GetConsumerBuffer().ReadAllAsync(token)) { - if (receivedData == null) { continue; } + if (receivedMessage == null) { continue; } _logger.LogDebug( ConsumerPipelines.ConsumerPipelineQueueing, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); await pipeline - .QueueForExecutionAsync(receivedData) + .QueueForExecutionAsync(receivedMessage) .ConfigureAwait(false); if (waitForCompletion) @@ -158,16 +158,16 @@ await pipeline _logger.LogTrace( ConsumerPipelines.ConsumerPipelineWaiting, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); - await receivedData + await receivedMessage .Completion .ConfigureAwait(false); _logger.LogTrace( ConsumerPipelines.ConsumerPipelineWaitingDone, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); } if (token.IsCancellationRequested) @@ -200,17 +200,17 @@ await _pipeExecLock { while (await Consumer.GetConsumerBuffer().WaitToReadAsync(token).ConfigureAwait(false)) { - while (Consumer.GetConsumerBuffer().TryRead(out var receivedData)) + while (Consumer.GetConsumerBuffer().TryRead(out var receivedMessage)) { - if (receivedData == null) { continue; } + if (receivedMessage == null) { continue; } _logger.LogDebug( ConsumerPipelines.ConsumerPipelineQueueing, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); await pipeline - .QueueForExecutionAsync(receivedData) + .QueueForExecutionAsync(receivedMessage) .ConfigureAwait(false); if (waitForCompletion) @@ -218,16 +218,16 @@ await pipeline _logger.LogTrace( ConsumerPipelines.ConsumerPipelineWaiting, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); - await receivedData + await receivedMessage .Completion .ConfigureAwait(false); _logger.LogTrace( ConsumerPipelines.ConsumerPipelineWaitingDone, ConsumerOptions.ConsumerName, - receivedData.DeliveryTag); + receivedMessage.DeliveryTag); } if (token.IsCancellationRequested) diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index 1ff0f9ab..5cb46b26 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -530,7 +530,7 @@ await _channelPool } /// - /// Acquires a channel from the channel pool, then publishes message based on the letter/envelope parameters. + /// Acquires a channel from the channel pool, then publishes message based on the message parameters. /// Only throws exception when failing to acquire channel or when creating a receipt after the ReceiptBuffer is closed. /// /// @@ -579,7 +579,7 @@ await _channelPool } /// - /// Acquires an ackable channel from the channel pool, then publishes message based on the letter/envelope parameters and waits for confirmation. + /// Acquires an ackable channel from the channel pool, then publishes message based on the message parameters and waits for confirmation. /// Only throws exception when failing to acquire channel or when creating a receipt after the ReceiptBuffer is closed. /// Not fully ready for production yet. /// @@ -676,7 +676,7 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, } /// - /// Use this method when a group of letters who have the same properties (deliverymode, messagetype, priority). + /// Use this method when a group of messages who have the same properties (deliverymode, messagetype, priority). /// Receipt with no error indicates that we successfully handed off to internal library, not necessarily published. /// /// diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json index 90ac5ee0..37d8b119 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json @@ -19,7 +19,7 @@ "UseTransientChannels": false }, "PublisherOptions": { - "LetterQueueBufferSize": 100, + "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, diff --git a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs index 28847bc0..98fba696 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs @@ -15,10 +15,10 @@ public static async Task RunConsumerTestAsync(ILogger logger, string configFileN { await consumer.StartConsumerAsync(); - await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) { - logger.LogInformation("Received message: [{message}]", Encoding.UTF8.GetString(receivedData.Data.Span)); - receivedData.AckMessage(); + logger.LogInformation("Received message: [{message}]", Encoding.UTF8.GetString(receivedMessage.Data.Span)); + receivedMessage.AckMessage(); } } catch (Exception ex) diff --git a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs index aa49c6d5..1ef374dc 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs @@ -28,24 +28,24 @@ private static async Task StartPublisherAsync(ILogger logger, IChannelPool chann try { await publisher.StartAutoPublishAsync(); - var letterTemplate = new Message("", Shared.QueueName, null, new Metadata()); + var messageTemplate = new Message("", Shared.QueueName, null, new Metadata()); for (var i = 0; i < testCount; i++) { - var letter = letterTemplate.Clone(); - letter.MessageId = Guid.NewGuid().ToString(); - letter.Body = Encoding.UTF8.GetBytes($"Hello World! {i}"); + var message = messageTemplate.Clone(); + message.MessageId = Guid.NewGuid().ToString(); + message.Body = Encoding.UTF8.GetBytes($"Hello World! {i}"); - await publisher.QueueMessageAsync(letter); - logger.LogInformation("Published message [Id: {MessageId}].", letter.MessageId); + await publisher.QueueMessageAsync(message); + logger.LogInformation("Published message [Id: {MessageId}].", message.MessageId); } - var exitLetter = letterTemplate.Clone(); - exitLetter.MessageId = "exit"; - exitLetter.Body = Encoding.UTF8.GetBytes("exit"); + var exitMessage = messageTemplate.Clone(); + exitMessage.MessageId = "exit"; + exitMessage.Body = Encoding.UTF8.GetBytes("exit"); logger.LogInformation("Publishing exit message."); - await publisher.PublishAsync(exitLetter, false); + await publisher.PublishAsync(exitMessage, false); logger.LogInformation("Stopping publisher."); await publisher.StopAutoPublishAsync(); @@ -66,27 +66,27 @@ private static async Task StartConsumerAsync(ILogger logger, IChannelPool channe { await consumer.StartConsumerAsync(); - await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) { try { - var letter = JsonSerializer.Deserialize(receivedData.Data.Span); - var dataAsString = Encoding.UTF8.GetString(letter.Body.Span); + var message = JsonSerializer.Deserialize(receivedMessage.Data.Span); + var dataAsString = Encoding.UTF8.GetString(message.Body.Span); if (dataAsString.StartsWith("exit")) { logger.LogInformation("Exit message received."); - receivedData.AckMessage(); + receivedMessage.AckMessage(); break; // Can leave messages in the internal queue, but we'll just break here. } else { logger.LogInformation( "Received message [Id: {MessageId}]: [{data}]", - receivedData.Message.MessageId, + receivedMessage.Message.MessageId, dataAsString); - receivedData.AckMessage(); + receivedMessage.AckMessage(); } } catch (Exception ex) @@ -123,14 +123,14 @@ private static async Task SendCountersToQueueSlowlyAsync(ILogger logger, IChanne try { await publisher.StartAutoPublishAsync(); - var letterTemplate = new Message("", Shared.QueueName, null, new Metadata()); + var messageTemplate = new Message("", Shared.QueueName, null, new Metadata()); for (var i = 0; i < testCount; i++) { - var letter = letterTemplate.Clone(); - letter.MessageId = Guid.NewGuid().ToString(); - letter.Body = Encoding.UTF8.GetBytes(i.ToString()); - await publisher.QueueMessageAsync(letter); + var message = messageTemplate.Clone(); + message.MessageId = Guid.NewGuid().ToString(); + message.Body = Encoding.UTF8.GetBytes(i.ToString()); + await publisher.QueueMessageAsync(message); await Task.Delay(delay); } @@ -154,10 +154,10 @@ private static async Task VerifyNoDuplicatesInQueueAsync(ILogger logger, IChanne { await consumer.StartConsumerAsync(); - await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) { - var letter = JsonSerializer.Deserialize(receivedData.Data.Span); - var number = Encoding.UTF8.GetString(letter.Body.Span); + var message = JsonSerializer.Deserialize(receivedMessage.Data.Span); + var number = Encoding.UTF8.GetString(message.Body.Span); if (!hashSet.Add(number)) { @@ -165,7 +165,7 @@ private static async Task VerifyNoDuplicatesInQueueAsync(ILogger logger, IChanne break; } - receivedData.AckMessage(); + receivedMessage.AckMessage(); if (hashSet.Count == testCount) { diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index 594dfaa0..c8b7b647 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -26,16 +26,16 @@ private static async Task StartPublisherAsync(ILogger logger, IChannelPool chann try { await publisher.StartAutoPublishAsync(); - var letterTemplate = new Message("", Shared.QueueName, null, new Metadata()); + var messageTemplate = new Message("", Shared.QueueName, null, new Metadata()); for (var i = 0; i < testCount; i++) { - var letter = letterTemplate.Clone(); - letter.MessageId = Guid.NewGuid().ToString(); - letter.Body = Encoding.UTF8.GetBytes($"Hello World! {i}"); + var message = messageTemplate.Clone(); + message.MessageId = Guid.NewGuid().ToString(); + message.Body = Encoding.UTF8.GetBytes($"Hello World! {i}"); - await publisher.QueueMessageAsync(letter); - logger.LogInformation("Published message [Id: {MessageId}].", letter.MessageId); + await publisher.QueueMessageAsync(message); + logger.LogInformation("Published message [Id: {MessageId}].", message.MessageId); await Task.Delay(delay); } diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index 5130004e..7da74c5a 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -15,26 +15,26 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger await consumer.StartConsumerAsync(); var dataAsBytes = rabbitService.SerializationProvider.Serialize(new { Name = "TestName", Age = 42 }); - var letter = new Message( + var message = new Message( exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, data: dataAsBytes, payloadId: Guid.NewGuid().ToString()); - await rabbitService.Publisher.QueueMessageAsync(letter); + await rabbitService.Publisher.QueueMessageAsync(message); // Ping pong the same message. - await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) { - if (receivedData?.Message is null) + if (receivedMessage?.Message is null) { - receivedData?.AckMessage(); + receivedMessage?.AckMessage(); continue; } - await rabbitService.DecomcryptAsync(receivedData.Message); - await rabbitService.Publisher.QueueMessageAsync(receivedData.Message); - receivedData.AckMessage(); + await rabbitService.DecomcryptAsync(receivedMessage.Message); + await rabbitService.Publisher.QueueMessageAsync(receivedMessage.Message); + receivedMessage.AckMessage(); } } @@ -46,26 +46,26 @@ public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory log await consumer.StartConsumerAsync(); var dataAsBytes = rabbitService.SerializationProvider.Serialize(new { Name = "TestName", Age = 42 }); - var letter = new Message( + var message = new Message( exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, data: dataAsBytes, payloadId: Guid.NewGuid().ToString()); - await rabbitService.Publisher.QueueMessageAsync(letter); + await rabbitService.Publisher.QueueMessageAsync(message); // Ping pong the same message. - await foreach (var receivedData in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) { - if (receivedData?.Message is null) + if (receivedMessage?.Message is null) { - receivedData?.AckMessage(); + receivedMessage?.AckMessage(); continue; } - await rabbitService.DecomcryptAsync(receivedData.Message); - rabbitService.Publisher.QueueMessage(receivedData.Message); - receivedData.AckMessage(); + await rabbitService.DecomcryptAsync(receivedMessage.Message); + rabbitService.Publisher.QueueMessage(receivedMessage.Message); + receivedMessage.AckMessage(); } } } diff --git a/version.props b/version.props index 42b37052..f2313121 100644 --- a/version.props +++ b/version.props @@ -1,7 +1,7 @@ - 3.3.0 - 3.3.0 - 3.3.0 + 4.0.0 + 4.0.0 + 4.0.0 From 244eef1714c10ac95816d37c7ad2cb55d4ea0137 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 10:38:20 -0500 Subject: [PATCH 11/47] Update ReceivedMessage.cs --- src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index 314f703c..f92f5325 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -73,7 +73,7 @@ public ReceivedMessage( Channel = channel; DeliveryTag = result.DeliveryTag; Properties = result.BasicProperties; - Data = result.Body.ToArray(); + Data = result.Body; ReadHeaders(); } @@ -88,7 +88,7 @@ public ReceivedMessage( ConsumerTag = args.ConsumerTag; DeliveryTag = args.DeliveryTag; Properties = args.BasicProperties; - Data = args.Body.ToArray(); + Data = args.Body; ReadHeaders(); } From bbcdd280cd2af1323e012b1afe8aa2098e7167a9 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 10:52:17 -0500 Subject: [PATCH 12/47] Sticking to Body vs. Data naming convention. --- .../Dataflows/ConsumerDataflow.cs | 10 +++++----- src/HouseofCat.RabbitMQ/Messages/Message.cs | 16 ++++++++-------- src/HouseofCat.RabbitMQ/Messages/Metadata.cs | 4 ++-- .../Messages/ReceivedMessage.cs | 18 +++++++++--------- .../Tests/ConsumerTests.cs | 2 +- .../Tests/PubSubTests.cs | 4 ++-- .../Tests/RabbitServiceTests.cs | 4 ++-- .../Program.cs | 2 +- 8 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 6a18475b..abc929f2 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -660,7 +660,7 @@ public virtual TState BuildStateAndPayload(string key, ReceivedMessage dat && data.ObjectType != Constants.HeaderValueForUnknownObjectType) { try - { state.Data[key] = _serializationProvider.Deserialize(data.Data); } + { state.Data[key] = _serializationProvider.Deserialize(data.Body); } catch { } } @@ -720,12 +720,12 @@ TState WrapAction(TState state) if (state.ReceivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType) { if (state.ReceivedMessage.Message == null) - { state.ReceivedMessage.Message = _serializationProvider.Deserialize(state.ReceivedMessage.Data); } + { state.ReceivedMessage.Message = _serializationProvider.Deserialize(state.ReceivedMessage.Body); } state.ReceivedMessage.Message.Body = action(state.ReceivedMessage.Message.Body).ToArray(); } else - { state.ReceivedMessage.Data = action(state.ReceivedMessage.Data).ToArray(); } + { state.ReceivedMessage.Body = action(state.ReceivedMessage.Body).ToArray(); } } return state; @@ -768,12 +768,12 @@ async Task WrapActionAsync(TState state) if (state.ReceivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType) { if (state.ReceivedMessage.Message == null) - { state.ReceivedMessage.Message = _serializationProvider.Deserialize(state.ReceivedMessage.Data); } + { state.ReceivedMessage.Message = _serializationProvider.Deserialize(state.ReceivedMessage.Body); } state.ReceivedMessage.Message.Body = await action(state.ReceivedMessage.Message.Body).ConfigureAwait(false); } else - { state.ReceivedMessage.Data = await action(state.ReceivedMessage.Data).ConfigureAwait(false); } + { state.ReceivedMessage.Body = await action(state.ReceivedMessage.Body).ConfigureAwait(false); } } return state; } diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index 0f326017..6dfcf187 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -39,7 +39,7 @@ public interface IMessage IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders); } -public class Message : IMessage +public sealed class Message : IMessage { public string MessageId { get; set; } @@ -51,7 +51,7 @@ public class Message : IMessage public bool Mandatory { get; set; } - // Max Priority Message level is 255, however, the max-queue priority though is 10, so > 10 is treated as 10. + // The max-queue priority though is 10, so > 10 is treated as 10. [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] public byte PriorityLevel { get; set; } @@ -80,20 +80,20 @@ public Message() { } public Message( string exchange, string routingKey, - ReadOnlyMemory data, + ReadOnlyMemory body, Metadata metadata = null) { Exchange = exchange; RoutingKey = routingKey; - Body = data; + Body = body; Metadata = metadata ?? new Metadata(); } - public Message(string exchange, string routingKey, ReadOnlyMemory data, string payloadId) + public Message(string exchange, string routingKey, ReadOnlyMemory body, string payloadId) { Exchange = exchange; RoutingKey = routingKey; - Body = data; + Body = body; if (!string.IsNullOrWhiteSpace(payloadId)) { Metadata = new Metadata { PayloadId = payloadId }; } @@ -101,11 +101,11 @@ public Message(string exchange, string routingKey, ReadOnlyMemory data, st { Metadata = new Metadata(); } } - public Message(string exchange, string routingKey, byte[] data, string payloadId, byte priority) + public Message(string exchange, string routingKey, byte[] body, string payloadId, byte priority) { Exchange = exchange; RoutingKey = routingKey; - Body = data; + Body = body; PriorityLevel = priority; if (!string.IsNullOrWhiteSpace(payloadId)) { Metadata = new Metadata { PayloadId = payloadId }; } diff --git a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs index 74cfe10f..cc409ab4 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs @@ -11,10 +11,10 @@ public interface IMetadata Dictionary CustomFields { get; set; } } -public class Metadata : IMetadata +public sealed class Metadata : IMetadata { /// - /// User supplied Id for tracking purposes. Allows identifying the inner payload without deserializing it first. + /// PayloadId is a user supplied Id for tracking purposes. Allows identifying the inner payload without needing to deserialize first. /// public string PayloadId { get; set; } public bool Encrypted { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index f92f5325..b7979d60 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -20,7 +20,7 @@ public interface IReceivedMessage bool Compressed { get; } string CompressionType { get; } - ReadOnlyMemory Data { get; set; } + ReadOnlyMemory Body { get; set; } string ConsumerTag { get; } ulong DeliveryTag { get; } Message Message { get; set; } @@ -35,7 +35,7 @@ public interface IReceivedMessage bool RejectMessage(bool requeue); } -public class ReceivedMessage : IReceivedMessage, IDisposable +public sealed class ReceivedMessage : IReceivedMessage, IDisposable { /// /// Indicates that the content was not deserializeable based on the provided headers. @@ -46,10 +46,10 @@ public class ReceivedMessage : IReceivedMessage, IDisposable public IModel Channel { get; set; } public string ConsumerTag { get; } public ulong DeliveryTag { get; } - public ReadOnlyMemory Data { get; set; } + public ReadOnlyMemory Body { get; set; } public Message Message { get; set; } - // Headers + // Retrieved From Headers public string ContentType { get; private set; } public string ObjectType { get; private set; } public bool Encrypted { get; private set; } @@ -73,7 +73,7 @@ public ReceivedMessage( Channel = channel; DeliveryTag = result.DeliveryTag; Properties = result.BasicProperties; - Data = result.Body; + Body = result.Body; ReadHeaders(); } @@ -88,7 +88,7 @@ public ReceivedMessage( ConsumerTag = args.ConsumerTag; DeliveryTag = args.DeliveryTag; Properties = args.BasicProperties; - Data = args.Body; + Body = args.Body; ReadHeaders(); } @@ -107,10 +107,10 @@ private void ReadHeaders() } if (ObjectType == Constants.HeaderValueForMessageObjectType - && Data.Length > 0) + && Body.Length > 0) { try - { Message = JsonSerializer.Deserialize(Data.Span); } + { Message = JsonSerializer.Deserialize(Body.Span); } catch { FailedToDeserialize = true; } } @@ -210,7 +210,7 @@ public bool RejectMessage(bool requeue) /// public void Complete() => _completionSource.SetResult(true); - protected virtual void Dispose(bool disposing) + private void Dispose(bool disposing) { if (!_disposedValue) { diff --git a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs index 98fba696..c843f10d 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs @@ -17,7 +17,7 @@ public static async Task RunConsumerTestAsync(ILogger logger, string configFileN await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) { - logger.LogInformation("Received message: [{message}]", Encoding.UTF8.GetString(receivedMessage.Data.Span)); + logger.LogInformation("Received message: [{message}]", Encoding.UTF8.GetString(receivedMessage.Body.Span)); receivedMessage.AckMessage(); } } diff --git a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs index 1ef374dc..26fcc757 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs @@ -70,7 +70,7 @@ private static async Task StartConsumerAsync(ILogger logger, IChannelPool channe { try { - var message = JsonSerializer.Deserialize(receivedMessage.Data.Span); + var message = JsonSerializer.Deserialize(receivedMessage.Body.Span); var dataAsString = Encoding.UTF8.GetString(message.Body.Span); if (dataAsString.StartsWith("exit")) @@ -156,7 +156,7 @@ private static async Task VerifyNoDuplicatesInQueueAsync(ILogger logger, IChanne await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) { - var message = JsonSerializer.Deserialize(receivedMessage.Data.Span); + var message = JsonSerializer.Deserialize(receivedMessage.Body.Span); var number = Encoding.UTF8.GetString(message.Body.Span); if (!hashSet.Add(number)) diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index 7da74c5a..f88e45d0 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -18,7 +18,7 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger var message = new Message( exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, - data: dataAsBytes, + body: dataAsBytes, payloadId: Guid.NewGuid().ToString()); await rabbitService.Publisher.QueueMessageAsync(message); @@ -49,7 +49,7 @@ public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory log var message = new Message( exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, - data: dataAsBytes, + body: dataAsBytes, payloadId: Guid.NewGuid().ToString()); await rabbitService.Publisher.QueueMessageAsync(message); diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index 6ca8ca5c..db464f74 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -26,7 +26,7 @@ "WriteToRabbitMessageToConsole", (state) => { - Console.WriteLine(Encoding.UTF8.GetString(state.ReceivedMessage.Data.Span)); + Console.WriteLine(Encoding.UTF8.GetString(state.ReceivedMessage.Body.Span)); return state; }); From 8b085beddc5c7ad93354a5971db68e2f582df1f0 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 11:06:42 -0500 Subject: [PATCH 13/47] Removing references to FactoryOptions (all in PoolOptions now). --- guides/csharp/TopLevel.md | 36 ++++----- guides/rabbitmq/AutoPublisher.md | 5 +- guides/rabbitmq/BasicConsume.md | 6 +- guides/rabbitmq/BasicGet.md | 4 +- guides/rabbitmq/BasicPublish.md | 4 +- guides/rabbitmq/ChannelPools.md | 4 +- guides/rabbitmq/ConnectionPools.md | 4 +- guides/rabbitmq/Serialization.md | 4 +- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 2 +- .../Options/FactoryOptions.cs | 78 ------------------- .../Options/PoolOptions.cs | 74 ++++++++++++++++++ .../Options/RabbitOptions.cs | 5 -- .../Pools/ConnectionPool.cs | 62 +++++++-------- src/HouseofCat.RabbitMQ/README.md | 4 +- .../SampleRabbitOptions.json | 6 +- .../RabbitMQ.BasicGetTests.json | 6 +- .../RabbitMQ.ConnectionPoolTests.json | 6 +- .../RabbitMQ.ConsumerTests.json | 6 +- .../RabbitMQ.PubSubTests.json | 6 +- .../RabbitMQ.PublisherTests.json | 6 +- .../RabbitMQ.RabbitServiceTests.json | 6 +- .../Tests/PublisherTests.cs | 5 +- .../RabbitMQ.ConsumerDataflows.json | 6 +- 23 files changed, 147 insertions(+), 198 deletions(-) delete mode 100644 src/HouseofCat.RabbitMQ/Options/FactoryOptions.cs diff --git a/guides/csharp/TopLevel.md b/guides/csharp/TopLevel.md index 8216128b..582b9657 100644 --- a/guides/csharp/TopLevel.md +++ b/guides/csharp/TopLevel.md @@ -117,28 +117,22 @@ Let me copy in a basic HoC config with our consumer settings in it. This file ne ```json { - "FactoryOptions": { - "Uri": "amqp://guest:guest@localhost:5672/", - "MaxChannelsPerConnection": 2000, - "HeartbeatInterval": 6, - "AutoRecovery": true, - "TopologyRecovery": true, - "NetRecoveryTimeout": 10, - "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true, - "SslOptions": { - "EnableSsl": false, - "CertServerName": "", - "LocalCertPath": "", - "LocalCertPassword": "", - "ProtocolVersions": 3072 - } - }, "PoolOptions": { - "ServiceName": "TopLevel-HoC-Consumer", - "MaxConnections": 5, - "MaxChannels": 25, - "SleepOnErrorInterval": 1000 + "Uri": "amqp://guest:guest@localhost:5672/", + "MaxChannelsPerConnection": 2000, + "HeartbeatInterval": 6, + "AutoRecovery": true, + "TopologyRecovery": true, + "NetRecoveryTimeout": 5, + "ContinuationTimeout": 10, + "EnableDispatchConsumersAsync": true, + "ServiceName": "HoC.RabbitMQ", + "MaxConnections": 2, + "MaxChannels": 10, + "MaxAckableChannels": 0, + "SleepOnErrorInterval": 5000, + "TansientChannelStartRange": 10000, + "UseTransientChannels": false }, "PublisherOptions": { "MessageQueueBufferSize": 100, diff --git a/guides/rabbitmq/AutoPublisher.md b/guides/rabbitmq/AutoPublisher.md index d0b02bd7..2889c146 100644 --- a/guides/rabbitmq/AutoPublisher.md +++ b/guides/rabbitmq/AutoPublisher.md @@ -20,12 +20,9 @@ using System.Text; // Step 1: Configure RabbitOptions (or load from file or IConfiguration). var rabbitOptions = new RabbitOptions { - FactoryOptions = new FactoryOptions - { - Uri = new Uri("amqp://guest:guest@localhost:5672"), - }, PoolOptions = new PoolOptions { + Uri = new Uri("amqp://guest:guest@localhost:5672"), ServiceName = "TestService", MaxConnections = 2, MaxChannels = 10, diff --git a/guides/rabbitmq/BasicConsume.md b/guides/rabbitmq/BasicConsume.md index fec77f6f..ee77625b 100644 --- a/guides/rabbitmq/BasicConsume.md +++ b/guides/rabbitmq/BasicConsume.md @@ -9,7 +9,7 @@ I will use this as a file named `SampleRabbitOptions.json` ```json { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -18,8 +18,6 @@ I will use this as a file named `SampleRabbitOptions.json` "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 2, @@ -64,7 +62,7 @@ had an error during a `BasicPublish` it is most likely permanently closed and yo You can find more information about that [here](https://www.rabbitmq.com/client-libraries/dotnet-api-guide#consuming-async). ***Note: The type of consumer has to be configured in the ConnectionFactory in the RabbitMQ.Client. See the -`EnableDispatchConsumersAsync` in the `FactoryOptions` in the `RabbitOptions`.*** +`EnableDispatchConsumersAsync` in the `PoolOptions` in the `RabbitOptions`.*** ```csharp using HouseofCat.RabbitMQ.Pools; diff --git a/guides/rabbitmq/BasicGet.md b/guides/rabbitmq/BasicGet.md index edb9506b..b146e9c0 100644 --- a/guides/rabbitmq/BasicGet.md +++ b/guides/rabbitmq/BasicGet.md @@ -9,15 +9,13 @@ I will use this as a file named `SampleRabbitOptions.json` ```json { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, "AutoRecovery": true, "TopologyRecovery": true, "NetRecoveryTimeout": 5 - }, - "PoolOptions": { "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxAckableChannels": 5, diff --git a/guides/rabbitmq/BasicPublish.md b/guides/rabbitmq/BasicPublish.md index 1f71247a..d60505d3 100644 --- a/guides/rabbitmq/BasicPublish.md +++ b/guides/rabbitmq/BasicPublish.md @@ -12,15 +12,13 @@ It really helps to have `RabbitOptions` already setup and ready to go. I will use this as a file named `SampleRabbitOptions.json` ```json { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, "AutoRecovery": true, "TopologyRecovery": true, "NetRecoveryTimeout": 5 - }, - "PoolOptions": { "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 5, diff --git a/guides/rabbitmq/ChannelPools.md b/guides/rabbitmq/ChannelPools.md index a9274e31..53e5ca5a 100644 --- a/guides/rabbitmq/ChannelPools.md +++ b/guides/rabbitmq/ChannelPools.md @@ -33,7 +33,7 @@ It really helps to have `RabbitOptions` already setup and ready to go. I will use this as a file named `SampleRabbitOptions.json`: ```json { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -42,8 +42,6 @@ I will use this as a file named `SampleRabbitOptions.json`: "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 5, diff --git a/guides/rabbitmq/ConnectionPools.md b/guides/rabbitmq/ConnectionPools.md index 5e6d57c3..d437efc6 100644 --- a/guides/rabbitmq/ConnectionPools.md +++ b/guides/rabbitmq/ConnectionPools.md @@ -37,7 +37,7 @@ It really helps to have `RabbitOptions` already setup and ready to go. I will use this as a file named `SampleRabbitOptions.json`: ```json { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -46,8 +46,6 @@ I will use this as a file named `SampleRabbitOptions.json`: "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 0, diff --git a/guides/rabbitmq/Serialization.md b/guides/rabbitmq/Serialization.md index 17c9f57e..0c8bd19e 100644 --- a/guides/rabbitmq/Serialization.md +++ b/guides/rabbitmq/Serialization.md @@ -10,15 +10,13 @@ It really helps to have `RabbitOptions` already setup and ready to go. I will use this as a file named `SampleRabbitOptions.json` ```json { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, "AutoRecovery": true, "TopologyRecovery": true, "NetRecoveryTimeout": 5 - }, - "PoolOptions": { "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 5, diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 8f512c07..4644e7f8 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -199,7 +199,7 @@ await _chanHost.WaitUntilChannelIsReadyAsync( { return false; } } - if (Options.FactoryOptions.EnableDispatchConsumersAsync) + if (Options.PoolOptions.EnableDispatchConsumersAsync) { if (_asyncConsumer != null) // Cleanup operation, this prevents an EventHandler leak. { diff --git a/src/HouseofCat.RabbitMQ/Options/FactoryOptions.cs b/src/HouseofCat.RabbitMQ/Options/FactoryOptions.cs deleted file mode 100644 index 18fdbb5b..00000000 --- a/src/HouseofCat.RabbitMQ/Options/FactoryOptions.cs +++ /dev/null @@ -1,78 +0,0 @@ -using RabbitMQ.Client; -using System; - -namespace HouseofCat.RabbitMQ; - -public class FactoryOptions -{ - /// - /// ConnectionFactory (RabbitMQ) Uri connection string. Set to null to use individual properties. - /// amqp(s)://guest:guest@localhost:5672/vhost - /// - public Uri Uri { get; set; } = new Uri("amqp://guest:guest@localhost:5672/"); - - /// - /// ConnectionFactory (RabbitMQ) virtual host property. Use in lieu of Uri connection string. - /// - public string VirtualHost { get; set; } = ""; - - /// - /// ConnectionFactory (RabbitMQ) username property. Use in lieu of Uri connection string. - /// - public string UserName { get; set; } = "guest"; - - /// - /// ConnectionFactory (RabbitMQ) password property. Use in lieu of Uri connection string. - /// - public string Password { get; set; } = "guest"; - - /// - /// ConnectionFactory (RabbitMQ) host name property. Use in lieu of Uri connection string. - /// - public string HostName { get; set; } = "localhost"; - - /// - /// ConnectionFactory (RabbitMQ) port property. Use in lieu of Uri connection string. - /// - public int Port { get; set; } = AmqpTcpEndpoint.UseDefaultPort; - - /// - /// ConnectionFactory (RabbitMQ) max connection property. - /// - public ushort MaxChannelsPerConnection { get; set; } = 100; - - /// - /// ConnectionFactory (RabbitMQ) timespan (in seconds) between heartbeats. More than two timeouts in a row trigger RabbitMQ AutoRecovery. - /// - public ushort HeartbeatInterval { get; set; } = 6; - - /// - /// ConnectionFactory (RabbitMQ) topology recovery property. - /// - public bool TopologyRecovery { get; set; } = true; - - /// - /// ConnectionFactory (RabbitMQ) the amount of time to wait before netrecovery begins (seconds). - /// - public ushort NetRecoveryTimeout { get; set; } = 5; - - /// - /// ConnectionFactory (RabbitMQ) specify the amount of time before timeout on protocol operations (seconds). - /// - public ushort ContinuationTimeout { get; set; } = 10; - - /// - /// ConnectionFactory (RabbitMQ) property to enable Async consumers. Can't be true and retrieve regular consumers. - /// - public bool EnableDispatchConsumersAsync { get; set; } - - /// - /// Class to hold settings for ChannelFactory/SSL (RabbitMQ) settings. - /// - public SslOptions SslOptions { get; set; } = new SslOptions(); - - /// - /// Class to hold settings for OAuth2 (RabbitMQ) settings. - /// - public OAuth2Options OAuth2Options { get; set; } = new OAuth2Options(); -} diff --git a/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs b/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs index 9d633519..ff25fa63 100644 --- a/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs @@ -1,7 +1,81 @@ +using RabbitMQ.Client; +using System; + namespace HouseofCat.RabbitMQ; public class PoolOptions { + /// + /// ConnectionFactory (RabbitMQ) Uri connection string. Set to null to use individual properties. + /// amqp(s)://guest:guest@localhost:5672/vhost + /// + public Uri Uri { get; set; } = new Uri("amqp://guest:guest@localhost:5672/"); + + /// + /// ConnectionFactory (RabbitMQ) virtual host property. Use in lieu of Uri connection string. + /// + public string VirtualHost { get; set; } = ""; + + /// + /// ConnectionFactory (RabbitMQ) username property. Use in lieu of Uri connection string. + /// + public string UserName { get; set; } = "guest"; + + /// + /// ConnectionFactory (RabbitMQ) password property. Use in lieu of Uri connection string. + /// + public string Password { get; set; } = "guest"; + + /// + /// ConnectionFactory (RabbitMQ) host name property. Use in lieu of Uri connection string. + /// + public string HostName { get; set; } = "localhost"; + + /// + /// ConnectionFactory (RabbitMQ) port property. Use in lieu of Uri connection string. + /// + public int Port { get; set; } = AmqpTcpEndpoint.UseDefaultPort; + + /// + /// ConnectionFactory (RabbitMQ) max connection property. + /// + public ushort MaxChannelsPerConnection { get; set; } = 100; + + /// + /// ConnectionFactory (RabbitMQ) timespan (in seconds) between heartbeats. More than two timeouts in a row trigger RabbitMQ AutoRecovery. + /// + public ushort HeartbeatInterval { get; set; } = 6; + + /// + /// ConnectionFactory (RabbitMQ) topology recovery property. + /// + public bool TopologyRecovery { get; set; } = true; + + /// + /// ConnectionFactory (RabbitMQ) the amount of time to wait before netrecovery begins (seconds). + /// + public ushort NetRecoveryTimeout { get; set; } = 5; + + /// + /// ConnectionFactory (RabbitMQ) specify the amount of time before timeout on protocol operations (seconds). + /// + public ushort ContinuationTimeout { get; set; } = 10; + + /// + /// ConnectionFactory (RabbitMQ) property to enable Async consumers. Can't be true and retrieve regular consumers. + /// + public bool EnableDispatchConsumersAsync { get; set; } + + /// + /// Class to hold settings for ChannelFactory/SSL (RabbitMQ) settings. + /// + public SslOptions SslOptions { get; set; } = new SslOptions(); + + /// + /// Class to hold settings for OAuth2 (RabbitMQ) settings. + /// + public OAuth2Options OAuth2Options { get; set; } = new OAuth2Options(); + /// /// Value to configure the ConnectionPool prefix for display names on RabbitMQ server. /// diff --git a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs index 25e2ba1e..d696c22f 100644 --- a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs @@ -6,11 +6,6 @@ namespace HouseofCat.RabbitMQ; public class RabbitOptions { - /// - /// Class to hold settings for ConnectionFactory (RabbitMQ) options. - /// - public FactoryOptions FactoryOptions { get; set; } = new FactoryOptions(); - /// /// Class to hold settings for Channel/ConnectionPool options. /// diff --git a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs index 03acdb88..af4a4a0e 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs @@ -74,40 +74,40 @@ protected virtual ConnectionFactory BuildConnectionFactory() var cf = new ConnectionFactory { AutomaticRecoveryEnabled = true, - TopologyRecoveryEnabled = Options.FactoryOptions.TopologyRecovery, - NetworkRecoveryInterval = TimeSpan.FromSeconds(Options.FactoryOptions.NetRecoveryTimeout), - ContinuationTimeout = TimeSpan.FromSeconds(Options.FactoryOptions.ContinuationTimeout), - RequestedHeartbeat = TimeSpan.FromSeconds(Options.FactoryOptions.HeartbeatInterval), - RequestedChannelMax = Options.FactoryOptions.MaxChannelsPerConnection, - DispatchConsumersAsync = Options.FactoryOptions.EnableDispatchConsumersAsync, + TopologyRecoveryEnabled = Options.PoolOptions.TopologyRecovery, + NetworkRecoveryInterval = TimeSpan.FromSeconds(Options.PoolOptions.NetRecoveryTimeout), + ContinuationTimeout = TimeSpan.FromSeconds(Options.PoolOptions.ContinuationTimeout), + RequestedHeartbeat = TimeSpan.FromSeconds(Options.PoolOptions.HeartbeatInterval), + RequestedChannelMax = Options.PoolOptions.MaxChannelsPerConnection, + DispatchConsumersAsync = Options.PoolOptions.EnableDispatchConsumersAsync, }; - if (Options.FactoryOptions.Uri != null) + if (Options.PoolOptions.Uri != null) { - cf.Uri = Options.FactoryOptions.Uri; + cf.Uri = Options.PoolOptions.Uri; } else { - cf.VirtualHost = Options.FactoryOptions.VirtualHost; - cf.HostName = Options.FactoryOptions.HostName; - cf.UserName = Options.FactoryOptions.UserName; - cf.Password = Options.FactoryOptions.Password; - if (Options.FactoryOptions.Port != AmqpTcpEndpoint.UseDefaultPort) + cf.VirtualHost = Options.PoolOptions.VirtualHost; + cf.HostName = Options.PoolOptions.HostName; + cf.UserName = Options.PoolOptions.UserName; + cf.Password = Options.PoolOptions.Password; + if (Options.PoolOptions.Port != AmqpTcpEndpoint.UseDefaultPort) { - cf.Port = Options.FactoryOptions.Port; + cf.Port = Options.PoolOptions.Port; } } - if (Options.FactoryOptions.SslOptions.EnableSsl) + if (Options.PoolOptions.SslOptions.EnableSsl) { cf.Ssl = new SslOption { - Enabled = Options.FactoryOptions.SslOptions.EnableSsl, - AcceptablePolicyErrors = Options.FactoryOptions.SslOptions.AcceptedPolicyErrors, - ServerName = Options.FactoryOptions.SslOptions.CertServerName, - CertPath = Options.FactoryOptions.SslOptions.LocalCertPath, - CertPassphrase = Options.FactoryOptions.SslOptions.LocalCertPassword, - Version = Options.FactoryOptions.SslOptions.ProtocolVersions + Enabled = Options.PoolOptions.SslOptions.EnableSsl, + AcceptablePolicyErrors = Options.PoolOptions.SslOptions.AcceptedPolicyErrors, + ServerName = Options.PoolOptions.SslOptions.CertServerName, + CertPath = Options.PoolOptions.SslOptions.LocalCertPath, + CertPassphrase = Options.PoolOptions.SslOptions.LocalCertPassword, + Version = Options.PoolOptions.SslOptions.ProtocolVersions }; } @@ -117,26 +117,26 @@ protected virtual ConnectionFactory BuildConnectionFactory() protected virtual ConnectionFactory BuildConnectionFactory(RabbitOptions options, HttpClientHandler oauth2ClientHandler) { var oAuth2ClientBuilder = new OAuth2ClientBuilder( - Options.FactoryOptions.OAuth2Options.ClientId, - Options.FactoryOptions.OAuth2Options.ClientSecret, - new Uri(Options.FactoryOptions.OAuth2Options.TokenEndpointUrl)); + Options.PoolOptions.OAuth2Options.ClientId, + Options.PoolOptions.OAuth2Options.ClientSecret, + new Uri(Options.PoolOptions.OAuth2Options.TokenEndpointUrl)); oAuth2ClientBuilder.SetHttpClientHandler(oauth2ClientHandler); var oAuth2Client = oAuth2ClientBuilder.Build(); var credentialsProvider = new OAuth2ClientCredentialsProvider( - Options.FactoryOptions.OAuth2Options.OAuth2ClientName, + Options.PoolOptions.OAuth2Options.OAuth2ClientName, oAuth2Client); var cf = new ConnectionFactory { AutomaticRecoveryEnabled = true, - TopologyRecoveryEnabled = Options.FactoryOptions.TopologyRecovery, - NetworkRecoveryInterval = TimeSpan.FromSeconds(Options.FactoryOptions.NetRecoveryTimeout), - ContinuationTimeout = TimeSpan.FromSeconds(Options.FactoryOptions.ContinuationTimeout), - RequestedHeartbeat = TimeSpan.FromSeconds(Options.FactoryOptions.HeartbeatInterval), - RequestedChannelMax = Options.FactoryOptions.MaxChannelsPerConnection, - DispatchConsumersAsync = Options.FactoryOptions.EnableDispatchConsumersAsync, + TopologyRecoveryEnabled = Options.PoolOptions.TopologyRecovery, + NetworkRecoveryInterval = TimeSpan.FromSeconds(Options.PoolOptions.NetRecoveryTimeout), + ContinuationTimeout = TimeSpan.FromSeconds(Options.PoolOptions.ContinuationTimeout), + RequestedHeartbeat = TimeSpan.FromSeconds(Options.PoolOptions.HeartbeatInterval), + RequestedChannelMax = Options.PoolOptions.MaxChannelsPerConnection, + DispatchConsumersAsync = Options.PoolOptions.EnableDispatchConsumersAsync, CredentialsProvider = credentialsProvider, CredentialsRefresher = new TimerBasedCredentialRefresher() }; diff --git a/src/HouseofCat.RabbitMQ/README.md b/src/HouseofCat.RabbitMQ/README.md index 7d22f9c1..677c6073 100644 --- a/src/HouseofCat.RabbitMQ/README.md +++ b/src/HouseofCat.RabbitMQ/README.md @@ -7,7 +7,7 @@ Review tutorials and documentation under `./guides/rabbitmq`. Sample Config File ```json { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -16,8 +16,6 @@ Sample Config File "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 10, diff --git a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json index 19642cef..07f26752 100644 --- a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json +++ b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -18,9 +18,7 @@ "LocalCertPath": "", "LocalCertPassword": "", "ProtocolVersions": 3072 - } - }, - "PoolOptions": { + }, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 0, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json index 1c1f8fbf..42fc73df 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -7,9 +7,7 @@ "TopologyRecovery": true, "NetRecoveryTimeout": 10, "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { + "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 1, "MaxChannels": 0, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json index 8c0f79d3..690fc30b 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -7,9 +7,7 @@ "TopologyRecovery": true, "NetRecoveryTimeout": 10, "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { + "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 0, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json index 8c26ce6d..a2c2833c 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -7,9 +7,7 @@ "TopologyRecovery": true, "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { + "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 0, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json index eecbca91..56198585 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -7,9 +7,7 @@ "TopologyRecovery": true, "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { + "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 10, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json index 37d8b119..6b2ecda9 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -7,9 +7,7 @@ "TopologyRecovery": true, "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { + "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 10, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json index ce2fbbf9..2c1f7471 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -7,9 +7,7 @@ "TopologyRecovery": true, "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { + "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 10, diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index c8b7b647..f33e9118 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -54,12 +54,9 @@ public static async Task RunAutoPublisherStandaloneAsync() // Step 1: Create RabbitOptions var rabbitOptions = new RabbitOptions { - FactoryOptions = new FactoryOptions - { - Uri = new Uri("amqp://guest:guest@localhost:5672"), - }, PoolOptions = new PoolOptions { + Uri = new Uri("amqp://guest:guest@localhost:5672"), ServiceName = "TestService", MaxConnections = 2, MaxChannels = 10, diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json index ce2fbbf9..2c1f7471 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json @@ -1,5 +1,5 @@ { - "FactoryOptions": { + "PoolOptions": { "Uri": "amqp://guest:guest@localhost:5672/", "MaxChannelsPerConnection": 2000, "HeartbeatInterval": 6, @@ -7,9 +7,7 @@ "TopologyRecovery": true, "NetRecoveryTimeout": 5, "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true - }, - "PoolOptions": { + "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", "MaxConnections": 2, "MaxChannels": 10, From fbc19e21359d02874ed8ac4460ede5de5e4bc13f Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 16:16:12 -0500 Subject: [PATCH 14/47] Returning back to IReceivedMessage usage. --- src/HouseofCat.RabbitMQ/Constants.cs | 4 +- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 239 +----------------- .../Consumer/Extensions/ConsumerExtensions.cs | 10 +- .../Dataflows/ConsumerDataflow.cs | 42 +-- .../Extensions/MessageExtensions.cs | 60 +++-- .../Extensions/MetadataExtensions.cs | 54 ++-- src/HouseofCat.RabbitMQ/Messages/Message.cs | 64 +---- src/HouseofCat.RabbitMQ/Messages/Metadata.cs | 6 +- .../Messages/PublishReceipt.cs | 5 +- .../Messages/ReceivedMessage.cs | 40 ++- .../Pipelines/ConsumerPipeline.cs | 15 +- .../Publisher/Publisher.cs | 41 ++- .../Services/RabbitService.cs | 70 +++-- 13 files changed, 203 insertions(+), 447 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index ab0b8134..862061b9 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -10,7 +10,9 @@ public static class Constants // Consumer public static string HeaderForContentType { get; set; } = "ContentType"; - public static string HeaderValueForContentTypeApplicationJson { get; set; } = "application/json;"; + public const string HeaderValueForContentTypeBinary = "application/octet-stream"; + public const string HeaderValueForContentTypePlainText = "text/plain"; + public const string HeaderValueForContentTypeJson = "application/json"; public static string HeaderForObjectType { get; set; } = "X-RD-OBJECTTYPE"; public static string HeaderValueForMessageObjectType { get; set; } = "IMESSAGE"; diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 4644e7f8..723b10e1 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -1,6 +1,6 @@ -using HouseofCat.Dataflows; using HouseofCat.RabbitMQ.Pools; using HouseofCat.Utilities.Errors; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.Events; @@ -10,7 +10,6 @@ using System.Threading.Channels; using System.Threading.Tasks; using static HouseofCat.RabbitMQ.LogMessages; -using HouseofCat.Utilities.Helpers; namespace HouseofCat.RabbitMQ; @@ -21,57 +20,22 @@ public interface IConsumer ConsumerOptions ConsumerOptions { get; } bool Started { get; } - Task DataflowExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - int boundedCapacity = 1000, - TaskScheduler taskScheduler = null, - CancellationToken token = default); - - Task DataflowExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - Func> preWorkBodyAsync = null, - Func postWorkBodyAsync = null, - int boundedCapacity = 1000, - TaskScheduler taskScheduler = null, - CancellationToken token = default); - - Task ChannelExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - Func postWorkBodyAsync = null, - int boundedCapacity = 1000, - TaskScheduler taskScheduler = null, - CancellationToken token = default); - - Task DirectChannelExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - TaskScheduler taskScheduler = null, - CancellationToken token = default); - ChannelReader GetConsumerBuffer(); ValueTask ReadAsync(); Task> ReadUntilEmptyAsync(); Task StartConsumerAsync(); Task StopConsumerAsync(bool immediate = false); IAsyncEnumerable StreamUntilConsumerStopAsync(); - IAsyncEnumerable StreamUntilQueueEmptyAsync(); } -public class Consumer : IConsumer, IDisposable +public class Consumer : IConsumer, IDisposable { private readonly ILogger _logger; private readonly SemaphoreSlim _conLock = new SemaphoreSlim(1, 1); private readonly SemaphoreSlim _executionLock = new SemaphoreSlim(1, 1); private IChannelHost _chanHost; private bool _disposedValue; - private Channel _consumerChannel; + private Channel _consumerChannel; public string ConsumerTag { get; private set; } private bool _shutdown; @@ -116,7 +80,7 @@ public async Task StartConsumerAsync() { await SetChannelHostAsync().ConfigureAwait(false); _shutdown = false; - _consumerChannel = Channel.CreateBounded( + _consumerChannel = Channel.CreateBounded( new BoundedChannelOptions(ConsumerOptions.BatchSize!.Value) { FullMode = ConsumerOptions.BehaviorWhenFull!.Value @@ -406,9 +370,9 @@ await _chanHost _chanHost.ChannelId); } - public ChannelReader GetConsumerBuffer() => _consumerChannel.Reader; + public ChannelReader GetConsumerBuffer() => _consumerChannel.Reader; - public async ValueTask ReadAsync() + public async ValueTask ReadAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); @@ -418,11 +382,11 @@ public async ValueTask ReadAsync() .ConfigureAwait(false); } - public async Task> ReadUntilEmptyAsync() + public async Task> ReadUntilEmptyAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); - var list = new List(); + var list = new List(); await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false); while (_consumerChannel.Reader.TryRead(out var message)) { @@ -433,19 +397,7 @@ public async Task> ReadUntilEmptyAsync() return list; } - public async IAsyncEnumerable StreamUntilQueueEmptyAsync() - { - if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); - - await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false); - while (_consumerChannel.Reader.TryRead(out var message)) - { - if (message == null) { break; } - yield return message; - } - } - - public async IAsyncEnumerable StreamUntilConsumerStopAsync() + public async IAsyncEnumerable StreamUntilConsumerStopAsync() { if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); @@ -455,179 +407,6 @@ public async IAsyncEnumerable StreamUntilConsumerStopAsync() } } - public async Task DataflowExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - int boundedCapacity = 1000, - TaskScheduler taskScheduler = null, - CancellationToken token = default) - { - var dataflowEngine = new DataflowEngine(workBodyAsync, maxDoP, ensureOrdered, null, null, boundedCapacity, taskScheduler); - - await TransferDataToDataflowEngine(dataflowEngine, token); - } - - public async Task DataflowExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - Func> preWorkBodyAsync = null, - Func postWorkBodyAsync = null, - int boundedCapacity = 1000, - TaskScheduler taskScheduler = null, - CancellationToken token = default) - { - var dataflowEngine = new DataflowEngine( - workBodyAsync, - maxDoP, - ensureOrdered, - preWorkBodyAsync, - postWorkBodyAsync, - boundedCapacity, - taskScheduler); - - await TransferDataToDataflowEngine(dataflowEngine, token); - } - - private async Task TransferDataToDataflowEngine( - DataflowEngine dataflowEngine, - CancellationToken token = default) - { - await _executionLock.WaitAsync(2000, token).ConfigureAwait(false); - - try - { - while (await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false)) - { - while (_consumerChannel.Reader.TryRead(out var receivedMessage)) - { - if (receivedMessage != null) - { - _logger.LogDebug( - Consumers.ConsumerDataflowQueueing, - ConsumerOptions.ConsumerName, - receivedMessage.DeliveryTag); - - await dataflowEngine - .EnqueueWorkAsync(receivedMessage) - .ConfigureAwait(false); - } - } - } - } - catch (OperationCanceledException) - { - _logger.LogWarning( - Consumers.ConsumerDataflowActionCancelled, - ConsumerOptions.ConsumerName); - } - catch (Exception ex) - { - _logger.LogError( - Consumers.ConsumerDataflowError, - ConsumerOptions.ConsumerName, - ex.Message); - } - finally { _executionLock.Release(); } - } - - public async Task ChannelExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - Func postWorkBodyAsync = null, - int boundedCapacity = 1000, - TaskScheduler taskScheduler = null, - CancellationToken token = default) - { - var channelBlockEngine = new ChannelBlockEngine( - workBodyAsync, - maxDoP, - ensureOrdered, - postWorkBodyAsync, - boundedCapacity, - taskScheduler, - token); - - await TransferDataToChannelBlockEngine(channelBlockEngine, token); - } - - public async Task DirectChannelExecutionEngineAsync( - Func> workBodyAsync, - int maxDoP = 4, - bool ensureOrdered = true, - TaskScheduler taskScheduler = null, - CancellationToken token = default) - { - _ = new ChannelBlockEngine( - _consumerChannel, workBodyAsync, maxDoP, ensureOrdered, taskScheduler, token); - - await _executionLock.WaitAsync(2000, token).ConfigureAwait(false); - - try - { - while (await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false)) - { - await Task.Delay(4, token); // sleep until channel is finished. - } - } - catch (OperationCanceledException) - { - _logger.LogWarning( - Consumers.ConsumerDataflowActionCancelled, - ConsumerOptions.ConsumerName); - } - catch (Exception ex) - { - _logger.LogError( - Consumers.ConsumerDataflowError, - ConsumerOptions.ConsumerName, - ex.Message); - } - finally { _executionLock.Release(); } - } - - private async Task TransferDataToChannelBlockEngine( - ChannelBlockEngine channelBlockEngine, - CancellationToken token = default) - { - await _executionLock.WaitAsync(2000, token).ConfigureAwait(false); - - try - { - while (await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false)) - { - var receivedMessage = await _consumerChannel.Reader.ReadAsync(token); - if (receivedMessage != null) - { - _logger.LogDebug( - Consumers.ConsumerDataflowQueueing, - ConsumerOptions.ConsumerName, - receivedMessage.DeliveryTag); - - await channelBlockEngine - .EnqueueWorkAsync(receivedMessage) - .ConfigureAwait(false); - } - } - } - catch (OperationCanceledException) - { - _logger.LogWarning( - Consumers.ConsumerDataflowActionCancelled, - ConsumerOptions.ConsumerName); - } - catch (Exception ex) - { - _logger.LogError( - Consumers.ConsumerDataflowError, - ConsumerOptions.ConsumerName, - ex.Message); - } - finally { _executionLock.Release(); } - } - protected virtual void Dispose(bool disposing) { if (!_disposedValue) diff --git a/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs b/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs index 0863eaa9..5de8c478 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs @@ -1,21 +1,21 @@ +using HouseofCat.Dataflows; +using HouseofCat.RabbitMQ.Dataflows; using System; using System.Threading; using System.Threading.Tasks; -using HouseofCat.Dataflows; -using HouseofCat.RabbitMQ.Dataflows; namespace HouseofCat.RabbitMQ; public static class ConsumerExtensions { public static ValueTask DirectChannelExecutionEngineAsync( - this IConsumer consumer, - Func> workBodyAsync, + this IConsumer consumer, + Func> workBodyAsync, Func postWorkBodyAsync = null, TaskScheduler taskScheduler = null, CancellationToken cancellationToken = default) { - var channelReaderBlockEngine = new ChannelReaderBlockEngine( + var channelReaderBlockEngine = new ChannelReaderBlockEngine( consumer.GetConsumerBuffer(), workBodyAsync, consumer.ConsumerOptions.ConsumerPipelineOptions.MaxDegreesOfParallelism, diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index abc929f2..814a0067 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -22,16 +22,16 @@ namespace HouseofCat.RabbitMQ.Dataflows; public string WorkflowName { get; } private readonly IRabbitService _rabbitService; - private readonly ICollection> _consumers; + private readonly ICollection> _consumers; private readonly ConsumerOptions _consumerOptions; private readonly TaskScheduler _taskScheduler; private readonly string _consumerName; private readonly int _consumerCount; // Main Flow - Ingestion - private readonly List> _consumerBlocks; - protected ITargetBlock _inputBuffer; - private TransformBlock _buildStateBlock; + private readonly List> _consumerBlocks; + protected ITargetBlock _inputBuffer; + private TransformBlock _buildStateBlock; private TransformBlock _createSendMessage; protected TransformBlock _decryptBlock; protected TransformBlock _decompressBlock; @@ -80,7 +80,7 @@ public ConsumerDataflow( TaskScheduler = _taskScheduler, }; - _consumerBlocks = new List>(); + _consumerBlocks = new List>(); } /// @@ -95,7 +95,7 @@ public ConsumerDataflow( public ConsumerDataflow( IRabbitService rabbitService, string workflowName, - ICollection> consumers, + ICollection> consumers, GlobalConsumerPipelineOptions globalConsumerPipelineOptions, TaskScheduler taskScheduler = null) : this( rabbitService, @@ -121,7 +121,7 @@ public ConsumerDataflow( public ConsumerDataflow( IRabbitService rabbitService, string workflowName, - ICollection> consumers, + ICollection> consumers, int maxDoP, bool ensureOrdered, TaskScheduler taskScheduler = null) @@ -146,12 +146,12 @@ public ConsumerDataflow( TaskScheduler = _taskScheduler, }; - _consumerBlocks = new List>(); + _consumerBlocks = new List>(); } - public virtual Task StartAsync() => StartAsync>(); + public virtual Task StartAsync() => StartAsync>(); - protected async Task StartAsync() where TConsumerBlock : ConsumerBlock, new() + protected async Task StartAsync() where TConsumerBlock : ConsumerBlock, new() { BuildLinkages(); @@ -481,13 +481,13 @@ public ConsumerDataflow WithSendStep( #region Step Linking protected virtual void BuildLinkages(DataflowLinkOptions overrideOptions = null) - where TConsumerBlock : ConsumerBlock, new() + where TConsumerBlock : ConsumerBlock, new() { Guard.AgainstNull(_buildStateBlock, nameof(_buildStateBlock)); // Create State Is Mandatory Guard.AgainstNull(_finalization, nameof(_finalization)); // Leaving The Workflow Is Mandatory Guard.AgainstNull(_errorAction, nameof(_errorAction)); // Processing Errors Is Mandatory - _inputBuffer ??= new BufferBlock(); + _inputBuffer ??= new BufferBlock(); _readyBuffer ??= new BufferBlock(); @@ -519,7 +519,7 @@ protected virtual void BuildLinkages(DataflowLinkOptions overrid } } - ((ISourceBlock)_inputBuffer).LinkTo(_buildStateBlock, overrideOptions ?? _linkStepOptions); + ((ISourceBlock)_inputBuffer).LinkTo(_buildStateBlock, overrideOptions ?? _linkStepOptions); _buildStateBlock.LinkTo(_errorBuffer, overrideOptions ?? _linkStepOptions, x => x == null); SetCurrentSourceBlock(_buildStateBlock); @@ -597,7 +597,7 @@ private void LinkWithFaultRoute(ISourceBlock source, IPropagatorBlock(string key, ReceivedMessage data) + public virtual TState BuildStateAndPayload(string key, IReceivedMessage data) { var state = new TState { @@ -667,11 +667,11 @@ public virtual TState BuildStateAndPayload(string key, ReceivedMessage dat return state; } - public TransformBlock GetBuildStateWithPayloadBlock( + public TransformBlock GetBuildStateWithPayloadBlock( string key, ExecutionDataflowBlockOptions options) { - TState BuildStateWrap(ReceivedMessage data) + TState BuildStateWrap(IReceivedMessage data) { try { return BuildStateAndPayload(key, data); } @@ -679,13 +679,13 @@ TState BuildStateWrap(ReceivedMessage data) { return null; } } - return new TransformBlock(BuildStateWrap, options); + return new TransformBlock(BuildStateWrap, options); } - public TransformBlock GetBuildStateBlock( + public TransformBlock GetBuildStateBlock( ExecutionDataflowBlockOptions options) { - TState BuildStateWrap(ReceivedMessage data) + TState BuildStateWrap(IReceivedMessage data) { try { return BuildState(data); } @@ -693,7 +693,7 @@ TState BuildStateWrap(ReceivedMessage data) { return null; } } - return new TransformBlock(BuildStateWrap, options); + return new TransformBlock(BuildStateWrap, options); } public TransformBlock GetByteManipulationTransformBlock( diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 8420958b..799fdc1c 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -10,29 +10,55 @@ public static class MessageExtensions { private static readonly XorShift XorShift = new XorShift(true); - public static TMessage Clone(this IMessage message) + public static Message Clone(this IMessage message) + { + return message.ShallowClone(); + } + + public static TMessage ShallowClone(this IMessage message) where TMessage : IMessage, new() { - return new TMessage + var clonedMessage = new TMessage { Exchange = new string(message.Exchange), RoutingKey = new string(message.RoutingKey), DeliveryMode = message.DeliveryMode, Mandatory = message.Mandatory, - PriorityLevel = message.PriorityLevel + PriorityLevel = message.PriorityLevel, + }; + + if (message.Metadata is not null) + { + clonedMessage.Metadata = new Metadata + { + PayloadId = new string(message.Metadata.PayloadId), + Encrypted = message.Metadata.Encrypted, + Compressed = message.Metadata.Compressed, + }; + + if (message.Metadata.Fields is not null) + { + clonedMessage.Metadata.Fields = new Dictionary(message.Metadata.Fields); + } + } + + return clonedMessage; } public static void UpsertHeader(this IMessage message, string key, object value) { - var metadata = message.CreateMetadataIfMissing(); - metadata.UpsertHeader(key, value); + message.Metadata ??= new Metadata(); + message.Metadata.UpsertHeader(key, value); } public static void WriteHeadersToMetadata(this IMessage message, IDictionary headers) { - var metadata = message.CreateMetadataIfMissing(); - metadata.WriteHeadersToMetadata(headers); + message.Metadata ??= new Metadata(); + foreach (var kvp in headers) + { + message.Metadata.UpsertHeader(kvp.Key, kvp.Value); + } } public static IBasicProperties CreateBasicProperties( @@ -41,32 +67,32 @@ public static IBasicProperties CreateBasicProperties( bool withOptionalHeaders, IMetadata metadata) { - var props = channelHost.GetChannel().CreateBasicProperties(); + var basicProperties = channelHost.GetChannel().CreateBasicProperties(); - props.DeliveryMode = message.DeliveryMode; - props.ContentType = new string(message.ContentType); - props.Priority = message.PriorityLevel; - props.MessageId = message.MessageId == null + basicProperties.DeliveryMode = message.DeliveryMode; + basicProperties.ContentType = new string(message.ContentType); + basicProperties.Priority = message.PriorityLevel; + basicProperties.MessageId = message.MessageId == null ? new string(message.MessageId) : Guid.NewGuid().ToString(); - if (!props.IsHeadersPresent()) + if (!basicProperties.IsHeadersPresent()) { - props.Headers = new Dictionary(); + basicProperties.Headers = new Dictionary(); } if (withOptionalHeaders && metadata != null) { - foreach (var kvp in metadata?.CustomFields) + foreach (var kvp in metadata?.Fields) { if (kvp.Key.StartsWith(Constants.HeaderPrefix, StringComparison.OrdinalIgnoreCase)) { - props.Headers[kvp.Key] = kvp.Value; + basicProperties.Headers[kvp.Key] = kvp.Value; } } } - return props; + return basicProperties; } public static IMessage CreateSimpleRandomMessage(string queueName, int bodySize = 1000) diff --git a/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs index a2c2fdbe..ef0b132d 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MetadataExtensions.cs @@ -6,29 +6,12 @@ namespace HouseofCat.RabbitMQ; public static class MetadataExtensions { - public static T Clone(this IMetadata metadata) - where T : IMetadata, new() - { - var clonedMetadata = new T - { - Compressed = metadata.Compressed, - Encrypted = metadata.Encrypted - }; - - foreach (var kvp in metadata.CustomFields) - { - clonedMetadata.CustomFields.Add(kvp.Key, kvp.Value); - } - - return clonedMetadata; - } - public static T GetHeader(this IMetadata metadata, string key) { Guard.AgainstNull(metadata, nameof(Metadata)); - Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(Metadata.CustomFields)); + Guard.AgainstNullOrEmpty(metadata.Fields, nameof(Metadata.Fields)); - if (metadata.CustomFields.TryGetValue(key, out object value)) + if (metadata.Fields.TryGetValue(key, out object value)) { if (value is T temp) { return temp; } @@ -40,48 +23,41 @@ public static T GetHeader(this IMetadata metadata, string key) public static void UpsertHeader(this IMetadata metadata, string key, object value) { - metadata.CustomFields ??= new Dictionary(); - - metadata.CustomFields[key] = value; + metadata.Fields ??= new Dictionary(); + metadata.Fields[key] = value; } public static bool RemoveHeader(this IMetadata metadata, string key) { Guard.AgainstNull(metadata, nameof(Metadata)); - Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(Metadata.CustomFields)); + Guard.AgainstNullOrEmpty(metadata.Fields, nameof(Metadata.Fields)); return metadata - .CustomFields + .Fields .Remove(key); } - public static IDictionary GetHeadersOutOfMetadata(this IMetadata metadata) + public static IDictionary GetHeadersFromMetadata(this IMetadata metadata) { Guard.AgainstNull(metadata, nameof(Metadata)); - Guard.AgainstNullOrEmpty(metadata.CustomFields, nameof(Metadata.CustomFields)); + Guard.AgainstNullOrEmpty(metadata.Fields, nameof(Metadata.Fields)); - var dict = new Dictionary(); - - foreach (var kvp in metadata.CustomFields) - { - if (kvp.Key.StartsWith(Constants.HeaderPrefix, StringComparison.OrdinalIgnoreCase)) - { - dict[kvp.Key] = kvp.Value; - } - } - - return dict; + return new Dictionary(metadata.Fields); } public static void WriteHeadersToMetadata(this IMetadata metadata, IDictionary headers) { - metadata.CustomFields ??= new Dictionary(); + if (metadata.Fields is null) + { + metadata.Fields ??= new Dictionary(headers); + return; + } foreach (var kvp in headers) { if (kvp.Key.StartsWith(Constants.HeaderPrefix, StringComparison.OrdinalIgnoreCase)) { - metadata.CustomFields[kvp.Key] = kvp.Value; + metadata.Fields[kvp.Key] = kvp.Value; } } } diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index 6dfcf187..54fe40d2 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -1,9 +1,7 @@ using HouseofCat.RabbitMQ.Pools; -using HouseofCat.Serialization; using HouseofCat.Utilities.Helpers; using RabbitMQ.Client; using System; -using System.Collections.Generic; using System.ComponentModel.DataAnnotations; namespace HouseofCat.RabbitMQ; @@ -24,15 +22,7 @@ public interface IMessage ReadOnlyMemory Body { get; set; } - IMetadata GetMetadata(); - - IMetadata CreateMetadataIfMissing(); - - T GetHeader(string key); - bool RemoveHeader(string key); - IDictionary GetHeadersOutOfMetadata(); - - ReadOnlyMemory GetBodyToPublish(ISerializationProvider serializationProvider); + IMetadata Metadata { get; set; } IPublishReceipt GetPublishReceipt(bool error); @@ -55,26 +45,11 @@ public sealed class Message : IMessage [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] public byte PriorityLevel { get; set; } - public string ContentType { get; set; } = Constants.HeaderValueForContentTypeApplicationJson; + public string ContentType { get; set; } = Constants.HeaderValueForContentTypeJson; public IMetadata Metadata { get; set; } public ReadOnlyMemory Body { get; set; } - public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders) - { - MessageId ??= Guid.NewGuid().ToString(); - - var basicProperties = this.CreateBasicProperties(channelHost, withOptionalHeaders, Metadata); - basicProperties.MessageId = MessageId; - - // Non-optional Header. - basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; - var openTelHeader = OpenTelemetryHelpers.CreateOpenTelemetryHeaderFromCurrentActivityOrDefault(); - basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; - - return basicProperties; - } - public Message() { } public Message( @@ -101,26 +76,20 @@ public Message(string exchange, string routingKey, ReadOnlyMemory body, st { Metadata = new Metadata(); } } - public Message(string exchange, string routingKey, byte[] body, string payloadId, byte priority) + public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders) { - Exchange = exchange; - RoutingKey = routingKey; - Body = body; - PriorityLevel = priority; - if (!string.IsNullOrWhiteSpace(payloadId)) - { Metadata = new Metadata { PayloadId = payloadId }; } - else - { Metadata = new Metadata(); } - } + MessageId ??= Guid.NewGuid().ToString(); - public Message Clone() - { - var clone = this.Clone(); - clone.Metadata = Metadata.Clone(); - return clone; - } + var basicProperties = this.CreateBasicProperties(channelHost, withOptionalHeaders, Metadata); + basicProperties.MessageId = MessageId; + + // Non-optional Header. + basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + var openTelHeader = OpenTelemetryHelpers.CreateOpenTelemetryHeaderFromCurrentActivityOrDefault(); + basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; - public IMetadata GetMetadata() => Metadata; + return basicProperties; + } public IMetadata CreateMetadataIfMissing() { @@ -128,13 +97,6 @@ public IMetadata CreateMetadataIfMissing() return Metadata; } - public T GetHeader(string key) => Metadata.GetHeader(key); - public bool RemoveHeader(string key) => Metadata.RemoveHeader(key); - public IDictionary GetHeadersOutOfMetadata() => Metadata.GetHeadersOutOfMetadata(); - - public ReadOnlyMemory GetBodyToPublish(ISerializationProvider serializationProvider) => - serializationProvider.Serialize(this).ToArray(); - public IPublishReceipt GetPublishReceipt(bool error) => new PublishReceipt { MessageId = MessageId, IsError = error, OriginalMessage = error ? this : null }; } diff --git a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs index cc409ab4..534f7906 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs @@ -8,17 +8,17 @@ public interface IMetadata bool Encrypted { get; set; } bool Compressed { get; set; } - Dictionary CustomFields { get; set; } + Dictionary Fields { get; set; } } public sealed class Metadata : IMetadata { /// - /// PayloadId is a user supplied Id for tracking purposes. Allows identifying the inner payload without needing to deserialize first. + /// PayloadId is a user supplied identifier that allows identifying the Body without needing to deserialize. /// public string PayloadId { get; set; } public bool Encrypted { get; set; } public bool Compressed { get; set; } - public Dictionary CustomFields { get; set; } = new Dictionary(); + public Dictionary Fields { get; set; } = new Dictionary(); } diff --git a/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs b/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs index f6d166ae..1707d367 100644 --- a/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs +++ b/src/HouseofCat.RabbitMQ/Messages/PublishReceipt.cs @@ -4,8 +4,7 @@ public interface IPublishReceipt { bool IsError { get; set; } string MessageId { get; set; } - - IMessage GetOriginalMessage(); + IMessage OriginalMessage { get; set; } } public struct PublishReceipt : IPublishReceipt @@ -13,6 +12,4 @@ public struct PublishReceipt : IPublishReceipt public bool IsError { get; set; } public string MessageId { get; set; } public IMessage OriginalMessage { get; set; } - - public readonly IMessage GetOriginalMessage() => OriginalMessage; } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index b7979d60..6a9e72ae 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -19,11 +19,12 @@ public interface IReceivedMessage DateTime EncryptedDateTime { get; } bool Compressed { get; } string CompressionType { get; } + public string TraceParentHeader { get; } ReadOnlyMemory Body { get; set; } string ConsumerTag { get; } ulong DeliveryTag { get; } - Message Message { get; set; } + IMessage Message { get; set; } IBasicProperties Properties { get; } @@ -37,19 +38,16 @@ public interface IReceivedMessage public sealed class ReceivedMessage : IReceivedMessage, IDisposable { - /// - /// Indicates that the content was not deserializeable based on the provided headers. - /// - public bool FailedToDeserialize { get; private set; } public IBasicProperties Properties { get; } public bool Ackable { get; } public IModel Channel { get; set; } public string ConsumerTag { get; } public ulong DeliveryTag { get; } public ReadOnlyMemory Body { get; set; } - public Message Message { get; set; } + public IMessage Message { get; set; } // Retrieved From Headers + public bool FailedToDeserialize { get; private set; } public string ContentType { get; private set; } public string ObjectType { get; private set; } public bool Encrypted { get; private set; } @@ -109,10 +107,23 @@ private void ReadHeaders() if (ObjectType == Constants.HeaderValueForMessageObjectType && Body.Length > 0) { - try - { Message = JsonSerializer.Deserialize(Body.Span); } - catch - { FailedToDeserialize = true; } + switch (ContentType) + { + case Constants.HeaderValueForContentTypeJson: + try + { Message = JsonSerializer.Deserialize(Body.Span); } + catch + { FailedToDeserialize = true; } + break; + + case Constants.HeaderValueForContentTypeBinary: + case Constants.HeaderValueForContentTypePlainText: + Message = new Message{ Body = Body }; + break; + + default: + break; + } } if (Properties.Headers.TryGetValue(Constants.HeaderForEncrypted, out object encryptedValue)) @@ -208,7 +219,13 @@ public bool RejectMessage(bool requeue) /// /// A way to indicate this message is fully finished with. /// - public void Complete() => _completionSource.SetResult(true); + public void Complete() + { + if (_completionSource.Task.Status < TaskStatus.RanToCompletion) + { + _completionSource.SetResult(true); + } + } private void Dispose(bool disposing) { @@ -216,6 +233,7 @@ private void Dispose(bool disposing) { if (disposing) { + Complete(); _completionSource.Task.Dispose(); } diff --git a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs index dec62575..55cee7c0 100644 --- a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs +++ b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs @@ -27,8 +27,8 @@ public class ConsumerPipeline : IConsumerPipeline, IDisposable where public bool Started { get; private set; } private readonly ILogger> _logger; - private IConsumer Consumer { get; } - private IPipeline Pipeline { get; } + private IConsumer Consumer { get; } + private IPipeline Pipeline { get; } private Task FeedPipelineWithDataTasks { get; set; } private TaskCompletionSource _completionSource; private CancellationTokenSource _cancellationTokenSource; @@ -37,8 +37,8 @@ public class ConsumerPipeline : IConsumerPipeline, IDisposable where private readonly SemaphoreSlim _pipeExecLock = new SemaphoreSlim(1, 1); public ConsumerPipeline( - IConsumer consumer, - IPipeline pipeline) + IConsumer consumer, + IPipeline pipeline) { _logger = LogHelpers.GetLogger>(); Pipeline = pipeline ?? throw new ArgumentNullException(nameof(pipeline)); @@ -130,7 +130,7 @@ await Consumer } public async Task PipelineStreamEngineAsync( - IPipeline pipeline, + IPipeline pipeline, bool waitForCompletion, CancellationToken token = default) { @@ -190,7 +190,10 @@ await receivedMessage finally { _pipeExecLock.Release(); } } - public async Task PipelineExecutionEngineAsync(IPipeline pipeline, bool waitForCompletion, CancellationToken token = default) + public async Task PipelineExecutionEngineAsync( + IPipeline pipeline, + bool waitForCompletion, + CancellationToken token = default) { await _pipeExecLock .WaitAsync(2000, token) diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index 5cb46b26..fc645681 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -232,8 +232,7 @@ public void QueueMessage(IMessage message) if (!AutoPublisherStarted) throw new InvalidOperationException(ExceptionMessages.AutoPublisherNotStartedError); Guard.AgainstNull(message, nameof(message)); - var metadata = message.GetMetadata(); - _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, metadata?.PayloadId); + _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, message.Metadata?.PayloadId); _messageQueue.Writer.TryWrite(message); } @@ -251,8 +250,7 @@ public async ValueTask QueueMessageAsync(IMessage message) throw new InvalidOperationException(ExceptionMessages.QueueChannelError); } - var metadata = message.GetMetadata(); - _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, metadata?.PayloadId); + _logger.LogDebug(LogMessages.AutoPublishers.MessageQueued, message.MessageId, message.Metadata?.PayloadId); await _messageQueue .Writer @@ -270,26 +268,26 @@ private async Task ProcessMessagesAsync(ChannelReader channelReader) if (message == null) { continue; } - var metadata = message.GetMetadata(); + message.Metadata ??= new Metadata(); if (_compress) { message.Body = _compressionProvider.Compress(message.Body).ToArray(); - metadata.Compressed = _compress; - metadata.CustomFields[Constants.HeaderForCompressed] = _compress; - metadata.CustomFields[Constants.HeaderForCompression] = _compressionProvider.Type; + message.Metadata.Compressed = _compress; + message.Metadata.Fields[Constants.HeaderForCompressed] = _compress; + message.Metadata.Fields[Constants.HeaderForCompression] = _compressionProvider.Type; } if (_encrypt) { message.Body = _encryptionProvider.Encrypt(message.Body).ToArray(); - metadata.Encrypted = _encrypt; - metadata.CustomFields[Constants.HeaderForEncrypted] = _encrypt; - metadata.CustomFields[Constants.HeaderForEncryption] = _encryptionProvider.Type; - metadata.CustomFields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.RFC3339Long); + message.Metadata.Encrypted = _encrypt; + message.Metadata.Fields[Constants.HeaderForEncrypted] = _encrypt; + message.Metadata.Fields[Constants.HeaderForEncryption] = _encryptionProvider.Type; + message.Metadata.Fields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.RFC3339Long); } - _logger.LogDebug(LogMessages.AutoPublishers.MessagePublished, message.MessageId, metadata?.PayloadId); + _logger.LogDebug(LogMessages.AutoPublishers.MessagePublished, message.MessageId, message.Metadata?.PayloadId); await PublishAsync(message, _createPublishReceipts, _withHeaders) .ConfigureAwait(false); @@ -309,21 +307,20 @@ private async Task ProcessReceiptsAsync(Func process // Super simple version to bake in requeueing of all failed to publish messages. private async ValueTask ProcessReceiptAsync(IPublishReceipt receipt) { - var originalMessage = receipt.GetOriginalMessage(); - if (receipt.IsError && originalMessage != null) + if (receipt.IsError && receipt.OriginalMessage != null) { if (AutoPublisherStarted) { - _logger.LogWarning($"Failed publish for message ({originalMessage.MessageId}). Retrying with AutoPublishing..."); + _logger.LogWarning($"Failed publish for message ({receipt.OriginalMessage.MessageId}). Retrying with AutoPublishing..."); try - { await QueueMessageAsync(receipt.GetOriginalMessage()); } + { await QueueMessageAsync(receipt.OriginalMessage); } catch (Exception ex) /* No-op */ { _logger.LogDebug("Error ({0}) occurred on retry, most likely because retry during shutdown.", ex.Message); } } else { - _logger.LogError($"Failed publish for message ({originalMessage.MessageId}). Unable to retry as the original message was not received."); + _logger.LogError($"Failed publish for message ({receipt.OriginalMessage.MessageId}). Unable to retry as the original message was not received."); } } } @@ -545,7 +542,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp try { - var body = message.GetBodyToPublish(_serializationProvider); + var body = _serializationProvider.Serialize(message.Body); chanHost .GetChannel() .BasicPublish( @@ -604,7 +601,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece message.RoutingKey, message.Mandatory, message.BuildProperties(chanHost, withOptionalHeaders), - message.GetBodyToPublish(_serializationProvider)); + _serializationProvider.Serialize(message.Body)); chanHost.GetChannel().WaitForConfirmsOrDie(_waitForConfirmation); } @@ -653,7 +650,7 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, messages[i].RoutingKey, messages[i].Mandatory, messages[i].BuildProperties(chanHost, withOptionalHeaders), - messages[i].GetBodyToPublish(_serializationProvider)); + _serializationProvider.Serialize(messages[i].Body)); } catch (Exception ex) { @@ -701,7 +698,7 @@ public async Task PublishManyAsBatchAsync(IList messages, bool createR messages[i].RoutingKey, messages[i].Mandatory, messages[i].BuildProperties(chanHost, withOptionalHeaders), - messages[i].GetBodyToPublish(_serializationProvider)); + _serializationProvider.Serialize(messages[i].Body)); if (createReceipt) { diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index 29824c64..53b93371 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -30,14 +30,14 @@ public interface IRabbitService IEncryptionProvider EncryptionProvider { get; } ICompressionProvider CompressionProvider { get; } - ConcurrentDictionary> Consumers { get; } + ConcurrentDictionary> Consumers { get; } Task ComcryptAsync(IMessage message); Task CompressAsync(IMessage message); - IConsumerPipeline CreateConsumerPipeline(string consumerName, int batchSize, bool? ensureOrdered, Func> pipelineBuilder) where TOut : RabbitWorkState; - IConsumerPipeline CreateConsumerPipeline(string consumerName, IPipeline pipeline) where TOut : RabbitWorkState; + IConsumerPipeline CreateConsumerPipeline(string consumerName, int batchSize, bool? ensureOrdered, Func> pipelineBuilder) where TOut : RabbitWorkState; + IConsumerPipeline CreateConsumerPipeline(string consumerName, IPipeline pipeline) where TOut : RabbitWorkState; - IConsumerPipeline CreateConsumerPipeline(string consumerName, Func> pipelineBuilder) where TOut : RabbitWorkState; + IConsumerPipeline CreateConsumerPipeline(string consumerName, Func> pipelineBuilder) where TOut : RabbitWorkState; Task DecomcryptAsync(IMessage message); Task DecompressAsync(IMessage message); @@ -45,8 +45,8 @@ public interface IRabbitService bool Encrypt(IMessage message); Task?> GetAsync(string queueName); Task GetAsync(string queueName); - IConsumer GetConsumer(string consumerName); - IConsumer GetConsumerByPipelineName(string consumerPipelineName); + IConsumer GetConsumer(string consumerName); + IConsumer GetConsumerByPipelineName(string consumerPipelineName); ValueTask ShutdownAsync(bool immediately); } @@ -65,7 +65,7 @@ public class RabbitService : IRabbitService, IDisposable public IEncryptionProvider EncryptionProvider { get; } public ICompressionProvider CompressionProvider { get; } - public ConcurrentDictionary> Consumers { get; private set; } = new ConcurrentDictionary>(); + public ConcurrentDictionary> Consumers { get; private set; } = new ConcurrentDictionary>(); private ConcurrentDictionary ConsumerPipelineNameToConsumerOptions { get; set; } = new ConcurrentDictionary(); public string TimeFormat { get; set; } = TimeHelpers.Formats.CatsAltFormat; @@ -283,7 +283,7 @@ public IConsumerPipeline CreateConsumerPipeline( string consumerName, int batchSize, bool? ensureOrdered, - Func> pipelineBuilder) + Func> pipelineBuilder) where TOut : RabbitWorkState { var consumer = GetConsumer(consumerName); @@ -294,7 +294,7 @@ public IConsumerPipeline CreateConsumerPipeline( public IConsumerPipeline CreateConsumerPipeline( string consumerName, - Func> pipelineBuilder) + Func> pipelineBuilder) where TOut : RabbitWorkState { var consumer = GetConsumer(consumerName); @@ -307,7 +307,7 @@ public IConsumerPipeline CreateConsumerPipeline( public IConsumerPipeline CreateConsumerPipeline( string consumerName, - IPipeline pipeline) + IPipeline pipeline) where TOut : RabbitWorkState { var consumer = GetConsumer(consumerName); @@ -315,14 +315,14 @@ public IConsumerPipeline CreateConsumerPipeline( return new ConsumerPipeline(consumer, pipeline); } - public IConsumer GetConsumer(string consumerName) + public IConsumer GetConsumer(string consumerName) { - if (!Consumers.TryGetValue(consumerName, out IConsumer value)) + if (!Consumers.TryGetValue(consumerName, out IConsumer value)) { throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerOptionsMessage, consumerName)); } return value; } - public IConsumer GetConsumerByPipelineName(string consumerPipelineName) + public IConsumer GetConsumerByPipelineName(string consumerPipelineName) { if (!ConsumerPipelineNameToConsumerOptions.TryGetValue(consumerPipelineName, out ConsumerOptions value)) { throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerPipelineOptionsMessage, consumerPipelineName)); } @@ -355,14 +355,13 @@ public async Task ComcryptAsync(IMessage message) // Returns Success public bool Encrypt(IMessage message) { - var metadata = message.GetMetadata(); - if (!metadata.Encrypted) + if (!message.Metadata.Encrypted) { message.Body = EncryptionProvider.Encrypt(message.Body); - metadata.Encrypted = true; - metadata.CustomFields[Constants.HeaderForEncrypted] = true; - metadata.CustomFields[Constants.HeaderForEncryption] = EncryptionProvider.Type; - metadata.CustomFields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeFormat); + message.Metadata.Encrypted = true; + message.Metadata.Fields[Constants.HeaderForEncrypted] = true; + message.Metadata.Fields[Constants.HeaderForEncryption] = EncryptionProvider.Type; + message.Metadata.Fields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeFormat); return true; } @@ -373,15 +372,14 @@ public bool Encrypt(IMessage message) // Returns Success public bool Decrypt(IMessage message) { - var metadata = message.GetMetadata(); - if (metadata.Encrypted) + if (message.Metadata.Encrypted) { message.Body = EncryptionProvider.Decrypt(message.Body); - metadata.Encrypted = false; - metadata.CustomFields[Constants.HeaderForEncrypted] = false; + message.Metadata.Encrypted = false; + message.Metadata.Fields[Constants.HeaderForEncrypted] = false; - metadata.CustomFields.Remove(Constants.HeaderForEncryption); - metadata.CustomFields.Remove(Constants.HeaderForEncryptDate); + message.Metadata.Fields.Remove(Constants.HeaderForEncryption); + message.Metadata.Fields.Remove(Constants.HeaderForEncryptDate); return true; } @@ -392,16 +390,15 @@ public bool Decrypt(IMessage message) // Returns Success public async Task CompressAsync(IMessage message) { - var metadata = message.GetMetadata(); - if (metadata.Encrypted) + if (message.Metadata.Encrypted) { return false; } // Don't compress after encryption. - if (!metadata.Compressed) + if (!message.Metadata.Compressed) { message.Body = (await CompressionProvider.CompressAsync(message.Body).ConfigureAwait(false)).ToArray(); - metadata.Compressed = true; - metadata.CustomFields[Constants.HeaderForCompressed] = true; - metadata.CustomFields[Constants.HeaderForCompression] = CompressionProvider.Type; + message.Metadata.Compressed = true; + message.Metadata.Fields[Constants.HeaderForCompressed] = true; + message.Metadata.Fields[Constants.HeaderForCompression] = CompressionProvider.Type; return true; } @@ -412,19 +409,18 @@ public async Task CompressAsync(IMessage message) // Returns Success public async Task DecompressAsync(IMessage message) { - var metadata = message.GetMetadata(); - if (metadata.Encrypted) + if (message.Metadata.Encrypted) { return false; } // Don't decompress before decryption. - if (metadata.Compressed) + if (message.Metadata.Compressed) { try { message.Body = (await CompressionProvider.DecompressAsync(message.Body).ConfigureAwait(false)).ToArray(); - metadata.Compressed = false; - metadata.CustomFields[Constants.HeaderForCompressed] = false; + message.Metadata.Compressed = false; + message.Metadata.Fields[Constants.HeaderForCompressed] = false; - metadata.CustomFields.Remove(Constants.HeaderForCompression); + message.Metadata.Fields.Remove(Constants.HeaderForCompression); } catch { return false; } } From cae88d46822081aad08c46fdac51b6333bf04fd7 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 16:27:56 -0500 Subject: [PATCH 15/47] Migrate from GetChannel() -> Channel directly. --- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 10 ++-- .../Consumer/Extensions/ConsumerExtensions.cs | 28 ----------- .../Dataflows/ConsumerDataflow.cs | 4 +- .../Extensions/MessageExtensions.cs | 2 +- src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs | 49 +++++++------------ .../Publisher/Publisher.cs | 26 +++++----- .../Services/MaintenanceService.cs | 28 +++++------ .../Services/RabbitService.cs | 4 +- src/HouseofCat.RabbitMQ/Topology/Topologer.cs | 16 +++--- .../Tests/BasicGetTests.cs | 9 ++-- tests/RabbitMQ.Console.Tests/Tests/Shared.cs | 7 ++- 11 files changed, 70 insertions(+), 113 deletions(-) delete mode 100644 src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 723b10e1..deb0d01e 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -229,8 +229,8 @@ private EventingBasicConsumer CreateConsumer() { EventingBasicConsumer consumer = null; - _chanHost.GetChannel().BasicQos(0, ConsumerOptions.BatchSize!.Value, false); - consumer = new EventingBasicConsumer(_chanHost.GetChannel()); + _chanHost.Channel.BasicQos(0, ConsumerOptions.BatchSize!.Value, false); + consumer = new EventingBasicConsumer(_chanHost.Channel); consumer.Received += ReceiveHandler; consumer.Shutdown += ConsumerShutdown; @@ -271,8 +271,8 @@ private AsyncEventingBasicConsumer CreateAsyncConsumer() { AsyncEventingBasicConsumer consumer = null; - _chanHost.GetChannel().BasicQos(0, ConsumerOptions.BatchSize!.Value, false); - consumer = new AsyncEventingBasicConsumer(_chanHost.GetChannel()); + _chanHost.Channel.BasicQos(0, ConsumerOptions.BatchSize!.Value, false); + consumer = new AsyncEventingBasicConsumer(_chanHost.Channel); consumer.Received += ReceiveHandlerAsync; consumer.Shutdown += ConsumerShutdownAsync; @@ -298,7 +298,7 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs { await _consumerChannel .Writer - .WriteAsync(new ReceivedMessage(_chanHost.GetChannel(), bdea, !(ConsumerOptions.AutoAck ?? false))) + .WriteAsync(new ReceivedMessage(_chanHost.Channel, bdea, !(ConsumerOptions.AutoAck ?? false))) .ConfigureAwait(false); return true; } diff --git a/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs b/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs deleted file mode 100644 index 5de8c478..00000000 --- a/src/HouseofCat.RabbitMQ/Consumer/Extensions/ConsumerExtensions.cs +++ /dev/null @@ -1,28 +0,0 @@ -using HouseofCat.Dataflows; -using HouseofCat.RabbitMQ.Dataflows; -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace HouseofCat.RabbitMQ; - -public static class ConsumerExtensions -{ - public static ValueTask DirectChannelExecutionEngineAsync( - this IConsumer consumer, - Func> workBodyAsync, - Func postWorkBodyAsync = null, - TaskScheduler taskScheduler = null, - CancellationToken cancellationToken = default) - { - var channelReaderBlockEngine = new ChannelReaderBlockEngine( - consumer.GetConsumerBuffer(), - workBodyAsync, - consumer.ConsumerOptions.ConsumerPipelineOptions.MaxDegreesOfParallelism, - consumer.ConsumerOptions.ConsumerPipelineOptions.EnsureOrdered, - postWorkBodyAsync, - taskScheduler); - - return channelReaderBlockEngine.ReadChannelAsync(cancellationToken); - } -} diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 814a0067..fec5ea8e 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -628,7 +628,7 @@ public virtual TState BuildState(IReceivedMessage data) return state; } - public virtual TState BuildStateAndPayload(string key, IReceivedMessage data) + public virtual TState BuildStateAndObjectFromPayload(string key, IReceivedMessage data) { var state = new TState { @@ -674,7 +674,7 @@ public TransformBlock GetBuildStateWithPayloadBlock(key, data); } + { return BuildStateAndObjectFromPayload(key, data); } catch { return null; } } diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 799fdc1c..5ae9509d 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -67,7 +67,7 @@ public static IBasicProperties CreateBasicProperties( bool withOptionalHeaders, IMetadata metadata) { - var basicProperties = channelHost.GetChannel().CreateBasicProperties(); + var basicProperties = channelHost.Channel.CreateBasicProperties(); basicProperties.DeliveryMode = message.DeliveryMode; basicProperties.ContentType = new string(message.ContentType); diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs index bbfd58af..ee00b416 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs @@ -14,12 +14,11 @@ public interface IChannelHost { bool Ackable { get; } ulong ChannelId { get; } + public IModel Channel { get; } bool Closed { get; } bool FlowControlled { get; } bool UsedByConsumer { get; } - IModel GetChannel(); - string StartConsuming(IBasicConsumer internalConsumer, ConsumerOptions options); Task StopConsumingAsync(); @@ -34,7 +33,7 @@ public interface IChannelHost public class ChannelHost : IChannelHost, IDisposable { private readonly ILogger _logger; - private IModel _channel { get; set; } + public IModel Channel { get; private set; } private IConnectionHost _connHost { get; set; } public ulong ChannelId { get; set; } @@ -59,17 +58,6 @@ public ChannelHost(ulong channelId, IConnectionHost connHost, bool ackable) BuildRabbitMQChannelAsync().GetAwaiter().GetResult(); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IModel GetChannel() - { - _hostLock.Wait(); - - try - { return _channel; } - finally - { _hostLock.Release(); } - } - private static readonly string _sleepingUntilConnectionHealthy = "ChannelHost [Id: {0}] is sleeping until Connection [Id: {1}] is healthy..."; public virtual async Task WaitUntilChannelIsReadyAsync(int sleepInterval, CancellationToken token = default) @@ -119,7 +107,7 @@ public async Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, return false; } - if (_channel != null) + if (Channel != null) { // One last check to see if the channel auto-recovered. var healthy = await ChannelHealthyAsync().ConfigureAwait(false); @@ -136,21 +124,21 @@ public async Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, return true; } - _channel.FlowControl -= FlowControl; - _channel.ModelShutdown -= ChannelClose; + Channel.FlowControl -= FlowControl; + Channel.ModelShutdown -= ChannelClose; Close(); - _channel = null; + Channel = null; } - _channel = _connHost.Connection.CreateModel(); + Channel = _connHost.Connection.CreateModel(); if (Ackable) { - _channel.ConfirmSelect(); + Channel.ConfirmSelect(); } - _channel.FlowControl += FlowControl; - _channel.ModelShutdown += ChannelClose; + Channel.FlowControl += FlowControl; + Channel.ModelShutdown += ChannelClose; _logger.LogDebug(_makeChannelSuccessful, ChannelId, _connHost.ConnectionId); @@ -159,7 +147,7 @@ public async Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, catch (Exception ex) { _logger.LogError(_makeChannelFailedError, ex.Message); - _channel = null; + Channel = null; return false; } finally @@ -192,7 +180,7 @@ public async Task ChannelHealthyAsync() { var connectionHealthy = await _connHost.HealthyAsync().ConfigureAwait(false); - return connectionHealthy && (_channel?.IsOpen ?? false); + return connectionHealthy && (Channel?.IsOpen ?? false); } public async Task ConnectionHealthyAsync() @@ -208,7 +196,7 @@ public string StartConsuming(IBasicConsumer internalConsumer, ConsumerOptions op Guard.AgainstNullOrEmpty(options.QueueName, nameof(options.QueueName)); Guard.AgainstNullOrEmpty(options.ConsumerName, nameof(options.ConsumerName)); - _consumerTag = GetChannel() + _consumerTag = Channel .BasicConsume( options.QueueName, options.AutoAck ?? false, @@ -235,7 +223,7 @@ public async Task StopConsumingAsync() if (healthy) { _logger.LogInformation(LogMessages.ChannelHosts.ConsumerStopConsumer, ChannelId, _consumerTag); - GetChannel().BasicCancel(_consumerTag); + Channel.BasicCancel(_consumerTag); } } catch (Exception ex) @@ -244,7 +232,6 @@ public async Task StopConsumingAsync() } } - private static readonly string _closeChannel = "Closing ChannelHost [Id: {0}]..."; private const int CloseCode = 200; private const string CloseMessage = "HouseofCat.RabbitMQ manual close Channelhost [Id: {0} - CN: {1}] initiated."; @@ -252,10 +239,10 @@ public void Close() { try { - _logger.LogInformation(_closeChannel, ChannelId); - _channel.Close( + _logger.LogInformation(CloseMessage, ChannelId, Channel.ChannelNumber); + Channel.Close( CloseCode, - string.Format(CloseMessage, ChannelId, _channel.ChannelNumber)); + string.Format(CloseMessage, ChannelId, Channel.ChannelNumber)); } catch { /* SWALLOW */ } } @@ -269,7 +256,7 @@ protected virtual void Dispose(bool disposing) _hostLock.Dispose(); } - _channel = null; + Channel = null; _connHost = null; _disposedValue = true; } diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index fc645681..281ac41a 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -344,7 +344,7 @@ public async Task PublishAsync( var channelHost = await _channelPool.GetChannelAsync().ConfigureAwait(false); if (basicProperties == null) { - basicProperties = channelHost.GetChannel().CreateBasicProperties(); + basicProperties = channelHost.Channel.CreateBasicProperties(); basicProperties.DeliveryMode = 2; basicProperties.MessageId = messageId ?? Guid.NewGuid().ToString(); @@ -359,7 +359,7 @@ public async Task PublishAsync( try { channelHost - .GetChannel() + .Channel .BasicPublish( exchange: exchangeName ?? string.Empty, routingKey: routingKey, @@ -399,7 +399,7 @@ public async Task PublishAsync( try { channelHost - .GetChannel() + .Channel .BasicPublish( exchange: exchangeName ?? string.Empty, routingKey: routingKey, @@ -440,7 +440,7 @@ public async Task PublishBatchAsync( var channelHost = await _channelPool.GetChannelAsync().ConfigureAwait(false); if (messageProperties == null) { - messageProperties = channelHost.GetChannel().CreateBasicProperties(); + messageProperties = channelHost.Channel.CreateBasicProperties(); messageProperties.DeliveryMode = 2; messageProperties.MessageId = Guid.NewGuid().ToString(); @@ -455,7 +455,7 @@ public async Task PublishBatchAsync( try { - var batch = channelHost.GetChannel().CreateBasicPublishBatch(); + var batch = channelHost.Channel.CreateBasicPublishBatch(); for (int i = 0; i < payloads.Count; i++) { @@ -499,7 +499,7 @@ public async Task PublishBatchAsync( try { - var batch = channelHost.GetChannel().CreateBasicPublishBatch(); + var batch = channelHost.Channel.CreateBasicPublishBatch(); for (int i = 0; i < payloads.Count; i++) { @@ -544,7 +544,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp { var body = _serializationProvider.Serialize(message.Body); chanHost - .GetChannel() + .Channel .BasicPublish( message.Exchange, message.RoutingKey, @@ -592,10 +592,10 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece try { - chanHost.GetChannel().WaitForConfirmsOrDie(_waitForConfirmation); + chanHost.Channel.WaitForConfirmsOrDie(_waitForConfirmation); chanHost - .GetChannel() + .Channel .BasicPublish( message.Exchange, message.RoutingKey, @@ -603,7 +603,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece message.BuildProperties(chanHost, withOptionalHeaders), _serializationProvider.Serialize(message.Body)); - chanHost.GetChannel().WaitForConfirmsOrDie(_waitForConfirmation); + chanHost.Channel.WaitForConfirmsOrDie(_waitForConfirmation); } catch (Exception ex) { @@ -645,7 +645,7 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, { try { - chanHost.GetChannel().BasicPublish( + chanHost.Channel.BasicPublish( messages[i].Exchange, messages[i].RoutingKey, messages[i].Mandatory, @@ -690,7 +690,7 @@ public async Task PublishManyAsBatchAsync(IList messages, bool createR { if (messages.Count > 0) { - var publishBatch = chanHost.GetChannel().CreateBasicPublishBatch(); + var publishBatch = chanHost.Channel.CreateBasicPublishBatch(); for (int i = 0; i < messages.Count; i++) { publishBatch.Add( @@ -761,7 +761,7 @@ private static IBasicProperties BuildProperties( byte? priority = 0, byte? deliveryMode = 2) { - var basicProperties = channelHost.GetChannel().CreateBasicProperties(); + var basicProperties = channelHost.Channel.CreateBasicProperties(); basicProperties.DeliveryMode = deliveryMode ?? 2; // Default Persisted basicProperties.Priority = priority ?? 0; // Default Priority basicProperties.MessageId = messageId ?? Guid.NewGuid().ToString(); diff --git a/src/HouseofCat.RabbitMQ/Services/MaintenanceService.cs b/src/HouseofCat.RabbitMQ/Services/MaintenanceService.cs index 3e664ec9..870fae1e 100644 --- a/src/HouseofCat.RabbitMQ/Services/MaintenanceService.cs +++ b/src/HouseofCat.RabbitMQ/Services/MaintenanceService.cs @@ -29,11 +29,11 @@ public async Task PurgeQueueAsync( try { - channelHost.GetChannel().QueuePurge(queueName); + channelHost.Channel.QueuePurge(queueName); if (deleteQueueAfter) { - channelHost.GetChannel().QueueDelete(queueName, false, false); + channelHost.Channel.QueueDelete(queueName, false, false); } } catch { error = true; } @@ -57,16 +57,16 @@ public async Task TransferMessageAsync( var error = false; var channelHost = await channelPool.GetChannelAsync().ConfigureAwait(false); - var properties = channelHost.GetChannel().CreateBasicProperties(); + var properties = channelHost.Channel.CreateBasicProperties(); properties.DeliveryMode = 2; try { - var result = channelHost.GetChannel().BasicGet(originQueueName, true); + var result = channelHost.Channel.BasicGet(originQueueName, true); if (result?.Body != null) { - channelHost.GetChannel().BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); + channelHost.Channel.BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); } } catch { error = true; } @@ -92,13 +92,13 @@ public async Task TransferMessageAsync( var error = false; var channelHost = await originChannelPool.GetChannelAsync().ConfigureAwait(false); - var properties = channelHost.GetChannel().CreateBasicProperties(); + var properties = channelHost.Channel.CreateBasicProperties(); properties.DeliveryMode = 2; BasicGetResult result = null; try { - result = channelHost.GetChannel().BasicGet(originQueueName, true); + result = channelHost.Channel.BasicGet(originQueueName, true); } catch { error = true; } finally @@ -112,7 +112,7 @@ await originChannelPool try { var targetChannelHost = await targetChannelPool.GetChannelAsync().ConfigureAwait(false); - targetChannelHost.GetChannel().BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); + targetChannelHost.Channel.BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); } catch { error = true; } finally @@ -136,7 +136,7 @@ public async Task TransferAllMessagesAsync( var error = false; var channelHost = await channelPool.GetChannelAsync().ConfigureAwait(false); - var properties = channelHost.GetChannel().CreateBasicProperties(); + var properties = channelHost.Channel.CreateBasicProperties(); properties.DeliveryMode = 2; try @@ -145,12 +145,12 @@ public async Task TransferAllMessagesAsync( while (true) { - result = channelHost.GetChannel().BasicGet(originQueueName, true); + result = channelHost.Channel.BasicGet(originQueueName, true); if (result == null) { break; } if (result?.Body != null) { - channelHost.GetChannel().BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); + channelHost.Channel.BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); } } } @@ -177,7 +177,7 @@ public async Task TransferAllMessagesAsync( var error = false; var channelHost = await originChannelPool.GetChannelAsync().ConfigureAwait(false); - var properties = channelHost.GetChannel().CreateBasicProperties(); + var properties = channelHost.Channel.CreateBasicProperties(); properties.DeliveryMode = 2; BasicGetResult result = null; @@ -186,7 +186,7 @@ public async Task TransferAllMessagesAsync( { try { - result = channelHost.GetChannel().BasicGet(originQueueName, true); + result = channelHost.Channel.BasicGet(originQueueName, true); if (result == null) { break; } } catch { error = true; } @@ -201,7 +201,7 @@ await originChannelPool try { var targetChannelHost = await targetChannelPool.GetChannelAsync().ConfigureAwait(false); - targetChannelHost.GetChannel().BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); + targetChannelHost.Channel.BasicPublish(string.Empty, targetQueueName, false, properties, result.Body); } catch { error = true; } finally diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index 53b93371..7c0f8f51 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -445,7 +445,7 @@ public async Task DecompressAsync(IMessage message) try { result = chanHost - .GetChannel() + .Channel .BasicGet(queueName, true); } catch { error = true; } @@ -481,7 +481,7 @@ public async Task GetAsync(string queueName) try { result = chanHost - .GetChannel() + .Channel .BasicGet(queueName, true); } catch { error = true; } diff --git a/src/HouseofCat.RabbitMQ/Topology/Topologer.cs b/src/HouseofCat.RabbitMQ/Topology/Topologer.cs index a03676e9..3476a8dd 100644 --- a/src/HouseofCat.RabbitMQ/Topology/Topologer.cs +++ b/src/HouseofCat.RabbitMQ/Topology/Topologer.cs @@ -146,7 +146,7 @@ public async Task CreateQueueAsync( try { - chanHost.GetChannel().QueueDeclare( + chanHost.Channel.QueueDeclare( queue: queueName, durable: durable, exclusive: exclusive, @@ -179,7 +179,7 @@ public async Task DeleteQueueAsync( try { - chanHost.GetChannel().QueueDelete( + chanHost.Channel.QueueDelete( queue: queueName, ifUnused: onlyIfUnused, ifEmpty: onlyIfEmpty); @@ -213,7 +213,7 @@ public async Task BindQueueToExchangeAsync( try { - chanHost.GetChannel().QueueBind( + chanHost.Channel.QueueBind( queue: queueName, exchange: exchangeName, routingKey: routingKey, @@ -248,7 +248,7 @@ public async Task UnbindQueueFromExchangeAsync( try { - chanHost.GetChannel().QueueUnbind( + chanHost.Channel.QueueUnbind( queue: queueName, exchange: exchangeName, routingKey: routingKey, @@ -284,7 +284,7 @@ public async Task CreateExchangeAsync( try { - chanHost.GetChannel().ExchangeDeclare( + chanHost.Channel.ExchangeDeclare( exchange: exchangeName, type: exchangeType, durable: durable, @@ -313,7 +313,7 @@ public async Task DeleteExchangeAsync(string exchangeName, bool onlyIfUnus try { - chanHost.GetChannel().ExchangeDelete( + chanHost.Channel.ExchangeDelete( exchange: exchangeName, ifUnused: onlyIfUnused); } @@ -346,7 +346,7 @@ public async Task BindExchangeToExchangeAsync( try { - chanHost.GetChannel().ExchangeBind( + chanHost.Channel.ExchangeBind( destination: childExchangeName, source: parentExchangeName, routingKey: routingKey, @@ -381,7 +381,7 @@ public async Task UnbindExchangeFromExchangeAsync( try { - chanHost.GetChannel().ExchangeUnbind( + chanHost.Channel.ExchangeUnbind( destination: childExchangeName, source: parentExchangeName, routingKey: routingKey, diff --git a/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs b/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs index 15939843..f744b87d 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/BasicGetTests.cs @@ -10,16 +10,15 @@ public static async Task RunBasicGetAsync(ILogger logger, string configFileNameP { var channelPool = await Shared.SetupTestsAsync(logger, configFileNamePath); var channelHost = await channelPool.GetTransientChannelAsync(true); - var channel = channelHost.GetChannel(); try { - var properties = channel.CreateBasicProperties(); + var properties = channelHost.Channel.CreateBasicProperties(); properties.DeliveryMode = 2; var messageAsBytes = Encoding.UTF8.GetBytes("Hello World"); - channel.BasicPublish(Shared.ExchangeName, Shared.RoutingKey, properties, messageAsBytes); + channelHost.Channel.BasicPublish(Shared.ExchangeName, Shared.RoutingKey, properties, messageAsBytes); logger.LogInformation( "Getting message from Queue [{queueName}]", @@ -28,12 +27,12 @@ public static async Task RunBasicGetAsync(ILogger logger, string configFileNameP BasicGetResult result = null; do { - result = channel.BasicGet(Shared.QueueName, false); + result = channelHost.Channel.BasicGet(Shared.QueueName, false); if (result is not null) { var message = Encoding.UTF8.GetString(result.Body.Span); logger.LogInformation("Received message: [{message}]", message); - channel.BasicAck(result.DeliveryTag, false); + channelHost.Channel.BasicAck(result.DeliveryTag, false); } else { diff --git a/tests/RabbitMQ.Console.Tests/Tests/Shared.cs b/tests/RabbitMQ.Console.Tests/Tests/Shared.cs index f6506ff2..fe12442d 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/Shared.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/Shared.cs @@ -24,13 +24,12 @@ public static async Task SetupTestsAsync(ILogger logger, string co var channelPool = new ChannelPool(rabbitOptions); var channelHost = await channelPool.GetTransientChannelAsync(true); - var channel = channelHost.GetChannel(); logger.LogInformation("Declaring Exchange: [{ExchangeName}]", ExchangeName); - channel.ExchangeDeclare(ExchangeName, ExchangeType.Direct, true, false, null); + channelHost.Channel.ExchangeDeclare(ExchangeName, ExchangeType.Direct, true, false, null); logger.LogInformation("Declaring Queue: [{QueueName}]", QueueName); - channel.QueueDeclare(QueueName, true, false, false, null); + channelHost.Channel.QueueDeclare(QueueName, true, false, false, null); logger.LogInformation( "Binding Queue [{queueName}] To Exchange: [{exchangeName}]. RoutingKey: [{routingKey}]", @@ -38,7 +37,7 @@ public static async Task SetupTestsAsync(ILogger logger, string co ExchangeName, RoutingKey); - channel.QueueBind(QueueName, ExchangeName, RoutingKey); + channelHost.Channel.QueueBind(QueueName, ExchangeName, RoutingKey); channelHost.Close(); From cf86570e53fbe0dfeca67fe6b22734fc2ba11f02 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 17:12:35 -0500 Subject: [PATCH 16/47] Removing locks from ChannelHost. --- src/HouseofCat.Dataflows/IWorkState.cs | 3 +- .../Dataflows/ConsumerDataflow.cs | 12 +++--- .../Dataflows/RabbitWorkState.cs | 3 +- .../Messages/ReceivedMessage.cs | 37 ++++++++++++------- src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs | 19 +--------- .../Tests/Shared.cs | 7 ++-- 6 files changed, 37 insertions(+), 44 deletions(-) diff --git a/src/HouseofCat.Dataflows/IWorkState.cs b/src/HouseofCat.Dataflows/IWorkState.cs index aef1bc59..c71dcb2b 100644 --- a/src/HouseofCat.Dataflows/IWorkState.cs +++ b/src/HouseofCat.Dataflows/IWorkState.cs @@ -1,4 +1,5 @@ using OpenTelemetry.Trace; +using System; using System.Collections.Generic; using System.Runtime.ExceptionServices; @@ -17,7 +18,7 @@ public interface IWorkState ExceptionDispatchInfo EDI { get; set; } // Outbound - byte[] SendData { get; set; } + ReadOnlyMemory SendData { get; set; } // RootSpan or ChildSpan derived from TraceParentHeader TelemetrySpan WorkflowSpan { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index fec5ea8e..d60fd659 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -710,10 +710,10 @@ TState WrapAction(TState state) { if (outbound) { - if (state.SendData?.Length > 0) - { state.SendData = action(state.SendData.AsMemory()).ToArray(); } + if (state.SendData.Length > 0) + { state.SendData = action(state.SendData); } else if (state.SendMessage.Body.Length > 0) - { state.SendMessage.Body = action(state.SendMessage.Body).ToArray(); } + { state.SendMessage.Body = action(state.SendMessage.Body); } } else if (predicate.Invoke(state)) { @@ -722,10 +722,10 @@ TState WrapAction(TState state) if (state.ReceivedMessage.Message == null) { state.ReceivedMessage.Message = _serializationProvider.Deserialize(state.ReceivedMessage.Body); } - state.ReceivedMessage.Message.Body = action(state.ReceivedMessage.Message.Body).ToArray(); + state.ReceivedMessage.Message.Body = action(state.ReceivedMessage.Message.Body); } else - { state.ReceivedMessage.Body = action(state.ReceivedMessage.Body).ToArray(); } + { state.ReceivedMessage.Body = action(state.ReceivedMessage.Body); } } return state; @@ -758,7 +758,7 @@ async Task WrapActionAsync(TState state) if (outbound) { - if (state.SendData?.Length > 0) + if (state.SendData.Length > 0) { state.SendData = await action(state.SendData).ConfigureAwait(false); } else if (state.SendMessage.Body.Length > 0) { state.SendMessage.Body = await action(state.SendMessage.Body).ConfigureAwait(false); } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index 1cfb23e2..1dfd3a9f 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -1,5 +1,6 @@ using HouseofCat.Dataflows; using OpenTelemetry.Trace; +using System; using System.Collections.Generic; using System.Runtime.ExceptionServices; using System.Runtime.Serialization; @@ -18,7 +19,7 @@ public abstract class RabbitWorkState : IRabbitWorkState [IgnoreDataMember] public virtual IReceivedMessage ReceivedMessage { get; set; } - public virtual byte[] SendData { get; set; } + public virtual ReadOnlyMemory SendData { get; set; } public virtual IMessage SendMessage { get; set; } public virtual bool SendMessageSent { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index 6a9e72ae..1b128170 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -9,24 +9,30 @@ namespace HouseofCat.RabbitMQ; public interface IReceivedMessage { + IMessage Message { get; set; } + ReadOnlyMemory Body { get; set; } + IBasicProperties Properties { get; } + + IModel Channel { get; } + bool Ackable { get; } - IModel Channel { get; set; } string ContentType { get; } string ObjectType { get; } + bool Encrypted { get; } string EncryptionType { get; } DateTime EncryptedDateTime { get; } + bool Compressed { get; } string CompressionType { get; } + public string TraceParentHeader { get; } - ReadOnlyMemory Body { get; set; } string ConsumerTag { get; } ulong DeliveryTag { get; } - IMessage Message { get; set; } - IBasicProperties Properties { get; } + bool FailedToDeserialize { get; } bool AckMessage(); void Complete(); @@ -38,25 +44,31 @@ public interface IReceivedMessage public sealed class ReceivedMessage : IReceivedMessage, IDisposable { + public IMessage Message { get; set; } + public ReadOnlyMemory Body { get; set; } public IBasicProperties Properties { get; } + + public IModel Channel { get; private set; } + public bool Ackable { get; } - public IModel Channel { get; set; } - public string ConsumerTag { get; } - public ulong DeliveryTag { get; } - public ReadOnlyMemory Body { get; set; } - public IMessage Message { get; set; } - // Retrieved From Headers - public bool FailedToDeserialize { get; private set; } public string ContentType { get; private set; } public string ObjectType { get; private set; } + public bool Encrypted { get; private set; } public string EncryptionType { get; private set; } public DateTime EncryptedDateTime { get; private set; } + public bool Compressed { get; private set; } public string CompressionType { get; private set; } + public string TraceParentHeader { get; private set; } + public string ConsumerTag { get; } + public ulong DeliveryTag { get; } + + public bool FailedToDeserialize { get; private set; } + private readonly TaskCompletionSource _completionSource = new TaskCompletionSource(); public Task Completion => _completionSource.Task; @@ -118,9 +130,6 @@ private void ReadHeaders() case Constants.HeaderValueForContentTypeBinary: case Constants.HeaderValueForContentTypePlainText: - Message = new Message{ Body = Body }; - break; - default: break; } diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs index ee00b416..b84c26c1 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs @@ -44,9 +44,6 @@ public class ChannelHost : IChannelHost, IDisposable public bool FlowControlled { get; private set; } public bool UsedByConsumer { get; private set; } - private readonly SemaphoreSlim _hostLock = new SemaphoreSlim(1, 1); - private bool _disposedValue; - public ChannelHost(ulong channelId, IConnectionHost connHost, bool ackable) { _logger = LogHelpers.GetLogger(); @@ -93,8 +90,6 @@ public virtual async Task WaitUntilChannelIsReadyAsync(int sleepInterval, Cancel [MethodImpl(MethodImplOptions.AggressiveInlining)] public async Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, CancellationToken token = default) { - await _hostLock.WaitAsync(token).ConfigureAwait(false); - try { var connectionHealthy = await _connHost @@ -150,30 +145,22 @@ public async Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, Channel = null; return false; } - finally - { _hostLock.Release(); } } protected virtual void ChannelClose(object sender, ShutdownEventArgs e) { - _hostLock.Wait(); _logger.LogDebug(e.ReplyText); Closed = true; - _hostLock.Release(); } protected virtual void FlowControl(object sender, FlowControlEventArgs e) { - _hostLock.Wait(); - if (e.Active) { _logger.LogWarning(LogMessages.ChannelHosts.FlowControlled, ChannelId); } else { _logger.LogInformation(LogMessages.ChannelHosts.FlowControlFinished, ChannelId); } FlowControlled = e.Active; - - _hostLock.Release(); } public async Task ChannelHealthyAsync() @@ -246,16 +233,12 @@ public void Close() } catch { /* SWALLOW */ } } + private bool _disposedValue; protected virtual void Dispose(bool disposing) { if (!_disposedValue) { - if (disposing) - { - _hostLock.Dispose(); - } - Channel = null; _connHost = null; _disposedValue = true; diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs index 5016fab4..03dfc0f3 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs @@ -27,13 +27,12 @@ public static async Task SetupTestsAsync(ILogger logger, string co var channelPool = new ChannelPool(rabbitOptions); var channelHost = await channelPool.GetTransientChannelAsync(true); - var channel = channelHost.GetChannel(); logger.LogInformation("Declaring Exchange: [{ExchangeName}]", ExchangeName); - channel.ExchangeDeclare(ExchangeName, ExchangeType.Direct, true, false, null); + channelHost.Channel.ExchangeDeclare(ExchangeName, ExchangeType.Direct, true, false, null); logger.LogInformation("Declaring Queue: [{QueueName}]", QueueName); - channel.QueueDeclare(QueueName, true, false, false, null); + channelHost.Channel.QueueDeclare(QueueName, true, false, false, null); logger.LogInformation( "Binding Queue [{queueName}] To Exchange: [{exchangeName}]. RoutingKey: [{routingKey}]", @@ -41,7 +40,7 @@ public static async Task SetupTestsAsync(ILogger logger, string co ExchangeName, RoutingKey); - channel.QueueBind(QueueName, ExchangeName, RoutingKey); + channelHost.Channel.QueueBind(QueueName, ExchangeName, RoutingKey); logger.LogInformation( "Publishing message to Exchange [{exchangeName}] with RoutingKey [{routingKey}]", From d8f636eb68634d39701b1ee705259fb999ad2d17 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 17:25:41 -0500 Subject: [PATCH 17/47] Remove IConnectionHost locks. --- RabbitMQ.Dataflows.sln | 7 - .../HouseofCat.Dataflows.csproj | 1 - .../HouseofCat.Metrics.csproj | 11 - src/HouseofCat.Metrics/IMetricsProvider.cs | 69 ----- src/HouseofCat.Metrics/NullMetricsProvider.cs | 92 ------- .../OpenTelemetryMetricsProvider.cs | 256 ------------------ .../Prometheus/PrometheusMetricsProvider.cs | 239 ---------------- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 7 +- src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs | 33 +-- src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs | 4 +- .../Pools/ConnectionHost.cs | 36 +-- .../Pools/ConnectionPool.cs | 3 +- .../{ExchangeConfig.cs => Exchange.cs} | 2 +- ...gBindingConfig.cs => ExchangingBinding.cs} | 2 +- .../Topology/{QueueConfig.cs => Queue.cs} | 2 +- ...{QueueBindingConfig.cs => QueueBinding.cs} | 2 +- src/HouseofCat.RabbitMQ/Topology/Topologer.cs | 3 +- .../Topology/TopologyConfig.cs | 10 +- 18 files changed, 38 insertions(+), 741 deletions(-) delete mode 100644 src/HouseofCat.Metrics/HouseofCat.Metrics.csproj delete mode 100644 src/HouseofCat.Metrics/IMetricsProvider.cs delete mode 100644 src/HouseofCat.Metrics/NullMetricsProvider.cs delete mode 100644 src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs delete mode 100644 src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs rename src/HouseofCat.RabbitMQ/Topology/{ExchangeConfig.cs => Exchange.cs} (90%) rename src/HouseofCat.RabbitMQ/Topology/{ExchangingBindingConfig.cs => ExchangingBinding.cs} (87%) rename src/HouseofCat.RabbitMQ/Topology/{QueueConfig.cs => Queue.cs} (91%) rename src/HouseofCat.RabbitMQ/Topology/{QueueBindingConfig.cs => QueueBinding.cs} (88%) diff --git a/RabbitMQ.Dataflows.sln b/RabbitMQ.Dataflows.sln index 1767474f..dc0ea98c 100644 --- a/RabbitMQ.Dataflows.sln +++ b/RabbitMQ.Dataflows.sln @@ -30,8 +30,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HouseofCat.Serialization", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HouseofCat.Hashing", "src\HouseofCat.Hashing\HouseofCat.Hashing.csproj", "{75C192B0-BCEC-4BAF-97F7-0F08209557FE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HouseofCat.Metrics", "src\HouseofCat.Metrics\HouseofCat.Metrics.csproj", "{B475ECF4-6CAD-4FB7-80CF-0B8326BAD718}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HouseofCat.Data", "src\HouseofCat.Data\HouseofCat.Data.csproj", "{2B4A8274-83BB-49FD-9E7C-AE1EB5C56B3A}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "guides", "guides", "{1AB5E832-AD36-40AF-A7E9-A105A778116C}" @@ -124,10 +122,6 @@ Global {75C192B0-BCEC-4BAF-97F7-0F08209557FE}.Debug|Any CPU.Build.0 = Debug|Any CPU {75C192B0-BCEC-4BAF-97F7-0F08209557FE}.Release|Any CPU.ActiveCfg = Release|Any CPU {75C192B0-BCEC-4BAF-97F7-0F08209557FE}.Release|Any CPU.Build.0 = Release|Any CPU - {B475ECF4-6CAD-4FB7-80CF-0B8326BAD718}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B475ECF4-6CAD-4FB7-80CF-0B8326BAD718}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B475ECF4-6CAD-4FB7-80CF-0B8326BAD718}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B475ECF4-6CAD-4FB7-80CF-0B8326BAD718}.Release|Any CPU.Build.0 = Release|Any CPU {2B4A8274-83BB-49FD-9E7C-AE1EB5C56B3A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {2B4A8274-83BB-49FD-9E7C-AE1EB5C56B3A}.Debug|Any CPU.Build.0 = Debug|Any CPU {2B4A8274-83BB-49FD-9E7C-AE1EB5C56B3A}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -160,7 +154,6 @@ Global {AF7AF28A-6704-4E62-9977-23E1FE1490D9} = {4FCA30F5-D19B-4756-BC68-6BA7E668F786} {6AE8BE48-BA11-4120-B8F8-3D42CF56DD46} = {4FCA30F5-D19B-4756-BC68-6BA7E668F786} {75C192B0-BCEC-4BAF-97F7-0F08209557FE} = {4FCA30F5-D19B-4756-BC68-6BA7E668F786} - {B475ECF4-6CAD-4FB7-80CF-0B8326BAD718} = {4FCA30F5-D19B-4756-BC68-6BA7E668F786} {2B4A8274-83BB-49FD-9E7C-AE1EB5C56B3A} = {4FCA30F5-D19B-4756-BC68-6BA7E668F786} {1AB5E832-AD36-40AF-A7E9-A105A778116C} = {29F4C0CA-2EA4-433C-8112-9A1336392E52} {018DB970-F843-4D69-AAE1-C3CDF5FC7093} = {1AB5E832-AD36-40AF-A7E9-A105A778116C} diff --git a/src/HouseofCat.Dataflows/HouseofCat.Dataflows.csproj b/src/HouseofCat.Dataflows/HouseofCat.Dataflows.csproj index 4cd60d26..70e2524d 100644 --- a/src/HouseofCat.Dataflows/HouseofCat.Dataflows.csproj +++ b/src/HouseofCat.Dataflows/HouseofCat.Dataflows.csproj @@ -3,7 +3,6 @@ - diff --git a/src/HouseofCat.Metrics/HouseofCat.Metrics.csproj b/src/HouseofCat.Metrics/HouseofCat.Metrics.csproj deleted file mode 100644 index b3a00626..00000000 --- a/src/HouseofCat.Metrics/HouseofCat.Metrics.csproj +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/src/HouseofCat.Metrics/IMetricsProvider.cs b/src/HouseofCat.Metrics/IMetricsProvider.cs deleted file mode 100644 index 1bfa668e..00000000 --- a/src/HouseofCat.Metrics/IMetricsProvider.cs +++ /dev/null @@ -1,69 +0,0 @@ -using OpenTelemetry.Trace; -using System; -using System.Collections.Generic; -using System.Diagnostics; - -namespace HouseofCat.Metrics; - -public interface IMetricsProvider -{ - void DecrementCounter(string name, string unit = null, string description = null); - void DecrementGauge(string name, string unit = null, string description = null); - void IncrementCounter(string name, string unit = null, string description = null); - void IncrementGauge(string name, string unit = null, string description = null); - void ObserveValue(string name, double value, string unit = null, string description = null); - void ObserveValueFluctuation(string name, double value, string unit = null, string description = null); - - IDisposable Duration(string name, bool microScale = false, string unit = null, string description = null); - - IDisposable Track(string name, string unit = null, string description = null); - - IDisposable TrackAndDuration( - string name, - bool microScale = false, - string unit = null, - string description = null, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null); - - IDisposable Trace( - string name, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null); - - /// - /// Creates a new active span. - /// - /// - /// - /// - /// - IDisposable GetSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null); - - /// - /// Attempts to create a new active child span from CURRENT active span or just a new active span if no current span. - /// - /// - /// - /// - /// - IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null); - - /// - /// Creates a new active child span from provided span context. - /// - /// - /// - /// - /// - IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - SpanContext parentSpanContext = default); -} diff --git a/src/HouseofCat.Metrics/NullMetricsProvider.cs b/src/HouseofCat.Metrics/NullMetricsProvider.cs deleted file mode 100644 index 30addafb..00000000 --- a/src/HouseofCat.Metrics/NullMetricsProvider.cs +++ /dev/null @@ -1,92 +0,0 @@ -using OpenTelemetry.Trace; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace HouseofCat.Metrics; - -public class NullMetricsProvider : IMetricsProvider -{ - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void DecrementCounter(string name, string unit = null, string description = null) - { /* NO-OP */ } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void DecrementGauge(string name, string unit = null, string description = null) - { /* NO-OP */ } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void IncrementCounter(string name, string unit = null, string description = null) - { /* NO-OP */ } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void IncrementGauge(string name, string unit = null, string description = null) - { /* NO-OP */ } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ObserveValue(string name, double value, string unit = null, string description = null) - { /* NO-OP */ } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ObserveValueFluctuation(string name, double value, string unit = null, string description = null) - { /* NO-OP */ } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Duration(string name, bool microScale = false, string unit = null, string description = null) - { - return null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Track(string name, string unit = null, string description = null) - { - return null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable TrackAndDuration( - string name, - bool microScale = false, - string unit = null, - string description = null, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null) - { - return null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Trace( - string name, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null) - { - return null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable GetSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null) - { - return null; - } - - public IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null) - { - return null; - } - - public IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - SpanContext parentSpanContext = default) - { - return null; - } -} diff --git a/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs b/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs deleted file mode 100644 index e728bc1c..00000000 --- a/src/HouseofCat.Metrics/OpenTelemetry/OpenTelemetryMetricsProvider.cs +++ /dev/null @@ -1,256 +0,0 @@ -using HouseofCat.Utilities.Errors; -using OpenTelemetry.Trace; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.Metrics; -using System.Runtime.CompilerServices; - -namespace HouseofCat.Metrics; - -public class OpenTelemetryMetricsProvider : IMetricsProvider, IDisposable -{ - private readonly IMeterFactory _factory; - private readonly Meter _meter; - private readonly string _meterName; - private readonly ActivitySource _activitySource; - private readonly Tracer _tracer; - - public ConcurrentDictionary Counters { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary Gauges { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary Histograms { get; } = new ConcurrentDictionary(); - - private bool _disposedValue; - - public OpenTelemetryMetricsProvider( - IMeterFactory meterFactory, - string meterName, - string activitySourceName = null, - string activityVersion = null) - { - Guard.AgainstNull(meterFactory, nameof(meterFactory)); - Guard.AgainstNullOrEmpty(meterName, nameof(meterName)); - - _factory = meterFactory; - _meterName = meterName; - - _meter = _factory.Create(_meterName); - _tracer = TracerProvider.Default.GetTracer(activitySourceName); - _activitySource = new ActivitySource(activitySourceName ?? "HouseofCat.Metrics", activityVersion); - } - - public Counter GetOrAddCounter(string name, string unit = null, string description = null) where T : struct - { - return (Counter)Counters.GetOrAdd(name, _meter.CreateCounter(name, unit, description)); - } - - public ObservableGauge GetOrAddGauge(string name, Func observableValue, string unit = null, string description = null) where T : struct - { - return (ObservableGauge)Gauges.GetOrAdd( - name, - _meter.CreateObservableGauge(name, observableValue, unit, description)); - } - - public Histogram GetOrAddHistogram(string name, string unit = null, string description = null) where T : struct - { - return (Histogram)Histograms.GetOrAdd(name, _meter.CreateHistogram(name, unit, description)); - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - _meter.Dispose(); - } - - _disposedValue = true; - } - } - - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - - #region IMetricsProvider Implementation - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ObserveValueFluctuation(string name, double value, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Histogram"; - GetOrAddHistogram(name, unit: unit, description: description) - .Record(value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ObserveValue(string name, double value, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Summary"; - GetOrAddHistogram(name, unit: unit, description: description) - .Record(value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void IncrementGauge(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Gauge"; - GetOrAddCounter(name, unit: unit, description: description) - .Add(1); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void DecrementGauge(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Gauge"; - GetOrAddCounter(name, unit: unit, description: description) - .Add(-1); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void IncrementCounter(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Counter"; - GetOrAddCounter(name, unit: unit, description: description) - .Add(1); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void DecrementCounter(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Counter"; - GetOrAddCounter(name, unit: unit, description: description) - .Add(-1); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Duration(string name, bool microScale = false, string unit = null, string description = null) - { - return null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Track(string name, string unit = null, string description = null) - { - return null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable TrackAndDuration( - string name, - bool microScale = false, - string unit = null, - string description = null, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary tags = null) - { - return GetActivity(name, activityKind, tags); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Trace( - string name, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null) - { - return GetActivity(name, activityKind, metricTags); - } - - private Activity GetActivity( - string name, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null) - { - var parentActivity = Activity.Current; - - Activity activity; - if (parentActivity is not null) - { activity = _activitySource?.StartActivity(name, activityKind, parentActivity.Context); } - else - { activity = _activitySource?.StartActivity(name, activityKind); } - - if (activity is null) return null; - - if (metricTags is not null && activity.IsAllDataRequested) - { - foreach (var tag in metricTags) - { - activity.SetTag(tag.Key, tag.Value); - } - } - - return activity; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable GetSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null) - { - return GetDefaultTelemetrySpan(name, false, spanKind, metricTags); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null) - { - return GetDefaultTelemetrySpan(name, true, spanKind, metricTags); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - SpanContext parentSpanContext = default) - { - return GetChildTelemetrySpan(name, spanKind, parentSpanContext); - } - - private TelemetrySpan GetDefaultTelemetrySpan( - string name, - bool childSpanOfCurrent = false, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null) - { - if (childSpanOfCurrent) - { - var parentSpan = Tracer.CurrentSpan; - if (parentSpan is not null) return _tracer?.StartActiveSpan(name, spanKind, parentSpan.Context); - } - - var span = _tracer?.StartActiveSpan(name); - if (span is null) return null; - - if (metricTags is not null) - { - foreach (var tag in metricTags) - { - span.SetAttribute(tag.Key, tag.Value); - } - } - - return span; - } - - private TelemetrySpan GetChildTelemetrySpan( - string name, - SpanKind spanKind = SpanKind.Internal, - SpanContext parentSpanContext = default) - { - return _tracer?.StartActiveSpan(name, spanKind, parentSpanContext); - } - - #endregion -} diff --git a/src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs b/src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs deleted file mode 100644 index 75465dbd..00000000 --- a/src/HouseofCat.Metrics/Prometheus/PrometheusMetricsProvider.cs +++ /dev/null @@ -1,239 +0,0 @@ -using HouseofCat.Utilities; -using HouseofCat.Utilities.Errors; -using OpenTelemetry.Trace; -using Prometheus; -using Prometheus.DotNetRuntime; -using Prometheus.DotNetRuntime.Metrics.Producers; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace HouseofCat.Metrics; - -public class PrometheusMetricsProvider : IMetricsProvider, IDisposable -{ - private readonly IMetricServer _server; - - public ConcurrentDictionary Counters { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary Gauges { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary Histograms { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary Summaries { get; } = new ConcurrentDictionary(); - - private IDisposable _collector; - private bool _disposedValue; - - /// - /// Use this constructor in AspNetCore setup (since AspNetCore Prometheus handles Server creation via middleware). - /// - public PrometheusMetricsProvider() - { } - - public PrometheusMetricsProvider(int port, string url, CollectorRegistry registry = null, bool useHttps = false) - { - _server = new MetricServer(port, url, registry, useHttps); - _server.Start(); - } - - public PrometheusMetricsProvider(string hostname, int port, string url, CollectorRegistry registry = null, bool useHttps = false) - { - _server = new MetricServer(hostname, port, url, registry, useHttps); - _server.Start(); - } - - public PrometheusMetricsProvider AddDotNetRuntimeStats( - CaptureLevel contentionCaptureLevel = CaptureLevel.Counters, - SampleEvery contentionSampleRate = SampleEvery.TenEvents, - CaptureLevel jitCaptureLevel = CaptureLevel.Counters, - SampleEvery jitSampleRate = SampleEvery.HundredEvents, - CaptureLevel threadPoolCaptureLevel = CaptureLevel.Counters, - ThreadPoolMetricsProducer.Options threadPoolOptions = null) - { - _collector ??= DotNetRuntimeStatsBuilder - .Customize() - .WithContentionStats(contentionCaptureLevel, contentionSampleRate) - .WithJitStats(jitCaptureLevel, jitSampleRate) - .WithThreadPoolStats(threadPoolCaptureLevel, threadPoolOptions) - .WithGcStats() - .StartCollecting(); - - return this; - } - - public PrometheusMetricsProvider AddDotNetRuntimeStats(DotNetRuntimeStatsBuilder.Builder builder) - { - _collector ??= builder.StartCollecting(); - - return this; - } - - public Counter GetOrAddCounter(string name, string description = null, CounterConfiguration config = null) - { - return Counters.GetOrAdd(name, Prometheus.Metrics.CreateCounter(name, description, config)); - } - - public Gauge GetOrAddGauge(string name, string description = null, GaugeConfiguration config = null) - { - return Gauges.GetOrAdd(name, Prometheus.Metrics.CreateGauge(name, description, config)); - } - - public Histogram GetOrAddHistogram(string name, string description = null, HistogramConfiguration config = null) - { - return Histograms.GetOrAdd(name, Prometheus.Metrics.CreateHistogram(name, description, config)); - } - - public Summary GetOrdAddSummary(string name, string description = null, SummaryConfiguration config = null) - { - return Summaries.GetOrAdd(name, Prometheus.Metrics.CreateSummary(name, description, config)); - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - _collector?.Dispose(); - _server.Stop(); - _server.Dispose(); - } - - _disposedValue = true; - } - } - - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - - #region IMetricsProvider Implementation - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ObserveValueFluctuation(string name, double value, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Histogram"; - GetOrAddHistogram(name, description: description).Observe(value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ObserveValue(string name, double value, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Summary"; - GetOrdAddSummary(name, description: description).Observe(value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void IncrementGauge(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Gauge"; - GetOrAddGauge(name, description: description).Inc(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void DecrementGauge(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Gauge"; - GetOrAddGauge(name, description: description).Dec(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void IncrementCounter(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Counter"; - GetOrAddCounter(name, description: description).Inc(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void DecrementCounter(string name, string unit = null, string description = null) - { - throw new NotImplementedException(); - } - - private double[] _durationMicroBuckets = Histogram.ExponentialBuckets(0.000_001, 2, 10); - private double[] _durationMilliBuckets = Histogram.ExponentialBuckets(0.001, 2, 16); - public void SetDurationHistogramBucketSize(double[] microBuckets, double[] milliBuckets) - { - _durationMicroBuckets = microBuckets; - _durationMilliBuckets = milliBuckets; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Duration(string name, bool microScale = false, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Timer"; - return GetOrAddHistogram( - name, - description: description, - new HistogramConfiguration - { - Buckets = microScale ? _durationMicroBuckets : _durationMilliBuckets - }).NewTimer(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Track(string name, string unit = null, string description = null) - { - Guard.AgainstNull(name, nameof(name)); - name = $"{name}_Track"; - return GetOrAddGauge(name, description).TrackInProgress(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable TrackAndDuration( - string name, - bool microScale = false, - string unit = null, - string description = null, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null) - { - var duration = Duration(name, microScale, description: description); - var track = Track(name, description: description); - return new MultiDispose(duration, track); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable Trace( - string name, - ActivityKind activityKind = ActivityKind.Internal, - IDictionary metricTags = null) - { - return null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IDisposable GetSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null) - { - return null; - } - - public IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - IDictionary metricTags = null) - { - return null; - } - - public IDisposable GetChildSpan( - string name, - SpanKind spanKind = SpanKind.Internal, - SpanContext parentSpanContext = default) - { - return null; - } - - #endregion -} diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index deb0d01e..8e1782e4 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -150,8 +150,7 @@ private async Task StartConsumingAsync() Consumers.StartingConsumer, ConsumerOptions.ConsumerName); - var healthy = await _chanHost.ChannelHealthyAsync(); - if (!healthy) + if (!_chanHost.ChannelHealthy()) { try { @@ -260,7 +259,7 @@ await HandleRecoverableShutdownAsync(e) .ConfigureAwait(false); } else - { await _chanHost.StopConsumingAsync(); } + { _chanHost.StopConsuming(); } } finally { _conLock.Release(); } @@ -330,7 +329,7 @@ await HandleRecoverableShutdownAsync(e) .ConfigureAwait(false); } else - { await _chanHost.StopConsumingAsync(); } + { _chanHost.StopConsuming(); } } finally { _conLock.Release(); } diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs index b84c26c1..edbf5260 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs @@ -20,14 +20,14 @@ public interface IChannelHost bool UsedByConsumer { get; } string StartConsuming(IBasicConsumer internalConsumer, ConsumerOptions options); - Task StopConsumingAsync(); + void StopConsuming(); Task WaitUntilChannelIsReadyAsync(int sleepInterval, CancellationToken token = default); Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, CancellationToken token = default); void Close(); - Task ChannelHealthyAsync(); - Task ConnectionHealthyAsync(); + bool ChannelHealthy(); + bool ConnectionHealthy(); } public class ChannelHost : IChannelHost, IDisposable @@ -59,7 +59,7 @@ public ChannelHost(ulong channelId, IConnectionHost connHost, bool ackable) public virtual async Task WaitUntilChannelIsReadyAsync(int sleepInterval, CancellationToken token = default) { - var connectionHealthy = await _connHost.HealthyAsync(); + var connectionHealthy = _connHost.Healthy(); if (!connectionHealthy) { _logger.LogInformation(_sleepingUntilConnectionHealthy, ChannelId, _connHost.ConnectionId); @@ -67,7 +67,7 @@ public virtual async Task WaitUntilChannelIsReadyAsync(int sleepInterval, Cancel { await Task.Delay(sleepInterval, token).ConfigureAwait(false); - connectionHealthy = await _connHost.HealthyAsync(); + connectionHealthy = _connHost.Healthy(); } } @@ -92,11 +92,7 @@ public async Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, { try { - var connectionHealthy = await _connHost - .HealthyAsync() - .ConfigureAwait(false); - - if (!connectionHealthy) + if (!_connHost.Healthy()) { _logger.LogError(_makeChannelConnectionUnhealthyError, ChannelId, _connHost.ConnectionId); return false; @@ -105,12 +101,12 @@ public async Task BuildRabbitMQChannelAsync(int autoRecoveryDelay = 1000, if (Channel != null) { // One last check to see if the channel auto-recovered. - var healthy = await ChannelHealthyAsync().ConfigureAwait(false); + var healthy = ChannelHealthy(); if (!healthy) { await Task.Delay(autoRecoveryDelay, token); - healthy = await ChannelHealthyAsync().ConfigureAwait(false); + healthy = ChannelHealthy(); } if (healthy) @@ -163,16 +159,16 @@ protected virtual void FlowControl(object sender, FlowControlEventArgs e) FlowControlled = e.Active; } - public async Task ChannelHealthyAsync() + public bool ChannelHealthy() { - var connectionHealthy = await _connHost.HealthyAsync().ConfigureAwait(false); + var connectionHealthy = _connHost.Healthy(); return connectionHealthy && (Channel?.IsOpen ?? false); } - public async Task ConnectionHealthyAsync() + public bool ConnectionHealthy() { - return await _connHost.HealthyAsync().ConfigureAwait(false); + return _connHost.Healthy(); } private string _consumerTag = null; @@ -200,14 +196,13 @@ public string StartConsuming(IBasicConsumer internalConsumer, ConsumerOptions op return _consumerTag; } - public async Task StopConsumingAsync() + public void StopConsuming() { if (string.IsNullOrEmpty(_consumerTag) || !UsedByConsumer) return; try { - var healthy = await ChannelHealthyAsync().ConfigureAwait(false); - if (healthy) + if (ChannelHealthy()) { _logger.LogInformation(LogMessages.ChannelHosts.ConsumerStopConsumer, ChannelId, _consumerTag); Channel.BasicCancel(_consumerTag); diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs index 414c122b..9d4d1d51 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs @@ -157,7 +157,7 @@ public async Task GetChannelAsync() .ReadAsync() .ConfigureAwait(false); - var healthy = await chanHost.ChannelHealthyAsync().ConfigureAwait(false); + var healthy = chanHost.ChannelHealthy(); var flagged = _flaggedChannels.ContainsKey(chanHost.ChannelId) && _flaggedChannels[chanHost.ChannelId]; if (flagged || !healthy) { @@ -222,7 +222,7 @@ public async Task GetAckChannelAsync() .ReadAsync() .ConfigureAwait(false); - var healthy = await chanHost.ChannelHealthyAsync().ConfigureAwait(false); + var healthy = chanHost.ChannelHealthy(); var flagged = _flaggedChannels.ContainsKey(chanHost.ChannelId) && _flaggedChannels[chanHost.ChannelId]; if (flagged || !healthy) { diff --git a/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs b/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs index 4ce1a479..7986d57a 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ConnectionHost.cs @@ -19,7 +19,7 @@ public interface IConnectionHost void AssignConnection(IConnection connection); void Close(); - Task HealthyAsync(); + bool Healthy(); } public class ConnectionHost : IConnectionHost, IDisposable @@ -32,8 +32,6 @@ public class ConnectionHost : IConnectionHost, IDisposable public bool Closed { get; private set; } private readonly ILogger _logger; - private readonly SemaphoreSlim _hostLock = new SemaphoreSlim(1, 1); - private bool _disposedValue; public ConnectionHost(ulong connectionId, IConnection connection) { @@ -45,8 +43,6 @@ public ConnectionHost(ulong connectionId, IConnection connection) public void AssignConnection(IConnection connection) { - _hostLock.Wait(); - if (Connection != null) { Connection.ConnectionBlocked -= ConnectionBlocked; @@ -65,32 +61,24 @@ public void AssignConnection(IConnection connection) Connection.ConnectionBlocked += ConnectionBlocked; Connection.ConnectionUnblocked += ConnectionUnblocked; Connection.ConnectionShutdown += ConnectionClosed; - - _hostLock.Release(); } protected virtual void ConnectionClosed(object sender, ShutdownEventArgs e) { - _hostLock.Wait(); _logger.LogWarning(e.ReplyText); Closed = true; - _hostLock.Release(); } protected virtual void ConnectionBlocked(object sender, ConnectionBlockedEventArgs e) { - _hostLock.Wait(); _logger.LogWarning(e.Reason); Blocked = true; - _hostLock.Release(); } protected virtual void ConnectionUnblocked(object sender, EventArgs e) { - _hostLock.Wait(); _logger.LogInformation("Connection unblocked!"); Blocked = false; - _hostLock.Release(); } private const int CloseCode = 200; @@ -104,31 +92,23 @@ protected virtual void ConnectionUnblocked(object sender, EventArgs e) /// AutoRecovery = False yields results like Closed, Dead, and IsOpen will be true, true, false or false, false, true. /// AutoRecovery = True, yields difficult results like Closed, Dead, And IsOpen will be false, false, false or true, true, true (and other variations). /// - public async Task HealthyAsync() + public bool Healthy() { - await _hostLock - .WaitAsync() - .ConfigureAwait(false); - - if (Closed && Connection.IsOpen) + var connectionOpen = (Connection?.IsOpen ?? false); + if (Closed && connectionOpen) { Closed = false; } // Means a Recovery took place. - else if (Dead && Connection.IsOpen) + else if (Dead && connectionOpen) { Dead = false; } // Means a Miracle took place. - _hostLock.Release(); - - return Connection.IsOpen && !Blocked; // TODO: See if we can incorporate Dead/Closed observations. + return connectionOpen && !Blocked; // TODO: See if we can incorporate Dead/Closed observations. } + private bool _disposedValue; + protected virtual void Dispose(bool disposing) { if (!_disposedValue) { - if (disposing) - { - _hostLock.Dispose(); - } - Connection = null; _disposedValue = true; } diff --git a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs index af4a4a0e..57149e17 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs @@ -193,8 +193,7 @@ public async ValueTask GetConnectionAsync() .ReadAsync().ConfigureAwait(false); // Connection Health Check - var healthy = await connHost.HealthyAsync().ConfigureAwait(false); - if (!healthy) + if (!connHost.Healthy()) { await ReturnConnectionAsync(connHost).ConfigureAwait(false); await Task.Delay(Options.PoolOptions.SleepOnErrorInterval).ConfigureAwait(false); diff --git a/src/HouseofCat.RabbitMQ/Topology/ExchangeConfig.cs b/src/HouseofCat.RabbitMQ/Topology/Exchange.cs similarity index 90% rename from src/HouseofCat.RabbitMQ/Topology/ExchangeConfig.cs rename to src/HouseofCat.RabbitMQ/Topology/Exchange.cs index c8189ca0..8ec82f0d 100644 --- a/src/HouseofCat.RabbitMQ/Topology/ExchangeConfig.cs +++ b/src/HouseofCat.RabbitMQ/Topology/Exchange.cs @@ -2,7 +2,7 @@ namespace HouseofCat.RabbitMQ; -public class ExchangeConfig +public sealed record Exchange { public string Name { get; set; } public string Type { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Topology/ExchangingBindingConfig.cs b/src/HouseofCat.RabbitMQ/Topology/ExchangingBinding.cs similarity index 87% rename from src/HouseofCat.RabbitMQ/Topology/ExchangingBindingConfig.cs rename to src/HouseofCat.RabbitMQ/Topology/ExchangingBinding.cs index c998403f..355aa323 100644 --- a/src/HouseofCat.RabbitMQ/Topology/ExchangingBindingConfig.cs +++ b/src/HouseofCat.RabbitMQ/Topology/ExchangingBinding.cs @@ -2,7 +2,7 @@ namespace HouseofCat.RabbitMQ; -public class ExchangeBindingConfig +public sealed record ExchangeBinding { public string ChildExchange { get; set; } public string ParentExchange { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Topology/QueueConfig.cs b/src/HouseofCat.RabbitMQ/Topology/Queue.cs similarity index 91% rename from src/HouseofCat.RabbitMQ/Topology/QueueConfig.cs rename to src/HouseofCat.RabbitMQ/Topology/Queue.cs index f7d5df8e..d45f2b0c 100644 --- a/src/HouseofCat.RabbitMQ/Topology/QueueConfig.cs +++ b/src/HouseofCat.RabbitMQ/Topology/Queue.cs @@ -2,7 +2,7 @@ namespace HouseofCat.RabbitMQ; -public class QueueConfig +public sealed record Queue { public string Name { get; set; } public bool Durable { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Topology/QueueBindingConfig.cs b/src/HouseofCat.RabbitMQ/Topology/QueueBinding.cs similarity index 88% rename from src/HouseofCat.RabbitMQ/Topology/QueueBindingConfig.cs rename to src/HouseofCat.RabbitMQ/Topology/QueueBinding.cs index d7231f5b..557bdfd0 100644 --- a/src/HouseofCat.RabbitMQ/Topology/QueueBindingConfig.cs +++ b/src/HouseofCat.RabbitMQ/Topology/QueueBinding.cs @@ -2,7 +2,7 @@ namespace HouseofCat.RabbitMQ; -public class QueueBindingConfig +public sealed record QueueBinding { public string QueueName { get; set; } public string ExchangeName { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Topology/Topologer.cs b/src/HouseofCat.RabbitMQ/Topology/Topologer.cs index 3476a8dd..8f26ca0a 100644 --- a/src/HouseofCat.RabbitMQ/Topology/Topologer.cs +++ b/src/HouseofCat.RabbitMQ/Topology/Topologer.cs @@ -3,7 +3,6 @@ using HouseofCat.Utilities.Errors; using System; using System.Collections.Generic; -using System.IO; using System.Threading.Tasks; namespace HouseofCat.RabbitMQ; @@ -24,7 +23,7 @@ public interface ITopologer Task UnbindQueueFromExchangeAsync(string queueName, string exchangeName, string routingKey = "", IDictionary args = null); } -public class Topologer : ITopologer +public sealed class Topologer : ITopologer { private readonly IChannelPool _channelPool; public RabbitOptions Options { get; } diff --git a/src/HouseofCat.RabbitMQ/Topology/TopologyConfig.cs b/src/HouseofCat.RabbitMQ/Topology/TopologyConfig.cs index 2a1a2342..e9028317 100644 --- a/src/HouseofCat.RabbitMQ/Topology/TopologyConfig.cs +++ b/src/HouseofCat.RabbitMQ/Topology/TopologyConfig.cs @@ -1,9 +1,9 @@ namespace HouseofCat.RabbitMQ; -public class TopologyConfig +public sealed record TopologyConfig { - public ExchangeConfig[] Exchanges { get; set; } - public QueueConfig[] Queues { get; set; } - public ExchangeBindingConfig[] ExchangeBindings { get; set; } - public QueueBindingConfig[] QueueBindings { get; set; } + public Exchange[] Exchanges { get; set; } + public Queue[] Queues { get; set; } + public ExchangeBinding[] ExchangeBindings { get; set; } + public QueueBinding[] QueueBindings { get; set; } } From 5f86c05e8c61a0a11dce6e500dd6a44d4e7925d0 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 17:29:26 -0500 Subject: [PATCH 18/47] Update ConsumerDataflow.cs --- .../Dataflows/ConsumerDataflow.cs | 72 ------------------- 1 file changed, 72 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index d60fd659..655e3e3e 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -5,7 +5,6 @@ using HouseofCat.RabbitMQ.Services; using HouseofCat.Serialization; using HouseofCat.Utilities.Errors; -using HouseofCat.Utilities.Helpers; using OpenTelemetry.Trace; using System; using System.Collections.Generic; @@ -345,23 +344,6 @@ public ConsumerDataflow WithBuildState( return this; } - public ConsumerDataflow WithBuildStateAndPayload( - string stateKey, - int? maxDoP = null, - bool? ensureOrdered = null, - int? boundedCapacity = null, - TaskScheduler taskScheduler = null) - { - Guard.AgainstNullOrEmpty(stateKey, nameof(stateKey)); - - if (_buildStateBlock == null) - { - var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _buildStateBlock = GetBuildStateWithPayloadBlock(stateKey, executionOptions); - } - return this; - } - public ConsumerDataflow WithDecryptionStep( int? maxDoP = null, bool? ensureOrdered = null, @@ -628,60 +610,6 @@ public virtual TState BuildState(IReceivedMessage data) return state; } - public virtual TState BuildStateAndObjectFromPayload(string key, IReceivedMessage data) - { - var state = new TState - { - ReceivedMessage = data, - Data = new Dictionary() - }; - - var attributes = new List>() - { - KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName) - }; - - if (state.ReceivedMessage?.Message?.MessageId is not null) - { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.MessageId), state.ReceivedMessage.Message.MessageId)); - } - if (state.ReceivedMessage?.Message?.Metadata?.PayloadId is not null) - { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.Metadata.PayloadId), state.ReceivedMessage.Message.Metadata.PayloadId)); - } - - state.StartWorkflowSpan( - WorkflowName, - spanKind: SpanKind.Consumer, - suppliedAttributes: attributes, - traceHeader: data.TraceParentHeader); - - if (_serializationProvider != null - && data.ObjectType != Constants.HeaderValueForUnknownObjectType) - { - try - { state.Data[key] = _serializationProvider.Deserialize(data.Body); } - catch { } - } - - return state; - } - - public TransformBlock GetBuildStateWithPayloadBlock( - string key, - ExecutionDataflowBlockOptions options) - { - TState BuildStateWrap(IReceivedMessage data) - { - try - { return BuildStateAndObjectFromPayload(key, data); } - catch - { return null; } - } - - return new TransformBlock(BuildStateWrap, options); - } - public TransformBlock GetBuildStateBlock( ExecutionDataflowBlockOptions options) { From cea1b5d5a2305a117e4048b0a6473bfc8b1273a5 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 18:18:45 -0500 Subject: [PATCH 19/47] Remove global Consumer options, streamlining the config and RabbitService. --- guides/csharp/TopLevel.md | 110 +++++++---------- guides/rabbitmq/AutoPublisher.md | 6 +- guides/rabbitmq/BasicConsume.md | 6 +- guides/rabbitmq/BasicGet.md | 4 +- guides/rabbitmq/BasicPublish.md | 4 +- guides/rabbitmq/ChannelPools.md | 4 +- guides/rabbitmq/ConnectionPools.md | 6 +- guides/rabbitmq/Serialization.md | 4 +- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 10 -- .../Dataflows/ConsumerBlock.cs | 17 +-- .../Dataflows/ConsumerDataflow.cs | 29 +---- .../Dataflows/RabbitWorkState.cs | 1 - .../Options/ConsumerOptions.cs | 56 ++++----- .../Options/ConsumerPipelineOptions.cs | 6 - .../Options/GlobalConsumerOptions.cs | 21 ---- .../Options/GlobalConsumerPipelineOptions.cs | 10 -- .../Options/PoolOptions.cs | 6 +- .../Options/RabbitOptions.cs | 20 --- .../Pipelines/ConsumerPipeline.cs | 9 +- src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs | 12 +- .../Pools/ConnectionPool.cs | 6 +- src/HouseofCat.RabbitMQ/README.md | 79 +++--------- .../SampleRabbitOptions.json | 86 +++---------- .../Extensions/RabbitServiceExtensions.cs | 68 +++++++++++ .../Services/RabbitService.cs | 114 +++--------------- .../RabbitMQ.BasicGetTests.json | 6 +- .../RabbitMQ.ConnectionPoolTests.json | 6 +- .../RabbitMQ.ConsumerTests.json | 36 +++--- .../RabbitMQ.PubSubTests.json | 39 +++--- .../RabbitMQ.PublisherTests.json | 6 +- .../RabbitMQ.RabbitServiceTests.json | 36 +++--- .../Tests/PublisherTests.cs | 6 +- .../RabbitMQ.ConsumerDataflows.json | 36 +++--- 33 files changed, 308 insertions(+), 557 deletions(-) delete mode 100644 src/HouseofCat.RabbitMQ/Options/ConsumerPipelineOptions.cs delete mode 100644 src/HouseofCat.RabbitMQ/Options/GlobalConsumerOptions.cs delete mode 100644 src/HouseofCat.RabbitMQ/Options/GlobalConsumerPipelineOptions.cs diff --git a/guides/csharp/TopLevel.md b/guides/csharp/TopLevel.md index 582b9657..61f99aa1 100644 --- a/guides/csharp/TopLevel.md +++ b/guides/csharp/TopLevel.md @@ -117,72 +117,52 @@ Let me copy in a basic HoC config with our consumer settings in it. This file ne ```json { - "PoolOptions": { - "Uri": "amqp://guest:guest@localhost:5672/", - "MaxChannelsPerConnection": 2000, - "HeartbeatInterval": 6, - "AutoRecovery": true, - "TopologyRecovery": true, - "NetRecoveryTimeout": 5, - "ContinuationTimeout": 10, - "EnableDispatchConsumersAsync": true, - "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 10, - "MaxAckableChannels": 0, - "SleepOnErrorInterval": 5000, - "TansientChannelStartRange": 10000, - "UseTransientChannels": false - }, - "PublisherOptions": { - "MessageQueueBufferSize": 100, - "PriorityMessageQueueBufferSize": 100, - "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 0, - "CreatePublishReceipts": true, - "Compress": false, - "Encrypt": false - }, - "GlobalConsumerOptions": { - "AggressiveOptions": { - "ErrorSuffix": "Error", - "BatchSize": 128, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": false, - "MaxDegreesOfParallelism": 64, - "EnsureOrdered": false - } - }, - "SingleThreadedOptions": { - "ErrorSuffix": "Error", - "BatchSize": 1, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 1, - "EnsureOrdered": true - } - } - }, - "ConsumerOptions": { - "HoC-Consumer": { - "Enabled": true, - "GlobalSettings": "AggressiveOptions", - "ConsumerName": "HoC-Consumer", - "QueueName": "HoC-ConsumerQueue" - } + "PoolOptions": { + "Uri": "amqp://guest:guest@localhost:5672/", + "MaxChannelsPerConnection": 2000, + "HeartbeatInterval": 6, + "AutoRecovery": true, + "TopologyRecovery": true, + "NetRecoveryTimeout": 5, + "ContinuationTimeout": 10, + "EnableDispatchConsumersAsync": true, + "ServiceName": "HoC.RabbitMQ", + "Connections": 2, + "Channels": 10, + "AckableChannels": 0, + "SleepOnErrorInterval": 5000, + "TansientChannelStartRange": 10000, + "UseTransientChannels": false + }, + "PublisherOptions": { + "MessageQueueBufferSize": 100, + "BehaviorWhenFull": 0, + "AutoPublisherSleepInterval": 0, + "CreatePublishReceipts": true, + "Compress": false, + "Encrypt": false + }, + "ConsumerOptions": { + "HoC-Consume": { + "Enabled": true, + "ConsumerName": "HoC-Consume", + "QueueName": "TestQueue", + "ErrorSuffix": "Error", + "BatchSize": 5, + "BehaviorWhenFull": 0, + "SleepOnIdleInterval": 0, + "UseTransientChannels": true, + "AutoAck": false, + "NoLocal": false, + "Exclusive": false, + "WorkflowName": "TestConsumerWorkflow", + "WorkflowMaxDegreesOfParallelism": 1, + "WorkflowConsumerCount": 1, + "WorkflowBatchSize": 5, + "WorkflowEnsureOrdered": false, + "WorkflowWaitForCompletion": false } + } } ``` diff --git a/guides/rabbitmq/AutoPublisher.md b/guides/rabbitmq/AutoPublisher.md index 2889c146..8e24c2df 100644 --- a/guides/rabbitmq/AutoPublisher.md +++ b/guides/rabbitmq/AutoPublisher.md @@ -24,9 +24,9 @@ var rabbitOptions = new RabbitOptions { Uri = new Uri("amqp://guest:guest@localhost:5672"), ServiceName = "TestService", - MaxConnections = 2, - MaxChannels = 10, - MaxAckableChannels = 0 + Connections = 2, + Channels = 10, + AckableChannels = 0 }, PublisherOptions = new PublisherOptions { diff --git a/guides/rabbitmq/BasicConsume.md b/guides/rabbitmq/BasicConsume.md index ee77625b..41bbc9d0 100644 --- a/guides/rabbitmq/BasicConsume.md +++ b/guides/rabbitmq/BasicConsume.md @@ -19,9 +19,9 @@ I will use this as a file named `SampleRabbitOptions.json` "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 2, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 2, + "AckableChannels": 0, "TansientChannelStartRange": 10000 } } diff --git a/guides/rabbitmq/BasicGet.md b/guides/rabbitmq/BasicGet.md index b146e9c0..94d8f729 100644 --- a/guides/rabbitmq/BasicGet.md +++ b/guides/rabbitmq/BasicGet.md @@ -17,8 +17,8 @@ I will use this as a file named `SampleRabbitOptions.json` "TopologyRecovery": true, "NetRecoveryTimeout": 5 "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxAckableChannels": 5, + "Connections": 2, + "AckableChannels": 5, "TansientChannelStartRange": 10000 } } diff --git a/guides/rabbitmq/BasicPublish.md b/guides/rabbitmq/BasicPublish.md index d60505d3..5f2cfbae 100644 --- a/guides/rabbitmq/BasicPublish.md +++ b/guides/rabbitmq/BasicPublish.md @@ -20,8 +20,8 @@ I will use this as a file named `SampleRabbitOptions.json` "TopologyRecovery": true, "NetRecoveryTimeout": 5 "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 5, + "Connections": 2, + "Channels": 5, "TansientChannelStartRange": 10000 } } diff --git a/guides/rabbitmq/ChannelPools.md b/guides/rabbitmq/ChannelPools.md index 53e5ca5a..5df62c7f 100644 --- a/guides/rabbitmq/ChannelPools.md +++ b/guides/rabbitmq/ChannelPools.md @@ -43,8 +43,8 @@ I will use this as a file named `SampleRabbitOptions.json`: "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 5, + "Connections": 2, + "Channels": 5, "MaxAckableChannels": 0, "SleepOnErrorInterval": 1000, "TansientChannelStartRange": 10000, diff --git a/guides/rabbitmq/ConnectionPools.md b/guides/rabbitmq/ConnectionPools.md index d437efc6..f783cdfb 100644 --- a/guides/rabbitmq/ConnectionPools.md +++ b/guides/rabbitmq/ConnectionPools.md @@ -47,9 +47,9 @@ I will use this as a file named `SampleRabbitOptions.json`: "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 0, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 0, + "AckableChannels": 0, "SleepOnErrorInterval": 1000, "TansientChannelStartRange": 10000, "UseTransientChannels": false diff --git a/guides/rabbitmq/Serialization.md b/guides/rabbitmq/Serialization.md index 0c8bd19e..e2562f5b 100644 --- a/guides/rabbitmq/Serialization.md +++ b/guides/rabbitmq/Serialization.md @@ -18,8 +18,8 @@ I will use this as a file named `SampleRabbitOptions.json` "TopologyRecovery": true, "NetRecoveryTimeout": 5 "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 5, + "Connections": 2, + "Channels": 5, "TansientChannelStartRange": 10000 } } diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 8e1782e4..623f4a3d 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -338,16 +338,6 @@ await HandleRecoverableShutdownAsync(e) private static readonly string _consumerShutdownExceptionMessage = "Consumer's ChannelHost {0} had an unhandled exception during recovery."; - /// - /// This method used to rebuild channels/connections for Consumers. Due to recent - /// changes in RabbitMQ.Client, it is now possible for the consumer to be in a state - /// of self-recovery. Unfortunately, there are still some edge cases where the channel - /// has exception and is closed server side and this library needs to be able to recover - /// from those events. - /// - /// Docs: https://www.rabbitmq.com/client-libraries/dotnet-api-guide#recovery - /// - /// protected virtual async Task HandleRecoverableShutdownAsync(ShutdownEventArgs e) { try diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs index f003fc00..aea2aff4 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs @@ -11,10 +11,11 @@ namespace HouseofCat.RabbitMQ.Dataflows; public class ConsumerBlock : ISourceBlock { public Task Completion { get; } - internal IConsumer Consumer { set => _consumer = value; } + + public IConsumer Consumer { get; set; } private readonly ILogger> _logger; - private IConsumer _consumer; + private readonly ITargetBlock _bufferBlock; private readonly ISourceBlock _sourceBufferBlock; @@ -27,7 +28,7 @@ public ConsumerBlock() : this(new BufferBlock()) public ConsumerBlock(IConsumer consumer) : this() { Guard.AgainstNull(consumer, nameof(consumer)); - _consumer = consumer; + Consumer = consumer; } protected ConsumerBlock(ITargetBlock bufferBlock) : this(bufferBlock, (ISourceBlock)bufferBlock) @@ -44,13 +45,13 @@ protected ConsumerBlock(ITargetBlock bufferBlock, ISourceBlock sourc public async Task StartConsumingAsync() { _cts = new CancellationTokenSource(); - await _consumer.StartConsumerAsync().ConfigureAwait(false); + await Consumer.StartConsumerAsync().ConfigureAwait(false); _bufferProcessor = PushToBufferBlockAsync(_cts.Token); } public async Task StopConsumingAsync(bool immediate = false) { - await _consumer.StopConsumerAsync(immediate).ConfigureAwait(false); + await Consumer.StopConsumerAsync(immediate).ConfigureAwait(false); _cts.Cancel(); await _bufferProcessor.ConfigureAwait(false); } @@ -95,9 +96,9 @@ protected virtual async Task PushToBufferBlockAsync(CancellationToken token = de { try { - while (await _consumer.GetConsumerBuffer().WaitToReadAsync(token).ConfigureAwait(false)) + while (await Consumer.GetConsumerBuffer().WaitToReadAsync(token).ConfigureAwait(false)) { - while (_consumer.GetConsumerBuffer().TryRead(out var message)) + while (Consumer.GetConsumerBuffer().TryRead(out var message)) { await _bufferBlock.SendAsync(message, token).ConfigureAwait(false); } @@ -116,7 +117,7 @@ protected virtual async Task StreamToBufferAsync(CancellationToken token = defau { try { - await foreach (var message in _consumer.GetConsumerBuffer().ReadAllAsync(token).ConfigureAwait(false)) + await foreach (var message in Consumer.GetConsumerBuffer().ReadAllAsync(token).ConfigureAwait(false)) { await _bufferBlock.SendAsync(message, token).ConfigureAwait(false); } diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 655e3e3e..3dbcf7ad 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -73,40 +73,15 @@ public ConsumerDataflow( _executeStepOptions = new ExecutionDataflowBlockOptions { - MaxDegreeOfParallelism = _consumerOptions.ConsumerPipelineOptions.MaxDegreesOfParallelism ?? 1, + MaxDegreeOfParallelism = _consumerOptions.WorkflowMaxDegreesOfParallelism, SingleProducerConstrained = true, - EnsureOrdered = _consumerOptions.ConsumerPipelineOptions.EnsureOrdered ?? true, + EnsureOrdered = _consumerOptions.WorkflowEnsureOrdered, TaskScheduler = _taskScheduler, }; _consumerBlocks = new List>(); } - /// - /// This constructor is used for when you want to supply Consumers manually, or custom Consumers without having to write a custom IRabbitService, - /// and have global consumer pipeline options to retrieve maxDoP and ensureOrdered from. - /// - /// - /// - /// - /// - /// - public ConsumerDataflow( - IRabbitService rabbitService, - string workflowName, - ICollection> consumers, - GlobalConsumerPipelineOptions globalConsumerPipelineOptions, - TaskScheduler taskScheduler = null) : this( - rabbitService, - workflowName, - consumers, - globalConsumerPipelineOptions?.MaxDegreesOfParallelism ?? 1, - globalConsumerPipelineOptions?.EnsureOrdered ?? true, - taskScheduler) - { - Guard.AgainstNull(globalConsumerPipelineOptions, nameof(globalConsumerPipelineOptions)); - } - /// /// This constructor is used for when you want to supply Consumers manually, or custom Consumers without having to write a custom IRabbitService, /// and want a custom maxDoP and/or ensureOrdered. diff --git a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs index 1dfd3a9f..0aca67f0 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/RabbitWorkState.cs @@ -21,7 +21,6 @@ public abstract class RabbitWorkState : IRabbitWorkState public virtual ReadOnlyMemory SendData { get; set; } public virtual IMessage SendMessage { get; set; } - public virtual bool SendMessageSent { get; set; } public virtual IDictionary Data { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs b/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs index 4da626f5..d4f72428 100644 --- a/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs @@ -1,11 +1,23 @@ +using HouseofCat.RabbitMQ.Dataflows; +using HouseofCat.RabbitMQ.Services; using System.Collections.Generic; +using System.Threading.Channels; namespace HouseofCat.RabbitMQ; -public class ConsumerOptions : GlobalConsumerOptions +public class ConsumerOptions { + public bool? NoLocal { get; set; } + public bool? Exclusive { get; set; } + public ushort? BatchSize { get; set; } = 5; + public bool? AutoAck { get; set; } + + public string ErrorSuffix { get; set; } + public string AltSuffix { get; set; } + + public BoundedChannelFullMode? BehaviorWhenFull { get; set; } = BoundedChannelFullMode.Wait; + public bool Enabled { get; set; } - public string GlobalSettings { get; set; } public string ConsumerName { get; set; } @@ -18,37 +30,11 @@ public class ConsumerOptions : GlobalConsumerOptions public string ErrorQueueName => $"{QueueName}.{ErrorSuffix ?? "Error"}"; public IDictionary ErrorQueueArgs { get; set; } - public string AltQueueName => $"{QueueName}.{AltSuffix ?? "Alt"}"; - public IDictionary AltQueueArgs { get; set; } - - public ConsumerPipelineOptions ConsumerPipelineOptions { get; set; } - - public void ApplyGlobalOptions(GlobalConsumerOptions globalConsumerOptions) - { - NoLocal = globalConsumerOptions.NoLocal ?? NoLocal; - Exclusive = globalConsumerOptions.Exclusive ?? Exclusive; - BatchSize = globalConsumerOptions.BatchSize ?? BatchSize; - - AutoAck = globalConsumerOptions.AutoAck ?? AutoAck; - ErrorSuffix = globalConsumerOptions.ErrorSuffix ?? ErrorSuffix; - AltSuffix = globalConsumerOptions.AltSuffix ?? AltSuffix; - BehaviorWhenFull = globalConsumerOptions.BehaviorWhenFull ?? BehaviorWhenFull; - - if (globalConsumerOptions.GlobalConsumerPipelineOptions != null) - { - ConsumerPipelineOptions ??= new ConsumerPipelineOptions(); - - ConsumerPipelineOptions.WaitForCompletion = - globalConsumerOptions.GlobalConsumerPipelineOptions.WaitForCompletion - ?? ConsumerPipelineOptions.WaitForCompletion; - - ConsumerPipelineOptions.MaxDegreesOfParallelism = - globalConsumerOptions.GlobalConsumerPipelineOptions.MaxDegreesOfParallelism - ?? ConsumerPipelineOptions.MaxDegreesOfParallelism; - - ConsumerPipelineOptions.EnsureOrdered = - globalConsumerOptions.GlobalConsumerPipelineOptions.EnsureOrdered - ?? ConsumerPipelineOptions.EnsureOrdered; - } - } + + public string WorkflowName { get; set; } + public int WorkflowMaxDegreesOfParallelism { get; set; } = 1; + public int WorkflowConsumerCount { get; set; } = 1; + public int WorkflowBatchSize { get; set; } = 5; + public bool WorkflowEnsureOrdered { get; set; } = true; + public bool WorkflowWaitForCompletion { get;set; } } diff --git a/src/HouseofCat.RabbitMQ/Options/ConsumerPipelineOptions.cs b/src/HouseofCat.RabbitMQ/Options/ConsumerPipelineOptions.cs deleted file mode 100644 index 7ca02a07..00000000 --- a/src/HouseofCat.RabbitMQ/Options/ConsumerPipelineOptions.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace HouseofCat.RabbitMQ; - -public class ConsumerPipelineOptions : GlobalConsumerPipelineOptions -{ - public string ConsumerPipelineName { get; set; } -} diff --git a/src/HouseofCat.RabbitMQ/Options/GlobalConsumerOptions.cs b/src/HouseofCat.RabbitMQ/Options/GlobalConsumerOptions.cs deleted file mode 100644 index a4cbcf7d..00000000 --- a/src/HouseofCat.RabbitMQ/Options/GlobalConsumerOptions.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Threading.Channels; - -namespace HouseofCat.RabbitMQ; - -/// -/// Global overrides for your consumers. -/// -public class GlobalConsumerOptions -{ - public bool? NoLocal { get; set; } - public bool? Exclusive { get; set; } - public ushort? BatchSize { get; set; } = 5; - public bool? AutoAck { get; set; } - - public string ErrorSuffix { get; set; } - public string AltSuffix { get; set; } - - public BoundedChannelFullMode? BehaviorWhenFull { get; set; } = BoundedChannelFullMode.Wait; - - public ConsumerPipelineOptions GlobalConsumerPipelineOptions { get; set; } -} diff --git a/src/HouseofCat.RabbitMQ/Options/GlobalConsumerPipelineOptions.cs b/src/HouseofCat.RabbitMQ/Options/GlobalConsumerPipelineOptions.cs deleted file mode 100644 index 7c0e4502..00000000 --- a/src/HouseofCat.RabbitMQ/Options/GlobalConsumerPipelineOptions.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; - -namespace HouseofCat.RabbitMQ; - -public class GlobalConsumerPipelineOptions -{ - public bool? WaitForCompletion { get; set; } - public int? MaxDegreesOfParallelism { get; set; } = Environment.ProcessorCount; - public bool? EnsureOrdered { get; set; } -} diff --git a/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs b/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs index ff25fa63..2441bc41 100644 --- a/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs @@ -85,19 +85,19 @@ public class PoolOptions /// Number of connections to be created in the ConnectionPool. Used in round-robin to create channels. /// Deafult valuse is 2. /// - public ushort MaxConnections { get; set; } = 2; + public ushort Connections { get; set; } = 2; /// /// Number of channels to keep in each of the channel pool. Used in round-robin to perform actions. /// Default value is 0. /// - public ushort MaxChannels { get; set; } + public ushort Channels { get; set; } /// /// Number of ackable channels to keep in each of the channel pool. Used in round-robin to perform actions. /// Default value is 10. /// - public ushort MaxAckableChannels { get; set; } = 10; + public ushort AckableChannels { get; set; } = 10; /// /// The time to sleep (in ms) when an error occurs on Channel or Connection creation. It's best not to be hyper aggressive with this value. diff --git a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs index d696c22f..a19800a1 100644 --- a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs @@ -16,11 +16,6 @@ public class RabbitOptions /// public PublisherOptions PublisherOptions { get; set; } = new PublisherOptions(); - /// - /// Class to hold the global Consumer options. Will apply these to every consumer. - /// - public IDictionary GlobalConsumerOptions { get; set; } = new Dictionary(); - /// /// Dictionary to hold all the ConsumerOptions using the ConsumerOptions class. /// @@ -31,19 +26,4 @@ public ConsumerOptions GetConsumerOptions(string consumerName) if (!ConsumerOptions.TryGetValue(consumerName, out ConsumerOptions value)) throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, ExceptionMessages.NoConsumerOptionsMessage, consumerName)); return value; } - - public void ApplyGlobalConsumerOptions() - { - foreach (var kvp in ConsumerOptions) - { - // Apply the global consumer settings and global consumer pipeline settings - // on top of (overriding) individual consumer settings. Opt out by not setting - // the global settings field. - if (!string.IsNullOrWhiteSpace(kvp.Value.GlobalSettings) - && GlobalConsumerOptions.TryGetValue(kvp.Value.GlobalSettings, out GlobalConsumerOptions value)) - { - kvp.Value.ApplyGlobalOptions(value); - } - } - } } diff --git a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs index 55cee7c0..6ac2d409 100644 --- a/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs +++ b/src/HouseofCat.RabbitMQ/Pipelines/ConsumerPipeline.cs @@ -44,10 +44,9 @@ public ConsumerPipeline( Pipeline = pipeline ?? throw new ArgumentNullException(nameof(pipeline)); Consumer = consumer ?? throw new ArgumentNullException(nameof(consumer)); ConsumerOptions = consumer.ConsumerOptions ?? throw new ArgumentNullException(nameof(consumer.Options)); - if (consumer.ConsumerOptions.ConsumerPipelineOptions == null) throw new ArgumentNullException(nameof(consumer.ConsumerOptions.ConsumerPipelineOptions)); - ConsumerPipelineName = !string.IsNullOrWhiteSpace(consumer.ConsumerOptions.ConsumerPipelineOptions.ConsumerPipelineName) - ? consumer.ConsumerOptions.ConsumerPipelineOptions.ConsumerPipelineName + ConsumerPipelineName = !string.IsNullOrEmpty(consumer.ConsumerOptions.WorkflowName) + ? consumer.ConsumerOptions.WorkflowName : "Unknown"; } @@ -74,7 +73,7 @@ await Consumer () => PipelineStreamEngineAsync( Pipeline, - ConsumerOptions.ConsumerPipelineOptions.WaitForCompletion!.Value, + ConsumerOptions.WorkflowWaitForCompletion, token.Equals(default) ? _cancellationTokenSource.Token : token), @@ -86,7 +85,7 @@ await Consumer () => PipelineExecutionEngineAsync( Pipeline, - ConsumerOptions.ConsumerPipelineOptions.WaitForCompletion!.Value, + ConsumerOptions.WorkflowWaitForCompletion, token.Equals(default) ? _cancellationTokenSource.Token : token), diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs index 9d4d1d51..80e0c161 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelPool.cs @@ -81,14 +81,14 @@ public ChannelPool(IConnectionPool connPool) _connectionPool = connPool; _flaggedChannels = new ConcurrentDictionary(); - if (Options.PoolOptions.MaxChannels > 0) + if (Options.PoolOptions.Channels > 0) { - _channels = Channel.CreateBounded(Options.PoolOptions.MaxChannels); + _channels = Channel.CreateBounded(Options.PoolOptions.Channels); } - if (Options.PoolOptions.MaxAckableChannels > 0) + if (Options.PoolOptions.AckableChannels > 0) { - _ackChannels = Channel.CreateBounded(Options.PoolOptions.MaxAckableChannels); + _ackChannels = Channel.CreateBounded(Options.PoolOptions.AckableChannels); } if (!Options.PoolOptions.OnlyTransientChannels) @@ -101,7 +101,7 @@ public ChannelPool(IConnectionPool connPool) private async Task CreateChannelsAsync() { - for (var i = 0; i < Options.PoolOptions.MaxChannels; i++) + for (var i = 0; i < Options.PoolOptions.Channels; i++) { var chanHost = await CreateChannelAsync(CurrentChannelId++, false).ConfigureAwait(false); @@ -110,7 +110,7 @@ await _channels .WriteAsync(chanHost); } - for (var i = 0; i < Options.PoolOptions.MaxAckableChannels; i++) + for (var i = 0; i < Options.PoolOptions.AckableChannels; i++) { var chanHost = await CreateChannelAsync(CurrentChannelId++, true).ConfigureAwait(false); diff --git a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs index 57149e17..ca95c1ad 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs @@ -42,7 +42,7 @@ public ConnectionPool(RabbitOptions options, HttpClientHandler oauth2ClientHandl _logger = LogHelpers.GetLogger(); - _connections = Channel.CreateBounded(Options.PoolOptions.MaxConnections); + _connections = Channel.CreateBounded(Options.PoolOptions.Connections); if (oauth2ClientHandler is not null) { @@ -63,7 +63,7 @@ public ConnectionPool(RabbitOptions options) _logger = LogHelpers.GetLogger(); - _connections = Channel.CreateBounded(Options.PoolOptions.MaxConnections); + _connections = Channel.CreateBounded(Options.PoolOptions.Connections); _connectionFactory = BuildConnectionFactory(); CreateConnectionsAsync().GetAwaiter().GetResult(); @@ -154,7 +154,7 @@ private async Task CreateConnectionsAsync() { _logger.LogTrace(LogMessages.ConnectionPools.CreateConnections); - for (int i = 0; i < Options.PoolOptions.MaxConnections; i++) + for (int i = 0; i < Options.PoolOptions.Connections; i++) { var serviceName = string.IsNullOrEmpty(Options.PoolOptions.ServiceName) ? $"HoC.RabbitMQ:{i}" diff --git a/src/HouseofCat.RabbitMQ/README.md b/src/HouseofCat.RabbitMQ/README.md index 677c6073..ea9cc4c6 100644 --- a/src/HouseofCat.RabbitMQ/README.md +++ b/src/HouseofCat.RabbitMQ/README.md @@ -17,9 +17,9 @@ Sample Config File "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 10, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 10, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false @@ -32,74 +32,25 @@ Sample Config File "Compress": true, "Encrypt": true }, - "GlobalConsumerOptions": { - "AggressiveSettings": { - "ErrorSuffix": "Error", - "BatchSize": 128, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": false, - "MaxDegreesOfParallelism": 64, - "EnsureOrdered": false - } - }, - "ModerateSettings": { - "ErrorSuffix": "Error", - "BatchSize": 48, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 100, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 24, - "EnsureOrdered": false - } - }, - "LightSettings": { - "ErrorSuffix": "Error", - "BatchSize": 8, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 100, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 4, - "EnsureOrdered": false - } - }, - "SingleThreaded": { + "ConsumerOptions": { + "TestConsumer": { + "Enabled": true, + "ConsumerName": "TestConsumer", + "QueueName": "TestQueue", "ErrorSuffix": "Error", - "BatchSize": 1, + "BatchSize": 5, "BehaviorWhenFull": 0, "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 1, - "EnsureOrdered": false - } - } - }, - "ConsumerOptions": { - "ConsumerFromConfig": { - "Enabled": true, - "GlobalSettings": "AggressiveSettings", - "ConsumerName": "ConsumerFromConfig", - "QueueName": "TestRabbitServiceQueue" + "WorkflowName": "TestConsumerWorkflow", + "WorkflowMaxDegreesOfParallelism": 1, + "WorkflowConsumerCount": 1, + "WorkflowBatchSize": 5, + "WorkflowEnsureOrdered": false, + "WorkflowWaitForCompletion": false } } } diff --git a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json index 07f26752..b7498e6e 100644 --- a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json +++ b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json @@ -20,9 +20,9 @@ "ProtocolVersions": 3072 }, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 0, - "MaxAckableChannels": 10, + "Connections": 2, + "Channels": 1, + "AckableChannels": 1, "SleepOnErrorInterval": 1000, "TansientChannelStartRange": 10000, "UseTransientChannels": false @@ -30,79 +30,31 @@ "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 0, + "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, "Compress": false, - "Encrypt": false + "Encrypt": false, + "WaitForConfirmationTimeoutInMilliseconds": 500 }, - "GlobalConsumerOptions": { - "AggressiveSettings": { - "ErrorSuffix": "Error", - "BatchSize": 128, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": false, - "MaxDegreesOfParallelism": 64, - "EnsureOrdered": false - } - }, - "ModerateSettings": { - "ErrorSuffix": "Error", - "BatchSize": 48, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 100, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 24, - "EnsureOrdered": false - } - }, - "LightSettings": { - "ErrorSuffix": "Error", - "BatchSize": 8, - "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 100, - "UseTransientChannels": true, - "AutoAck": false, - "NoLocal": false, - "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 4, - "EnsureOrdered": false - } - }, - "SingleThreaded": { + "ConsumerOptions": { + "TestConsumer": { + "Enabled": true, + "ConsumerName": "TestConsumer", + "QueueName": "TestQueue", "ErrorSuffix": "Error", - "BatchSize": 1, + "BatchSize": 5, "BehaviorWhenFull": 0, "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 1, - "EnsureOrdered": false - } - } - }, - "ConsumerOptions": { - "ConsumerFromConfig": { - "Enabled": true, - "GlobalSettings": "AggressiveSettings", - "ConsumerName": "ConsumerFromConfig", - "QueueName": "TestRabbitServiceQueue" + "WorkflowName": "TestConsumerWorkflow", + "WorkflowMaxDegreesOfParallelism": 1, + "WorkflowConsumerCount": 1, + "WorkflowBatchSize": 5, + "WorkflowEnsureOrdered": false, + "WorkflowWaitForCompletion": false } } -} \ No newline at end of file +} diff --git a/src/HouseofCat.RabbitMQ/Services/Extensions/RabbitServiceExtensions.cs b/src/HouseofCat.RabbitMQ/Services/Extensions/RabbitServiceExtensions.cs index 129476a4..947b939b 100644 --- a/src/HouseofCat.RabbitMQ/Services/Extensions/RabbitServiceExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Services/Extensions/RabbitServiceExtensions.cs @@ -1,12 +1,16 @@ using HouseofCat.Compression; using HouseofCat.Compression.Recyclable; +using HouseofCat.Dataflows.Pipelines; using HouseofCat.Encryption; using HouseofCat.Hashing; +using HouseofCat.RabbitMQ.Dataflows; +using HouseofCat.RabbitMQ.Pipelines; using HouseofCat.Serialization; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Threading.Tasks; namespace HouseofCat.RabbitMQ.Services.Extensions; @@ -103,4 +107,68 @@ public static async Task BuildRabbitServiceAsync( return rabbitService; } + + public static IConsumerPipeline CreateConsumerPipeline( + this IRabbitService rabbitService, + string consumerName, + int maxDoP, + int batchSize, + bool? ensureOrdered, + Func> pipelineBuilder) + where TOut : RabbitWorkState + { + var consumer = rabbitService.GetConsumer(consumerName); + var pipeline = pipelineBuilder.Invoke(maxDoP, batchSize, ensureOrdered); + + return new ConsumerPipeline(consumer, pipeline); + } + + public static IConsumerPipeline CreateConsumerPipeline( + this IRabbitService rabbitService, + string consumerName, + Func> pipelineBuilder) + where TOut : RabbitWorkState + { + if (rabbitService.ConsumerOptions.TryGetValue(consumerName, out var options)) + { + return rabbitService.CreateConsumerPipeline( + consumerName, + options.WorkflowMaxDegreesOfParallelism, + options.WorkflowBatchSize, + options.WorkflowEnsureOrdered, + pipelineBuilder); + } + + throw new InvalidOperationException($"ConsumerOptions for {consumerName} not found."); + } + + public static IConsumerPipeline CreateConsumerPipeline( + this IRabbitService rabbitService, + string consumerName, + IPipeline pipeline) + where TOut : RabbitWorkState + { + var consumer = rabbitService.GetConsumer(consumerName); + + return new ConsumerPipeline(consumer, pipeline); + } + + public static ConsumerDataflow BuildConsumerDataflow( + this IRabbitService rabbitService, + string consumerName, + TaskScheduler taskScheduler = null) + where TState : class, IRabbitWorkState, new() + { + if (rabbitService.ConsumerOptions.TryGetValue(consumerName, out var options)) + { + return new ConsumerDataflow( + rabbitService, + options.WorkflowName, + options.ConsumerName, + options.WorkflowConsumerCount, + taskScheduler); + } + + throw new InvalidOperationException($"ConsumerOptions for {consumerName} not found."); + } } diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index 7c0f8f51..3f9d24d7 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -19,7 +19,6 @@ namespace HouseofCat.RabbitMQ.Services; public interface IRabbitService { - IPublisher Publisher { get; } IChannelPool ChannelPool { get; } ITopologer Topologer { get; } RabbitOptions Options { get; } @@ -30,23 +29,23 @@ public interface IRabbitService IEncryptionProvider EncryptionProvider { get; } ICompressionProvider CompressionProvider { get; } + IPublisher Publisher { get; } + ConcurrentDictionary> Consumers { get; } + ConcurrentDictionary ConsumerOptions { get; } Task ComcryptAsync(IMessage message); Task CompressAsync(IMessage message); - IConsumerPipeline CreateConsumerPipeline(string consumerName, int batchSize, bool? ensureOrdered, Func> pipelineBuilder) where TOut : RabbitWorkState; - IConsumerPipeline CreateConsumerPipeline(string consumerName, IPipeline pipeline) where TOut : RabbitWorkState; - - IConsumerPipeline CreateConsumerPipeline(string consumerName, Func> pipelineBuilder) where TOut : RabbitWorkState; Task DecomcryptAsync(IMessage message); Task DecompressAsync(IMessage message); + bool Decrypt(IMessage message); bool Encrypt(IMessage message); - Task?> GetAsync(string queueName); + + Task> GetAsync(string queueName); Task GetAsync(string queueName); IConsumer GetConsumer(string consumerName); - IConsumer GetConsumerByPipelineName(string consumerPipelineName); ValueTask ShutdownAsync(bool immediately); } @@ -66,7 +65,7 @@ public class RabbitService : IRabbitService, IDisposable public ICompressionProvider CompressionProvider { get; } public ConcurrentDictionary> Consumers { get; private set; } = new ConcurrentDictionary>(); - private ConcurrentDictionary ConsumerPipelineNameToConsumerOptions { get; set; } = new ConcurrentDictionary(); + public ConcurrentDictionary ConsumerOptions { get; private set; } = new ConcurrentDictionary(); public string TimeFormat { get; set; } = TimeHelpers.Formats.CatsAltFormat; @@ -129,7 +128,6 @@ public RabbitService( Topologer = new Topologer(ChannelPool); - Options.ApplyGlobalConsumerOptions(); BuildConsumers(); Publisher.StartAutoPublish(processReceiptAsync); @@ -173,13 +171,10 @@ await kvp private void BuildConsumers() { - foreach (var consumerSetting in Options.ConsumerOptions) + foreach (var options in Options.ConsumerOptions) { - if (!string.IsNullOrEmpty(consumerSetting.Value.ConsumerPipelineOptions.ConsumerPipelineName)) - { - ConsumerPipelineNameToConsumerOptions.TryAdd(consumerSetting.Value.ConsumerPipelineOptions.ConsumerPipelineName, consumerSetting.Value); - } - Consumers.TryAdd(consumerSetting.Value.ConsumerName, new Consumer(ChannelPool, consumerSetting.Value)); + ConsumerOptions.TryAdd(options.Value.ConsumerName, options.Value); + Consumers.TryAdd(options.Value.ConsumerName, new Consumer(ChannelPool, options.Value)); } } @@ -253,68 +248,9 @@ await Topologer .ConfigureAwait(false); } } - - if (!string.IsNullOrWhiteSpace(consumer.Value.ConsumerOptions.AltSuffix) - && !string.IsNullOrWhiteSpace(consumer.Value.ConsumerOptions.AltQueueName)) - { - if (consumer.Value.ConsumerOptions.AltQueueArgs == null - || consumer.Value.ConsumerOptions.AltQueueArgs.Count == 0) - { - await Topologer - .CreateQueueAsync(consumer.Value.ConsumerOptions.AltQueueName) - .ConfigureAwait(false); - } - else - { - await Topologer - .CreateQueueAsync( - consumer.Value.ConsumerOptions.AltQueueName, - true, - false, - false, - consumer.Value.ConsumerOptions.AltQueueArgs) - .ConfigureAwait(false); - } - } } } - public IConsumerPipeline CreateConsumerPipeline( - string consumerName, - int batchSize, - bool? ensureOrdered, - Func> pipelineBuilder) - where TOut : RabbitWorkState - { - var consumer = GetConsumer(consumerName); - var pipeline = pipelineBuilder.Invoke(batchSize, ensureOrdered); - - return new ConsumerPipeline(consumer, pipeline); - } - - public IConsumerPipeline CreateConsumerPipeline( - string consumerName, - Func> pipelineBuilder) - where TOut : RabbitWorkState - { - var consumer = GetConsumer(consumerName); - var pipeline = pipelineBuilder.Invoke( - consumer.ConsumerOptions.ConsumerPipelineOptions.MaxDegreesOfParallelism ?? 1, - consumer.ConsumerOptions.ConsumerPipelineOptions.EnsureOrdered ?? true); - - return new ConsumerPipeline(consumer, pipeline); - } - - public IConsumerPipeline CreateConsumerPipeline( - string consumerName, - IPipeline pipeline) - where TOut : RabbitWorkState - { - var consumer = GetConsumer(consumerName); - - return new ConsumerPipeline(consumer, pipeline); - } - public IConsumer GetConsumer(string consumerName) { if (!Consumers.TryGetValue(consumerName, out IConsumer value)) @@ -322,15 +258,6 @@ public IConsumer GetConsumer(string consumerName) return value; } - public IConsumer GetConsumerByPipelineName(string consumerPipelineName) - { - if (!ConsumerPipelineNameToConsumerOptions.TryGetValue(consumerPipelineName, out ConsumerOptions value)) - { throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerPipelineOptionsMessage, consumerPipelineName)); } - if (!Consumers.TryGetValue(value.ConsumerName, out var consumer)) - { throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerOptionsMessage, value.ConsumerName)); } - return consumer; - } - public async Task DecomcryptAsync(IMessage message) { if (message is null) return; @@ -352,7 +279,6 @@ public async Task ComcryptAsync(IMessage message) Encrypt(message); } - // Returns Success public bool Encrypt(IMessage message) { if (!message.Metadata.Encrypted) @@ -369,7 +295,6 @@ public bool Encrypt(IMessage message) return false; } - // Returns Success public bool Decrypt(IMessage message) { if (message.Metadata.Encrypted) @@ -387,7 +312,6 @@ public bool Decrypt(IMessage message) return false; } - // Returns Success public async Task CompressAsync(IMessage message) { if (message.Metadata.Encrypted) @@ -406,7 +330,6 @@ public async Task CompressAsync(IMessage message) return true; } - // Returns Success public async Task DecompressAsync(IMessage message) { if (message.Metadata.Encrypted) @@ -428,7 +351,7 @@ public async Task DecompressAsync(IMessage message) return true; } - public async Task?> GetAsync(string queueName) + public async Task> GetAsync(string queueName) { IChannelHost chanHost; @@ -447,6 +370,8 @@ public async Task DecompressAsync(IMessage message) result = chanHost .Channel .BasicGet(queueName, true); + + return result.Body; } catch { error = true; } finally @@ -456,14 +381,9 @@ await ChannelPool .ConfigureAwait(false); } - return result?.Body; + return default; } - /// - /// Simple retrieve message (byte[]) from queue and convert to type T. Default (assumed null) if nothing was available (or on transmission error). - /// AutoAcks message. - /// - /// public async Task GetAsync(string queueName) { IChannelHost chanHost; @@ -492,7 +412,9 @@ await ChannelPool .ConfigureAwait(false); } - return result != null ? SerializationProvider.Deserialize(result.Body) : default; + return result != null + ? SerializationProvider.Deserialize(result.Body) + : default; } protected virtual void Dispose(bool disposing) @@ -505,7 +427,7 @@ protected virtual void Dispose(bool disposing) } Consumers = null; - ConsumerPipelineNameToConsumerOptions = null; + ConsumerOptions = null; _disposedValue = true; } } diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json index 42fc73df..c85b063c 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.BasicGetTests.json @@ -9,9 +9,9 @@ "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 1, - "MaxChannels": 0, - "MaxAckableChannels": 0, + "Connections": 1, + "Channels": 0, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json index 690fc30b..2a5e1f27 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConnectionPoolTests.json @@ -9,9 +9,9 @@ "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 0, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 0, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json index a2c2833c..8e6d9543 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json @@ -9,36 +9,32 @@ "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 0, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 0, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false }, - "GlobalConsumerOptions": { - "SingleThreaded": { + "ConsumerOptions": { + "TestConsumer": { + "Enabled": true, + "ConsumerName": "TestConsumer", + "QueueName": "TestQueue", "ErrorSuffix": "Error", - "BatchSize": 1, + "BatchSize": 5, "BehaviorWhenFull": 0, "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 1, - "EnsureOrdered": false - } - } - }, - "ConsumerOptions": { - "TestConsumer": { - "Enabled": true, - "GlobalSettings": "SingleThreaded", - "ConsumerName": "TestConsumer", - "QueueName": "TestQueue" + "WorkflowName": "TestConsumerWorkflow", + "WorkflowMaxDegreesOfParallelism": 1, + "WorkflowConsumerCount": 1, + "WorkflowBatchSize": 5, + "WorkflowEnsureOrdered": false, + "WorkflowWaitForCompletion": false } } -} \ No newline at end of file +} diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json index 56198585..b8f3fd2b 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json @@ -9,9 +9,9 @@ "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 10, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 10, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false @@ -22,31 +22,28 @@ "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, "Compress": false, - "Encrypt": false + "Encrypt": false, + "WaitForConfirmationTimeoutInMilliseconds": 500 }, - "GlobalConsumerOptions": { - "SingleThreaded": { + "ConsumerOptions": { + "TestConsumer": { + "Enabled": true, + "ConsumerName": "TestConsumer", + "QueueName": "TestQueue", "ErrorSuffix": "Error", - "BatchSize": 1, + "BatchSize": 5, "BehaviorWhenFull": 0, "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 1, - "EnsureOrdered": false - } - } - }, - "ConsumerOptions": { - "TestConsumer": { - "Enabled": true, - "GlobalSettings": "SingleThreaded", - "ConsumerName": "TestConsumer", - "QueueName": "TestQueue" + "WorkflowName": "TestConsumerWorkflow", + "WorkflowMaxDegreesOfParallelism": 1, + "WorkflowConsumerCount": 1, + "WorkflowBatchSize": 5, + "WorkflowEnsureOrdered": false, + "WorkflowWaitForCompletion": false } } -} \ No newline at end of file +} diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json index 6b2ecda9..7011e318 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json @@ -9,9 +9,9 @@ "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 10, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 10, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json index 2c1f7471..6128d6fa 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json @@ -9,9 +9,9 @@ "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 10, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 10, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false @@ -24,29 +24,25 @@ "Compress": true, "Encrypt": true }, - "GlobalConsumerOptions": { - "ModerateSettings": { + "ConsumerOptions": { + "TestConsumer": { + "Enabled": true, + "ConsumerName": "TestConsumer", + "QueueName": "TestQueue", "ErrorSuffix": "Error", - "BatchSize": 12, + "BatchSize": 5, "BehaviorWhenFull": 0, "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 4, - "EnsureOrdered": false - } - } - }, - "ConsumerOptions": { - "TestConsumer": { - "Enabled": true, - "GlobalSettings": "ModerateSettings", - "ConsumerName": "TestConsumer", - "QueueName": "TestQueue" + "WorkflowName": "TestConsumerWorkflow", + "WorkflowMaxDegreesOfParallelism": 1, + "WorkflowConsumerCount": 1, + "WorkflowBatchSize": 5, + "WorkflowEnsureOrdered": false, + "WorkflowWaitForCompletion": false } } -} \ No newline at end of file +} diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index f33e9118..36031fe5 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -58,9 +58,9 @@ public static async Task RunAutoPublisherStandaloneAsync() { Uri = new Uri("amqp://guest:guest@localhost:5672"), ServiceName = "TestService", - MaxConnections = 2, - MaxChannels = 10, - MaxAckableChannels = 0 + Connections = 2, + Channels = 10, + AckableChannels = 0 }, PublisherOptions = new PublisherOptions { diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json index 2c1f7471..6128d6fa 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json @@ -9,9 +9,9 @@ "ContinuationTimeout": 10, "EnableDispatchConsumersAsync": true, "ServiceName": "HoC.RabbitMQ", - "MaxConnections": 2, - "MaxChannels": 10, - "MaxAckableChannels": 0, + "Connections": 2, + "Channels": 10, + "AckableChannels": 0, "SleepOnErrorInterval": 5000, "TansientChannelStartRange": 10000, "UseTransientChannels": false @@ -24,29 +24,25 @@ "Compress": true, "Encrypt": true }, - "GlobalConsumerOptions": { - "ModerateSettings": { + "ConsumerOptions": { + "TestConsumer": { + "Enabled": true, + "ConsumerName": "TestConsumer", + "QueueName": "TestQueue", "ErrorSuffix": "Error", - "BatchSize": 12, + "BatchSize": 5, "BehaviorWhenFull": 0, "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, - "GlobalConsumerPipelineOptions": { - "WaitForCompletion": true, - "MaxDegreesOfParallelism": 4, - "EnsureOrdered": false - } - } - }, - "ConsumerOptions": { - "TestConsumer": { - "Enabled": true, - "GlobalSettings": "ModerateSettings", - "ConsumerName": "TestConsumer", - "QueueName": "TestQueue" + "WorkflowName": "TestConsumerWorkflow", + "WorkflowMaxDegreesOfParallelism": 1, + "WorkflowConsumerCount": 1, + "WorkflowBatchSize": 5, + "WorkflowEnsureOrdered": false, + "WorkflowWaitForCompletion": false } } -} \ No newline at end of file +} From c7ffe11dcdf2080aa71a41c04a49b871bae8f781 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 18:22:39 -0500 Subject: [PATCH 20/47] Make ConsumerOption fields non-nullable. --- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 13 ++++++------- src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs | 12 +++++------- src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs | 6 +++--- src/HouseofCat.RabbitMQ/Services/RabbitService.cs | 3 +-- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 623f4a3d..1cf6acc0 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -81,9 +81,9 @@ public async Task StartConsumerAsync() await SetChannelHostAsync().ConfigureAwait(false); _shutdown = false; _consumerChannel = Channel.CreateBounded( - new BoundedChannelOptions(ConsumerOptions.BatchSize!.Value) + new BoundedChannelOptions(ConsumerOptions.BatchSize) { - FullMode = ConsumerOptions.BehaviorWhenFull!.Value + FullMode = ConsumerOptions.BehaviorWhenFull.Value }); await Task.Yield(); @@ -211,11 +211,10 @@ await _chanHost.WaitUntilChannelIsReadyAsync( protected virtual async Task SetChannelHostAsync() { - var autoAck = ConsumerOptions.AutoAck ?? false; _logger.LogTrace(Consumers.GettingTransientChannelHost, ConsumerOptions.ConsumerName); _chanHost = await ChannelPool - .GetTransientChannelAsync(!autoAck) + .GetTransientChannelAsync(!ConsumerOptions.AutoAck) .ConfigureAwait(false); _logger.LogDebug( @@ -228,7 +227,7 @@ private EventingBasicConsumer CreateConsumer() { EventingBasicConsumer consumer = null; - _chanHost.Channel.BasicQos(0, ConsumerOptions.BatchSize!.Value, false); + _chanHost.Channel.BasicQos(0, ConsumerOptions.BatchSize, false); consumer = new EventingBasicConsumer(_chanHost.Channel); consumer.Received += ReceiveHandler; @@ -270,7 +269,7 @@ private AsyncEventingBasicConsumer CreateAsyncConsumer() { AsyncEventingBasicConsumer consumer = null; - _chanHost.Channel.BasicQos(0, ConsumerOptions.BatchSize!.Value, false); + _chanHost.Channel.BasicQos(0, ConsumerOptions.BatchSize, false); consumer = new AsyncEventingBasicConsumer(_chanHost.Channel); consumer.Received += ReceiveHandlerAsync; @@ -297,7 +296,7 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs { await _consumerChannel .Writer - .WriteAsync(new ReceivedMessage(_chanHost.Channel, bdea, !(ConsumerOptions.AutoAck ?? false))) + .WriteAsync(new ReceivedMessage(_chanHost.Channel, bdea, !ConsumerOptions.AutoAck)) .ConfigureAwait(false); return true; } diff --git a/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs b/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs index d4f72428..f92c74b9 100644 --- a/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs @@ -1,5 +1,3 @@ -using HouseofCat.RabbitMQ.Dataflows; -using HouseofCat.RabbitMQ.Services; using System.Collections.Generic; using System.Threading.Channels; @@ -7,10 +5,10 @@ namespace HouseofCat.RabbitMQ; public class ConsumerOptions { - public bool? NoLocal { get; set; } - public bool? Exclusive { get; set; } - public ushort? BatchSize { get; set; } = 5; - public bool? AutoAck { get; set; } + public bool NoLocal { get; set; } + public bool Exclusive { get; set; } + public ushort BatchSize { get; set; } = 5; + public bool AutoAck { get; set; } public string ErrorSuffix { get; set; } public string AltSuffix { get; set; } @@ -36,5 +34,5 @@ public class ConsumerOptions public int WorkflowConsumerCount { get; set; } = 1; public int WorkflowBatchSize { get; set; } = 5; public bool WorkflowEnsureOrdered { get; set; } = true; - public bool WorkflowWaitForCompletion { get;set; } + public bool WorkflowWaitForCompletion { get; set; } } diff --git a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs index edbf5260..02554d81 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ChannelHost.cs @@ -182,10 +182,10 @@ public string StartConsuming(IBasicConsumer internalConsumer, ConsumerOptions op _consumerTag = Channel .BasicConsume( options.QueueName, - options.AutoAck ?? false, + options.AutoAck, options.ConsumerName, - options.NoLocal ?? false, - options.Exclusive ?? false, + options.NoLocal, + options.Exclusive, null, internalConsumer); diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index 3f9d24d7..56ac175c 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -363,11 +363,10 @@ public async Task> GetAsync(string queueName) } catch { return default; } - BasicGetResult result = null; var error = false; try { - result = chanHost + var result = chanHost .Channel .BasicGet(queueName, true); From 62b78cb1d680e619a6f0c383d8cc2c5039d8af48 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 18:44:23 -0500 Subject: [PATCH 21/47] More ConsumerOption cleanup. --- guides/csharp/TopLevel.md | 27 +++--- .../Options/ConsumerOptions.cs | 11 +-- .../Options/PoolOptions.cs | 4 +- src/HouseofCat.RabbitMQ/README.md | 21 +++-- .../SampleRabbitOptions.json | 16 ++-- .../Services/RabbitService.cs | 84 +++++++------------ .../RabbitMQ.ConsumerTests.json | 13 ++- .../RabbitMQ.PubSubTests.json | 14 +++- .../RabbitMQ.PublisherTests.json | 4 +- .../RabbitMQ.RabbitServiceTests.json | 19 +++-- .../RabbitMQ.ConsumerDataflows.json | 19 +++-- 11 files changed, 126 insertions(+), 106 deletions(-) diff --git a/guides/csharp/TopLevel.md b/guides/csharp/TopLevel.md index 61f99aa1..b841d166 100644 --- a/guides/csharp/TopLevel.md +++ b/guides/csharp/TopLevel.md @@ -137,28 +137,35 @@ Let me copy in a basic HoC config with our consumer settings in it. This file ne "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 0, - "CreatePublishReceipts": true, + "CreatePublishReceipts": false, "Compress": false, - "Encrypt": false + "Encrypt": false, + "WaitForConfirmationTimeoutInMilliseconds": 500 }, "ConsumerOptions": { - "HoC-Consume": { + "HoC-Consumer": { "Enabled": true, - "ConsumerName": "HoC-Consume", - "QueueName": "TestQueue", - "ErrorSuffix": "Error", + "ConsumerName": "HoC-Consumer", "BatchSize": 5, "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, + "QueueName": "TestQueue", + "QueueArguments": null, + "TargetQueueName": "TestTargetQueue", + "TargetQueueArgs": null, + "ErrorQueueName": "TestQueue.Error", + "ErrorQueueArgs": null, + "BuildQueues": true, + "BuildQueueDurable": true, + "BuildQueueExclusive": false, + "BuildQueueAutoDelete": false, "WorkflowName": "TestConsumerWorkflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, - "WorkflowBatchSize": 5, + "WorkflowBatchSize": 5, "WorkflowEnsureOrdered": false, "WorkflowWaitForCompletion": false } @@ -253,7 +260,7 @@ await foreach (var receivedMessage in consumer.StreamOutUntilClosedAsync()) await rabbitService.ShutdownAsync(immediately: false); -void ProcessMessage(IreceivedMessage receivedMessage) +void ProcessMessage(IReceivedMessage receivedMessage) { try { diff --git a/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs b/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs index f92c74b9..dd1cd874 100644 --- a/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/ConsumerOptions.cs @@ -10,9 +10,6 @@ public class ConsumerOptions public ushort BatchSize { get; set; } = 5; public bool AutoAck { get; set; } - public string ErrorSuffix { get; set; } - public string AltSuffix { get; set; } - public BoundedChannelFullMode? BehaviorWhenFull { get; set; } = BoundedChannelFullMode.Wait; public bool Enabled { get; set; } @@ -24,11 +21,15 @@ public class ConsumerOptions public string TargetQueueName { get; set; } public IDictionary TargetQueueArgs { get; set; } - public Dictionary TargetQueues { get; set; } = new Dictionary(); - public string ErrorQueueName => $"{QueueName}.{ErrorSuffix ?? "Error"}"; + public string ErrorQueueName { get; set; } public IDictionary ErrorQueueArgs { get; set; } + public bool BuildQueues { get; set; } = true; + public bool BuildQueueDurable { get; set; } = true; + public bool BuildQueueExclusive { get; set; } + public bool BuildQueueAutoDelete { get; set; } + public string WorkflowName { get; set; } public int WorkflowMaxDegreesOfParallelism { get; set; } = 1; public int WorkflowConsumerCount { get; set; } = 1; diff --git a/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs b/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs index 2441bc41..644eebb7 100644 --- a/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/PoolOptions.cs @@ -91,13 +91,13 @@ public class PoolOptions /// Number of channels to keep in each of the channel pool. Used in round-robin to perform actions. /// Default value is 0. /// - public ushort Channels { get; set; } + public ushort Channels { get; set; } = 1; /// /// Number of ackable channels to keep in each of the channel pool. Used in round-robin to perform actions. /// Default value is 10. /// - public ushort AckableChannels { get; set; } = 10; + public ushort AckableChannels { get; set; } = 1; /// /// The time to sleep (in ms) when an error occurs on Channel or Connection creation. It's best not to be hyper aggressive with this value. diff --git a/src/HouseofCat.RabbitMQ/README.md b/src/HouseofCat.RabbitMQ/README.md index ea9cc4c6..06a2db05 100644 --- a/src/HouseofCat.RabbitMQ/README.md +++ b/src/HouseofCat.RabbitMQ/README.md @@ -27,28 +27,35 @@ Sample Config File "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, - "Compress": true, - "Encrypt": true + "Compress": false, + "Encrypt": false, + "WaitForConfirmationTimeoutInMilliseconds": 500 }, "ConsumerOptions": { "TestConsumer": { "Enabled": true, "ConsumerName": "TestConsumer", - "QueueName": "TestQueue", - "ErrorSuffix": "Error", "BatchSize": 5, "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, + "QueueName": "TestQueue", + "QueueArguments": null, + "TargetQueueName": "TestTargetQueue", + "TargetQueueArgs": null, + "ErrorQueueName": "TestQueue.Error", + "ErrorQueueArgs": null, + "BuildQueues": true, + "BuildQueueDurable": true, + "BuildQueueExclusive": false, + "BuildQueueAutoDelete": false, "WorkflowName": "TestConsumerWorkflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, - "WorkflowBatchSize": 5, + "WorkflowBatchSize": 5, "WorkflowEnsureOrdered": false, "WorkflowWaitForCompletion": false } diff --git a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json index b7498e6e..ef78c4ad 100644 --- a/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json +++ b/src/HouseofCat.RabbitMQ/SampleRabbitOptions.json @@ -30,7 +30,6 @@ "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, "Compress": false, "Encrypt": false, @@ -40,19 +39,26 @@ "TestConsumer": { "Enabled": true, "ConsumerName": "TestConsumer", - "QueueName": "TestQueue", - "ErrorSuffix": "Error", "BatchSize": 5, "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, + "QueueName": "TestQueue", + "QueueArguments": null, + "TargetQueueName": "TestTargetQueue", + "TargetQueueArgs": null, + "ErrorQueueName": "TestQueue.Error", + "ErrorQueueArgs": null, + "BuildQueues": true, + "BuildQueueDurable": true, + "BuildQueueExclusive": false, + "BuildQueueAutoDelete": false, "WorkflowName": "TestConsumerWorkflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, - "WorkflowBatchSize": 5, + "WorkflowBatchSize": 5, "WorkflowEnsureOrdered": false, "WorkflowWaitForCompletion": false } diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index 56ac175c..05f0563a 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -182,71 +182,43 @@ private async Task BuildConsumerTopologyAsync() { foreach (var consumer in Consumers) { + if (!consumer.Value.ConsumerOptions.BuildQueues) + { continue; } + if (!string.IsNullOrWhiteSpace(consumer.Value.ConsumerOptions.QueueName)) { - if (consumer.Value.ConsumerOptions.QueueArgs == null - || consumer.Value.ConsumerOptions.QueueArgs.Count == 0) - { - await Topologer - .CreateQueueAsync(consumer.Value.ConsumerOptions.QueueName) - .ConfigureAwait(false); - } - else - { - await Topologer - .CreateQueueAsync( - consumer.Value.ConsumerOptions.QueueName, - true, - false, - false, - consumer.Value.ConsumerOptions.QueueArgs) - .ConfigureAwait(false); - } + await Topologer + .CreateQueueAsync( + consumer.Value.ConsumerOptions.QueueName, + consumer.Value.ConsumerOptions.BuildQueueDurable, + consumer.Value.ConsumerOptions.BuildQueueExclusive, + consumer.Value.ConsumerOptions.BuildQueueAutoDelete, + consumer.Value.ConsumerOptions.QueueArgs) + .ConfigureAwait(false); } if (!string.IsNullOrWhiteSpace(consumer.Value.ConsumerOptions.TargetQueueName)) { - if (consumer.Value.ConsumerOptions.TargetQueueArgs == null - || consumer.Value.ConsumerOptions.TargetQueueArgs.Count == 0) - { - await Topologer - .CreateQueueAsync(consumer.Value.ConsumerOptions.TargetQueueName) - .ConfigureAwait(false); - } - else - { - await Topologer - .CreateQueueAsync( - consumer.Value.ConsumerOptions.TargetQueueName, - true, - false, - false, - consumer.Value.ConsumerOptions.TargetQueueArgs) - .ConfigureAwait(false); - } + await Topologer + .CreateQueueAsync( + consumer.Value.ConsumerOptions.TargetQueueName, + consumer.Value.ConsumerOptions.BuildQueueDurable, + consumer.Value.ConsumerOptions.BuildQueueExclusive, + consumer.Value.ConsumerOptions.BuildQueueAutoDelete, + consumer.Value.ConsumerOptions.TargetQueueArgs) + .ConfigureAwait(false); } - if (!string.IsNullOrWhiteSpace(consumer.Value.ConsumerOptions.ErrorSuffix) - && !string.IsNullOrWhiteSpace(consumer.Value.ConsumerOptions.ErrorQueueName)) + if (!string.IsNullOrWhiteSpace(consumer.Value.ConsumerOptions.ErrorQueueName)) { - if (consumer.Value.ConsumerOptions.ErrorQueueArgs == null - || consumer.Value.ConsumerOptions.ErrorQueueArgs.Count == 0) - { - await Topologer - .CreateQueueAsync(consumer.Value.ConsumerOptions.ErrorQueueName) - .ConfigureAwait(false); - } - else - { - await Topologer - .CreateQueueAsync( - consumer.Value.ConsumerOptions.ErrorQueueName, - true, - false, - false, - consumer.Value.ConsumerOptions.ErrorQueueArgs) - .ConfigureAwait(false); - } + await Topologer + .CreateQueueAsync( + consumer.Value.ConsumerOptions.ErrorQueueName, + consumer.Value.ConsumerOptions.BuildQueueDurable, + consumer.Value.ConsumerOptions.BuildQueueExclusive, + consumer.Value.ConsumerOptions.BuildQueueAutoDelete, + consumer.Value.ConsumerOptions.ErrorQueueArgs) + .ConfigureAwait(false); } } } diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json index 8e6d9543..42a8ef37 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.ConsumerTests.json @@ -20,15 +20,22 @@ "TestConsumer": { "Enabled": true, "ConsumerName": "TestConsumer", - "QueueName": "TestQueue", - "ErrorSuffix": "Error", "BatchSize": 5, "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, + "QueueName": "TestQueue", + "QueueArguments": null, + "TargetQueueName": "TestTargetQueue", + "TargetQueueArgs": null, + "ErrorQueueName": "TestQueue.Error", + "ErrorQueueArgs": null, + "BuildQueues": true, + "BuildQueueDurable": true, + "BuildQueueExclusive": false, + "BuildQueueAutoDelete": false, "WorkflowName": "TestConsumerWorkflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json index b8f3fd2b..c68f9568 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PubSubTests.json @@ -19,7 +19,6 @@ "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, "Compress": false, "Encrypt": false, @@ -29,15 +28,22 @@ "TestConsumer": { "Enabled": true, "ConsumerName": "TestConsumer", - "QueueName": "TestQueue", - "ErrorSuffix": "Error", "BatchSize": 5, "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, + "QueueName": "TestQueue", + "QueueArguments": null, + "TargetQueueName": "TestTargetQueue", + "TargetQueueArgs": null, + "ErrorQueueName": "TestQueue.Error", + "ErrorQueueArgs": null, + "BuildQueues": true, + "BuildQueueDurable": true, + "BuildQueueExclusive": false, + "BuildQueueAutoDelete": false, "WorkflowName": "TestConsumerWorkflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json index 7011e318..dd91c8d5 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.PublisherTests.json @@ -19,9 +19,9 @@ "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, "Compress": false, - "Encrypt": false + "Encrypt": false, + "WaitForConfirmationTimeoutInMilliseconds": 500 } } diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json index 6128d6fa..c68f9568 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json @@ -19,24 +19,31 @@ "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, - "Compress": true, - "Encrypt": true + "Compress": false, + "Encrypt": false, + "WaitForConfirmationTimeoutInMilliseconds": 500 }, "ConsumerOptions": { "TestConsumer": { "Enabled": true, "ConsumerName": "TestConsumer", - "QueueName": "TestQueue", - "ErrorSuffix": "Error", "BatchSize": 5, "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, + "QueueName": "TestQueue", + "QueueArguments": null, + "TargetQueueName": "TestTargetQueue", + "TargetQueueArgs": null, + "ErrorQueueName": "TestQueue.Error", + "ErrorQueueArgs": null, + "BuildQueues": true, + "BuildQueueDurable": true, + "BuildQueueExclusive": false, + "BuildQueueAutoDelete": false, "WorkflowName": "TestConsumerWorkflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json index 6128d6fa..c68f9568 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json @@ -19,24 +19,31 @@ "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "AutoPublisherSleepInterval": 1, "CreatePublishReceipts": true, - "Compress": true, - "Encrypt": true + "Compress": false, + "Encrypt": false, + "WaitForConfirmationTimeoutInMilliseconds": 500 }, "ConsumerOptions": { "TestConsumer": { "Enabled": true, "ConsumerName": "TestConsumer", - "QueueName": "TestQueue", - "ErrorSuffix": "Error", "BatchSize": 5, "BehaviorWhenFull": 0, - "SleepOnIdleInterval": 0, "UseTransientChannels": true, "AutoAck": false, "NoLocal": false, "Exclusive": false, + "QueueName": "TestQueue", + "QueueArguments": null, + "TargetQueueName": "TestTargetQueue", + "TargetQueueArgs": null, + "ErrorQueueName": "TestQueue.Error", + "ErrorQueueArgs": null, + "BuildQueues": true, + "BuildQueueDurable": true, + "BuildQueueExclusive": false, + "BuildQueueAutoDelete": false, "WorkflowName": "TestConsumerWorkflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, From 54ef069f82d7e70698f459ad4dc882ee486bae6f Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 20:48:05 -0500 Subject: [PATCH 22/47] Streamlining OpenTelemetry. --- .../Mappers/GeometryPointTypeHandler.cs | 3 +- .../Extensions/WorkStateExtensions.cs | 30 +- .../Extensions/MessageExtensions.cs | 3 +- src/HouseofCat.RabbitMQ/Messages/Message.cs | 30 +- .../Messages/ReceivedMessage.cs | 22 +- .../Options/RabbitOptions.cs | 5 +- .../Publisher/Publisher.cs | 14 +- .../Extensions/AssemblyExtensions.cs | 6 +- .../Helpers/AppHelpers.cs | 6 +- .../Helpers/OpenTelemetryHelpers.cs | 289 +++++++++--------- .../RabbitMQ.RabbitServiceTests.json | 6 +- .../Tests/PublisherTests.cs | 2 +- .../Program.cs | 6 +- 13 files changed, 190 insertions(+), 232 deletions(-) diff --git a/src/HouseofCat.Data/Database/Dapper/Mappers/GeometryPointTypeHandler.cs b/src/HouseofCat.Data/Database/Dapper/Mappers/GeometryPointTypeHandler.cs index d949a4fe..46cfcf76 100644 --- a/src/HouseofCat.Data/Database/Dapper/Mappers/GeometryPointTypeHandler.cs +++ b/src/HouseofCat.Data/Database/Dapper/Mappers/GeometryPointTypeHandler.cs @@ -2,7 +2,6 @@ using Microsoft.Spatial; using System; using System.Data; -using System.Globalization; using System.Text.RegularExpressions; namespace HouseofCat.Database.Dapper; @@ -44,6 +43,6 @@ public override void SetValue(IDbDataParameter parameter, GeometryPoint value) private static double ConvertToDouble(string value) { - return double.Parse(value, CultureInfo.InvariantCulture); + return double.Parse(value); } } diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 12afd13f..7c48a1e4 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -1,9 +1,7 @@ -using HouseofCat.Utilities.Extensions; -using HouseofCat.Utilities.Helpers; +using HouseofCat.Utilities.Helpers; using OpenTelemetry.Trace; using System.Collections.Generic; using System.Diagnostics; -using System.Reflection; namespace HouseofCat.Dataflows.Extensions; @@ -78,27 +76,18 @@ public static void SetSpanAsError(this IWorkState state, TelemetrySpan span, str public static void StartWorkflowSpan( this IWorkState state, string workflowName, - string workflowVersion = null, SpanKind spanKind = SpanKind.Internal, IEnumerable> suppliedAttributes = null, string traceHeader = null) { if (state is null) return; - if (string.IsNullOrWhiteSpace(workflowVersion)) - { - var assembly = Assembly.GetExecutingAssembly(); - workflowVersion = assembly.GetExecutingSemanticVersion(); - } - state.Data[DefaultWorkflowNameKey] = workflowName; - state.Data[DefaultWorkflowVersionKey] = workflowVersion; var spanName = string.Format(DefaultSpanNameFormat, workflowName); var attributes = new SpanAttributes(); attributes.Add(DefaultWorkflowNameKey, workflowName); - attributes.Add(DefaultWorkflowVersionKey, workflowVersion); if (suppliedAttributes is not null) { @@ -110,15 +99,13 @@ public static void StartWorkflowSpan( if (traceHeader is not null) { - var spanContext = OpenTelemetryHelpers.ExtractSpanContext(traceHeader); + var spanContext = OpenTelemetryHelpers.ExtractSpanContextFromTraceHeader(traceHeader); if (spanContext.HasValue) { state.WorkflowSpan = OpenTelemetryHelpers .StartActiveChildSpan( - workflowName?.ToString(), spanName, spanContext.Value, - workflowVersion?.ToString(), spanKind: spanKind, attributes: attributes); } @@ -126,9 +113,7 @@ public static void StartWorkflowSpan( state.WorkflowSpan = OpenTelemetryHelpers .StartRootSpan( - workflowName?.ToString(), spanName, - workflowVersion?.ToString(), spanKind, attributes: attributes); } @@ -149,13 +134,7 @@ public static TelemetrySpan CreateActiveSpan( { if (state?.Data is null) return null; - state.Data.TryGetValue(DefaultWorkflowNameKey, out var workflowName); - state.Data.TryGetValue(DefaultWorkflowVersionKey, out var workflowVersion); - - if (workflowName is null) return null; - var attributes = new SpanAttributes(); - if (suppliedAttributes is not null) { foreach (var kvp in suppliedAttributes) @@ -166,9 +145,7 @@ public static TelemetrySpan CreateActiveSpan( return OpenTelemetryHelpers .StartActiveSpan( - workflowName?.ToString(), spanName, - workflowVersion?.ToString(), spanKind, parentContext: state.WorkflowSpan?.Context ?? default, attributes: attributes); @@ -192,16 +169,13 @@ public static TelemetrySpan CreateActiveChildSpan( if (state?.Data is null) return null; state.Data.TryGetValue(DefaultWorkflowNameKey, out var workflowName); - state.Data.TryGetValue(DefaultWorkflowVersionKey, out var workflowVersion); var childSpanName = string.Format(DefaultChildSpanNameFormat, workflowName, spanName); return OpenTelemetryHelpers .StartActiveChildSpan( - workflowName?.ToString(), childSpanName, spanContext, - workflowVersion?.ToString(), spanKind, attributes: attributes); } diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 5ae9509d..74aa792a 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -72,7 +72,8 @@ public static IBasicProperties CreateBasicProperties( basicProperties.DeliveryMode = message.DeliveryMode; basicProperties.ContentType = new string(message.ContentType); basicProperties.Priority = message.PriorityLevel; - basicProperties.MessageId = message.MessageId == null + + basicProperties.MessageId = string.IsNullOrEmpty(message.MessageId) ? new string(message.MessageId) : Guid.NewGuid().ToString(); diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index 54fe40d2..559237cb 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -3,13 +3,16 @@ using RabbitMQ.Client; using System; using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; namespace HouseofCat.RabbitMQ; public interface IMessage { string MessageId { get; set; } + string Exchange { get; set; } + string RoutingKey { get; set; } byte DeliveryMode { get; set; } @@ -22,7 +25,7 @@ public interface IMessage ReadOnlyMemory Body { get; set; } - IMetadata Metadata { get; set; } + Metadata Metadata { get; set; } IPublishReceipt GetPublishReceipt(bool error); @@ -33,7 +36,12 @@ public sealed class Message : IMessage { public string MessageId { get; set; } + public Metadata Metadata { get; set; } + + public ReadOnlyMemory Body { get; set; } + public string Exchange { get; set; } + public string RoutingKey { get; set; } [Range(1, 2, ErrorMessage = Constants.RangeErrorMessage)] @@ -47,10 +55,10 @@ public sealed class Message : IMessage public string ContentType { get; set; } = Constants.HeaderValueForContentTypeJson; - public IMetadata Metadata { get; set; } - public ReadOnlyMemory Body { get; set; } - - public Message() { } + public Message() + { + MessageId ??= Guid.NewGuid().ToString(); + } public Message( string exchange, @@ -58,6 +66,7 @@ public Message( ReadOnlyMemory body, Metadata metadata = null) { + MessageId ??= Guid.NewGuid().ToString(); Exchange = exchange; RoutingKey = routingKey; Body = body; @@ -66,6 +75,7 @@ public Message( public Message(string exchange, string routingKey, ReadOnlyMemory body, string payloadId) { + MessageId ??= Guid.NewGuid().ToString(); Exchange = exchange; RoutingKey = routingKey; Body = body; @@ -78,25 +88,17 @@ public Message(string exchange, string routingKey, ReadOnlyMemory body, st public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders) { - MessageId ??= Guid.NewGuid().ToString(); - var basicProperties = this.CreateBasicProperties(channelHost, withOptionalHeaders, Metadata); basicProperties.MessageId = MessageId; // Non-optional Header. basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; - var openTelHeader = OpenTelemetryHelpers.CreateOpenTelemetryHeaderFromCurrentActivityOrDefault(); + var openTelHeader = OpenTelemetryHelpers.GetOrCreateTraceHeaderFromCurrentActivity(); basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; return basicProperties; } - public IMetadata CreateMetadataIfMissing() - { - Metadata ??= new Metadata(); - return Metadata; - } - public IPublishReceipt GetPublishReceipt(bool error) => new PublishReceipt { MessageId = MessageId, IsError = error, OriginalMessage = error ? this : null }; } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index 1b128170..d7afb786 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -17,7 +17,6 @@ public interface IReceivedMessage bool Ackable { get; } - string ContentType { get; } string ObjectType { get; } bool Encrypted { get; } @@ -52,7 +51,6 @@ public sealed class ReceivedMessage : IReceivedMessage, IDisposable public bool Ackable { get; } - public string ContentType { get; private set; } public string ObjectType { get; private set; } public bool Encrypted { get; private set; } @@ -111,26 +109,28 @@ private void ReadHeaders() { ObjectType = Encoding.UTF8.GetString((byte[])objectType); - if (Properties.Headers.TryGetValue(Constants.HeaderForObjectType, out object contentType)) - { - ContentType = Encoding.UTF8.GetString((byte[])contentType); - } - if (ObjectType == Constants.HeaderValueForMessageObjectType && Body.Length > 0) { - switch (ContentType) + switch (Properties.ContentType) { case Constants.HeaderValueForContentTypeJson: try - { Message = JsonSerializer.Deserialize(Body.Span); } + { + Message = JsonSerializer.Deserialize(Body.Span); + } catch { FailedToDeserialize = true; } break; case Constants.HeaderValueForContentTypeBinary: case Constants.HeaderValueForContentTypePlainText: + break; default: + try + { Message = JsonSerializer.Deserialize(Body.Span); } + catch + { FailedToDeserialize = true; } break; } } @@ -153,10 +153,6 @@ private void ReadHeaders() else { ObjectType = Constants.HeaderValueForUnknownObjectType; - if (Properties.Headers.TryGetValue(Constants.HeaderForContentType, out object contentType)) - { - ContentType = Encoding.UTF8.GetString((byte[])contentType); - } } if (Properties.Headers.TryGetValue(Constants.HeaderForTraceParent, out object traceParentHeader)) diff --git a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs index a19800a1..7978f064 100644 --- a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs @@ -23,7 +23,10 @@ public class RabbitOptions public ConsumerOptions GetConsumerOptions(string consumerName) { - if (!ConsumerOptions.TryGetValue(consumerName, out ConsumerOptions value)) throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, ExceptionMessages.NoConsumerOptionsMessage, consumerName)); + if (!ConsumerOptions.TryGetValue(consumerName, out ConsumerOptions value)) + { + throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerOptionsMessage, consumerName)); + } return value; } } diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index 281ac41a..acb5d927 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -182,9 +182,11 @@ private void SetupPublisher(Func processReceiptAsync _publishingTask = ProcessMessagesAsync(_messageQueue.Reader); - processReceiptAsync ??= ProcessReceiptAsync; - - _processReceiptsAsync ??= ProcessReceiptsAsync(processReceiptAsync); + if (Options.PublisherOptions.CreatePublishReceipts) + { + processReceiptAsync ??= ProcessReceiptAsync; + _processReceiptsAsync ??= ProcessReceiptsAsync(processReceiptAsync); + } AutoPublisherStarted = true; } @@ -542,7 +544,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp try { - var body = _serializationProvider.Serialize(message.Body); + var body = _serializationProvider.Serialize(message); chanHost .Channel .BasicPublish( @@ -601,7 +603,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece message.RoutingKey, message.Mandatory, message.BuildProperties(chanHost, withOptionalHeaders), - _serializationProvider.Serialize(message.Body)); + _serializationProvider.Serialize(message)); chanHost.Channel.WaitForConfirmsOrDie(_waitForConfirmation); } @@ -741,7 +743,7 @@ await _receiptBuffer private static void SetMandatoryHeaders(IBasicProperties basicProperties) { basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; - var openTelHeader = OpenTelemetryHelpers.CreateOpenTelemetryHeaderFromCurrentActivityOrDefault(); + var openTelHeader = OpenTelemetryHelpers.GetOrCreateTraceHeaderFromCurrentActivity(); basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; } diff --git a/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs b/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs index 9d80b48a..7ae2c409 100644 --- a/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs +++ b/src/HouseofCat.Utilities/Extensions/AssemblyExtensions.cs @@ -8,17 +8,15 @@ public static class AssemblyExtensions { private static readonly ConcurrentDictionary _assemblyVersion = new ConcurrentDictionary(); - public static string GetExecutingSemanticVersion(this Assembly assembly) + public static string GetSemanticVersion(this Assembly assembly) { if (_assemblyVersion.TryGetValue(assembly, out var cachedVersion)) { return cachedVersion; } - var version = AppHelpers.GetFlexibleSemVersion(Assembly.GetExecutingAssembly().GetName()); + var version = AppHelpers.GetFlexibleSemVersion(Assembly.GetEntryAssembly().GetName()); _assemblyVersion.TryAdd(assembly, version); return version; } - - } diff --git a/src/HouseofCat.Utilities/Helpers/AppHelpers.cs b/src/HouseofCat.Utilities/Helpers/AppHelpers.cs index dc052230..2dc789e2 100644 --- a/src/HouseofCat.Utilities/Helpers/AppHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/AppHelpers.cs @@ -35,9 +35,9 @@ public static string GetFlexibleSemVersion(AssemblyName assemblyName) return (versionParts?.Length ?? 0) switch { 0 => null, - 1 => versionParts[0], - 2 => $"{versionParts[0]}.{versionParts[1]}", - _ => $"{versionParts[0]}.{versionParts[1]}.{versionParts[2]}" + 1 => $"v{versionParts[0]}", + 2 => $"v{versionParts[0]}.{versionParts[1]}", + _ => $"v{versionParts[0]}.{versionParts[1]}.{versionParts[2]}" }; } } diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs index 7d66f190..a67fba11 100644 --- a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -1,57 +1,172 @@ -using OpenTelemetry.Trace; +using HouseofCat.Utilities.Extensions; +using OpenTelemetry.Trace; using System; using System.Collections.Generic; using System.Diagnostics; +using System.Reflection; namespace HouseofCat.Utilities.Helpers; public static class OpenTelemetryHelpers { - public static string CreateOpenTelemetryHeader( + private static string _sourceName; + private static string _sourceVersion; + + static OpenTelemetryHelpers() + { + SetEntryAssemblyAsSourceForTelemetry(); + } + + public static void SetEntryAssemblyAsSourceForTelemetry() + { + var assembly = Assembly.GetEntryAssembly(); + _sourceName = assembly.GetName().Name; + _sourceVersion = assembly.GetSemanticVersion(); + } + + #region Span Helpers + + public static TelemetrySpan StartRootSpan( + string spanName, + SpanKind spanKind = SpanKind.Internal, + SpanAttributes attributes = null, + IEnumerable links = null, + DateTimeOffset startTime = default) + { + var provider = TracerProvider + .Default + .GetTracer( + _sourceName, + _sourceVersion); + + if (provider is null) return null; + + return provider.StartRootSpan( + spanName, + spanKind, + initialAttributes: attributes, + links: links, + startTime: startTime); + } + + public static TelemetrySpan StartActiveSpan( + string spanName, + SpanKind spanKind = SpanKind.Internal, + SpanContext parentContext = default, + SpanAttributes attributes = null, + IEnumerable links = null, + DateTimeOffset startTime = default) + { + var provider = TracerProvider + .Default + .GetTracer( + _sourceName, + _sourceVersion); + + if (provider is null) return null; + + return provider.StartActiveSpan( + spanName, + spanKind, + initialAttributes: attributes, + parentContext: parentContext, + links: links, + startTime: startTime); + } + + public static TelemetrySpan StartActiveChildSpan( + string spanName, + SpanContext parentContext, + SpanKind spanKind = SpanKind.Internal, + SpanAttributes attributes = null) + { + var provider = TracerProvider + .Default + .GetTracer( + _sourceName, + _sourceVersion); + + if (provider is null) return null; + + return provider.StartActiveSpan( + spanName, + spanKind, + initialAttributes: attributes, + parentContext: parentContext); + } + + #endregion + + #region Current and ParentId Helpers + + public static string GetCurrentActivityId() + { + return Activity.Current?.Id; + } + + public static ActivitySpanId? GetCurrentActivitySpanId() + { + return Activity.Current?.SpanId; + } + + public static void SetCurrentActivityParentId(string parentId) + { + Activity.Current?.SetParentId(parentId); + } + + public static void SetCurrentActivityParentId( string traceId, string spanId, - string version = "00", ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) { - return $"{version}-{traceId}-{spanId}-{(int)activityTraceFlags:00}"; + Activity.Current?.SetParentId( + ActivityTraceId.CreateFromString(traceId), + ActivitySpanId.CreateFromString(spanId), + activityTraceFlags); } - public static string CreateOpenTelemetryHeaderFromCurrentActivityOrDefault( + public static void SetCurrentActivityParentId( + ActivityTraceId traceId, + ActivitySpanId spanId, + ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) + { + Activity.Current?.SetParentId(traceId, spanId, activityTraceFlags); + } + + #endregion + + #region Header Helpers + + public static string GetOrCreateTraceHeaderFromCurrentActivity( ActivityTraceFlags flags = ActivityTraceFlags.None) { var activity = Activity.Current; if (activity is null) { - return CreateOpenTelemetryHeader( + return FormatOpenTelemetryHeader( ActivityTraceId.CreateRandom().ToHexString(), ActivitySpanId.CreateRandom().ToHexString(), "00", flags); } - return CreateOpenTelemetryHeader( + return FormatOpenTelemetryHeader( activity.TraceId.ToHexString(), activity.SpanId.ToHexString(), "00", activity.ActivityTraceFlags); } - public static string GetCurrentActivityId() - { - return Activity.Current?.Id; - } - - public static ActivitySpanId? GetCurrentActivitySpanId() - { - return Activity.Current?.SpanId; - } - - public static void SetCurrentActivityParentId(string parentId) + public static string FormatOpenTelemetryHeader( + string traceId, + string spanId, + string version = "00", + ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) { - Activity.Current?.SetParentId(parentId); + return $"{version}-{traceId}-{spanId}-{(int)activityTraceFlags:00}"; } - public static SpanContext? ExtractSpanContext(string traceHeader) + public static SpanContext? ExtractSpanContextFromTraceHeader(string traceHeader) { if (string.IsNullOrEmpty(traceHeader)) return default; @@ -102,137 +217,5 @@ public static void SetCurrentActivityParentId(string parentId) } } - public static void SetCurrentActivityParentIdFromTraceHeader(string traceheader) - { - if (string.IsNullOrEmpty(traceheader)) return; - - var activityTraceFlags = ActivityTraceFlags.None; - var split = traceheader.Split('-'); - - // With Version - ActivityTraceId activityTraceId; - ActivitySpanId activitySpanId; - if (split.Length == 4 && split[3].Length == 2) - { - activityTraceId = ActivityTraceId.CreateFromString(split[1].AsSpan()); - activitySpanId = ActivitySpanId.CreateFromString(split[2].AsSpan()); - if (int.TryParse(split[3], out var flagsAsInt)) - { - activityTraceFlags = (ActivityTraceFlags)flagsAsInt; - } - } - // Without Version Pre-Appended - else if (split.Length == 3 && split[0].Length > 2) - { - activityTraceId = ActivityTraceId.CreateFromString(split[0].AsSpan()); - activitySpanId = ActivitySpanId.CreateFromString(split[1].AsSpan()); - if (int.TryParse(split[2], out var flagsAsInt)) - { - activityTraceFlags = (ActivityTraceFlags)flagsAsInt; - } - } - else - { - return; - } - - Activity.Current?.SetParentId( - activityTraceId, - activitySpanId, - activityTraceFlags); - } - - public static void SetCurrentActivityParentId( - string traceId, - string spanId, - ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) - { - Activity.Current?.SetParentId( - ActivityTraceId.CreateFromString(traceId), - ActivitySpanId.CreateFromString(spanId), - activityTraceFlags); - } - - public static void SetCurrentActivityParentId( - ActivityTraceId traceId, - ActivitySpanId spanId, - ActivityTraceFlags activityTraceFlags = ActivityTraceFlags.None) - { - Activity.Current?.SetParentId(traceId, spanId, activityTraceFlags); - } - - public static TelemetrySpan StartRootSpan( - string sourceName, - string spanName, - string sourceVersion = null, - SpanKind spanKind = SpanKind.Internal, - SpanAttributes attributes = null, - IEnumerable links = null, - DateTimeOffset startTime = default) - { - var provider = TracerProvider - .Default - .GetTracer( - sourceName, - sourceVersion); - - if (provider is null) return null; - - return provider.StartRootSpan( - spanName, - spanKind, - initialAttributes: attributes, - links: links, - startTime: startTime); - } - - public static TelemetrySpan StartActiveSpan( - string sourceName, - string spanName, - string sourceVersion = null, - SpanKind spanKind = SpanKind.Internal, - SpanContext parentContext = default, - SpanAttributes attributes = null, - IEnumerable links = null, - DateTimeOffset startTime = default) - { - var provider = TracerProvider - .Default - .GetTracer( - sourceName, - sourceVersion); - - if (provider is null) return null; - - return provider.StartActiveSpan( - spanName, - spanKind, - initialAttributes: attributes, - parentContext: parentContext, - links: links, - startTime: startTime); - } - - public static TelemetrySpan StartActiveChildSpan( - string sourceName, - string spanName, - SpanContext parentContext, - string sourceVersion = null, - SpanKind spanKind = SpanKind.Internal, - SpanAttributes attributes = null) - { - var provider = TracerProvider - .Default - .GetTracer( - sourceName, - sourceVersion); - - if (provider is null) return null; - - return provider.StartActiveSpan( - spanName, - spanKind, - initialAttributes: attributes, - parentContext: parentContext); - } + #endregion } diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json index c68f9568..285ec1b8 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.RabbitServiceTests.json @@ -19,9 +19,9 @@ "PublisherOptions": { "MessageQueueBufferSize": 100, "BehaviorWhenFull": 0, - "CreatePublishReceipts": true, - "Compress": false, - "Encrypt": false, + "CreatePublishReceipts": false, + "Compress": true, + "Encrypt": true, "WaitForConfirmationTimeoutInMilliseconds": 500 }, "ConsumerOptions": { diff --git a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs index 36031fe5..024947d0 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PublisherTests.cs @@ -88,7 +88,7 @@ await publisher.StartAutoPublishAsync( // Step 4: Create Message var data = Encoding.UTF8.GetBytes("Hello, RabbitMQ!"); - var message = new Message(Shared.ExchangeName, Shared.RoutingKey, data, Guid.NewGuid().ToString()) + var message = new Message(Shared.ExchangeName, Shared.RoutingKey, data) { // DeliveryId for tracking/routing through Publisher/Consumer. MessageId = Guid.NewGuid().ToString(), diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index db464f74..2d65b847 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -10,12 +10,12 @@ LogHelpers.LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger(); -var applicationName = "RabbitMQ.ConsumerDataflow.Tests"; +var applicationName = "RabbitMQ.ConsumerDataflows.Tests"; using var traceProvider = Sdk .CreateTracerProviderBuilder() - .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(applicationName)) - .AddSource(Shared.ConsumerWorkflowName) + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(applicationName, serviceVersion: "v1.0.0")) + .AddSource(applicationName) .AddConsoleExporter() .Build(); From 2d81452a0344f7af34dd13f325559d963faa3f85 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 31 Mar 2024 20:58:26 -0500 Subject: [PATCH 23/47] Migrate more telemetry static methods to OpenTelemetryHelpers vs. Extension methods. --- .../Extensions/WorkStateExtensions.cs | 20 +-------- .../Helpers/OpenTelemetryHelpers.cs | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 7c48a1e4..1155731b 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -1,7 +1,6 @@ using HouseofCat.Utilities.Helpers; using OpenTelemetry.Trace; using System.Collections.Generic; -using System.Diagnostics; namespace HouseofCat.Dataflows.Extensions; @@ -18,27 +17,12 @@ public static void SetOpenTelemetryError(this IWorkState state, string message = public static void SetCurrentActivityAsError(this IWorkState state, string message = null) { - var activity = Activity.Current; - if (activity is null) return; - - activity.SetStatus(ActivityStatusCode.Error, message ?? state.EDI.SourceException?.Message); - if (state?.EDI is not null) - { - if (state.EDI.SourceException is not null) - { - activity.RecordException(state.EDI.SourceException); - } - } - else - { - activity.SetStatus(ActivityStatusCode.Error, message); - } + OpenTelemetryHelpers.SetCurrentActivityAsError(state?.EDI?.SourceException, message); } public static void SetCurrentSpanAsError(this IWorkState state, string message = null) { - var span = Tracer.CurrentSpan; - state.SetSpanAsError(span, message); + OpenTelemetryHelpers.SetCurrentSpanAsError(state?.EDI?.SourceException, message); } public static void SetSpanAsError(this IWorkState state, TelemetrySpan span, string message = null) diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs index a67fba11..10170645 100644 --- a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -218,4 +218,48 @@ public static string FormatOpenTelemetryHeader( } #endregion + + #region Error Handling + + public static void SetCurrentActivityAsError(Exception ex, string message = null) + { + var activity = Activity.Current; + if (activity is null) return; + + SetActivityAsError(activity, ex, message); + } + + public static void SetActivityAsError(Activity activity, Exception ex, string message = null) + { + if (activity is null) return; + + if (ex is not null) + { + activity.RecordException(ex); + } + + activity.SetStatus(ActivityStatusCode.Error, message); + } + + public static void SetCurrentSpanAsError(Exception ex, string message = null) + { + var span = Tracer.CurrentSpan; + if (span is null) return; + + SetSpanAsError(span, ex, message); + } + + public static void SetSpanAsError(TelemetrySpan span, Exception ex, string message = null) + { + if (span is null) return; + + if (ex is not null) + { + span.RecordException(ex); + } + + span.SetStatus(Status.Error.WithDescription(message)); + } + + #endregion } From 47bc7f24560ff9655d3e3960ce9b4d61cd97c33a Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Thu, 11 Apr 2024 20:27:51 -0500 Subject: [PATCH 24/47] Adding open telemetry to Publishing. --- guides/csharp/TasksBranchExecution.md | 2 +- .../Extensions/WorkStateExtensions.cs | 8 +- src/HouseofCat.Dataflows/Pipeline.cs | 2 +- src/HouseofCat.RabbitMQ/Constants.cs | 32 +++- .../Dataflows/ConsumerDataflow.cs | 8 +- .../Extensions/MessageExtensions.cs | 34 +++- src/HouseofCat.RabbitMQ/Messages/Message.cs | 6 +- src/HouseofCat.RabbitMQ/Messages/Metadata.cs | 68 ++++++- .../Messages/ReceivedMessage.cs | 6 - .../Pools/ConnectionPool.cs | 2 +- .../Publisher/Publisher.cs | 173 +++++++++++++----- .../Services/RabbitService.cs | 16 +- .../ISerializationProvider.cs | 1 + src/HouseofCat.Serialization/JsonProvider.cs | 2 + .../Helpers/OpenTelemetryHelpers.cs | 44 ++--- src/HouseofCat.Utilities/Random/RandomData.cs | 6 +- 16 files changed, 297 insertions(+), 113 deletions(-) diff --git a/guides/csharp/TasksBranchExecution.md b/guides/csharp/TasksBranchExecution.md index 6b1298fb..85c4c22f 100644 --- a/guides/csharp/TasksBranchExecution.md +++ b/guides/csharp/TasksBranchExecution.md @@ -154,7 +154,7 @@ namespace Tasks // All my messages, assigned to a Task assigned inside an array. // Execution of each task begins when created but there is no blocking here. var tasks = new Task[myStrings.Count]; - for (int i = 0; i < myStrings.Count; i++) + for (var i = 0; i < myStrings.Count; i++) { tasks[i] = ProcessMessageAsync(myStrings[i]); } diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 1155731b..397bc706 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -87,10 +87,10 @@ public static void StartWorkflowSpan( if (spanContext.HasValue) { state.WorkflowSpan = OpenTelemetryHelpers - .StartActiveChildSpan( + .StartActiveSpan( spanName, - spanContext.Value, spanKind: spanKind, + spanContext.Value, attributes: attributes); } } @@ -157,10 +157,10 @@ public static TelemetrySpan CreateActiveChildSpan( var childSpanName = string.Format(DefaultChildSpanNameFormat, workflowName, spanName); return OpenTelemetryHelpers - .StartActiveChildSpan( + .StartActiveSpan( childSpanName, - spanContext, spanKind, + spanContext, attributes: attributes); } diff --git a/src/HouseofCat.Dataflows/Pipeline.cs b/src/HouseofCat.Dataflows/Pipeline.cs index bccb081a..20d47df2 100644 --- a/src/HouseofCat.Dataflows/Pipeline.cs +++ b/src/HouseofCat.Dataflows/Pipeline.cs @@ -218,7 +218,7 @@ public void AddSteps( { if (Ready) throw new InvalidOperationException(_invalidAddError); - for (int i = 0; i < stepFunctions.Count; i++) + for (var i = 0; i < stepFunctions.Count; i++) { AddStep(stepFunctions[i], localMaxDoP, ensureOrdered, bufferSizeOverride); } diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index 862061b9..e78ad473 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -2,9 +2,6 @@ namespace HouseofCat.RabbitMQ; public static class Constants { - // RabbitService - public static int EncryptionKeySize { get; set; } = 32; // AES256 - // Publisher public static string HeaderPrefix { get; set; } = "X-"; @@ -27,8 +24,33 @@ public static class Constants public const string RangeErrorMessage = "Value for {0} must be between {1} and {2}."; - // Pipeline - public static string DefaultPipelineName { get; set; } = "NoNameProvided"; + public static string MessagingSystemKey { get; set; } = "messaging.system"; + public static string MessagingSystemValue { get; set; } = "rabbitmq"; + + public static string MessagingOperationKey { get; set; } = "messaging.operation"; + public static string MessagingOperationPublishValue { get; set; } = "publish"; + public static string MessagingOperationConsumeValue { get; set; } = "receive"; + public static string MessagingOperationProcessValue { get; set; } = "process"; + + public static string MessagingDestinationNameKey { get; set; } = "messaging.destination.name"; + public static string MessagingMessageMessageIdKey { get; set; } = "messaging.message.id"; + + public static string MessagingMessageRoutingKeyKey { get; set; } = "messaging.rabbitmq.message.routing_key"; + public static string MessagingMessageDeliveryTagIdKey { get; set; } = "messaging.rabbitmq.message.delivery_tag"; + + public static string MessagingMessageDeliveryModeKey { get; set; } = "messaging.rabbitmq.message.delivery_mode"; + public static string MessagingMessagePriorityKey { get; set; } = "messaging.rabbitmq.message.priority"; + public static string MessagingMessageContentTypeKey { get; set; } = "messaging.rabbitmq.message.content_type"; + public static string MessagingMessageMandatoryKey { get; set; } = "messaging.rabbitmq.message.mandatory"; + + public static string MessagingBatchProcessValue { get; set; } = "messaging.batch.message_count"; + + public static string MessagingMessagePayloadIdKey { get; set; } = "messaging.rabbitmq.message.payload_id"; + public static string MessagingMessageEncryptedKey { get; set; } = "messaging.rabbitmq.message.encrypted"; + public static string MessagingMessageEncryptedDateKey { get; set; } = "messaging.rabbitmq.message.encrypted_date"; + public static string MessagingMessageEncryptionKey { get; set; } = "messaging.rabbitmq.message.encryption"; + public static string MessagingMessageCompressedKey { get; set; } = "messaging.rabbitmq.message.compressed"; + public static string MessagingMessageCompressionKey { get; set; } = "messaging.rabbitmq.message.compression"; } public static class ExceptionMessages diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 3dbcf7ad..5a02930a 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -569,11 +569,15 @@ public virtual TState BuildState(IReceivedMessage data) if (state.ReceivedMessage?.Message?.MessageId is not null) { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.MessageId), state.ReceivedMessage.Message.MessageId)); + attributes.Add(KeyValuePair.Create(Constants.MessagingMessageMessageIdKey, state.ReceivedMessage.Message.MessageId)); } if (state.ReceivedMessage?.Message?.Metadata?.PayloadId is not null) { - attributes.Add(KeyValuePair.Create(nameof(state.ReceivedMessage.Message.Metadata.PayloadId), state.ReceivedMessage.Message.Metadata.PayloadId)); + attributes.Add(KeyValuePair.Create(Constants.MessagingMessagePayloadIdKey, state.ReceivedMessage.Message.Metadata.PayloadId)); + } + if (state.ReceivedMessage?.DeliveryTag is not null) + { + attributes.Add(KeyValuePair.Create(Constants.MessagingMessageDeliveryTagIdKey, state.ReceivedMessage.DeliveryTag.ToString())); } state.StartWorkflowSpan( diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 74aa792a..18a4a6f6 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -1,5 +1,6 @@ using HouseofCat.RabbitMQ.Pools; using HouseofCat.Utilities.Random; +using OpenTelemetry.Trace; using RabbitMQ.Client; using System; using System.Collections.Generic; @@ -32,9 +33,7 @@ public static TMessage ShallowClone(this IMessage message) { clonedMessage.Metadata = new Metadata { - PayloadId = new string(message.Metadata.PayloadId), - Encrypted = message.Metadata.Encrypted, - Compressed = message.Metadata.Compressed, + PayloadId = new string(message.Metadata.PayloadId) }; if (message.Metadata.Fields is not null) @@ -138,4 +137,33 @@ public static IList CreateManySimpleRandomMessages(string queueName, i return messages; } + + public static void EnrichSpanWithTags(this IMessage message, TelemetrySpan span) + { + if (message == null || span == null) return; + + span.SetAttribute(Constants.MessagingSystemKey, Constants.MessagingSystemValue); + span.SetAttribute(Constants.MessagingDestinationNameKey, message.Exchange); + + span.SetAttribute(Constants.MessagingMessageMessageIdKey, message.MessageId); + span.SetAttribute(Constants.MessagingMessageRoutingKeyKey, message.RoutingKey); + span.SetAttribute(Constants.MessagingMessageDeliveryModeKey, message.DeliveryMode); + span.SetAttribute(Constants.MessagingMessagePriorityKey, message.PriorityLevel); + span.SetAttribute(Constants.MessagingMessageContentTypeKey, message.ContentType); + span.SetAttribute(Constants.MessagingMessageMandatoryKey, message.Mandatory); + + span.SetAttribute(Constants.MessagingMessagePayloadIdKey, message.Metadata?.PayloadId); + + if (message.Metadata?.Encrypted() != null) + { + span.SetAttribute(Constants.MessagingMessageEncryptedKey, "true"); + span.SetAttribute(Constants.MessagingMessageEncryptedDateKey, message.Metadata?.EncryptedDate()); + span.SetAttribute(Constants.MessagingMessageEncryptionKey, message.Metadata?.EncryptionType()); + } + if (message.Metadata?.Compressed() != null) + { + span.SetAttribute(Constants.MessagingMessageCompressedKey, "true"); + span.SetAttribute(Constants.MessagingMessageCompressionKey, message.Metadata?.CompressionType()); + } + } } diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index 559237cb..c6694105 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -3,7 +3,6 @@ using RabbitMQ.Client; using System; using System.ComponentModel.DataAnnotations; -using System.Text.Json.Serialization; namespace HouseofCat.RabbitMQ; @@ -29,7 +28,7 @@ public interface IMessage IPublishReceipt GetPublishReceipt(bool error); - IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders); + IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders, string contentType); } public sealed class Message : IMessage @@ -86,13 +85,14 @@ public Message(string exchange, string routingKey, ReadOnlyMemory body, st { Metadata = new Metadata(); } } - public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders) + public IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders, string contentType) { var basicProperties = this.CreateBasicProperties(channelHost, withOptionalHeaders, Metadata); basicProperties.MessageId = MessageId; // Non-optional Header. basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + basicProperties.Headers[Constants.HeaderForContentType] = contentType; var openTelHeader = OpenTelemetryHelpers.GetOrCreateTraceHeaderFromCurrentActivity(); basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; diff --git a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs index 534f7906..29db2d4e 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; namespace HouseofCat.RabbitMQ; @@ -5,8 +6,6 @@ namespace HouseofCat.RabbitMQ; public interface IMetadata { string PayloadId { get; } - bool Encrypted { get; set; } - bool Compressed { get; set; } Dictionary Fields { get; set; } } @@ -17,8 +16,69 @@ public sealed class Metadata : IMetadata /// PayloadId is a user supplied identifier that allows identifying the Body without needing to deserialize. /// public string PayloadId { get; set; } - public bool Encrypted { get; set; } - public bool Compressed { get; set; } public Dictionary Fields { get; set; } = new Dictionary(); + + public bool Encrypted() + { + if (Fields == null) return false; + + return Fields.TryGetValue(Constants.HeaderForEncrypted, out var value) && (bool)value; + } + + public string EncryptionType() + { + if (Fields == null) return null; + + if (Fields.TryGetValue(Constants.HeaderForEncryption, out var value)) + { + return (string)value; + } + + return null; + } + + public string EncryptedDate() + { + if (Fields == null) return null; + + if (Fields.TryGetValue(Constants.HeaderForEncryptDate, out var value)) + { + return (string)value; + } + + return null; + } + + public DateTime? EncryptedDateTime() + { + if (Fields == null) return null; + + if (Fields.TryGetValue(Constants.HeaderForEncryptDate, out var value) + && DateTime.TryParse((string)value, out var dateTime)) + { + return dateTime; + } + + return null; + } + + public bool Compressed() + { + if (Fields == null) return false; + + return Fields.TryGetValue(Constants.HeaderForCompressed, out var value) && (bool)value; + } + + public string CompressionType() + { + if (Fields == null) return null; + + if (Fields.TryGetValue(Constants.HeaderForCompression, out var value)) + { + return (string)value; + } + + return null; + } } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index d7afb786..ca34d9a4 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -122,15 +122,9 @@ private void ReadHeaders() catch { FailedToDeserialize = true; } break; - case Constants.HeaderValueForContentTypeBinary: case Constants.HeaderValueForContentTypePlainText: - break; default: - try - { Message = JsonSerializer.Deserialize(Body.Span); } - catch - { FailedToDeserialize = true; } break; } } diff --git a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs index ca95c1ad..451ffcbe 100644 --- a/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs +++ b/src/HouseofCat.RabbitMQ/Pools/ConnectionPool.cs @@ -154,7 +154,7 @@ private async Task CreateConnectionsAsync() { _logger.LogTrace(LogMessages.ConnectionPools.CreateConnections); - for (int i = 0; i < Options.PoolOptions.Connections; i++) + for (var i = 0; i < Options.PoolOptions.Connections; i++) { var serviceName = string.IsNullOrEmpty(Options.PoolOptions.ServiceName) ? $"HoC.RabbitMQ:{i}" diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index acb5d927..6b5f7012 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -5,6 +5,7 @@ using HouseofCat.Utilities.Errors; using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; +using OpenTelemetry.Trace; using RabbitMQ.Client; using System; using System.Collections.Generic; @@ -22,17 +23,28 @@ public interface IPublisher string TimeFormat { get; set; } + void StartAutoPublish(Func processReceiptAsync = null); + Task StartAutoPublishAsync(Func processReceiptAsync = null); + Task StopAutoPublishAsync(bool immediately = false); + + void QueueMessage(IMessage message); + ValueTask QueueMessageAsync(IMessage message); + ChannelReader GetReceiptBufferReader(); Task PublishAsync(IMessage message, bool createReceipt, bool withHeaders = true); Task PublishWithConfirmationAsync(IMessage message, bool createReceipt, bool withOptionalHeaders = true); + Task PublishManyAsBatchAsync(IList messages, bool createReceipt, bool withHeaders = true); + Task PublishManyAsync(IList messages, bool createReceipt, bool withHeaders = true); + Task PublishAsync( string exchangeName, string routingKey, ReadOnlyMemory payload, bool mandatory = false, IBasicProperties messageProperties = null, - string messageId = null); + string messageId = null, + string contentType = null); Task PublishAsync( string exchangeName, @@ -41,14 +53,17 @@ Task PublishAsync( IDictionary headers = null, string messageId = null, byte? priority = 0, - bool mandatory = false); + byte? deliveryMode = 2, + bool mandatory = false, + string contentType = null); Task PublishBatchAsync( string exchangeName, string routingKey, IList> payloads, bool mandatory = false, - IBasicProperties messageProperties = null); + IBasicProperties messageProperties = null, + string contentType = null); Task PublishBatchAsync( string exchangeName, @@ -56,16 +71,9 @@ Task PublishBatchAsync( IList> payloads, IDictionary headers = null, byte? priority = 0, - bool mandatory = false); - - Task PublishManyAsBatchAsync(IList messages, bool createReceipt, bool withHeaders = true); - Task PublishManyAsync(IList messages, bool createReceipt, bool withHeaders = true); - - void QueueMessage(IMessage message); - ValueTask QueueMessageAsync(IMessage message); - void StartAutoPublish(Func processReceiptAsync = null); - Task StartAutoPublishAsync(Func processReceiptAsync = null); - Task StopAutoPublishAsync(bool immediately = false); + byte? deliveryMode = 2, + bool mandatory = false, + string contentType = null); } public class Publisher : IPublisher, IDisposable @@ -93,7 +101,7 @@ public class Publisher : IPublisher, IDisposable private Task _processReceiptsAsync; private bool _disposedValue; - public string TimeFormat { get; set; } = TimeHelpers.Formats.CatsAltFormat; + public string TimeFormat { get; set; } = TimeHelpers.Formats.RFC3339Long; public Publisher( RabbitOptions options, @@ -156,11 +164,18 @@ public Publisher( _waitForConfirmation = TimeSpan.FromMilliseconds(Options.PublisherOptions.WaitForConfirmationTimeoutInMilliseconds); } + public ChannelReader GetReceiptBufferReader() + { + return _receiptBuffer.Reader; + } + + #region AutoPublisher + public void StartAutoPublish(Func processReceiptAsync = null) { _pubLock.Wait(); - try { SetupPublisher(processReceiptAsync); } + try { SetupAutoPublisher(processReceiptAsync); } finally { _pubLock.Release(); } } @@ -168,17 +183,19 @@ public async Task StartAutoPublishAsync(Func process { await _pubLock.WaitAsync().ConfigureAwait(false); - try { SetupPublisher(processReceiptAsync); } + try { SetupAutoPublisher(processReceiptAsync); } finally { _pubLock.Release(); } } - private void SetupPublisher(Func processReceiptAsync = null) + private void SetupAutoPublisher(Func processReceiptAsync = null) { + if (AutoPublisherStarted) return; + _messageQueue = Channel.CreateBounded( - new BoundedChannelOptions(Options.PublisherOptions.MessageQueueBufferSize) - { - FullMode = Options.PublisherOptions.BehaviorWhenFull - }); + new BoundedChannelOptions(Options.PublisherOptions.MessageQueueBufferSize) + { + FullMode = Options.PublisherOptions.BehaviorWhenFull + }); _publishingTask = ProcessMessagesAsync(_messageQueue.Reader); @@ -222,13 +239,6 @@ await _messageQueue { _pubLock.Release(); } } - public ChannelReader GetReceiptBufferReader() - { - return _receiptBuffer.Reader; - } - - #region AutoPublisher - public void QueueMessage(IMessage message) { if (!AutoPublisherStarted) throw new InvalidOperationException(ExceptionMessages.AutoPublisherNotStartedError); @@ -275,7 +285,6 @@ private async Task ProcessMessagesAsync(ChannelReader channelReader) if (_compress) { message.Body = _compressionProvider.Compress(message.Body).ToArray(); - message.Metadata.Compressed = _compress; message.Metadata.Fields[Constants.HeaderForCompressed] = _compress; message.Metadata.Fields[Constants.HeaderForCompression] = _compressionProvider.Type; } @@ -283,7 +292,6 @@ private async Task ProcessMessagesAsync(ChannelReader channelReader) if (_encrypt) { message.Body = _encryptionProvider.Encrypt(message.Body).ToArray(); - message.Metadata.Encrypted = _encrypt; message.Metadata.Fields[Constants.HeaderForEncrypted] = _encrypt; message.Metadata.Fields[Constants.HeaderForEncryption] = _encryptionProvider.Type; message.Metadata.Fields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.RFC3339Long); @@ -338,7 +346,8 @@ public async Task PublishAsync( ReadOnlyMemory payload, bool mandatory = false, IBasicProperties basicProperties = null, - string messageId = null) + string messageId = null, + string contentType = null) { Guard.AgainstBothNullOrEmpty(exchangeName, nameof(exchangeName), routingKey, nameof(routingKey)); @@ -356,7 +365,7 @@ public async Task PublishAsync( } } - SetMandatoryHeaders(basicProperties); + SetMandatoryHeaders(basicProperties, contentType); try { @@ -391,10 +400,15 @@ public async Task PublishAsync( IDictionary headers = null, string messageId = null, byte? priority = 0, - bool mandatory = false) + byte? deliveryMode = 2, + bool mandatory = false, + string contentType = null) { Guard.AgainstBothNullOrEmpty(exchangeName, nameof(exchangeName), routingKey, nameof(routingKey)); + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishBatchAsync), SpanKind.Producer); + EnrichSpanWithTags(span, exchangeName, routingKey); + var error = false; var channelHost = await _channelPool.GetChannelAsync().ConfigureAwait(false); @@ -406,7 +420,7 @@ public async Task PublishAsync( exchange: exchangeName ?? string.Empty, routingKey: routingKey, mandatory: mandatory, - basicProperties: BuildProperties(headers, channelHost, messageId, priority), + basicProperties: BuildProperties(headers, channelHost, messageId, priority, deliveryMode, contentType), body: payload); } catch (Exception ex) @@ -433,11 +447,14 @@ public async Task PublishBatchAsync( string routingKey, IList> payloads, bool mandatory = false, - IBasicProperties messageProperties = null) + IBasicProperties messageProperties = null, + string contentType = null) { Guard.AgainstBothNullOrEmpty(exchangeName, nameof(exchangeName), routingKey, nameof(routingKey)); Guard.AgainstNullOrEmpty(payloads, nameof(payloads)); + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishBatchAsync), SpanKind.Producer); + var error = false; var channelHost = await _channelPool.GetChannelAsync().ConfigureAwait(false); if (messageProperties == null) @@ -453,14 +470,16 @@ public async Task PublishBatchAsync( } // Non-optional Header. - messageProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + SetMandatoryHeaders(messageProperties, contentType); try { var batch = channelHost.Channel.CreateBasicPublishBatch(); - for (int i = 0; i < payloads.Count; i++) + for (var i = 0; i < payloads.Count; i++) { + using var innerSpan = OpenTelemetryHelpers.StartActiveSpan("IBasicPublishBatch.Add", SpanKind.Producer); + EnrichSpanWithTags(span, exchangeName, routingKey); batch.Add(exchangeName, routingKey, mandatory, messageProperties, payloads[i]); } @@ -491,11 +510,16 @@ public async Task PublishBatchAsync( IList> payloads, IDictionary headers = null, byte? priority = 0, - bool mandatory = false) + byte? deliveryMode = 2, + bool mandatory = false, + string contentType = null) { Guard.AgainstBothNullOrEmpty(exchangeName, nameof(exchangeName), routingKey, nameof(routingKey)); Guard.AgainstNullOrEmpty(payloads, nameof(payloads)); + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishBatchAsync), SpanKind.Producer); + span.SetAttribute(Constants.MessagingBatchProcessValue, payloads.Count); + var error = false; var channelHost = await _channelPool.GetChannelAsync().ConfigureAwait(false); @@ -503,9 +527,13 @@ public async Task PublishBatchAsync( { var batch = channelHost.Channel.CreateBasicPublishBatch(); - for (int i = 0; i < payloads.Count; i++) + for (var i = 0; i < payloads.Count; i++) { - batch.Add(exchangeName, routingKey, mandatory, BuildProperties(headers, channelHost, null, priority), payloads[i]); + using var innerSpan = OpenTelemetryHelpers.StartActiveSpan("IBasicPublishBatch.Add", SpanKind.Producer); + EnrichSpanWithTags(span, exchangeName, routingKey); + + var properties = BuildProperties(headers, channelHost, null, priority, deliveryMode, contentType); + batch.Add(exchangeName, routingKey, mandatory, properties, payloads[i]); } batch.Publish(); @@ -537,6 +565,9 @@ await _channelPool /// public async Task PublishAsync(IMessage message, bool createReceipt, bool withOptionalHeaders = true) { + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishAsync), SpanKind.Producer); + message.EnrichSpanWithTags(span); + var error = false; var chanHost = await _channelPool .GetChannelAsync() @@ -551,7 +582,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp message.Exchange, message.RoutingKey, message.Mandatory, - message.BuildProperties(chanHost, withOptionalHeaders), + message.BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), body); } catch (Exception ex) @@ -587,6 +618,9 @@ await _channelPool /// public async Task PublishWithConfirmationAsync(IMessage message, bool createReceipt, bool withOptionalHeaders = true) { + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishWithConfirmationAsync), SpanKind.Producer); + message.EnrichSpanWithTags(span); + var error = false; var chanHost = await _channelPool .GetAckChannelAsync() @@ -602,7 +636,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece message.Exchange, message.RoutingKey, message.Mandatory, - message.BuildProperties(chanHost, withOptionalHeaders), + message.BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), _serializationProvider.Serialize(message)); chanHost.Channel.WaitForConfirmsOrDie(_waitForConfirmation); @@ -638,20 +672,30 @@ await _channelPool /// public async Task PublishManyAsync(IList messages, bool createReceipt, bool withOptionalHeaders = true) { + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishManyAsync), SpanKind.Producer); + span?.SetAttribute(Constants.MessagingBatchProcessValue, messages.Count); + var error = false; var chanHost = await _channelPool .GetChannelAsync() .ConfigureAwait(false); - for (int i = 0; i < messages.Count; i++) + for (var i = 0; i < messages.Count; i++) { try { + using var innerSpan = OpenTelemetryHelpers.StartActiveSpan( + nameof(chanHost.Channel.BasicPublish), + SpanKind.Producer, + span?.Context ?? default); + + messages[i].EnrichSpanWithTags(innerSpan); + chanHost.Channel.BasicPublish( messages[i].Exchange, messages[i].RoutingKey, messages[i].Mandatory, - messages[i].BuildProperties(chanHost, withOptionalHeaders), + messages[i].BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), _serializationProvider.Serialize(messages[i].Body)); } catch (Exception ex) @@ -683,6 +727,9 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, /// public async Task PublishManyAsBatchAsync(IList messages, bool createReceipt, bool withOptionalHeaders = true) { + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishManyAsBatchAsync), SpanKind.Producer); + span?.SetAttribute(Constants.MessagingBatchProcessValue, messages.Count); + var error = false; var chanHost = await _channelPool .GetChannelAsync() @@ -693,13 +740,20 @@ public async Task PublishManyAsBatchAsync(IList messages, bool createR if (messages.Count > 0) { var publishBatch = chanHost.Channel.CreateBasicPublishBatch(); - for (int i = 0; i < messages.Count; i++) + for (var i = 0; i < messages.Count; i++) { + using var innerSpan = OpenTelemetryHelpers.StartActiveSpan( + "IBasicPublishBatch.Add", + SpanKind.Producer, + span?.Context ?? default); + + messages[i].EnrichSpanWithTags(innerSpan); + publishBatch.Add( messages[i].Exchange, messages[i].RoutingKey, messages[i].Mandatory, - messages[i].BuildProperties(chanHost, withOptionalHeaders), + messages[i].BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), _serializationProvider.Serialize(messages[i].Body)); if (createReceipt) @@ -740,9 +794,13 @@ await _receiptBuffer .ConfigureAwait(false); } - private static void SetMandatoryHeaders(IBasicProperties basicProperties) + private static void SetMandatoryHeaders(IBasicProperties basicProperties, string contentType) { basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + if (!string.IsNullOrEmpty(contentType)) + { + basicProperties.Headers[Constants.HeaderForContentType] = contentType; + } var openTelHeader = OpenTelemetryHelpers.GetOrCreateTraceHeaderFromCurrentActivity(); basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; } @@ -761,7 +819,8 @@ private static IBasicProperties BuildProperties( IChannelHost channelHost, string messageId = null, byte? priority = 0, - byte? deliveryMode = 2) + byte? deliveryMode = 2, + string contentType = null) { var basicProperties = channelHost.Channel.CreateBasicProperties(); basicProperties.DeliveryMode = deliveryMode ?? 2; // Default Persisted @@ -784,11 +843,27 @@ private static IBasicProperties BuildProperties( } } - SetMandatoryHeaders(basicProperties); + SetMandatoryHeaders(basicProperties, contentType); return basicProperties; } + public static void EnrichSpanWithTags( + TelemetrySpan span, + string exchangeName, + string routingKey, + string messageId = null) + { + span.SetAttribute(Constants.MessagingSystemKey, Constants.MessagingSystemValue); + span.SetAttribute(Constants.MessagingDestinationNameKey, exchangeName); + span.SetAttribute(Constants.MessagingMessageRoutingKeyKey, routingKey); + + if (!string.IsNullOrEmpty(messageId)) + { + span.SetAttribute(Constants.MessagingMessageMessageIdKey, messageId); + } + } + protected virtual void Dispose(bool disposing) { if (!_disposedValue) diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index 05f0563a..d4b17073 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -253,10 +253,9 @@ public async Task ComcryptAsync(IMessage message) public bool Encrypt(IMessage message) { - if (!message.Metadata.Encrypted) + if (!message.Metadata.Encrypted()) { message.Body = EncryptionProvider.Encrypt(message.Body); - message.Metadata.Encrypted = true; message.Metadata.Fields[Constants.HeaderForEncrypted] = true; message.Metadata.Fields[Constants.HeaderForEncryption] = EncryptionProvider.Type; message.Metadata.Fields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeFormat); @@ -269,10 +268,9 @@ public bool Encrypt(IMessage message) public bool Decrypt(IMessage message) { - if (message.Metadata.Encrypted) + if (message.Metadata.Encrypted()) { message.Body = EncryptionProvider.Decrypt(message.Body); - message.Metadata.Encrypted = false; message.Metadata.Fields[Constants.HeaderForEncrypted] = false; message.Metadata.Fields.Remove(Constants.HeaderForEncryption); @@ -286,13 +284,12 @@ public bool Decrypt(IMessage message) public async Task CompressAsync(IMessage message) { - if (message.Metadata.Encrypted) + if (message.Metadata.Encrypted()) { return false; } // Don't compress after encryption. - if (!message.Metadata.Compressed) + if (!message.Metadata.Compressed()) { message.Body = (await CompressionProvider.CompressAsync(message.Body).ConfigureAwait(false)).ToArray(); - message.Metadata.Compressed = true; message.Metadata.Fields[Constants.HeaderForCompressed] = true; message.Metadata.Fields[Constants.HeaderForCompression] = CompressionProvider.Type; @@ -304,15 +301,14 @@ public async Task CompressAsync(IMessage message) public async Task DecompressAsync(IMessage message) { - if (message.Metadata.Encrypted) + if (message.Metadata.Encrypted()) { return false; } // Don't decompress before decryption. - if (message.Metadata.Compressed) + if (message.Metadata.Compressed()) { try { message.Body = (await CompressionProvider.DecompressAsync(message.Body).ConfigureAwait(false)).ToArray(); - message.Metadata.Compressed = false; message.Metadata.Fields[Constants.HeaderForCompressed] = false; message.Metadata.Fields.Remove(Constants.HeaderForCompression); diff --git a/src/HouseofCat.Serialization/ISerializationProvider.cs b/src/HouseofCat.Serialization/ISerializationProvider.cs index e508c3c9..b4ac2b04 100644 --- a/src/HouseofCat.Serialization/ISerializationProvider.cs +++ b/src/HouseofCat.Serialization/ISerializationProvider.cs @@ -6,6 +6,7 @@ namespace HouseofCat.Serialization; public interface ISerializationProvider { + string ContentType { get; } TOut Deserialize(string input); TOut Deserialize(ReadOnlyMemory input); TOut Deserialize(Stream inputStream); diff --git a/src/HouseofCat.Serialization/JsonProvider.cs b/src/HouseofCat.Serialization/JsonProvider.cs index b4196f0e..39132900 100644 --- a/src/HouseofCat.Serialization/JsonProvider.cs +++ b/src/HouseofCat.Serialization/JsonProvider.cs @@ -9,6 +9,8 @@ namespace HouseofCat.Serialization; public class JsonProvider : ISerializationProvider { + public string ContentType { get; private set; } = "application/json"; + private readonly JsonSerializerOptions _options; public JsonProvider(JsonSerializerOptions options = null) diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs index 10170645..8de3a4f6 100644 --- a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -24,6 +24,12 @@ public static void SetEntryAssemblyAsSourceForTelemetry() _sourceVersion = assembly.GetSemanticVersion(); } + public static void SetAssemblyAsSourceForTelemetry(Assembly assembly) + { + _sourceName = assembly.GetName().Name; + _sourceVersion = assembly.GetSemanticVersion(); + } + #region Span Helpers public static TelemetrySpan StartRootSpan( @@ -49,6 +55,17 @@ public static TelemetrySpan StartRootSpan( startTime: startTime); } + /// + /// Starts a new active span, automatically setting the parent ontext if there is a current span + /// create a new active child span automatically. + /// + /// + /// + /// + /// + /// + /// + /// public static TelemetrySpan StartActiveSpan( string spanName, SpanKind spanKind = SpanKind.Internal, @@ -65,6 +82,12 @@ public static TelemetrySpan StartActiveSpan( if (provider is null) return null; + var currentSpan = Tracer.CurrentSpan; + if (parentContext == default && currentSpan != null) + { + parentContext = currentSpan.Context; + } + return provider.StartActiveSpan( spanName, spanKind, @@ -74,27 +97,6 @@ public static TelemetrySpan StartActiveSpan( startTime: startTime); } - public static TelemetrySpan StartActiveChildSpan( - string spanName, - SpanContext parentContext, - SpanKind spanKind = SpanKind.Internal, - SpanAttributes attributes = null) - { - var provider = TracerProvider - .Default - .GetTracer( - _sourceName, - _sourceVersion); - - if (provider is null) return null; - - return provider.StartActiveSpan( - spanName, - spanKind, - initialAttributes: attributes, - parentContext: parentContext); - } - #endregion #region Current and ParentId Helpers diff --git a/src/HouseofCat.Utilities/Random/RandomData.cs b/src/HouseofCat.Utilities/Random/RandomData.cs index 4d599c60..55c2cf86 100644 --- a/src/HouseofCat.Utilities/Random/RandomData.cs +++ b/src/HouseofCat.Utilities/Random/RandomData.cs @@ -24,7 +24,7 @@ public static async Task RandomStringAsync(int minLength, int maxLength) int length = Rand.Next(minLength, maxLength + 1); - for (int i = 0; i < length; ++i) + for (var i = 0; i < length; ++i) { chars[i] = AllowedChars[Rand.Next(setLength)]; } @@ -45,7 +45,7 @@ public static string RandomString(int minLength, int maxLength) int length = Rand.Next(minLength, maxLength + 1); - for (int i = 0; i < length; ++i) + for (var i = 0; i < length; ++i) { chars[i] = AllowedChars[Rand.Next(setLength)]; } @@ -58,7 +58,7 @@ public static int[] CreateRandomInts(int count, int maxValue = int.MaxValue) var ints = new int[count]; var validatedInt = ((maxValue < 2 ? 2 : maxValue) / 2) - 1; - for (int i = 0; i < count; i++) + for (var i = 0; i < count; i++) { ints[i] = Rand.Next(validatedInt) - Rand.Next(validatedInt); } From 8e7a6c17ce7533b89781c45e7382ce5101a2e237 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Fri, 12 Apr 2024 04:38:09 -0500 Subject: [PATCH 25/47] Capture publish exceptions in OpenTelemetry. --- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 2 ++ .../Extensions/MessageExtensions.cs | 34 ++++++++++++++----- .../Publisher/Publisher.cs | 28 +++++++++++---- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 1cf6acc0..129a73ee 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -92,6 +92,8 @@ public async Task StartConsumerAsync() { _logger.LogTrace(Consumers.StartingConsumerLoop, ConsumerOptions.ConsumerName); success = await StartConsumingAsync().ConfigureAwait(false); + if (!success) + { await Task.Delay(Options.PoolOptions.SleepOnErrorInterval); } } _logger.LogDebug(Consumers.Started, ConsumerOptions.ConsumerName); diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index 18a4a6f6..d1e54d5c 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -140,27 +140,45 @@ public static IList CreateManySimpleRandomMessages(string queueName, i public static void EnrichSpanWithTags(this IMessage message, TelemetrySpan span) { - if (message == null || span == null) return; + if (message == null || span == null || !span.IsRecording) return; span.SetAttribute(Constants.MessagingSystemKey, Constants.MessagingSystemValue); - span.SetAttribute(Constants.MessagingDestinationNameKey, message.Exchange); - span.SetAttribute(Constants.MessagingMessageMessageIdKey, message.MessageId); - span.SetAttribute(Constants.MessagingMessageRoutingKeyKey, message.RoutingKey); + if (!string.IsNullOrEmpty(message.Exchange)) + { + span.SetAttribute(Constants.MessagingDestinationNameKey, message.Exchange); + } + if (!string.IsNullOrEmpty(message.MessageId)) + { + span.SetAttribute(Constants.MessagingMessageMessageIdKey, message.MessageId); + } + if (!string.IsNullOrEmpty(message.RoutingKey)) + { + span.SetAttribute(Constants.MessagingMessageRoutingKeyKey, message.RoutingKey); + } + if (!string.IsNullOrEmpty(message.ContentType)) + { + span.SetAttribute(Constants.MessagingMessageContentTypeKey, message.ContentType); + } + span.SetAttribute(Constants.MessagingMessageDeliveryModeKey, message.DeliveryMode); span.SetAttribute(Constants.MessagingMessagePriorityKey, message.PriorityLevel); - span.SetAttribute(Constants.MessagingMessageContentTypeKey, message.ContentType); span.SetAttribute(Constants.MessagingMessageMandatoryKey, message.Mandatory); - span.SetAttribute(Constants.MessagingMessagePayloadIdKey, message.Metadata?.PayloadId); + if (!string.IsNullOrEmpty(message.Metadata?.PayloadId)) + { + span.SetAttribute(Constants.MessagingMessagePayloadIdKey, message.Metadata?.PayloadId); + } - if (message.Metadata?.Encrypted() != null) + var encrypted = message.Metadata?.Encrypted(); + if (encrypted.HasValue && encrypted.Value) { span.SetAttribute(Constants.MessagingMessageEncryptedKey, "true"); span.SetAttribute(Constants.MessagingMessageEncryptedDateKey, message.Metadata?.EncryptedDate()); span.SetAttribute(Constants.MessagingMessageEncryptionKey, message.Metadata?.EncryptionType()); } - if (message.Metadata?.Compressed() != null) + var compressed = message.Metadata?.Compressed(); + if (compressed.HasValue && compressed.Value) { span.SetAttribute(Constants.MessagingMessageCompressedKey, "true"); span.SetAttribute(Constants.MessagingMessageCompressionKey, message.Metadata?.CompressionType()); diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index 6b5f7012..bac491fc 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -351,6 +351,9 @@ public async Task PublishAsync( { Guard.AgainstBothNullOrEmpty(exchangeName, nameof(exchangeName), routingKey, nameof(routingKey)); + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishAsync), SpanKind.Producer); + EnrichSpanWithTags(span, exchangeName, routingKey, messageId); + var error = false; var channelHost = await _channelPool.GetChannelAsync().ConfigureAwait(false); if (basicProperties == null) @@ -380,6 +383,7 @@ public async Task PublishAsync( } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); _logger.LogDebug(LogMessages.Publishers.PublishFailed, $"{exchangeName}->{routingKey}", ex.Message); error = true; } @@ -406,7 +410,7 @@ public async Task PublishAsync( { Guard.AgainstBothNullOrEmpty(exchangeName, nameof(exchangeName), routingKey, nameof(routingKey)); - using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishBatchAsync), SpanKind.Producer); + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishAsync), SpanKind.Producer); EnrichSpanWithTags(span, exchangeName, routingKey); var error = false; @@ -425,6 +429,7 @@ public async Task PublishAsync( } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); _logger.LogDebug( LogMessages.Publishers.PublishFailed, $"{exchangeName}->{routingKey}", @@ -487,6 +492,7 @@ public async Task PublishBatchAsync( } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); _logger.LogDebug( LogMessages.Publishers.PublishFailed, $"{exchangeName}->{routingKey}", @@ -540,6 +546,8 @@ public async Task PublishBatchAsync( } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); + _logger.LogDebug( LogMessages.Publishers.PublishFailed, $"{exchangeName}->{routingKey}", @@ -587,6 +595,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); _logger.LogDebug( LogMessages.Publishers.PublishMessageFailed, $"{message.Exchange}->{message.RoutingKey}", @@ -643,6 +652,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); _logger.LogDebug( LogMessages.Publishers.PublishMessageFailed, $"{message.Exchange}->{message.RoutingKey}", @@ -700,6 +710,7 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); _logger.LogDebug( LogMessages.Publishers.PublishMessageFailed, $"{messages[i].Exchange}->{messages[i].RoutingKey}", @@ -767,6 +778,7 @@ public async Task PublishManyAsBatchAsync(IList messages, bool createR } catch (Exception ex) { + OpenTelemetryHelpers.SetSpanAsError(span, ex); _logger.LogDebug( LogMessages.Publishers.PublishBatchFailed, ex.Message); @@ -854,14 +866,18 @@ public static void EnrichSpanWithTags( string routingKey, string messageId = null) { + if (span == null || !span.IsRecording) return; + span.SetAttribute(Constants.MessagingSystemKey, Constants.MessagingSystemValue); - span.SetAttribute(Constants.MessagingDestinationNameKey, exchangeName); - span.SetAttribute(Constants.MessagingMessageRoutingKeyKey, routingKey); + + if (!string.IsNullOrEmpty(exchangeName)) + { span.SetAttribute(Constants.MessagingDestinationNameKey, exchangeName); } + + if (!string.IsNullOrEmpty(routingKey)) + { span.SetAttribute(Constants.MessagingMessageRoutingKeyKey, routingKey); } if (!string.IsNullOrEmpty(messageId)) - { - span.SetAttribute(Constants.MessagingMessageMessageIdKey, messageId); - } + { span.SetAttribute(Constants.MessagingMessageMessageIdKey, messageId); } } protected virtual void Dispose(bool disposing) From 184395de12ab99de26cb453a3e811c08e3c689be Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Fri, 12 Apr 2024 04:48:35 -0500 Subject: [PATCH 26/47] Allow ReceiveMessage to auto deserialize Json/Messagepack. --- src/HouseofCat.RabbitMQ/Constants.cs | 1 + src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs | 9 +++++++++ src/HouseofCat.Serialization/MessagePackProvider.cs | 9 +++++++++ src/HouseofCat.Serialization/NewtonsoftJsonProvider.cs | 2 ++ 4 files changed, 21 insertions(+) diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index e78ad473..3412034b 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -10,6 +10,7 @@ public static class Constants public const string HeaderValueForContentTypeBinary = "application/octet-stream"; public const string HeaderValueForContentTypePlainText = "text/plain"; public const string HeaderValueForContentTypeJson = "application/json"; + public const string HeaderValueForContentTypeMessagePack = "application/msgpack"; public static string HeaderForObjectType { get; set; } = "X-RD-OBJECTTYPE"; public static string HeaderValueForMessageObjectType { get; set; } = "IMESSAGE"; diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index ca34d9a4..156ac855 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -1,3 +1,4 @@ +using HouseofCat.Serialization; using RabbitMQ.Client; using RabbitMQ.Client.Events; using System; @@ -122,6 +123,14 @@ private void ReadHeaders() catch { FailedToDeserialize = true; } break; + case Constants.HeaderValueForContentTypeMessagePack: + try + { + Message = MessagePackProvider.GlobalDeserialize(Body); + } + catch + { FailedToDeserialize = true; } + break; case Constants.HeaderValueForContentTypeBinary: case Constants.HeaderValueForContentTypePlainText: default: diff --git a/src/HouseofCat.Serialization/MessagePackProvider.cs b/src/HouseofCat.Serialization/MessagePackProvider.cs index 7e1d4d07..4065ff0d 100644 --- a/src/HouseofCat.Serialization/MessagePackProvider.cs +++ b/src/HouseofCat.Serialization/MessagePackProvider.cs @@ -10,11 +10,20 @@ public class MessagePackProvider : ISerializationProvider { private readonly MessagePackSerializerOptions _options; + // After nearly a decade, they haven't made their mind up. Tired of waiting. + // https://github.com/msgpack/msgpack/issues/194 + public string ContentType { get; private set; } = "application/msgpack"; + public MessagePackProvider(MessagePackSerializerOptions options = null) { _options = options; } + public static TOut GlobalDeserialize(ReadOnlyMemory input, MessagePackSerializerOptions options = null) + { + return MessagePackSerializer.Deserialize(input, options); + } + public ReadOnlyMemory Serialize(TIn input) { return MessagePackSerializer.Serialize(input, _options); diff --git a/src/HouseofCat.Serialization/NewtonsoftJsonProvider.cs b/src/HouseofCat.Serialization/NewtonsoftJsonProvider.cs index 70d66bb2..b615bb80 100644 --- a/src/HouseofCat.Serialization/NewtonsoftJsonProvider.cs +++ b/src/HouseofCat.Serialization/NewtonsoftJsonProvider.cs @@ -11,6 +11,8 @@ public sealed class NewtonsoftJsonProvider : ISerializationProvider { private readonly JsonSerializer _jsonSerializer = new JsonSerializer(); + public string ContentType { get; private set; } = "application/json"; + public TOut Deserialize(ReadOnlyMemory input) { Guard.AgainstEmpty(input, nameof(input)); From edc50c3dfe01b8cf862597e2fce591ccd805b3f1 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Fri, 12 Apr 2024 11:09:51 -0500 Subject: [PATCH 27/47] Migrate auto-deserialization to the Consumer not the ReceivedMessage. --- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 58 ++++++++++++++----- .../Messages/ReceivedMessage.cs | 30 +--------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 129a73ee..b06416d7 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -1,4 +1,5 @@ using HouseofCat.RabbitMQ.Pools; +using HouseofCat.Serialization; using HouseofCat.Utilities.Errors; using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; @@ -6,6 +7,7 @@ using RabbitMQ.Client.Events; using System; using System.Collections.Generic; +using System.Text.Json; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -31,14 +33,13 @@ public interface IConsumer public class Consumer : IConsumer, IDisposable { private readonly ILogger _logger; - private readonly SemaphoreSlim _conLock = new SemaphoreSlim(1, 1); - private readonly SemaphoreSlim _executionLock = new SemaphoreSlim(1, 1); - private IChannelHost _chanHost; - private bool _disposedValue; - private Channel _consumerChannel; + protected readonly SemaphoreSlim _conLock = new SemaphoreSlim(1, 1); + protected IChannelHost _chanHost; + protected bool _disposedValue; + protected Channel _consumerChannel; public string ConsumerTag { get; private set; } - private bool _shutdown; + protected bool _shutdown; public RabbitOptions Options { get; } public ConsumerOptions ConsumerOptions { get; } @@ -139,12 +140,12 @@ await _consumerChannel finally { _conLock.Release(); } } - private AsyncEventingBasicConsumer _asyncConsumer; - private EventingBasicConsumer _consumer; + protected AsyncEventingBasicConsumer _asyncConsumer; + protected EventingBasicConsumer _consumer; - private CancellationTokenSource _cts; + protected CancellationTokenSource _cts; - private async Task StartConsumingAsync() + protected async Task StartConsumingAsync() { if (_shutdown) return false; @@ -248,7 +249,7 @@ protected virtual async void ReceiveHandler(object _, BasicDeliverEventArgs bdea await HandleMessageAsync(bdea).ConfigureAwait(false); } - private async void ConsumerShutdown(object sender, ShutdownEventArgs e) + protected async void ConsumerShutdown(object sender, ShutdownEventArgs e) { if (await _conLock.WaitAsync(0)) { @@ -267,7 +268,7 @@ await HandleRecoverableShutdownAsync(e) } } - private AsyncEventingBasicConsumer CreateAsyncConsumer() + protected AsyncEventingBasicConsumer CreateAsyncConsumer() { AsyncEventingBasicConsumer consumer = null; @@ -296,9 +297,37 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs try { + var receivedMessage = new ReceivedMessage(_chanHost.Channel, bdea, !ConsumerOptions.AutoAck); + if (receivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType + && receivedMessage.Body.Length > 0) + { + switch (receivedMessage.Properties.ContentType) + { + case Constants.HeaderValueForContentTypeJson: + try + { + receivedMessage.Message = JsonSerializer.Deserialize(receivedMessage.Body.Span); + } + catch + { receivedMessage.FailedToDeserialize = true; } + break; + case Constants.HeaderValueForContentTypeMessagePack: + try + { + receivedMessage.Message = MessagePackProvider.GlobalDeserialize(receivedMessage.Body); + } + catch + { receivedMessage.FailedToDeserialize = true; } + break; + case Constants.HeaderValueForContentTypeBinary: + case Constants.HeaderValueForContentTypePlainText: + default: + break; + } + } await _consumerChannel .Writer - .WriteAsync(new ReceivedMessage(_chanHost.Channel, bdea, !ConsumerOptions.AutoAck)) + .WriteAsync(receivedMessage) .ConfigureAwait(false); return true; } @@ -337,7 +366,7 @@ await HandleRecoverableShutdownAsync(e) } } - private static readonly string _consumerShutdownExceptionMessage = "Consumer's ChannelHost {0} had an unhandled exception during recovery."; + protected static readonly string _consumerShutdownExceptionMessage = "Consumer's ChannelHost {0} had an unhandled exception during recovery."; protected virtual async Task HandleRecoverableShutdownAsync(ShutdownEventArgs e) { @@ -403,7 +432,6 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - _executionLock.Dispose(); _conLock.Dispose(); } diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index 156ac855..4c944869 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -66,7 +66,7 @@ public sealed class ReceivedMessage : IReceivedMessage, IDisposable public string ConsumerTag { get; } public ulong DeliveryTag { get; } - public bool FailedToDeserialize { get; private set; } + public bool FailedToDeserialize { get; set; } private readonly TaskCompletionSource _completionSource = new TaskCompletionSource(); public Task Completion => _completionSource.Task; @@ -110,34 +110,6 @@ private void ReadHeaders() { ObjectType = Encoding.UTF8.GetString((byte[])objectType); - if (ObjectType == Constants.HeaderValueForMessageObjectType - && Body.Length > 0) - { - switch (Properties.ContentType) - { - case Constants.HeaderValueForContentTypeJson: - try - { - Message = JsonSerializer.Deserialize(Body.Span); - } - catch - { FailedToDeserialize = true; } - break; - case Constants.HeaderValueForContentTypeMessagePack: - try - { - Message = MessagePackProvider.GlobalDeserialize(Body); - } - catch - { FailedToDeserialize = true; } - break; - case Constants.HeaderValueForContentTypeBinary: - case Constants.HeaderValueForContentTypePlainText: - default: - break; - } - } - if (Properties.Headers.TryGetValue(Constants.HeaderForEncrypted, out object encryptedValue)) { Encrypted = (bool)encryptedValue; } From d157e2e1adf48e24c11ef375a549432f92870411 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Fri, 12 Apr 2024 15:48:26 -0500 Subject: [PATCH 28/47] More granular activity/spans for consumer and workflow process. --- .../Extensions/WorkStateExtensions.cs | 20 +++--- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 67 +++++++++++-------- .../Dataflows/ConsumerDataflow.cs | 34 +++++++--- src/HouseofCat.RabbitMQ/Messages/Message.cs | 12 ++++ .../Messages/ReceivedMessage.cs | 10 ++- .../Helpers/OpenTelemetryHelpers.cs | 1 + 6 files changed, 92 insertions(+), 52 deletions(-) diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 397bc706..146b85b7 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -62,7 +62,7 @@ public static void StartWorkflowSpan( string workflowName, SpanKind spanKind = SpanKind.Internal, IEnumerable> suppliedAttributes = null, - string traceHeader = null) + SpanContext? parentSpanContext = null) { if (state is null) return; @@ -81,18 +81,14 @@ public static void StartWorkflowSpan( } } - if (traceHeader is not null) + if (parentSpanContext.HasValue) { - var spanContext = OpenTelemetryHelpers.ExtractSpanContextFromTraceHeader(traceHeader); - if (spanContext.HasValue) - { - state.WorkflowSpan = OpenTelemetryHelpers - .StartActiveSpan( - spanName, - spanKind: spanKind, - spanContext.Value, - attributes: attributes); - } + state.WorkflowSpan = OpenTelemetryHelpers + .StartActiveSpan( + spanName, + spanKind: spanKind, + parentSpanContext.Value, + attributes: attributes); } state.WorkflowSpan = OpenTelemetryHelpers diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index b06416d7..96651ca2 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -3,6 +3,8 @@ using HouseofCat.Utilities.Errors; using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; +using OpenTelemetry.Trace; +using Org.BouncyCastle.Asn1.Cms; using RabbitMQ.Client; using RabbitMQ.Client.Events; using System; @@ -298,33 +300,13 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs try { var receivedMessage = new ReceivedMessage(_chanHost.Channel, bdea, !ConsumerOptions.AutoAck); - if (receivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType - && receivedMessage.Body.Length > 0) - { - switch (receivedMessage.Properties.ContentType) - { - case Constants.HeaderValueForContentTypeJson: - try - { - receivedMessage.Message = JsonSerializer.Deserialize(receivedMessage.Body.Span); - } - catch - { receivedMessage.FailedToDeserialize = true; } - break; - case Constants.HeaderValueForContentTypeMessagePack: - try - { - receivedMessage.Message = MessagePackProvider.GlobalDeserialize(receivedMessage.Body); - } - catch - { receivedMessage.FailedToDeserialize = true; } - break; - case Constants.HeaderValueForContentTypeBinary: - case Constants.HeaderValueForContentTypePlainText: - default: - break; - } - } + using var span = OpenTelemetryHelpers.StartActiveSpan( + nameof(HandleMessageAsync), + SpanKind.Consumer, + receivedMessage.ParentSpanContext ?? default); + + AutoDeserialize(receivedMessage); + await _consumerChannel .Writer .WriteAsync(receivedMessage) @@ -341,6 +323,37 @@ await _consumerChannel } } + protected virtual void AutoDeserialize(ReceivedMessage receivedMessage) + { + if (receivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType + && receivedMessage.Body.Length > 0) + { + switch (receivedMessage.Properties.ContentType) + { + case Constants.HeaderValueForContentTypeJson: + try + { + receivedMessage.Message = JsonSerializer.Deserialize(receivedMessage.Body.Span); + } + catch + { receivedMessage.FailedToDeserialize = true; } + break; + case Constants.HeaderValueForContentTypeMessagePack: + try + { + receivedMessage.Message = MessagePackProvider.GlobalDeserialize(receivedMessage.Body); + } + catch + { receivedMessage.FailedToDeserialize = true; } + break; + case Constants.HeaderValueForContentTypeBinary: + case Constants.HeaderValueForContentTypePlainText: + default: + break; + } + } + } + protected async Task ConsumerShutdownAsync(object sender, ShutdownEventArgs e) { if (await _conLock.WaitAsync(0)) diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 5a02930a..bb0f7b43 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -543,7 +543,11 @@ private void LinkPostProcessing(DataflowLinkOptions overrideOptions = null) _currentBlock.LinkTo(_finalization, overrideOptions ?? _linkStepOptions); // Last Action } - private void LinkWithFaultRoute(ISourceBlock source, IPropagatorBlock target, Predicate faultPredicate, DataflowLinkOptions overrideOptions = null) + private void LinkWithFaultRoute( + ISourceBlock source, + IPropagatorBlock target, + Predicate faultPredicate, + DataflowLinkOptions overrideOptions = null) { source.LinkTo(target, overrideOptions ?? _linkStepOptions); target.LinkTo(_errorBuffer, overrideOptions ?? _linkStepOptions, faultPredicate); // Fault Linkage @@ -554,17 +558,31 @@ private void LinkWithFaultRoute(ISourceBlock source, IPropagatorBlock() }; + var attributes = GetSpanAttributes(state, receivedMessage); + + state.StartWorkflowSpan( + WorkflowName, + spanKind: SpanKind.Internal, + suppliedAttributes: attributes, + parentSpanContext: receivedMessage.ParentSpanContext); + + return state; + } + + protected virtual List> GetSpanAttributes(TState state, IReceivedMessage receivedMessage) + { var attributes = new List>() { - KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName) + KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName), + KeyValuePair.Create(Constants.MessagingOperationKey, Constants.MessagingOperationProcessValue) }; if (state.ReceivedMessage?.Message?.MessageId is not null) @@ -580,13 +598,7 @@ public virtual TState BuildState(IReceivedMessage data) attributes.Add(KeyValuePair.Create(Constants.MessagingMessageDeliveryTagIdKey, state.ReceivedMessage.DeliveryTag.ToString())); } - state.StartWorkflowSpan( - WorkflowName, - spanKind: SpanKind.Consumer, - suppliedAttributes: attributes, - traceHeader: data.TraceParentHeader); - - return state; + return attributes; } public TransformBlock GetBuildStateBlock( diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index c6694105..feb8facb 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -3,6 +3,7 @@ using RabbitMQ.Client; using System; using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; namespace HouseofCat.RabbitMQ; @@ -10,16 +11,22 @@ public interface IMessage { string MessageId { get; set; } + [JsonIgnore] string Exchange { get; set; } + [JsonIgnore] string RoutingKey { get; set; } + [JsonIgnore] byte DeliveryMode { get; set; } + [JsonIgnore] bool Mandatory { get; set; } + [JsonIgnore] byte PriorityLevel { get; set; } + [JsonIgnore] string ContentType { get; set; } ReadOnlyMemory Body { get; set; } @@ -39,19 +46,24 @@ public sealed class Message : IMessage public ReadOnlyMemory Body { get; set; } + [JsonIgnore] public string Exchange { get; set; } + [JsonIgnore] public string RoutingKey { get; set; } + [JsonIgnore] [Range(1, 2, ErrorMessage = Constants.RangeErrorMessage)] public byte DeliveryMode { get; set; } = 2; + [JsonIgnore] public bool Mandatory { get; set; } // The max-queue priority though is 10, so > 10 is treated as 10. [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] public byte PriorityLevel { get; set; } + [JsonIgnore] public string ContentType { get; set; } = Constants.HeaderValueForContentTypeJson; public Message() diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index 4c944869..e6c11a5c 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -1,9 +1,9 @@ -using HouseofCat.Serialization; +using HouseofCat.Utilities.Helpers; +using OpenTelemetry.Trace; using RabbitMQ.Client; using RabbitMQ.Client.Events; using System; using System.Text; -using System.Text.Json; using System.Threading.Tasks; namespace HouseofCat.RabbitMQ; @@ -28,6 +28,7 @@ public interface IReceivedMessage string CompressionType { get; } public string TraceParentHeader { get; } + public SpanContext? ParentSpanContext { get; } string ConsumerTag { get; } ulong DeliveryTag { get; } @@ -62,6 +63,7 @@ public sealed class ReceivedMessage : IReceivedMessage, IDisposable public string CompressionType { get; private set; } public string TraceParentHeader { get; private set; } + public SpanContext? ParentSpanContext { get; private set; } public string ConsumerTag { get; } public ulong DeliveryTag { get; } @@ -133,6 +135,10 @@ private void ReadHeaders() if (Properties.Headers.TryGetValue(Constants.HeaderForTraceParent, out object traceParentHeader)) { TraceParentHeader = Encoding.UTF8.GetString((byte[])traceParentHeader); + if (!string.IsNullOrEmpty(TraceParentHeader)) + { + ParentSpanContext = OpenTelemetryHelpers.ExtractSpanContextFromTraceHeader(TraceParentHeader); + } } } diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs index 8de3a4f6..2ff46eb1 100644 --- a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -173,6 +173,7 @@ public static string FormatOpenTelemetryHeader( if (string.IsNullOrEmpty(traceHeader)) return default; var split = traceHeader.Split('-'); + if (split.Length < 3) return default; try { From feb6e66a5f856a696f88c56508284664f47ad3d5 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 09:10:12 -0500 Subject: [PATCH 29/47] Add a flexible object convert to Consumer initial message deserialization. --- .editorconfig | 6 +++ src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 34 ++++++++++---- src/HouseofCat.RabbitMQ/Messages/Metadata.cs | 14 +++++- .../Publisher/Publisher.cs | 6 +-- .../Json/FlexibleObjectConverter.cs | 45 +++++++++++++++++++ tests/RabbitMQ.Console.Tests/Program.cs | 4 +- .../Tests/RabbitServiceTests.cs | 8 ++++ 7 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs diff --git a/.editorconfig b/.editorconfig index c7cbe0f3..2d9d8141 100644 --- a/.editorconfig +++ b/.editorconfig @@ -42,6 +42,12 @@ csharp_prefer_simple_default_expression = true:suggestion # CA1859: Use concrete types when possible for improved performance dotnet_diagnostic.CA1859.severity = silent +# IDE0008: Use explicit type +dotnet_diagnostic.IDE0008.severity = none + +# IDE0019: Use pattern matching +csharp_style_pattern_matching_over_as_with_null_check = false + [*.{cs,vb}] #### Naming styles #### diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 96651ca2..f169fc89 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -2,9 +2,9 @@ using HouseofCat.Serialization; using HouseofCat.Utilities.Errors; using HouseofCat.Utilities.Helpers; +using HouseofCat.Utilities.Json; using Microsoft.Extensions.Logging; using OpenTelemetry.Trace; -using Org.BouncyCastle.Asn1.Cms; using RabbitMQ.Client; using RabbitMQ.Client.Events; using System; @@ -49,20 +49,27 @@ public class Consumer : IConsumer, IDisposable public IChannelPool ChannelPool { get; } public bool Started { get; private set; } - public Consumer(RabbitOptions options, string consumerName) - : this(new ChannelPool(options), consumerName) + public Consumer( + RabbitOptions options, + string consumerName, + JsonSerializerOptions jsonOptions = null) + : this(new ChannelPool(options), consumerName, jsonOptions) { } - public Consumer(IChannelPool channelPool, string consumerName) - : this( - channelPool, - channelPool.Options.GetConsumerOptions(consumerName)) + public Consumer( + IChannelPool channelPool, + string consumerName, + JsonSerializerOptions jsonOptions = null) + : this(channelPool, channelPool.Options.GetConsumerOptions(consumerName), jsonOptions) { Guard.AgainstNull(channelPool, nameof(channelPool)); Guard.AgainstNullOrEmpty(consumerName, nameof(consumerName)); } - public Consumer(IChannelPool channelPool, ConsumerOptions consumerOptions) + public Consumer( + IChannelPool channelPool, + ConsumerOptions consumerOptions, + JsonSerializerOptions jsonOptions = null) { Guard.AgainstNull(channelPool, nameof(channelPool)); Guard.AgainstNull(consumerOptions, nameof(consumerOptions)); @@ -71,6 +78,13 @@ public Consumer(IChannelPool channelPool, ConsumerOptions consumerOptions) Options = channelPool.Options; ChannelPool = channelPool; ConsumerOptions = consumerOptions; + + _defaultOptions = jsonOptions ?? new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + }; + + _defaultOptions.Converters.Add(new FlexibleObjectReaderConverter()); } public async Task StartConsumerAsync() @@ -323,6 +337,8 @@ await _consumerChannel } } + protected JsonSerializerOptions _defaultOptions; + protected virtual void AutoDeserialize(ReceivedMessage receivedMessage) { if (receivedMessage.ObjectType == Constants.HeaderValueForMessageObjectType @@ -333,7 +349,7 @@ protected virtual void AutoDeserialize(ReceivedMessage receivedMessage) case Constants.HeaderValueForContentTypeJson: try { - receivedMessage.Message = JsonSerializer.Deserialize(receivedMessage.Body.Span); + receivedMessage.Message = JsonSerializer.Deserialize(receivedMessage.Body.Span, _defaultOptions); } catch { receivedMessage.FailedToDeserialize = true; } diff --git a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs index 29db2d4e..f7f5db8a 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs @@ -23,7 +23,12 @@ public bool Encrypted() { if (Fields == null) return false; - return Fields.TryGetValue(Constants.HeaderForEncrypted, out var value) && (bool)value; + if (Fields.TryGetValue(Constants.HeaderForEncrypted, out var value)) + { + return (bool)value; + } + + return false; } public string EncryptionType() @@ -67,7 +72,12 @@ public bool Compressed() { if (Fields == null) return false; - return Fields.TryGetValue(Constants.HeaderForCompressed, out var value) && (bool)value; + if (Fields.TryGetValue(Constants.HeaderForCompressed, out var value)) + { + return (bool)value; + } + + return false; } public string CompressionType() diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index bac491fc..99f52656 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -173,7 +173,7 @@ public ChannelReader GetReceiptBufferReader() public void StartAutoPublish(Func processReceiptAsync = null) { - _pubLock.Wait(); + if (!_pubLock.Wait(0)) return; try { SetupAutoPublisher(processReceiptAsync); } finally { _pubLock.Release(); } @@ -181,7 +181,7 @@ public void StartAutoPublish(Func processReceiptAsyn public async Task StartAutoPublishAsync(Func processReceiptAsync = null) { - await _pubLock.WaitAsync().ConfigureAwait(false); + if (!await _pubLock.WaitAsync(0).ConfigureAwait(false)) return; try { SetupAutoPublisher(processReceiptAsync); } finally { _pubLock.Release(); } @@ -210,7 +210,7 @@ private void SetupAutoPublisher(Func processReceiptA public async Task StopAutoPublishAsync(bool immediately = false) { - await _pubLock.WaitAsync().ConfigureAwait(false); + if (!await _pubLock.WaitAsync(0).ConfigureAwait(false)) return; try { diff --git a/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs b/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs new file mode 100644 index 00000000..9289ebc1 --- /dev/null +++ b/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs @@ -0,0 +1,45 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace HouseofCat.Utilities.Json; + +public class FlexibleObjectReaderConverter : JsonConverter +{ + public override object Read(ref Utf8JsonReader reader, Type _, JsonSerializerOptions options) + { + return reader.TokenType switch + { + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartArray => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Number => reader.TryGetInt64(out long l) ? l : reader.GetDouble(), + JsonTokenType.String => reader.TryGetDateTime(out DateTime datetime) ? datetime : reader.GetString(), + _ => GetObjectFromJsonDocument(ref reader) + }; + } + + private static object GetObjectFromJsonDocument(ref Utf8JsonReader reader) + { + using JsonDocument document = JsonDocument.ParseValue(ref reader); + return document.RootElement.Clone(); + } + + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + if (value is null) + { + writer.WriteNullValue(); + } + else if (value.GetType() == typeof(object)) + { + writer.WriteStartObject(); + writer.WriteEndObject(); + } + else + { + JsonSerializer.Serialize(writer, value, value.GetType(), options); + } + } +} diff --git a/tests/RabbitMQ.Console.Tests/Program.cs b/tests/RabbitMQ.Console.Tests/Program.cs index e7e06fbc..7af4aeb5 100644 --- a/tests/RabbitMQ.Console.Tests/Program.cs +++ b/tests/RabbitMQ.Console.Tests/Program.cs @@ -21,8 +21,8 @@ //await PubSubTests.RunPubSubCheckForDuplicateTestAsync(logger, "./RabbitMQ.PubSubTests.json"); // RabbitService Tests -//await RabbitServiceTests.RunRabbitServicePingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); -await RabbitServiceTests.RunRabbitServiceAltPingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); +await RabbitServiceTests.RunRabbitServicePingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); +//await RabbitServiceTests.RunRabbitServiceAltPingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); logger.LogInformation("Tests complete! Press return to exit...."); diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index f88e45d0..f8c39158 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -33,6 +33,10 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger } await rabbitService.DecomcryptAsync(receivedMessage.Message); + + receivedMessage.Message.Exchange = Shared.ExchangeName; + receivedMessage.Message.RoutingKey = Shared.RoutingKey; + receivedMessage.Message.Metadata.PayloadId = Guid.NewGuid().ToString(); await rabbitService.Publisher.QueueMessageAsync(receivedMessage.Message); receivedMessage.AckMessage(); } @@ -64,6 +68,10 @@ public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory log } await rabbitService.DecomcryptAsync(receivedMessage.Message); + + receivedMessage.Message.Exchange = Shared.ExchangeName; + receivedMessage.Message.RoutingKey = Shared.RoutingKey; + receivedMessage.Message.Metadata.PayloadId = Guid.NewGuid().ToString(); rabbitService.Publisher.QueueMessage(receivedMessage.Message); receivedMessage.AckMessage(); } From 89508fb7c8ad62d92ee89a9fbaf2e36305134e5a Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 09:17:10 -0500 Subject: [PATCH 30/47] Small adjustment to FlexibleObjectJsonConverter. --- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 2 +- .../Json/FlexibleObjectConverter.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index f169fc89..f3c9f388 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -84,7 +84,7 @@ public Consumer( PropertyNameCaseInsensitive = true, }; - _defaultOptions.Converters.Add(new FlexibleObjectReaderConverter()); + _defaultOptions.Converters.Add(new FlexibleObjectJsonConverter()); } public async Task StartConsumerAsync() diff --git a/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs b/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs index 9289ebc1..35c7b3d8 100644 --- a/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs +++ b/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs @@ -4,18 +4,18 @@ namespace HouseofCat.Utilities.Json; -public class FlexibleObjectReaderConverter : JsonConverter +public class FlexibleObjectJsonConverter : JsonConverter { - public override object Read(ref Utf8JsonReader reader, Type _, JsonSerializerOptions options) + public override object Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { return reader.TokenType switch { - JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, options), - JsonTokenType.StartArray => JsonSerializer.Deserialize(ref reader, options), + JsonTokenType.StartObject => JsonSerializer.Deserialize(ref reader, typeToConvert, options), + JsonTokenType.StartArray => JsonSerializer.Deserialize(ref reader, typeToConvert, options), JsonTokenType.True => true, JsonTokenType.False => false, JsonTokenType.Number => reader.TryGetInt64(out long l) ? l : reader.GetDouble(), - JsonTokenType.String => reader.TryGetDateTime(out DateTime datetime) ? datetime : reader.GetString(), + JsonTokenType.String => reader.GetString(), _ => GetObjectFromJsonDocument(ref reader) }; } From 50fdf5e4f59d38a8e6750429292e64330fb67ef3 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 09:17:37 -0500 Subject: [PATCH 31/47] Filename match Class name. --- ...{FlexibleObjectConverter.cs => FlexibleObjectJsonConverter.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/HouseofCat.Utilities/Json/{FlexibleObjectConverter.cs => FlexibleObjectJsonConverter.cs} (100%) diff --git a/src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs b/src/HouseofCat.Utilities/Json/FlexibleObjectJsonConverter.cs similarity index 100% rename from src/HouseofCat.Utilities/Json/FlexibleObjectConverter.cs rename to src/HouseofCat.Utilities/Json/FlexibleObjectJsonConverter.cs From 05e58edd791684da96303b991b5a179468bab0e1 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 09:59:52 -0500 Subject: [PATCH 32/47] Adding helper method too bootstrap TraceProvider. --- src/HouseofCat.Dataflows/BaseDataflow.cs | 10 ++--- .../Dataflows/ConsumerDataflow.cs | 4 +- .../Helpers/OpenTelemetryHelpers.cs | 44 +++++++++++++++++++ .../HouseofCat.Utilities.csproj | 2 + .../OpenTelemetry.Console.Tests.csproj | 5 --- .../Tests/RabbitServiceTests.cs | 8 +++- .../Program.cs | 12 +---- .../RabbitMQ.ConsumerDataflows.Tests.csproj | 5 --- 8 files changed, 61 insertions(+), 29 deletions(-) diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index ccfbbbb3..28ff93af 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -64,7 +64,7 @@ public TransformBlock GetWrappedTransformBlock( { TState WrapAction(TState state) { - using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { @@ -90,7 +90,7 @@ public TransformBlock GetWrappedTransformBlock( { async Task WrapActionAsync(TState state) { - using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { return await action(state).ConfigureAwait(false); @@ -115,7 +115,7 @@ public ActionBlock GetLastWrappedActionBlock( { void WrapAction(TState state) { - var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { action(state); @@ -140,7 +140,7 @@ public ActionBlock GetLastWrappedActionBlock( { void WrapAction(TState state) { - var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { action(state); @@ -165,7 +165,7 @@ public ActionBlock GetLastWrappedActionBlock( { async Task WrapActionAsync(TState state) { - var childSpan = state.CreateActiveSpan(spanName, SpanKind.Consumer); + var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { await action(state).ConfigureAwait(false); diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index bb0f7b43..1ebac366 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -624,7 +624,7 @@ public TransformBlock GetByteManipulationTransformBlock( { TState WrapAction(TState state) { - using var childSpan = state.CreateActiveChildSpan(spanName, state.WorkflowSpan.Context, SpanKind.Consumer); + using var childSpan = state.CreateActiveChildSpan(spanName, state.WorkflowSpan.Context, SpanKind.Internal); try { if (outbound) @@ -671,7 +671,7 @@ public TransformBlock GetByteManipulationTransformBlock( { async Task WrapActionAsync(TState state) { - using var childSpan = state.CreateActiveChildSpan(spanName, state.WorkflowSpan.Context, SpanKind.Consumer); + using var childSpan = state.CreateActiveChildSpan(spanName, state.WorkflowSpan.Context, SpanKind.Internal); try { diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs index 2ff46eb1..ce577ccc 100644 --- a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -1,4 +1,6 @@ using HouseofCat.Utilities.Extensions; +using OpenTelemetry; +using OpenTelemetry.Resources; using OpenTelemetry.Trace; using System; using System.Collections.Generic; @@ -30,6 +32,48 @@ public static void SetAssemblyAsSourceForTelemetry(Assembly assembly) _sourceVersion = assembly.GetSemanticVersion(); } + public static TracerProvider CreateTraceProvider( + string appName = null, + string appVersion = null, + string sourceName = null, + bool addConsoleExporter = false) + { + var builder = Sdk + .CreateTracerProviderBuilder() + .SetResourceBuilder( + ResourceBuilder + .CreateDefault() + .AddService( + appName ?? _sourceName, + serviceVersion: appVersion ?? _sourceVersion)) + .AddSource(sourceName ?? _sourceName); + + if (addConsoleExporter) + { + builder.AddConsoleExporter(); + } + + return builder.Build(); + } + + public static TracerProvider CreateTraceProvider( + ResourceBuilder resourceBuilder, + bool addConsoleExporter = false, + params string[] sourceNames) + { + var builder = Sdk + .CreateTracerProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddSource(sourceNames); + + if (addConsoleExporter) + { + builder.AddConsoleExporter(); + } + + return builder.Build(); + } + #region Span Helpers public static TelemetrySpan StartRootSpan( diff --git a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj index 164178b9..694cc473 100644 --- a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj +++ b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj @@ -5,7 +5,9 @@ + + diff --git a/tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj b/tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj index f9523c37..2c860313 100644 --- a/tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj +++ b/tests/OpenTelemetry.Console.Tests/OpenTelemetry.Console.Tests.csproj @@ -6,11 +6,6 @@ enable - - - - - diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index f8c39158..0a3b39a5 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -42,6 +42,12 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger } } + /// + /// Test for synchronous publish message queueing. + /// + /// + /// + /// public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory loggerFactory, string configFileNamePath) { var rabbitService = await Shared.SetupRabbitServiceAsync(loggerFactory, configFileNamePath); @@ -56,7 +62,7 @@ public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory log body: dataAsBytes, payloadId: Guid.NewGuid().ToString()); - await rabbitService.Publisher.QueueMessageAsync(message); + rabbitService.Publisher.QueueMessage(message); // Ping pong the same message. await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index 2d65b847..e3c6acce 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -1,8 +1,5 @@ using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; -using OpenTelemetry; -using OpenTelemetry.Resources; -using OpenTelemetry.Trace; using RabbitMQ.ConsumerDataflows.Tests; using System.Text; @@ -10,14 +7,7 @@ LogHelpers.LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger(); -var applicationName = "RabbitMQ.ConsumerDataflows.Tests"; - -using var traceProvider = Sdk - .CreateTracerProviderBuilder() - .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(applicationName, serviceVersion: "v1.0.0")) - .AddSource(applicationName) - .AddConsoleExporter() - .Build(); +using var traceProvider = OpenTelemetryHelpers.CreateTraceProvider(addConsoleExporter: true); var rabbitService = await Shared.SetupRabbitServiceAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); var dataflowService = new ConsumerDataflowService(rabbitService); diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj index 50a4a09e..60332993 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj @@ -6,11 +6,6 @@ enable - - - - - From ae0258c16b6185832145370afee7d0f3a8763d4f Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 11:29:27 -0500 Subject: [PATCH 33/47] Converting RabbitMQ ConsumerDataflow to full test application for OpenTelemetry. --- .editorconfig | 6 ++ src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 92 +++++++++---------- .../Dataflows/ConsumerDataflow.cs | 8 +- src/HouseofCat.RabbitMQ/Messages/Message.cs | 1 + src/HouseofCat.RabbitMQ/Messages/Metadata.cs | 2 +- .../Services/RabbitService.cs | 43 +++++---- .../Extensions/ServiceCollectionExtensions.cs | 62 +++++++++++++ .../HouseofCat.Utilities.csproj | 10 +- .../Tests/ConsumerTests.cs | 2 +- .../Tests/PubSubTests.cs | 4 +- .../Tests/RabbitServiceTests.cs | 6 +- .../Program.cs | 35 ++++--- .../RabbitMQ.ConsumerDataflows.Tests.csproj | 3 + .../Tests/Shared.cs | 4 +- .../appsettings.json | 12 +++ 15 files changed, 201 insertions(+), 89 deletions(-) create mode 100644 src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs create mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json diff --git a/.editorconfig b/.editorconfig index 2d9d8141..58e018f8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -48,6 +48,12 @@ dotnet_diagnostic.IDE0008.severity = none # IDE0019: Use pattern matching csharp_style_pattern_matching_over_as_with_null_check = false +# IDE0058: Expression value is never used +csharp_style_unused_value_expression_statement_preference = unused_local_variable + +# CA1806: Do not ignore method results +dotnet_diagnostic.CA1806.severity = none + [*.{cs,vb}] #### Naming styles #### diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index f3c9f388..1800a274 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -9,6 +9,7 @@ using RabbitMQ.Client.Events; using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Channels; @@ -26,10 +27,11 @@ public interface IConsumer ChannelReader GetConsumerBuffer(); ValueTask ReadAsync(); - Task> ReadUntilEmptyAsync(); Task StartConsumerAsync(); Task StopConsumerAsync(bool immediate = false); - IAsyncEnumerable StreamUntilConsumerStopAsync(); + + Task> ReadUntilEmptyAsync(CancellationToken token = default); + IAsyncEnumerable ReadUntilStopAsync(CancellationToken token = default); } public class Consumer : IConsumer, IDisposable @@ -123,7 +125,7 @@ public async Task StartConsumerAsync() public async Task StopConsumerAsync(bool immediate = false) { - await _conLock.WaitAsync(); + if (!await _conLock.WaitAsync(0).ConfigureAwait(false)) return; _logger.LogDebug(Consumers.StopConsumer, ConsumerOptions.ConsumerName); @@ -267,21 +269,20 @@ protected virtual async void ReceiveHandler(object _, BasicDeliverEventArgs bdea protected async void ConsumerShutdown(object sender, ShutdownEventArgs e) { - if (await _conLock.WaitAsync(0)) + if (!await _conLock.WaitAsync(0).ConfigureAwait(false)) return; + + try { - try + if (!_shutdown) { - if (!_shutdown) - { - await HandleRecoverableShutdownAsync(e) - .ConfigureAwait(false); - } - else - { _chanHost.StopConsuming(); } + await HandleRecoverableShutdownAsync(e) + .ConfigureAwait(false); } - finally - { _conLock.Release(); } + else + { _chanHost.StopConsuming(); } } + finally + { _conLock.Release(); } } protected AsyncEventingBasicConsumer CreateAsyncConsumer() @@ -372,27 +373,26 @@ protected virtual void AutoDeserialize(ReceivedMessage receivedMessage) protected async Task ConsumerShutdownAsync(object sender, ShutdownEventArgs e) { - if (await _conLock.WaitAsync(0)) + if (!await _conLock.WaitAsync(0).ConfigureAwait(false)) return; + + try { - try + if (!_shutdown) { - if (!_shutdown) - { - _logger.LogInformation( - Consumers.ConsumerShutdownEvent, - ConsumerOptions.ConsumerName, - _chanHost.ChannelId, - e.ReplyText); - - await HandleRecoverableShutdownAsync(e) - .ConfigureAwait(false); - } - else - { _chanHost.StopConsuming(); } + _logger.LogInformation( + Consumers.ConsumerShutdownEvent, + ConsumerOptions.ConsumerName, + _chanHost.ChannelId, + e.ReplyText); + + await HandleRecoverableShutdownAsync(e) + .ConfigureAwait(false); } - finally - { _conLock.Release(); } + else + { _chanHost.StopConsuming(); } } + finally + { _conLock.Release(); } } protected static readonly string _consumerShutdownExceptionMessage = "Consumer's ChannelHost {0} had an unhandled exception during recovery."; @@ -430,12 +430,12 @@ public async ValueTask ReadAsync() .ConfigureAwait(false); } - public async Task> ReadUntilEmptyAsync() + public async Task> ReadUntilEmptyAsync(CancellationToken token = default) { - if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); + if (!await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); var list = new List(); - await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false); + await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false); while (_consumerChannel.Reader.TryRead(out var message)) { if (message == null) { break; } @@ -445,11 +445,12 @@ public async Task> ReadUntilEmptyAsync() return list; } - public async IAsyncEnumerable StreamUntilConsumerStopAsync() + public async IAsyncEnumerable ReadUntilStopAsync( + [EnumeratorCancellation]CancellationToken token = default) { - if (!await _consumerChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); + if (!await _consumerChannel.Reader.WaitToReadAsync(token).ConfigureAwait(false)) throw new InvalidOperationException(ExceptionMessages.ChannelReadErrorMessage); - await foreach (var receivedMessage in _consumerChannel.Reader.ReadAllAsync()) + await foreach (var receivedMessage in _consumerChannel.Reader.ReadAllAsync(token)) { yield return receivedMessage; } @@ -457,17 +458,16 @@ public async IAsyncEnumerable StreamUntilConsumerStopAsync() protected virtual void Dispose(bool disposing) { - if (!_disposedValue) - { - if (disposing) - { - _conLock.Dispose(); - } + if (_disposedValue) return; - _consumerChannel = null; - _chanHost = null; - _disposedValue = true; + if (disposing) + { + _conLock.Dispose(); } + + _consumerChannel = null; + _chanHost = null; + _disposedValue = true; } public void Dispose() diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 1ebac366..208e5dbe 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -73,7 +73,9 @@ public ConsumerDataflow( _executeStepOptions = new ExecutionDataflowBlockOptions { - MaxDegreeOfParallelism = _consumerOptions.WorkflowMaxDegreesOfParallelism, + MaxDegreeOfParallelism = _consumerOptions.WorkflowMaxDegreesOfParallelism < 1 + ? 1 + : _consumerOptions.WorkflowMaxDegreesOfParallelism, SingleProducerConstrained = true, EnsureOrdered = _consumerOptions.WorkflowEnsureOrdered, TaskScheduler = _taskScheduler, @@ -114,7 +116,9 @@ public ConsumerDataflow( _executeStepOptions = new ExecutionDataflowBlockOptions { - MaxDegreeOfParallelism = maxDoP, + MaxDegreeOfParallelism = maxDoP < 1 + ? 1 + : maxDoP, SingleProducerConstrained = true, EnsureOrdered = ensureOrdered, TaskScheduler = _taskScheduler, diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index feb8facb..224669b6 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -60,6 +60,7 @@ public sealed class Message : IMessage public bool Mandatory { get; set; } // The max-queue priority though is 10, so > 10 is treated as 10. + [JsonIgnore] [Range(0, 10, ErrorMessage = Constants.RangeErrorMessage)] public byte PriorityLevel { get; set; } diff --git a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs index f7f5db8a..0b8583a6 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Metadata.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Metadata.cs @@ -5,7 +5,7 @@ namespace HouseofCat.RabbitMQ; public interface IMetadata { - string PayloadId { get; } + string PayloadId { get; set; } Dictionary Fields { get; set; } } diff --git a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs index d4b17073..4f8902f9 100644 --- a/src/HouseofCat.RabbitMQ/Services/RabbitService.cs +++ b/src/HouseofCat.RabbitMQ/Services/RabbitService.cs @@ -9,6 +9,7 @@ using HouseofCat.Utilities.Errors; using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; +using OpenTelemetry.Trace; using RabbitMQ.Client; using System; using System.Collections.Concurrent; @@ -139,7 +140,7 @@ public RabbitService( public async ValueTask ShutdownAsync(bool immediately) { - await _serviceLock.WaitAsync().ConfigureAwait(false); + if (await _serviceLock.WaitAsync(0).ConfigureAwait(false)) return; try { @@ -321,15 +322,13 @@ public async Task DecompressAsync(IMessage message) public async Task> GetAsync(string queueName) { - IChannelHost chanHost; + using var span = OpenTelemetryHelpers.StartActiveSpan( + nameof(GetAsync), + SpanKind.Consumer); - try - { - chanHost = await ChannelPool - .GetChannelAsync() - .ConfigureAwait(false); - } - catch { return default; } + var chanHost = await ChannelPool + .GetChannelAsync() + .ConfigureAwait(false); var error = false; try @@ -340,7 +339,11 @@ public async Task> GetAsync(string queueName) return result.Body; } - catch { error = true; } + catch (Exception ex) + { + OpenTelemetryHelpers.SetSpanAsError(span, ex); + error = true; + } finally { await ChannelPool @@ -353,15 +356,13 @@ await ChannelPool public async Task GetAsync(string queueName) { - IChannelHost chanHost; + using var span = OpenTelemetryHelpers.StartActiveSpan( + nameof(GetAsync), + SpanKind.Consumer); - try - { - chanHost = await ChannelPool - .GetChannelAsync() - .ConfigureAwait(false); - } - catch { return default; } + var chanHost = await ChannelPool + .GetChannelAsync() + .ConfigureAwait(false); BasicGetResult result = null; var error = false; @@ -371,7 +372,11 @@ public async Task GetAsync(string queueName) .Channel .BasicGet(queueName, true); } - catch { error = true; } + catch (Exception ex) + { + OpenTelemetryHelpers.SetSpanAsError(span, ex); + error = true; + } finally { await ChannelPool diff --git a/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..4475fbf7 --- /dev/null +++ b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,62 @@ +using HouseofCat.Utilities.Errors; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Exporter; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using System; +using System.Reflection; + +namespace HouseofCat.Utilities.Extensions; + +public static class ServiceCollectionExtensions +{ + public static void AddOpenTelemetryExporter( + this IServiceCollection services, + IConfiguration config) + { + var assembly = Assembly.GetEntryAssembly(); + var sourceName = assembly.GetName().Name; + var sourceVersion = assembly.GetSemanticVersion(); + + bool.TryParse(config["OpenTelemetry:Enabled"] ?? "false", out var enabled); + if (!enabled) return; + + var otlpServiceName = config["OpenTelemetry:ServiceName"] ?? sourceName; + var otlpServiceNamespace = config["OpenTelemetry:ServiceNamespace"]; + var otlpServiceVersion = config["OpenTelemetry:ServiceVersion"] ?? sourceVersion; + + var otlpEndpoint = config["OpenTelemetry:EndpointUrl"]; + + var otlpHeaderFormat = config["OpenTelemetry:HeaderFormat"] ?? "{0}={1}"; + var otlpHeaderKey = config["OpenTelemetry:HeaderKey"]; + var otlpApiKey = config["OpenTelemetry:ApiKey"]; + + Guard.AgainstNull(otlpEndpoint, nameof(otlpEndpoint)); + + var otlpBuilder = services.AddOpenTelemetry() + .ConfigureResource( + resource => resource.AddService( + serviceName: otlpServiceName, + serviceNamespace: otlpServiceNamespace, + serviceVersion: otlpServiceVersion)); + + otlpBuilder + .WithTracing( + tracing => tracing + .AddAspNetCoreInstrumentation() + .AddOtlpExporter( + otlpOptions => + { + otlpOptions.Endpoint = new Uri(otlpEndpoint); + otlpOptions.Protocol = OtlpExportProtocol.Grpc; + otlpOptions.Headers = string.Format(otlpHeaderFormat, otlpHeaderKey, otlpApiKey); + }) +#if DEBUG + .AddSource(sourceName) + .AddConsoleExporter()); +#else + .AddSource(sourceName)); +#endif + } +} diff --git a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj index 694cc473..676d00a3 100644 --- a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj +++ b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj @@ -5,9 +5,13 @@ - - - + + + + + + + diff --git a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs index c843f10d..a6fbcdc8 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/ConsumerTests.cs @@ -15,7 +15,7 @@ public static async Task RunConsumerTestAsync(ILogger logger, string configFileN { await consumer.StartConsumerAsync(); - await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.ReadUntilStopAsync()) { logger.LogInformation("Received message: [{message}]", Encoding.UTF8.GetString(receivedMessage.Body.Span)); receivedMessage.AckMessage(); diff --git a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs index 26fcc757..df82148d 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/PubSubTests.cs @@ -66,7 +66,7 @@ private static async Task StartConsumerAsync(ILogger logger, IChannelPool channe { await consumer.StartConsumerAsync(); - await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.ReadUntilStopAsync()) { try { @@ -154,7 +154,7 @@ private static async Task VerifyNoDuplicatesInQueueAsync(ILogger logger, IChanne { await consumer.StartConsumerAsync(); - await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.ReadUntilStopAsync()) { var message = JsonSerializer.Deserialize(receivedMessage.Body.Span); var number = Encoding.UTF8.GetString(message.Body.Span); diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index 0a3b39a5..7c230015 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -24,7 +24,7 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger await rabbitService.Publisher.QueueMessageAsync(message); // Ping pong the same message. - await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.ReadUntilStopAsync()) { if (receivedMessage?.Message is null) { @@ -43,7 +43,7 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger } /// - /// Test for synchronous publish message queueing. + /// Test for sync publish message queueing. /// /// /// @@ -65,7 +65,7 @@ public static async Task RunRabbitServiceAltPingPongTestAsync(ILoggerFactory log rabbitService.Publisher.QueueMessage(message); // Ping pong the same message. - await foreach (var receivedMessage in consumer.StreamUntilConsumerStopAsync()) + await foreach (var receivedMessage in consumer.ReadUntilStopAsync()) { if (receivedMessage?.Message is null) { diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index e3c6acce..cc22c151 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -1,4 +1,7 @@ -using HouseofCat.Utilities.Helpers; +using HouseofCat.Utilities.Extensions; +using HouseofCat.Utilities.Helpers; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using RabbitMQ.ConsumerDataflows.Tests; using System.Text; @@ -7,7 +10,15 @@ LogHelpers.LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger(); -using var traceProvider = OpenTelemetryHelpers.CreateTraceProvider(addConsoleExporter: true); +var builder = WebApplication.CreateBuilder(args); +var configuration = new ConfigurationBuilder() + .SetBasePath(AppContext.BaseDirectory) + .AddJsonFile("appsettings.json") + .Build(); + +builder.Services.AddOpenTelemetryExporter(configuration); + +using var app = builder.Build(); var rabbitService = await Shared.SetupRabbitServiceAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); var dataflowService = new ConsumerDataflowService(rabbitService); @@ -38,18 +49,20 @@ await dataflowService.StartAsync(); -logger.LogInformation("Listening for Messages! Press Return to stop consumer..."); - -Console.ReadLine(); +logger.LogInformation("Listening for Messages! Press CTRL+C to initiate graceful shutdown and stop consumer..."); -logger.LogInformation("ConsumerService stopping..."); +app.Lifetime.ApplicationStopping.Register( + async () => + { + logger.LogInformation("ConsumerService stopping..."); -await dataflowService.StopAsync(); + await dataflowService.StopAsync(); -logger.LogInformation("RabbitMQ AutoPublish stopping..."); + logger.LogInformation("RabbitMQ AutoPublish stopping..."); -await rabbitService.Publisher.StopAutoPublishAsync(); + await rabbitService.Publisher.StopAutoPublishAsync(); -logger.LogInformation("All stopped! Press return to exit..."); + logger.LogInformation("All stopped! Press return to exit..."); + }); -Console.ReadLine(); +await app.RunAsync(); \ No newline at end of file diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj index 60332993..7276ce58 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj @@ -13,6 +13,9 @@ + + Always + Always diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs index 03dfc0f3..f4731a56 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs @@ -56,7 +56,9 @@ public static async Task SetupTestsAsync(ILogger logger, string co public static readonly string EncryptionSalt = "SaltySaltSalt"; public static readonly int KeySize = 32; - public static async Task SetupRabbitServiceAsync(ILoggerFactory loggerFactory, string configFileNamePath) + public static async Task SetupRabbitServiceAsync( + ILoggerFactory loggerFactory, + string configFileNamePath) { var rabbitOptions = await RabbitExtensions.GetRabbitOptionsFromJsonFileAsync(configFileNamePath); var jsonProvider = new JsonProvider(); diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json b/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json new file mode 100644 index 00000000..2a404744 --- /dev/null +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json @@ -0,0 +1,12 @@ +{ + "OpenTelemetry": { + "Enabled": true, + "ServiceName": "RabbitMQ.ConsumerDataflows.Tests", + "ServiceVersion": "v1.0.0", + "ServiceNamespace": "HoC", + "EndpointUrl": "https://ingest.us.signoz.cloud:443", + "HeaderFormat": "{0}={1}", + "HeaderKey": "signoz-access-token", + "ApiKey": "*" + } +} From d317dbc66d62ea73f8bbac632a9ab7c2256b618c Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 14:27:03 -0500 Subject: [PATCH 34/47] Telemetry adjustments. --- src/HouseofCat.Dataflows/BaseDataflow.cs | 12 ++++++------ .../Extensions/WorkStateExtensions.cs | 9 ++++++--- src/HouseofCat.RabbitMQ/Constants.cs | 1 + src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 4 ++++ .../Dataflows/ConsumerDataflow.cs | 19 ++++++++++--------- .../Messages/ReceivedMessage.cs | 4 ++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index 28ff93af..88a87dc2 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -115,7 +115,7 @@ public ActionBlock GetLastWrappedActionBlock( { void WrapAction(TState state) { - var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { action(state); @@ -126,7 +126,7 @@ void WrapAction(TState state) childSpan?.RecordException(ex); } - childSpan?.Dispose(); + childSpan?.End(); state.EndRootSpan(); } @@ -140,7 +140,7 @@ public ActionBlock GetLastWrappedActionBlock( { void WrapAction(TState state) { - var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { action(state); @@ -151,7 +151,7 @@ void WrapAction(TState state) childSpan?.RecordException(ex); } - childSpan?.Dispose(); + childSpan?.End(); state.EndRootSpan(); } @@ -165,7 +165,7 @@ public ActionBlock GetLastWrappedActionBlock( { async Task WrapActionAsync(TState state) { - var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); + using var childSpan = state.CreateActiveSpan(spanName, SpanKind.Internal); try { await action(state).ConfigureAwait(false); @@ -176,7 +176,7 @@ async Task WrapActionAsync(TState state) childSpan?.RecordException(ex); } - childSpan?.Dispose(); + childSpan?.End(); state.EndRootSpan(); } diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 146b85b7..48f078f7 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -43,10 +43,11 @@ public static void SetSpanAsError(this IWorkState state, TelemetrySpan span, str } } - public static string DefaultSpanNameFormat { get; set; } = "{0}.workflow"; - public static string DefaultChildSpanNameFormat { get; set; } = "{0}.{1}.workflow.step"; + public static string DefaultSpanNameFormat { get; set; } = "{0}"; + public static string DefaultChildSpanNameFormat { get; set; } = "{0}.{1}"; public static string DefaultWorkflowNameKey { get; set; } = "hoc.workflow.name"; - public static string DefaultWorkflowVersionKey { get; set; } = "hoc.workflow.version"; + + public static string DefaultWorkflowStepIdKey { get; set; } = "hoc.workflow.step.id"; /// /// Start a new RootSpan/ChildSpan with respect to WorkState. @@ -121,6 +122,8 @@ public static TelemetrySpan CreateActiveSpan( { attributes.Add(kvp.Key, kvp.Value); } + + attributes.Add(DefaultWorkflowStepIdKey, spanName); } return OpenTelemetryHelpers diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index 3412034b..fe6906af 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -34,6 +34,7 @@ public static class Constants public static string MessagingOperationProcessValue { get; set; } = "process"; public static string MessagingDestinationNameKey { get; set; } = "messaging.destination.name"; + public static string MessagingConsumerNameKey { get; set; } = "messaging.rabbitmq.consumer.name"; public static string MessagingMessageMessageIdKey { get; set; } = "messaging.message.id"; public static string MessagingMessageRoutingKeyKey { get; set; } = "messaging.rabbitmq.message.routing_key"; diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 1800a274..909a6294 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -320,12 +320,16 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs SpanKind.Consumer, receivedMessage.ParentSpanContext ?? default); + receivedMessage.ParentSpanContext = span.Context; + AutoDeserialize(receivedMessage); await _consumerChannel .Writer .WriteAsync(receivedMessage) .ConfigureAwait(false); + + span.End(); return true; } catch (Exception ex) diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 208e5dbe..64cbe4bb 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -585,7 +585,7 @@ protected virtual List> GetSpanAttributes(TState st { var attributes = new List>() { - KeyValuePair.Create(nameof(_consumerOptions.ConsumerName), _consumerOptions.ConsumerName), + KeyValuePair.Create(Constants.MessagingConsumerNameKey, _consumerOptions.ConsumerName), KeyValuePair.Create(Constants.MessagingOperationKey, Constants.MessagingOperationProcessValue) }; @@ -650,8 +650,6 @@ TState WrapAction(TState state) else { state.ReceivedMessage.Body = action(state.ReceivedMessage.Body); } } - - return state; } catch (Exception ex) { @@ -659,8 +657,10 @@ TState WrapAction(TState state) childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); - return state; } + + childSpan?.End(); + return state; } return new TransformBlock(WrapAction, options); @@ -698,7 +698,6 @@ async Task WrapActionAsync(TState state) else { state.ReceivedMessage.Body = await action(state.ReceivedMessage.Body).ConfigureAwait(false); } } - return state; } catch (Exception ex) { @@ -706,8 +705,10 @@ async Task WrapActionAsync(TState state) childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); - return state; } + + childSpan?.End(); + return state; } return new TransformBlock(WrapActionAsync, options); @@ -725,8 +726,6 @@ async Task WrapPublishAsync(TState state) { await service.Publisher.PublishAsync(state.SendMessage, true, true).ConfigureAwait(false); state.SendMessageSent = true; - - return state; } catch (Exception ex) { @@ -734,8 +733,10 @@ async Task WrapPublishAsync(TState state) childSpan?.RecordException(ex); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); - return state; } + + childSpan?.End(); + return state; } return new TransformBlock(WrapPublishAsync, options); diff --git a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs index e6c11a5c..7cf84e61 100644 --- a/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs +++ b/src/HouseofCat.RabbitMQ/Messages/ReceivedMessage.cs @@ -28,7 +28,7 @@ public interface IReceivedMessage string CompressionType { get; } public string TraceParentHeader { get; } - public SpanContext? ParentSpanContext { get; } + public SpanContext? ParentSpanContext { get; set; } string ConsumerTag { get; } ulong DeliveryTag { get; } @@ -63,7 +63,7 @@ public sealed class ReceivedMessage : IReceivedMessage, IDisposable public string CompressionType { get; private set; } public string TraceParentHeader { get; private set; } - public SpanContext? ParentSpanContext { get; private set; } + public SpanContext? ParentSpanContext { get; set; } public string ConsumerTag { get; } public ulong DeliveryTag { get; } From d0021f8828cbde3fe4b1cc8e75444a12d73414b1 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 16:53:05 -0500 Subject: [PATCH 35/47] Integrating Publish and Consuming telemetry. --- RabbitMQ.Dataflows.sln | 2 +- src/HouseofCat.Dataflows/BaseDataflow.cs | 6 +- .../Extensions/WorkStateExtensions.cs | 22 ++--- src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 47 +++++++++- .../Dataflows/ConsumerBlock.cs | 9 +- .../Dataflows/ConsumerDataflow.cs | 35 ++++--- src/HouseofCat.RabbitMQ/Messages/Message.cs | 5 + .../Options/RabbitOptions.cs | 7 +- .../Publisher/Publisher.cs | 18 +++- .../Services/ConsumerDataflowService.cs | 93 +++++++++++++++++++ .../Extensions/ServiceCollectionExtensions.cs | 2 +- .../Helpers/OpenTelemetryHelpers.cs | 13 ++- .../Program.cs | 38 +++++++- ...> RabbitMQ.ConsumerDataflowService.csproj} | 2 +- .../RabbitMQ.ConsumerDataflows.json | 2 +- .../Tests/ConsumerDataflowOptions.cs | 27 ------ .../Tests/ConsumerDataflowService.cs | 92 ------------------ .../Tests/CustomWorkState.cs | 2 +- .../Tests/Shared.cs | 6 +- 19 files changed, 255 insertions(+), 173 deletions(-) create mode 100644 src/HouseofCat.RabbitMQ/Services/ConsumerDataflowService.cs rename tests/RabbitMQ.ConsumerDataflows.Tests/{RabbitMQ.ConsumerDataflows.Tests.csproj => RabbitMQ.ConsumerDataflowService.csproj} (93%) delete mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs delete mode 100644 tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs diff --git a/RabbitMQ.Dataflows.sln b/RabbitMQ.Dataflows.sln index dc0ea98c..62aed2c6 100644 --- a/RabbitMQ.Dataflows.sln +++ b/RabbitMQ.Dataflows.sln @@ -86,7 +86,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "rabbitmq", "rabbitmq", "{5C EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Console.Tests", "tests\OpenTelemetry.Console.Tests\OpenTelemetry.Console.Tests.csproj", "{077E07C3-9A35-42A0-8228-E9778F02DFCE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RabbitMQ.ConsumerDataflows.Tests", "tests\RabbitMQ.ConsumerDataflows.Tests\RabbitMQ.ConsumerDataflows.Tests.csproj", "{F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RabbitMQ.ConsumerDataflowService", "tests\RabbitMQ.ConsumerDataflows.Tests\RabbitMQ.ConsumerDataflowService.csproj", "{F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index 88a87dc2..74a8ba4f 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -27,7 +27,11 @@ protected void SetCurrentSourceBlock(IDataflowBlock block) _currentBlock = (ISourceBlock)block; } - protected ExecutionDataflowBlockOptions GetExecuteStepOptions(int? maxDoP, bool? ensureOrdered, int? boundedCapacity, TaskScheduler taskScheduler = null) + protected ExecutionDataflowBlockOptions GetExecuteStepOptions( + int? maxDoP, + bool? ensureOrdered, + int? boundedCapacity, + TaskScheduler taskScheduler = null) { if (maxDoP.HasValue || ensureOrdered.HasValue || boundedCapacity.HasValue) { diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 48f078f7..1494bd8a 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -8,11 +8,8 @@ public static class WorkStateExtensions { public static void SetOpenTelemetryError(this IWorkState state, string message = null) { - if (state is null) return; - state.SetCurrentActivityAsError(message); - - if (state.WorkflowSpan is null) return; - state.SetCurrentSpanAsError(message); + if (state.WorkflowSpan is null || !state.WorkflowSpan.IsRecording) return; + state.SetSpanAsError(state.WorkflowSpan, message); } public static void SetCurrentActivityAsError(this IWorkState state, string message = null) @@ -91,12 +88,14 @@ public static void StartWorkflowSpan( parentSpanContext.Value, attributes: attributes); } - - state.WorkflowSpan = OpenTelemetryHelpers - .StartRootSpan( - spanName, - spanKind, - attributes: attributes); + else + { + state.WorkflowSpan = OpenTelemetryHelpers + .StartRootSpan( + spanName, + spanKind, + attributes: attributes); + } } /// @@ -173,6 +172,7 @@ public static void EndRootSpan( { state.SetOpenTelemetryError(); } + state.WorkflowSpan?.End(); state.WorkflowSpan?.Dispose(); } } diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 909a6294..21863c2c 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -308,6 +308,8 @@ protected virtual async Task ReceiveHandlerAsync(object _, BasicDeliverEventArgs await HandleMessageAsync(bdea).ConfigureAwait(false); } + private static readonly string _consumerSpanNameFormat = "{0}.{1}"; + protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs bdea) { if (!await _consumerChannel.Writer.WaitToWriteAsync().ConfigureAwait(false)) return false; @@ -316,11 +318,13 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs { var receivedMessage = new ReceivedMessage(_chanHost.Channel, bdea, !ConsumerOptions.AutoAck); using var span = OpenTelemetryHelpers.StartActiveSpan( - nameof(HandleMessageAsync), + string.Format(_consumerSpanNameFormat, ConsumerOptions.ConsumerName, nameof(HandleMessageAsync)), SpanKind.Consumer, receivedMessage.ParentSpanContext ?? default); - receivedMessage.ParentSpanContext = span.Context; + EnrichSpanWithTags(span, receivedMessage); + + receivedMessage.ParentSpanContext = span?.Context; AutoDeserialize(receivedMessage); @@ -342,6 +346,45 @@ await _consumerChannel } } + protected void EnrichSpanWithTags(TelemetrySpan span, IReceivedMessage receivedMessage) + { + if (span == null || !span.IsRecording) return; + + span.SetAttribute(Constants.MessagingSystemKey, Constants.MessagingSystemValue); + + if (!string.IsNullOrEmpty(receivedMessage?.Message?.MessageId)) + { + span.SetAttribute(Constants.MessagingMessageMessageIdKey, receivedMessage?.Message.MessageId); + } + if (!string.IsNullOrEmpty(ConsumerOptions.ConsumerName)) + { + span.SetAttribute(Constants.MessagingConsumerNameKey, ConsumerOptions.ConsumerName); + } + if (!string.IsNullOrEmpty(ConsumerOptions.QueueName)) + { + span.SetAttribute(Constants.MessagingMessageRoutingKeyKey, ConsumerOptions.QueueName); + } + + if (!string.IsNullOrEmpty(receivedMessage?.Message?.Metadata?.PayloadId)) + { + span.SetAttribute(Constants.MessagingMessagePayloadIdKey, receivedMessage?.Message?.Metadata?.PayloadId); + } + + var encrypted = receivedMessage?.Message?.Metadata?.Encrypted(); + if (encrypted.HasValue && encrypted.Value) + { + span.SetAttribute(Constants.MessagingMessageEncryptedKey, "true"); + span.SetAttribute(Constants.MessagingMessageEncryptedDateKey, receivedMessage?.Message?.Metadata?.EncryptedDate()); + span.SetAttribute(Constants.MessagingMessageEncryptionKey, receivedMessage?.Message?.Metadata?.EncryptionType()); + } + var compressed = receivedMessage?.Message?.Metadata?.Compressed(); + if (compressed.HasValue && compressed.Value) + { + span.SetAttribute(Constants.MessagingMessageCompressedKey, "true"); + span.SetAttribute(Constants.MessagingMessageCompressionKey, receivedMessage?.Message?.Metadata?.CompressionType()); + } + } + protected JsonSerializerOptions _defaultOptions; protected virtual void AutoDeserialize(ReceivedMessage receivedMessage) diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs index aea2aff4..5f36725e 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerBlock.cs @@ -95,15 +95,14 @@ public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock WithReadyToProcessBuffer(int boundedCapacity, Ta return this; } + protected static readonly string _defaultSpanNameFormat = "{0}.{1}"; + protected static readonly string _defaultStepSpanNameFormat = "{0}.{1}.{2}"; + + protected string GetSpanName(string stepName) + { + return string.Format(_defaultSpanNameFormat, WorkflowName, stepName); + } + + protected string GetStepSpanName(string stepName) + { + return string.Format(_defaultStepSpanNameFormat, WorkflowName, _suppliedTransforms.Count, stepName); + } + public ConsumerDataflow AddStep( Func suppliedStep, - string spanName, + string stepName, int? maxDoP = null, bool? ensureOrdered = null, int? boundedCapacity = null, @@ -251,13 +264,13 @@ public ConsumerDataflow AddStep( Guard.AgainstNull(suppliedStep, nameof(suppliedStep)); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, spanName)); + _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, GetStepSpanName(stepName))); return this; } public ConsumerDataflow AddStep( Func> suppliedStep, - string spanName, + string stepName, int? maxDoP = null, bool? ensureOrdered = null, int? boundedCapacity = null, @@ -266,7 +279,7 @@ public ConsumerDataflow AddStep( Guard.AgainstNull(suppliedStep, nameof(suppliedStep)); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, spanName)); + _suppliedTransforms.Add(GetWrappedTransformBlock(suppliedStep, executionOptions, GetStepSpanName(stepName))); return this; } @@ -288,7 +301,7 @@ public ConsumerDataflow WithFinalization( if (_finalization == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.Finalization"); + _finalization = GetLastWrappedActionBlock(action, executionOptions, GetSpanName("finalization")); } return this; } @@ -304,7 +317,7 @@ public ConsumerDataflow WithFinalization( if (_finalization == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _finalization = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.Finalization"); + _finalization = GetLastWrappedActionBlock(action, executionOptions, GetSpanName("finalization")); } return this; } @@ -339,7 +352,7 @@ public ConsumerDataflow WithDecryptionStep( executionOptions, false, x => x.ReceivedMessage.Encrypted, - $"{WorkflowName}_Decrypt"); + GetSpanName("decrypt")); } return this; } @@ -360,7 +373,7 @@ public ConsumerDataflow WithDecompressionStep( executionOptions, false, x => x.ReceivedMessage.Compressed, - $"{WorkflowName}_Decompress"); + GetSpanName("decompress")); } return this; @@ -376,7 +389,7 @@ public ConsumerDataflow WithCreateSendMessage( if (_createSendMessage == null) { var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _createSendMessage = GetWrappedTransformBlock(createMessage, executionOptions, $"{WorkflowName}_CreateSendMessage"); + _createSendMessage = GetWrappedTransformBlock(createMessage, executionOptions, GetSpanName("create_send_message")); } return this; } @@ -397,7 +410,7 @@ public ConsumerDataflow WithCompression( executionOptions, true, x => !x.ReceivedMessage.Compressed, - $"{WorkflowName}_Compress"); + GetSpanName("compress")); } return this; } @@ -418,7 +431,7 @@ public ConsumerDataflow WithEncryption( executionOptions, true, x => !x.ReceivedMessage.Encrypted, - $"{WorkflowName}_Encrypt"); + GetSpanName("encrypt")); } return this; } diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index 224669b6..2b36a0ba 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -1,5 +1,6 @@ using HouseofCat.RabbitMQ.Pools; using HouseofCat.Utilities.Helpers; +using OpenTelemetry.Trace; using RabbitMQ.Client; using System; using System.ComponentModel.DataAnnotations; @@ -36,6 +37,8 @@ public interface IMessage IPublishReceipt GetPublishReceipt(bool error); IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders, string contentType); + + public SpanContext? ParentSpanContext { get; set; } } public sealed class Message : IMessage @@ -67,6 +70,8 @@ public sealed class Message : IMessage [JsonIgnore] public string ContentType { get; set; } = Constants.HeaderValueForContentTypeJson; + public SpanContext? ParentSpanContext { get; set; } + public Message() { MessageId ??= Guid.NewGuid().ToString(); diff --git a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs index 7978f064..0e375198 100644 --- a/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs +++ b/src/HouseofCat.RabbitMQ/Options/RabbitOptions.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Globalization; namespace HouseofCat.RabbitMQ; @@ -23,10 +22,10 @@ public class RabbitOptions public ConsumerOptions GetConsumerOptions(string consumerName) { - if (!ConsumerOptions.TryGetValue(consumerName, out ConsumerOptions value)) + if (ConsumerOptions.TryGetValue(consumerName, out ConsumerOptions value)) { - throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerOptionsMessage, consumerName)); + return value; } - return value; + throw new ArgumentException(string.Format(ExceptionMessages.NoConsumerOptionsMessage, consumerName)); } } diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index 99f52656..c0b94c0c 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -564,6 +564,8 @@ await _channelPool return error; } + private static readonly string _defaultSpanName = "messaging.rabbitmq.publisher"; + /// /// Acquires a channel from the channel pool, then publishes message based on the message parameters. /// Only throws exception when failing to acquire channel or when creating a receipt after the ReceiptBuffer is closed. @@ -573,7 +575,11 @@ await _channelPool /// public async Task PublishAsync(IMessage message, bool createReceipt, bool withOptionalHeaders = true) { - using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishAsync), SpanKind.Producer); + using var span = OpenTelemetryHelpers.StartActiveSpan( + _defaultSpanName, + SpanKind.Producer, + message.ParentSpanContext ?? default); + message.EnrichSpanWithTags(span); var error = false; @@ -614,6 +620,8 @@ await CreateReceiptAsync(message, error) await _channelPool .ReturnChannelAsync(chanHost, error); + + span?.End(); } } @@ -627,7 +635,11 @@ await _channelPool /// public async Task PublishWithConfirmationAsync(IMessage message, bool createReceipt, bool withOptionalHeaders = true) { - using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishWithConfirmationAsync), SpanKind.Producer); + using var span = OpenTelemetryHelpers.StartActiveSpan( + _defaultSpanName, + SpanKind.Producer, + message.ParentSpanContext ?? default); + message.EnrichSpanWithTags(span); var error = false; @@ -671,6 +683,8 @@ await CreateReceiptAsync(message, error) await _channelPool .ReturnChannelAsync(chanHost, error); + + span?.End(); } } diff --git a/src/HouseofCat.RabbitMQ/Services/ConsumerDataflowService.cs b/src/HouseofCat.RabbitMQ/Services/ConsumerDataflowService.cs new file mode 100644 index 00000000..b2bc9b99 --- /dev/null +++ b/src/HouseofCat.RabbitMQ/Services/ConsumerDataflowService.cs @@ -0,0 +1,93 @@ +using HouseofCat.RabbitMQ.Dataflows; +using System; +using System.Threading.Tasks; + +namespace HouseofCat.RabbitMQ.Services; + +public class ConsumerDataflowService where TState : class, IRabbitWorkState, new() +{ + private readonly ConsumerDataflow _workflow; + private readonly ConsumerOptions _options; + private readonly IRabbitService _rabbitService; + + public ConsumerDataflowService( + IRabbitService rabbitService, + string consumerName) + { + _rabbitService = rabbitService; + _options = _rabbitService.Options.GetConsumerOptions(consumerName); + + _workflow = new ConsumerDataflow( + _rabbitService, + _options.WorkflowName, + _options.ConsumerName, + _options.WorkflowConsumerCount) + .WithBuildState(); + } + + public void AddStep(string stepName, Func step) + { + _workflow.AddStep( + step, + stepName, + _options.WorkflowMaxDegreesOfParallelism, + _options.WorkflowEnsureOrdered, + _options.WorkflowBatchSize); + } + + public void AddStep(string stepName, Func> step) + { + _workflow.AddStep( + step, + stepName, + _options.WorkflowMaxDegreesOfParallelism, + _options.WorkflowEnsureOrdered, + _options.WorkflowBatchSize); + } + + public void AddFinalization(Action step) + { + _workflow.WithFinalization( + step, + _options.WorkflowMaxDegreesOfParallelism, + _options.WorkflowEnsureOrdered, + _options.WorkflowBatchSize); + } + + public void AddFinalization(Func step) + { + _workflow.WithFinalization( + step, + _options.WorkflowMaxDegreesOfParallelism, + _options.WorkflowEnsureOrdered, + _options.WorkflowBatchSize); + } + + public void AddErrorHandling(Action step) + { + _workflow.WithErrorHandling( + step, + _options.WorkflowBatchSize, + _options.WorkflowMaxDegreesOfParallelism, + _options.WorkflowEnsureOrdered); + } + + public void AddErrorHandling(Func step) + { + _workflow.WithErrorHandling( + step, + _options.WorkflowBatchSize, + _options.WorkflowMaxDegreesOfParallelism, + _options.WorkflowEnsureOrdered); + } + + public async Task StartAsync() + { + await _workflow.StartAsync(); + } + + public async Task StopAsync() + { + await _workflow.StopAsync(); + } +} diff --git a/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs index 4475fbf7..f543e388 100644 --- a/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs +++ b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs @@ -23,7 +23,7 @@ public static void AddOpenTelemetryExporter( if (!enabled) return; var otlpServiceName = config["OpenTelemetry:ServiceName"] ?? sourceName; - var otlpServiceNamespace = config["OpenTelemetry:ServiceNamespace"]; + var otlpServiceNamespace = config["OpenTelemetry:ServiceNamespace"] ?? "hoc"; var otlpServiceVersion = config["OpenTelemetry:ServiceVersion"] ?? sourceVersion; var otlpEndpoint = config["OpenTelemetry:EndpointUrl"]; diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs index ce577ccc..3fd98693 100644 --- a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -126,19 +126,24 @@ public static TelemetrySpan StartActiveSpan( if (provider is null) return null; - var currentSpan = Tracer.CurrentSpan; - if (parentContext == default && currentSpan != null) + if (parentContext == default) { - parentContext = currentSpan.Context; + var currentSpan = Tracer.CurrentSpan; + if (currentSpan != null) + { + parentContext = currentSpan.Context; + } } - return provider.StartActiveSpan( + var span = provider.StartActiveSpan( spanName, spanKind, initialAttributes: attributes, parentContext: parentContext, links: links, startTime: startTime); + + return span; } #endregion diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index cc22c151..e7b3d3bb 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -1,9 +1,11 @@ -using HouseofCat.Utilities.Extensions; +using HouseofCat.RabbitMQ; +using HouseofCat.RabbitMQ.Services; +using HouseofCat.Utilities.Extensions; using HouseofCat.Utilities.Helpers; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using RabbitMQ.ConsumerDataflows.Tests; +using RabbitMQ.ConsumerDataflowService; using System.Text; var loggerFactory = LogHelpers.CreateConsoleLoggerFactory(LogLevel.Information); @@ -20,17 +22,43 @@ using var app = builder.Build(); -var rabbitService = await Shared.SetupRabbitServiceAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); -var dataflowService = new ConsumerDataflowService(rabbitService); +var rabbitService = await Shared.SetupRabbitServiceAsync(loggerFactory, "RabbitMQ.ConsumerDataflows.json"); +var dataflowService = new ConsumerDataflowService(rabbitService, "TestConsumer"); dataflowService.AddStep( - "WriteToRabbitMessageToConsole", + "write_message_to_console", (state) => { Console.WriteLine(Encoding.UTF8.GetString(state.ReceivedMessage.Body.Span)); return state; }); +dataflowService.AddStep( + "create_new_message", + (state) => + { + state.SendMessage = new Message + { + Exchange = "", + RoutingKey = "TestTargetQueue", + Body = Encoding.UTF8.GetBytes("Test New Message"), + Metadata = new Metadata + { + PayloadId = Guid.NewGuid().ToString(), + }, + ParentSpanContext = state.WorkflowSpan?.Context, + }; + return state; + }); + +dataflowService.AddStep( + "queue_new_message", + async (state) => + { + await rabbitService.Publisher.QueueMessageAsync(state.SendMessage); + return state; + }); + dataflowService.AddFinalization( (state) => { diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflowService.csproj similarity index 93% rename from tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj rename to tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflowService.csproj index 7276ce58..e9ccb55c 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.Tests.csproj +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflowService.csproj @@ -16,7 +16,7 @@ Always - + Always diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json index c68f9568..4606c863 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json @@ -44,7 +44,7 @@ "BuildQueueDurable": true, "BuildQueueExclusive": false, "BuildQueueAutoDelete": false, - "WorkflowName": "TestConsumerWorkflow", + "WorkflowName": "test_workflow", "WorkflowMaxDegreesOfParallelism": 1, "WorkflowConsumerCount": 1, "WorkflowBatchSize": 5, diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs deleted file mode 100644 index 2451c5c7..00000000 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowOptions.cs +++ /dev/null @@ -1,27 +0,0 @@ -using HouseofCat.RabbitMQ; -using HouseofCat.RabbitMQ.Dataflows; -using HouseofCat.RabbitMQ.Services; - -namespace RabbitMQ.ConsumerDataflows.Tests; - -public sealed class ConsumerDataflowOptions -{ - public string WorkflowName { get; set; } - public string WorkStateKey { get; set; } = "State"; - public string ConsumerName { get; set; } - public int ConsumerCount { get; set; } = 1; - public int MaxDegreeOfParallelism { get; set; } = 1; - public bool EnsureOrdered { get; set; } = true; - public int Capacity { get; set; } = 1000; - public string ErrorQueueName { get; set; } - - public ConsumerDataflow BuildConsumerDataflow(IRabbitService rabbitService) - { - return new ConsumerDataflow( - rabbitService, - WorkflowName, - ConsumerName, - ConsumerCount) - .WithBuildState(); - } -} diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs deleted file mode 100644 index 1f4fd230..00000000 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/ConsumerDataflowService.cs +++ /dev/null @@ -1,92 +0,0 @@ -using HouseofCat.RabbitMQ.Dataflows; -using HouseofCat.RabbitMQ.Services; - -namespace RabbitMQ.ConsumerDataflows.Tests; - -public class ConsumerDataflowService -{ - private readonly ConsumerDataflowOptions _consumerWorkflowOptions; - private readonly ConsumerDataflow _workflow; - - public ConsumerDataflowService(IRabbitService rabbitService) - { - _consumerWorkflowOptions = new ConsumerDataflowOptions - { - WorkflowName = Shared.ConsumerWorkflowName, - ConsumerName = Shared.ConsumerName, - ConsumerCount = 1, - MaxDegreeOfParallelism = 1, - EnsureOrdered = true, - Capacity = 1000, - ErrorQueueName = Shared.ErrorQueue - }; - - _workflow = _consumerWorkflowOptions.BuildConsumerDataflow(rabbitService); - } - - public void AddStep(string stepName, Func step) - { - _workflow.AddStep( - step, - stepName, - _consumerWorkflowOptions.MaxDegreeOfParallelism, - _consumerWorkflowOptions.EnsureOrdered, - _consumerWorkflowOptions.Capacity); - } - - public void AddStep(string stepName, Func> step) - { - _workflow.AddStep( - step, - stepName, - _consumerWorkflowOptions.MaxDegreeOfParallelism, - _consumerWorkflowOptions.EnsureOrdered, - _consumerWorkflowOptions.Capacity); - } - - public void AddFinalization(Action step) - { - _workflow.WithFinalization( - step, - _consumerWorkflowOptions.MaxDegreeOfParallelism, - _consumerWorkflowOptions.EnsureOrdered, - _consumerWorkflowOptions.Capacity); - } - - public void AddFinalization(Func step) - { - _workflow.WithFinalization( - step, - _consumerWorkflowOptions.MaxDegreeOfParallelism, - _consumerWorkflowOptions.EnsureOrdered, - _consumerWorkflowOptions.Capacity); - } - - public void AddErrorHandling(Action step) - { - _workflow.WithErrorHandling( - step, - _consumerWorkflowOptions.Capacity, - _consumerWorkflowOptions.MaxDegreeOfParallelism, - _consumerWorkflowOptions.EnsureOrdered); - } - - public void AddErrorHandling(Func step) - { - _workflow.WithErrorHandling( - step, - _consumerWorkflowOptions.Capacity, - _consumerWorkflowOptions.MaxDegreeOfParallelism, - _consumerWorkflowOptions.EnsureOrdered); - } - - public async Task StartAsync() - { - await _workflow.StartAsync(); - } - - public async Task StopAsync() - { - await _workflow.StopAsync(); - } -} diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs index 3f2436f7..4247de19 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs @@ -1,6 +1,6 @@ using HouseofCat.RabbitMQ.Dataflows; -namespace RabbitMQ.ConsumerDataflows.Tests; +namespace RabbitMQ.ConsumerDataflowService; public sealed class CustomWorkState : RabbitWorkState { diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs index f4731a56..37b07a1d 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using RabbitMQ.Client; -namespace RabbitMQ.ConsumerDataflows.Tests; +namespace RabbitMQ.ConsumerDataflowService; public static class Shared { @@ -21,7 +21,7 @@ public static class Shared public static readonly string ConsumerName = "TestConsumer"; public static readonly string ErrorQueue = "TestQueue.Error"; - public static async Task SetupTestsAsync(ILogger logger, string configFileNamePath) + public static async Task SetupAsync(ILogger logger, string configFileNamePath) { var rabbitOptions = await RabbitExtensions.GetRabbitOptionsFromJsonFileAsync(configFileNamePath); var channelPool = new ChannelPool(rabbitOptions); @@ -75,8 +75,6 @@ public static async Task SetupRabbitServiceAsync( gzipProvider, loggerFactory); - await rabbitService.Publisher.StartAutoPublishAsync(); - return rabbitService; } } From 8dc16552b8e1ccbfe8cf241ef6e9b513e5fd8243 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 16:57:58 -0500 Subject: [PATCH 36/47] Don't send SpanContext with Message (use header). --- src/HouseofCat.RabbitMQ/Messages/Message.cs | 2 ++ tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Messages/Message.cs b/src/HouseofCat.RabbitMQ/Messages/Message.cs index 2b36a0ba..2fa5dc59 100644 --- a/src/HouseofCat.RabbitMQ/Messages/Message.cs +++ b/src/HouseofCat.RabbitMQ/Messages/Message.cs @@ -38,6 +38,7 @@ public interface IMessage IBasicProperties BuildProperties(IChannelHost channelHost, bool withOptionalHeaders, string contentType); + [JsonIgnore] public SpanContext? ParentSpanContext { get; set; } } @@ -70,6 +71,7 @@ public sealed class Message : IMessage [JsonIgnore] public string ContentType { get; set; } = Constants.HeaderValueForContentTypeJson; + [JsonIgnore] public SpanContext? ParentSpanContext { get; set; } public Message() diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index e7b3d3bb..12597cb2 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -34,20 +34,24 @@ }); dataflowService.AddStep( - "create_new_message", - (state) => + "create_new_secret_message", + async (state) => { - state.SendMessage = new Message + var message = new Message { Exchange = "", RoutingKey = "TestTargetQueue", - Body = Encoding.UTF8.GetBytes("Test New Message"), + Body = Encoding.UTF8.GetBytes("Secret Message"), Metadata = new Metadata { PayloadId = Guid.NewGuid().ToString(), }, ParentSpanContext = state.WorkflowSpan?.Context, }; + + await rabbitService.ComcryptAsync(message); + + state.SendMessage = message; return state; }); From c138511bf41e83289004d609928e98093c5c79a4 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 17:31:21 -0500 Subject: [PATCH 37/47] Additional testing and adjustments. --- src/HouseofCat.Dataflows/BaseDataflow.cs | 16 +++--- .../Extensions/WorkStateExtensions.cs | 5 ++ .../Dataflows/ConsumerDataflow.cs | 30 +++++------ .../Extensions/ServiceCollectionExtensions.cs | 11 +++- tests/RabbitMQ.Console.Tests/Program.cs | 51 ++++++++++++------- .../RabbitMQ.Console.Tests.csproj | 3 ++ tests/RabbitMQ.Console.Tests/appsettings.json | 11 ++++ .../Program.cs | 8 ++- 8 files changed, 92 insertions(+), 43 deletions(-) create mode 100644 tests/RabbitMQ.Console.Tests/appsettings.json diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index 74a8ba4f..e1576fbe 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -76,8 +76,8 @@ TState WrapAction(TState state) } catch (Exception ex) { - childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); return state; @@ -101,8 +101,8 @@ async Task WrapActionAsync(TState state) } catch (Exception ex) { - childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); state.IsFaulted = true; state.EDI = ExceptionDispatchInfo.Capture(ex); return state; @@ -126,12 +126,12 @@ void WrapAction(TState state) } catch (Exception ex) { - childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); } childSpan?.End(); - state.EndRootSpan(); + state.EndRootSpan(true); } return new ActionBlock(WrapAction, options); @@ -151,12 +151,12 @@ void WrapAction(TState state) } catch (Exception ex) { - childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); } childSpan?.End(); - state.EndRootSpan(); + state.EndRootSpan(true); } return new ActionBlock(WrapAction, options); @@ -176,12 +176,12 @@ async Task WrapActionAsync(TState state) } catch (Exception ex) { - childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); childSpan?.RecordException(ex); + childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); } childSpan?.End(); - state.EndRootSpan(); + state.EndRootSpan(true); } return new ActionBlock(WrapActionAsync, options); diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 1494bd8a..3cae7d54 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -162,6 +162,11 @@ public static TelemetrySpan CreateActiveChildSpan( attributes: attributes); } + public static void AddEvent(this IWorkState state, string name, string description) + { + state.AddEvent(name, description); + } + public static void EndRootSpan( this IWorkState state, bool includeErrorWhenFaulted = false) diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index e1f67129..384b8b10 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -191,6 +191,19 @@ public ConsumerDataflow SetEncryptionProvider(IEncryptionProvider provid #region Step Adders + protected static readonly string _defaultSpanNameFormat = "{0}.{1}"; + protected static readonly string _defaultStepSpanNameFormat = "{0}.{1}.{2}"; + + protected string GetSpanName(string stepName) + { + return string.Format(_defaultSpanNameFormat, WorkflowName, stepName); + } + + protected string GetStepSpanName(string stepName) + { + return string.Format(_defaultStepSpanNameFormat, WorkflowName, _suppliedTransforms.Count, stepName); + } + protected virtual ITargetBlock CreateTargetBlock( int boundedCapacity, TaskScheduler taskScheduler = null) => new BufferBlock( @@ -212,7 +225,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.ErrorHandler"); + _errorAction = GetLastWrappedActionBlock(action, executionOptions, GetSpanName("error_handler")); } return this; } @@ -229,7 +242,7 @@ public ConsumerDataflow WithErrorHandling( { _errorBuffer = CreateTargetBlock(boundedCapacity, taskScheduler); var executionOptions = GetExecuteStepOptions(maxDoP, ensureOrdered, boundedCapacity, taskScheduler ?? _taskScheduler); - _errorAction = GetLastWrappedActionBlock(action, executionOptions, $"{WorkflowName}.ErrorHandler"); + _errorAction = GetLastWrappedActionBlock(action, executionOptions, GetSpanName("error_handler")); } return this; } @@ -240,19 +253,6 @@ public ConsumerDataflow WithReadyToProcessBuffer(int boundedCapacity, Ta return this; } - protected static readonly string _defaultSpanNameFormat = "{0}.{1}"; - protected static readonly string _defaultStepSpanNameFormat = "{0}.{1}.{2}"; - - protected string GetSpanName(string stepName) - { - return string.Format(_defaultSpanNameFormat, WorkflowName, stepName); - } - - protected string GetStepSpanName(string stepName) - { - return string.Format(_defaultStepSpanNameFormat, WorkflowName, _suppliedTransforms.Count, stepName); - } - public ConsumerDataflow AddStep( Func suppliedStep, string stepName, diff --git a/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs index f543e388..d16cbcd3 100644 --- a/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs +++ b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs @@ -5,12 +5,15 @@ using OpenTelemetry.Resources; using OpenTelemetry.Trace; using System; +using System.Collections.Generic; using System.Reflection; namespace HouseofCat.Utilities.Extensions; public static class ServiceCollectionExtensions { + public static string DeploymentEnvironmentKey { get; set; } = "deployment.environment"; + public static void AddOpenTelemetryExporter( this IServiceCollection services, IConfiguration config) @@ -22,6 +25,7 @@ public static void AddOpenTelemetryExporter( bool.TryParse(config["OpenTelemetry:Enabled"] ?? "false", out var enabled); if (!enabled) return; + var environment = config["OpenTelemetry:Environment"] ?? "Dev"; var otlpServiceName = config["OpenTelemetry:ServiceName"] ?? sourceName; var otlpServiceNamespace = config["OpenTelemetry:ServiceNamespace"] ?? "hoc"; var otlpServiceVersion = config["OpenTelemetry:ServiceVersion"] ?? sourceVersion; @@ -39,7 +43,12 @@ public static void AddOpenTelemetryExporter( resource => resource.AddService( serviceName: otlpServiceName, serviceNamespace: otlpServiceNamespace, - serviceVersion: otlpServiceVersion)); + serviceVersion: otlpServiceVersion) + .AddAttributes( + new[] + { + new KeyValuePair(DeploymentEnvironmentKey, environment) + })); otlpBuilder .WithTracing( diff --git a/tests/RabbitMQ.Console.Tests/Program.cs b/tests/RabbitMQ.Console.Tests/Program.cs index 7af4aeb5..ca642c6a 100644 --- a/tests/RabbitMQ.Console.Tests/Program.cs +++ b/tests/RabbitMQ.Console.Tests/Program.cs @@ -1,29 +1,46 @@ -using RabbitMQ.ConsoleTests; -using Microsoft.Extensions.Logging; +using HouseofCat.Utilities.Extensions; using HouseofCat.Utilities.Helpers; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using RabbitMQ.ConsoleTests; var loggerFactory = LogHelpers.CreateConsoleLoggerFactory(LogLevel.Information); LogHelpers.LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger(); -// Basic Tests -//await BasicGetTests.RunBasicGetAsync(logger, "./RabbitMQ.BasicGetTests.json"); +var builder = WebApplication.CreateBuilder(args); +var configuration = new ConfigurationBuilder() + .SetBasePath(AppContext.BaseDirectory) + .AddJsonFile("appsettings.json") + .Build(); + +builder.Services.AddOpenTelemetryExporter(configuration); + +using var app = builder.Build(); + +logger.LogInformation("Tests complete! Press CTRL+C to gracefully exit...."); -// Publisher Tests -//await PublisherTests.RunSlowPublisherTestAsync(logger, "./RabbitMQ.PublisherTests.json"); -//await PublisherTests.RunAutoPublisherStandaloneAsync(); +app.Lifetime.ApplicationStarted.Register( + async () => + { + // Basic Tests + //await BasicGetTests.RunBasicGetAsync(logger, "./RabbitMQ.BasicGetTests.json"); -// Consumer Tests -//await ConsumerTests.RunConsumerTestAsync(logger, "./RabbitMQ.ConsumerTests.json"); + // Publisher Tests + //await PublisherTests.RunSlowPublisherTestAsync(logger, "./RabbitMQ.PublisherTests.json"); + //await PublisherTests.RunAutoPublisherStandaloneAsync(); -// PubSub Tests -//await PubSubTests.RunPubSubTestAsync(logger, "./RabbitMQ.PubSubTests.json"); -//await PubSubTests.RunPubSubCheckForDuplicateTestAsync(logger, "./RabbitMQ.PubSubTests.json"); + // Consumer Tests + //await ConsumerTests.RunConsumerTestAsync(logger, "./RabbitMQ.ConsumerTests.json"); -// RabbitService Tests -await RabbitServiceTests.RunRabbitServicePingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); -//await RabbitServiceTests.RunRabbitServiceAltPingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); + // PubSub Tests + //await PubSubTests.RunPubSubTestAsync(logger, "./RabbitMQ.PubSubTests.json"); + //await PubSubTests.RunPubSubCheckForDuplicateTestAsync(logger, "./RabbitMQ.PubSubTests.json"); -logger.LogInformation("Tests complete! Press return to exit...."); + // RabbitService Tests + await RabbitServiceTests.RunRabbitServicePingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); + //await RabbitServiceTests.RunRabbitServiceAltPingPongTestAsync(loggerFactory, "./RabbitMQ.RabbitServiceTests.json"); + }); -Console.ReadLine(); +await app.RunAsync(); diff --git a/tests/RabbitMQ.Console.Tests/RabbitMQ.Console.Tests.csproj b/tests/RabbitMQ.Console.Tests/RabbitMQ.Console.Tests.csproj index de53682f..cf12ea46 100644 --- a/tests/RabbitMQ.Console.Tests/RabbitMQ.Console.Tests.csproj +++ b/tests/RabbitMQ.Console.Tests/RabbitMQ.Console.Tests.csproj @@ -13,6 +13,9 @@ + + Always + Always diff --git a/tests/RabbitMQ.Console.Tests/appsettings.json b/tests/RabbitMQ.Console.Tests/appsettings.json new file mode 100644 index 00000000..59492b1a --- /dev/null +++ b/tests/RabbitMQ.Console.Tests/appsettings.json @@ -0,0 +1,11 @@ +{ + "OpenTelemetry": { + "Enabled": false, + "ServiceVersion": "v1.0.0", + "Environment": "production", + "EndpointUrl": "https://ingest.us.signoz.cloud:443", + "HeaderFormat": "{0}={1}", + "HeaderKey": "signoz-access-token", + "ApiKey": "*" + } +} diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs index 12597cb2..91e17df8 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs @@ -29,7 +29,12 @@ "write_message_to_console", (state) => { - Console.WriteLine(Encoding.UTF8.GetString(state.ReceivedMessage.Body.Span)); + var message = Encoding.UTF8.GetString(state.ReceivedMessage.Body.Span); + if (message == "throw") + { + throw new Exception("Throwing an exception!"); + } + Console.WriteLine(message); return state; }); @@ -75,7 +80,6 @@ (state) => { logger.LogError(state?.EDI?.SourceException, "Error Step!"); - state?.ReceivedMessage?.NackMessage(requeue: true); state?.ReceivedMessage?.Complete(); }); From 4c0d6868b8522c4d6b70a2b00f1e3436c53360c8 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 17:31:56 -0500 Subject: [PATCH 38/47] Cleanup appsettings. --- tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json b/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json index 2a404744..59de588f 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json @@ -1,9 +1,8 @@ { "OpenTelemetry": { "Enabled": true, - "ServiceName": "RabbitMQ.ConsumerDataflows.Tests", "ServiceVersion": "v1.0.0", - "ServiceNamespace": "HoC", + "Environment": "Dev", "EndpointUrl": "https://ingest.us.signoz.cloud:443", "HeaderFormat": "{0}={1}", "HeaderKey": "signoz-access-token", From 938f89a877ee26bc9d300c07aaa3e669e7667712 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 17:33:14 -0500 Subject: [PATCH 39/47] Cleanup --- .../Extensions/ServiceCollectionExtensions.cs | 4 ++-- tests/RabbitMQ.Console.Tests/appsettings.json | 3 +-- tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs index d16cbcd3..6451724f 100644 --- a/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs +++ b/src/HouseofCat.Utilities/Extensions/ServiceCollectionExtensions.cs @@ -62,10 +62,10 @@ public static void AddOpenTelemetryExporter( otlpOptions.Headers = string.Format(otlpHeaderFormat, otlpHeaderKey, otlpApiKey); }) #if DEBUG - .AddSource(sourceName) + .AddSource(otlpServiceName) .AddConsoleExporter()); #else - .AddSource(sourceName)); + .AddSource(otlpServiceName)); #endif } } diff --git a/tests/RabbitMQ.Console.Tests/appsettings.json b/tests/RabbitMQ.Console.Tests/appsettings.json index 59492b1a..bb8887e2 100644 --- a/tests/RabbitMQ.Console.Tests/appsettings.json +++ b/tests/RabbitMQ.Console.Tests/appsettings.json @@ -1,8 +1,7 @@ { "OpenTelemetry": { "Enabled": false, - "ServiceVersion": "v1.0.0", - "Environment": "production", + "Environment": "Dev", "EndpointUrl": "https://ingest.us.signoz.cloud:443", "HeaderFormat": "{0}={1}", "HeaderKey": "signoz-access-token", diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json b/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json index 59de588f..be926f86 100644 --- a/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json +++ b/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json @@ -1,7 +1,6 @@ { "OpenTelemetry": { "Enabled": true, - "ServiceVersion": "v1.0.0", "Environment": "Dev", "EndpointUrl": "https://ingest.us.signoz.cloud:443", "HeaderFormat": "{0}={1}", From 70028adedd9e14eac48524681f0212f8afd4fff4 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Sun, 14 Apr 2024 17:35:25 -0500 Subject: [PATCH 40/47] Finish project renaming. --- RabbitMQ.Dataflows.sln | 2 +- TesseractLogo.svg | 3 --- .../Program.cs | 0 .../RabbitMQ.ConsumerDataflowService.csproj | 0 .../RabbitMQ.ConsumerDataflows.json | 0 .../Tests/CustomWorkState.cs | 0 .../Tests/Shared.cs | 0 .../appsettings.json | 0 8 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 TesseractLogo.svg rename tests/{RabbitMQ.ConsumerDataflows.Tests => RabbitMQ.ConsumerDataflowService}/Program.cs (100%) rename tests/{RabbitMQ.ConsumerDataflows.Tests => RabbitMQ.ConsumerDataflowService}/RabbitMQ.ConsumerDataflowService.csproj (100%) rename tests/{RabbitMQ.ConsumerDataflows.Tests => RabbitMQ.ConsumerDataflowService}/RabbitMQ.ConsumerDataflows.json (100%) rename tests/{RabbitMQ.ConsumerDataflows.Tests => RabbitMQ.ConsumerDataflowService}/Tests/CustomWorkState.cs (100%) rename tests/{RabbitMQ.ConsumerDataflows.Tests => RabbitMQ.ConsumerDataflowService}/Tests/Shared.cs (100%) rename tests/{RabbitMQ.ConsumerDataflows.Tests => RabbitMQ.ConsumerDataflowService}/appsettings.json (100%) diff --git a/RabbitMQ.Dataflows.sln b/RabbitMQ.Dataflows.sln index 62aed2c6..4210b747 100644 --- a/RabbitMQ.Dataflows.sln +++ b/RabbitMQ.Dataflows.sln @@ -86,7 +86,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "rabbitmq", "rabbitmq", "{5C EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Console.Tests", "tests\OpenTelemetry.Console.Tests\OpenTelemetry.Console.Tests.csproj", "{077E07C3-9A35-42A0-8228-E9778F02DFCE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RabbitMQ.ConsumerDataflowService", "tests\RabbitMQ.ConsumerDataflows.Tests\RabbitMQ.ConsumerDataflowService.csproj", "{F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RabbitMQ.ConsumerDataflowService", "tests\RabbitMQ.ConsumerDataflowService\RabbitMQ.ConsumerDataflowService.csproj", "{F6C0B657-E70B-4DE9-96AE-E7612C89AF5F}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/TesseractLogo.svg b/TesseractLogo.svg deleted file mode 100644 index 4168d596..00000000 --- a/TesseractLogo.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs b/tests/RabbitMQ.ConsumerDataflowService/Program.cs similarity index 100% rename from tests/RabbitMQ.ConsumerDataflows.Tests/Program.cs rename to tests/RabbitMQ.ConsumerDataflowService/Program.cs diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflowService.csproj b/tests/RabbitMQ.ConsumerDataflowService/RabbitMQ.ConsumerDataflowService.csproj similarity index 100% rename from tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflowService.csproj rename to tests/RabbitMQ.ConsumerDataflowService/RabbitMQ.ConsumerDataflowService.csproj diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json b/tests/RabbitMQ.ConsumerDataflowService/RabbitMQ.ConsumerDataflows.json similarity index 100% rename from tests/RabbitMQ.ConsumerDataflows.Tests/RabbitMQ.ConsumerDataflows.json rename to tests/RabbitMQ.ConsumerDataflowService/RabbitMQ.ConsumerDataflows.json diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs b/tests/RabbitMQ.ConsumerDataflowService/Tests/CustomWorkState.cs similarity index 100% rename from tests/RabbitMQ.ConsumerDataflows.Tests/Tests/CustomWorkState.cs rename to tests/RabbitMQ.ConsumerDataflowService/Tests/CustomWorkState.cs diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs b/tests/RabbitMQ.ConsumerDataflowService/Tests/Shared.cs similarity index 100% rename from tests/RabbitMQ.ConsumerDataflows.Tests/Tests/Shared.cs rename to tests/RabbitMQ.ConsumerDataflowService/Tests/Shared.cs diff --git a/tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json b/tests/RabbitMQ.ConsumerDataflowService/appsettings.json similarity index 100% rename from tests/RabbitMQ.ConsumerDataflows.Tests/appsettings.json rename to tests/RabbitMQ.ConsumerDataflowService/appsettings.json From 0c858e52f457ca5b75d299993ce895a375c295ea Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Wed, 17 Apr 2024 09:47:34 -0500 Subject: [PATCH 41/47] Cleanup and OpenTelemetry tweaks. --- src/HouseofCat.Dataflows/BaseDataflow.cs | 6 +- .../Extensions/WorkStateExtensions.cs | 2 +- src/HouseofCat.RabbitMQ/Constants.cs | 3 + src/HouseofCat.RabbitMQ/Consumer/Consumer.cs | 5 +- .../Dataflows/ConsumerDataflow.cs | 6 +- .../Extensions/MessageExtensions.cs | 1 + .../Publisher/Publisher.cs | 124 +++++++++++------- .../Helpers/OpenTelemetryHelpers.cs | 8 +- .../Tests/RabbitServiceTests.cs | 28 +++- 9 files changed, 120 insertions(+), 63 deletions(-) diff --git a/src/HouseofCat.Dataflows/BaseDataflow.cs b/src/HouseofCat.Dataflows/BaseDataflow.cs index e1576fbe..e2cb5913 100644 --- a/src/HouseofCat.Dataflows/BaseDataflow.cs +++ b/src/HouseofCat.Dataflows/BaseDataflow.cs @@ -130,7 +130,7 @@ void WrapAction(TState state) childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); } - childSpan?.End(); + childSpan.End(); state.EndRootSpan(true); } @@ -155,7 +155,7 @@ void WrapAction(TState state) childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); } - childSpan?.End(); + childSpan.End(); state.EndRootSpan(true); } @@ -180,7 +180,7 @@ async Task WrapActionAsync(TState state) childSpan?.SetStatus(Status.Error.WithDescription(ex.Message)); } - childSpan?.End(); + childSpan.End(); state.EndRootSpan(true); } diff --git a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs index 3cae7d54..972b997c 100644 --- a/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs +++ b/src/HouseofCat.Dataflows/Extensions/WorkStateExtensions.cs @@ -177,7 +177,7 @@ public static void EndRootSpan( { state.SetOpenTelemetryError(); } - state.WorkflowSpan?.End(); + state.WorkflowSpan.End(); state.WorkflowSpan?.Dispose(); } } diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index fe6906af..775a4caa 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -37,6 +37,9 @@ public static class Constants public static string MessagingConsumerNameKey { get; set; } = "messaging.rabbitmq.consumer.name"; public static string MessagingMessageMessageIdKey { get; set; } = "messaging.message.id"; + public static string MessagingMessageBodySizeKey { get; set; } = "messaging.rabbitmq.message.body.size"; + public static string MessagingMessageEnvelopeSizeKey { get; set; } = "messaging.rabbitmq.message.envelope.size"; + public static string MessagingMessageRoutingKeyKey { get; set; } = "messaging.rabbitmq.message.routing_key"; public static string MessagingMessageDeliveryTagIdKey { get; set; } = "messaging.rabbitmq.message.delivery_tag"; diff --git a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs index 21863c2c..0a71e233 100644 --- a/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs +++ b/src/HouseofCat.RabbitMQ/Consumer/Consumer.cs @@ -105,7 +105,6 @@ public async Task StartConsumerAsync() FullMode = ConsumerOptions.BehaviorWhenFull.Value }); - await Task.Yield(); var success = false; while (!success) { @@ -308,7 +307,7 @@ protected virtual async Task ReceiveHandlerAsync(object _, BasicDeliverEventArgs await HandleMessageAsync(bdea).ConfigureAwait(false); } - private static readonly string _consumerSpanNameFormat = "{0}.{1}"; + private static readonly string _consumerSpanNameFormat = "messaging.rabbitmq.consumer receive"; protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs bdea) { @@ -318,7 +317,7 @@ protected virtual async ValueTask HandleMessageAsync(BasicDeliverEventArgs { var receivedMessage = new ReceivedMessage(_chanHost.Channel, bdea, !ConsumerOptions.AutoAck); using var span = OpenTelemetryHelpers.StartActiveSpan( - string.Format(_consumerSpanNameFormat, ConsumerOptions.ConsumerName, nameof(HandleMessageAsync)), + _consumerSpanNameFormat, SpanKind.Consumer, receivedMessage.ParentSpanContext ?? default); diff --git a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs index 384b8b10..193dd7c0 100644 --- a/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs +++ b/src/HouseofCat.RabbitMQ/Dataflows/ConsumerDataflow.cs @@ -672,7 +672,7 @@ TState WrapAction(TState state) state.EDI = ExceptionDispatchInfo.Capture(ex); } - childSpan?.End(); + childSpan.End(); return state; } @@ -720,7 +720,7 @@ async Task WrapActionAsync(TState state) state.EDI = ExceptionDispatchInfo.Capture(ex); } - childSpan?.End(); + childSpan.End(); return state; } @@ -748,7 +748,7 @@ async Task WrapPublishAsync(TState state) state.EDI = ExceptionDispatchInfo.Capture(ex); } - childSpan?.End(); + childSpan.End(); return state; } diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index d1e54d5c..e35b5a4f 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -161,6 +161,7 @@ public static void EnrichSpanWithTags(this IMessage message, TelemetrySpan span) span.SetAttribute(Constants.MessagingMessageContentTypeKey, message.ContentType); } + span.SetAttribute(Constants.MessagingMessageBodySizeKey, message.Body.Length); span.SetAttribute(Constants.MessagingMessageDeliveryModeKey, message.DeliveryMode); span.SetAttribute(Constants.MessagingMessagePriorityKey, message.PriorityLevel); span.SetAttribute(Constants.MessagingMessageMandatoryKey, message.Mandatory); diff --git a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs index c0b94c0c..70228430 100644 --- a/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs +++ b/src/HouseofCat.RabbitMQ/Publisher/Publisher.cs @@ -270,6 +270,10 @@ await _messageQueue .ConfigureAwait(false); } + private static readonly string _defaultAutoPublisherSpanName = "messaging.rabbitmq.autopublisher process"; + private static readonly string _compressEventName = "compressed"; + private static readonly string _encryptEventName = "encrypted"; + private async Task ProcessMessagesAsync(ChannelReader channelReader) { await Task.Yield(); @@ -280,13 +284,25 @@ private async Task ProcessMessagesAsync(ChannelReader channelReader) if (message == null) { continue; } + using var span = OpenTelemetryHelpers.StartActiveSpan( + _defaultAutoPublisherSpanName, + SpanKind.Internal, + message.ParentSpanContext ?? default); + message.Metadata ??= new Metadata(); + // If parent span context is not set, set it to the current span. + if (message.ParentSpanContext == default) + { + message.ParentSpanContext = span.Context; + } + if (_compress) { message.Body = _compressionProvider.Compress(message.Body).ToArray(); message.Metadata.Fields[Constants.HeaderForCompressed] = _compress; message.Metadata.Fields[Constants.HeaderForCompression] = _compressionProvider.Type; + span?.AddEvent(_compressEventName); } if (_encrypt) @@ -295,12 +311,15 @@ private async Task ProcessMessagesAsync(ChannelReader channelReader) message.Metadata.Fields[Constants.HeaderForEncrypted] = _encrypt; message.Metadata.Fields[Constants.HeaderForEncryption] = _encryptionProvider.Type; message.Metadata.Fields[Constants.HeaderForEncryptDate] = TimeHelpers.GetDateTimeNow(TimeHelpers.Formats.RFC3339Long); + span?.AddEvent(_encryptEventName); } _logger.LogDebug(LogMessages.AutoPublishers.MessagePublished, message.MessageId, message.Metadata?.PayloadId); await PublishAsync(message, _createPublishReceipts, _withHeaders) .ConfigureAwait(false); + + span.End(); } } } @@ -389,6 +408,7 @@ public async Task PublishAsync( } finally { + span.End(); await _channelPool .ReturnChannelAsync(channelHost, error); } @@ -439,6 +459,7 @@ public async Task PublishAsync( } finally { + span.End(); await _channelPool .ReturnChannelAsync(channelHost, error); } @@ -502,6 +523,7 @@ public async Task PublishBatchAsync( } finally { + span.End(); await _channelPool .ReturnChannelAsync(channelHost, error); } @@ -557,6 +579,7 @@ public async Task PublishBatchAsync( } finally { + span.End(); await _channelPool .ReturnChannelAsync(channelHost, error); } @@ -564,7 +587,7 @@ await _channelPool return error; } - private static readonly string _defaultSpanName = "messaging.rabbitmq.publisher"; + private static readonly string _defaultPublishSpanName = "messaging.rabbitmq.publisher publish"; /// /// Acquires a channel from the channel pool, then publishes message based on the message parameters. @@ -576,7 +599,7 @@ await _channelPool public async Task PublishAsync(IMessage message, bool createReceipt, bool withOptionalHeaders = true) { using var span = OpenTelemetryHelpers.StartActiveSpan( - _defaultSpanName, + _defaultPublishSpanName, SpanKind.Producer, message.ParentSpanContext ?? default); @@ -590,6 +613,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp try { var body = _serializationProvider.Serialize(message); + span?.SetAttribute(Constants.MessagingMessageEnvelopeSizeKey, body.Length); chanHost .Channel .BasicPublish( @@ -612,6 +636,7 @@ public async Task PublishAsync(IMessage message, bool createReceipt, bool withOp } finally { + span.End(); if (createReceipt) { await CreateReceiptAsync(message, error) @@ -620,8 +645,6 @@ await CreateReceiptAsync(message, error) await _channelPool .ReturnChannelAsync(chanHost, error); - - span?.End(); } } @@ -636,7 +659,7 @@ await _channelPool public async Task PublishWithConfirmationAsync(IMessage message, bool createReceipt, bool withOptionalHeaders = true) { using var span = OpenTelemetryHelpers.StartActiveSpan( - _defaultSpanName, + _defaultPublishSpanName, SpanKind.Producer, message.ParentSpanContext ?? default); @@ -649,7 +672,8 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece try { - chanHost.Channel.WaitForConfirmsOrDie(_waitForConfirmation); + var body = _serializationProvider.Serialize(message); + span?.SetAttribute(Constants.MessagingMessageEnvelopeSizeKey, body.Length); chanHost .Channel @@ -658,7 +682,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece message.RoutingKey, message.Mandatory, message.BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), - _serializationProvider.Serialize(message)); + body); chanHost.Channel.WaitForConfirmsOrDie(_waitForConfirmation); } @@ -675,6 +699,7 @@ public async Task PublishWithConfirmationAsync(IMessage message, bool createRece } finally { + span.End(); if (createReceipt) { await CreateReceiptAsync(message, error) @@ -683,11 +708,11 @@ await CreateReceiptAsync(message, error) await _channelPool .ReturnChannelAsync(chanHost, error); - - span?.End(); } } + private static readonly string _defaultBatchPublishSpanName = "messaging.rabbitmq.publisher batch"; + /// /// Use this method to sequentially publish all messages in a list in the order received. /// @@ -696,7 +721,7 @@ await _channelPool /// public async Task PublishManyAsync(IList messages, bool createReceipt, bool withOptionalHeaders = true) { - using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishManyAsync), SpanKind.Producer); + using var span = OpenTelemetryHelpers.StartActiveSpan(_defaultBatchPublishSpanName, SpanKind.Producer); span?.SetAttribute(Constants.MessagingBatchProcessValue, messages.Count); var error = false; @@ -709,18 +734,21 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, try { using var innerSpan = OpenTelemetryHelpers.StartActiveSpan( - nameof(chanHost.Channel.BasicPublish), + _defaultPublishSpanName, SpanKind.Producer, span?.Context ?? default); messages[i].EnrichSpanWithTags(innerSpan); + var body = _serializationProvider.Serialize(messages[i].Body); + innerSpan?.SetAttribute(Constants.MessagingMessageEnvelopeSizeKey, body.Length); + chanHost.Channel.BasicPublish( messages[i].Exchange, messages[i].RoutingKey, messages[i].Mandatory, messages[i].BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), - _serializationProvider.Serialize(messages[i].Body)); + body); } catch (Exception ex) { @@ -740,6 +768,7 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, if (error) { break; } } + span.End(); await _channelPool.ReturnChannelAsync(chanHost, error).ConfigureAwait(false); } @@ -752,6 +781,8 @@ public async Task PublishManyAsync(IList messages, bool createReceipt, /// public async Task PublishManyAsBatchAsync(IList messages, bool createReceipt, bool withOptionalHeaders = true) { + if (messages.Count < 1) return; + using var span = OpenTelemetryHelpers.StartActiveSpan(nameof(PublishManyAsBatchAsync), SpanKind.Producer); span?.SetAttribute(Constants.MessagingBatchProcessValue, messages.Count); @@ -762,33 +793,33 @@ public async Task PublishManyAsBatchAsync(IList messages, bool createR try { - if (messages.Count > 0) + var publishBatch = chanHost.Channel.CreateBasicPublishBatch(); + for (var i = 0; i < messages.Count; i++) { - var publishBatch = chanHost.Channel.CreateBasicPublishBatch(); - for (var i = 0; i < messages.Count; i++) - { - using var innerSpan = OpenTelemetryHelpers.StartActiveSpan( - "IBasicPublishBatch.Add", - SpanKind.Producer, - span?.Context ?? default); + using var innerSpan = OpenTelemetryHelpers.StartActiveSpan( + _defaultPublishSpanName, + SpanKind.Producer, + span?.Context ?? default); - messages[i].EnrichSpanWithTags(innerSpan); + messages[i].EnrichSpanWithTags(innerSpan); - publishBatch.Add( - messages[i].Exchange, - messages[i].RoutingKey, - messages[i].Mandatory, - messages[i].BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), - _serializationProvider.Serialize(messages[i].Body)); + var body = _serializationProvider.Serialize(messages[i].Body); + innerSpan?.SetAttribute(Constants.MessagingMessageEnvelopeSizeKey, body.Length); - if (createReceipt) - { - await CreateReceiptAsync(messages[i], error).ConfigureAwait(false); - } - } + publishBatch.Add( + messages[i].Exchange, + messages[i].RoutingKey, + messages[i].Mandatory, + messages[i].BuildProperties(chanHost, withOptionalHeaders, _serializationProvider.ContentType), + body); - publishBatch.Publish(); + if (createReceipt) + { + await CreateReceiptAsync(messages[i], error).ConfigureAwait(false); + } } + + publishBatch.Publish(); } catch (Exception ex) { @@ -800,7 +831,10 @@ public async Task PublishManyAsBatchAsync(IList messages, bool createR error = true; } finally - { await _channelPool.ReturnChannelAsync(chanHost, error).ConfigureAwait(false); } + { + span.End(); + await _channelPool.ReturnChannelAsync(chanHost, error).ConfigureAwait(false); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -820,17 +854,6 @@ await _receiptBuffer .ConfigureAwait(false); } - private static void SetMandatoryHeaders(IBasicProperties basicProperties, string contentType) - { - basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; - if (!string.IsNullOrEmpty(contentType)) - { - basicProperties.Headers[Constants.HeaderForContentType] = contentType; - } - var openTelHeader = OpenTelemetryHelpers.GetOrCreateTraceHeaderFromCurrentActivity(); - basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; - } - /// /// Intended to bring feature parity and include properties when publishing byte[], which you get for free when publishing with IMessage objects. /// @@ -874,6 +897,17 @@ private static IBasicProperties BuildProperties( return basicProperties; } + private static void SetMandatoryHeaders(IBasicProperties basicProperties, string contentType) + { + basicProperties.Headers[Constants.HeaderForObjectType] = Constants.HeaderValueForMessageObjectType; + if (!string.IsNullOrEmpty(contentType)) + { + basicProperties.Headers[Constants.HeaderForContentType] = contentType; + } + var openTelHeader = OpenTelemetryHelpers.GetOrCreateTraceHeaderFromCurrentActivity(); + basicProperties.Headers[Constants.HeaderForTraceParent] = openTelHeader; + } + public static void EnrichSpanWithTags( TelemetrySpan span, string exchangeName, diff --git a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs index 3fd98693..2089e7f8 100644 --- a/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs +++ b/src/HouseofCat.Utilities/Helpers/OpenTelemetryHelpers.cs @@ -276,14 +276,14 @@ public static string FormatOpenTelemetryHeader( public static void SetCurrentActivityAsError(Exception ex, string message = null) { var activity = Activity.Current; - if (activity is null) return; + if (activity is null || !activity.IsAllDataRequested) return; SetActivityAsError(activity, ex, message); } public static void SetActivityAsError(Activity activity, Exception ex, string message = null) { - if (activity is null) return; + if (activity is null || !activity.IsAllDataRequested) return; if (ex is not null) { @@ -296,14 +296,14 @@ public static void SetActivityAsError(Activity activity, Exception ex, string me public static void SetCurrentSpanAsError(Exception ex, string message = null) { var span = Tracer.CurrentSpan; - if (span is null) return; + if (span is null || !span.IsRecording) return; SetSpanAsError(span, ex, message); } public static void SetSpanAsError(TelemetrySpan span, Exception ex, string message = null) { - if (span is null) return; + if (span is null || !span.IsRecording) return; if (ex is not null) { diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index 7c230015..bf3ffc4b 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -1,5 +1,8 @@ using HouseofCat.RabbitMQ; +using HouseofCat.RabbitMQ.Services; +using HouseofCat.Utilities.Helpers; using Microsoft.Extensions.Logging; +using OpenTelemetry.Trace; namespace RabbitMQ.ConsoleTests; @@ -14,6 +17,7 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger var consumer = rabbitService.GetConsumer(Shared.ConsumerName); await consumer.StartConsumerAsync(); + using var preProcessSpan = OpenTelemetryHelpers.StartActiveSpan("messaging.rabbitmq.publisher create message", SpanKind.Internal); var dataAsBytes = rabbitService.SerializationProvider.Serialize(new { Name = "TestName", Age = 42 }); var message = new Message( exchange: Shared.ExchangeName, @@ -21,11 +25,19 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger body: dataAsBytes, payloadId: Guid.NewGuid().ToString()); + message.ParentSpanContext = preProcessSpan.Context; + await rabbitService.Publisher.QueueMessageAsync(message); + preProcessSpan.End(); // Ping pong the same message. await foreach (var receivedMessage in consumer.ReadUntilStopAsync()) { + using var consumerSpan = OpenTelemetryHelpers.StartActiveSpan( + "messaging.rabbitmq.consumer process", + SpanKind.Consumer, + parentContext: receivedMessage.ParentSpanContext ?? default); + if (receivedMessage?.Message is null) { receivedMessage?.AckMessage(); @@ -33,15 +45,23 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger } await rabbitService.DecomcryptAsync(receivedMessage.Message); + //await RequeueMessageAsync(rabbitService, receivedMessage); - receivedMessage.Message.Exchange = Shared.ExchangeName; - receivedMessage.Message.RoutingKey = Shared.RoutingKey; - receivedMessage.Message.Metadata.PayloadId = Guid.NewGuid().ToString(); - await rabbitService.Publisher.QueueMessageAsync(receivedMessage.Message); receivedMessage.AckMessage(); + consumerSpan.End(); } } + private static async Task RequeueMessageAsync( + IRabbitService rabbitService, + IReceivedMessage receivedMessage) + { + receivedMessage.Message.Exchange = Shared.ExchangeName; + receivedMessage.Message.RoutingKey = Shared.RoutingKey; + receivedMessage.Message.Metadata.PayloadId = Guid.NewGuid().ToString(); + await rabbitService.Publisher.QueueMessageAsync(receivedMessage.Message); + } + /// /// Test for sync publish message queueing. /// From 7886c86b8091651530bb4ff3c785e7a965e186f7 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Thu, 18 Apr 2024 06:37:13 -0500 Subject: [PATCH 42/47] Removing unhelpful attributes for OpenTelemetry. --- src/HouseofCat.RabbitMQ/Constants.cs | 4 ---- src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/HouseofCat.RabbitMQ/Constants.cs b/src/HouseofCat.RabbitMQ/Constants.cs index 775a4caa..fdadc563 100644 --- a/src/HouseofCat.RabbitMQ/Constants.cs +++ b/src/HouseofCat.RabbitMQ/Constants.cs @@ -42,11 +42,7 @@ public static class Constants public static string MessagingMessageRoutingKeyKey { get; set; } = "messaging.rabbitmq.message.routing_key"; public static string MessagingMessageDeliveryTagIdKey { get; set; } = "messaging.rabbitmq.message.delivery_tag"; - - public static string MessagingMessageDeliveryModeKey { get; set; } = "messaging.rabbitmq.message.delivery_mode"; - public static string MessagingMessagePriorityKey { get; set; } = "messaging.rabbitmq.message.priority"; public static string MessagingMessageContentTypeKey { get; set; } = "messaging.rabbitmq.message.content_type"; - public static string MessagingMessageMandatoryKey { get; set; } = "messaging.rabbitmq.message.mandatory"; public static string MessagingBatchProcessValue { get; set; } = "messaging.batch.message_count"; diff --git a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs index e35b5a4f..90cde0a8 100644 --- a/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs +++ b/src/HouseofCat.RabbitMQ/Extensions/MessageExtensions.cs @@ -162,9 +162,6 @@ public static void EnrichSpanWithTags(this IMessage message, TelemetrySpan span) } span.SetAttribute(Constants.MessagingMessageBodySizeKey, message.Body.Length); - span.SetAttribute(Constants.MessagingMessageDeliveryModeKey, message.DeliveryMode); - span.SetAttribute(Constants.MessagingMessagePriorityKey, message.PriorityLevel); - span.SetAttribute(Constants.MessagingMessageMandatoryKey, message.Mandatory); if (!string.IsNullOrEmpty(message.Metadata?.PayloadId)) { From d76df2af90e58bd2e375a71d39e12286cf749217 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Thu, 18 Apr 2024 23:49:21 -0500 Subject: [PATCH 43/47] Update JsonProvider.cs --- src/HouseofCat.Serialization/JsonProvider.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/HouseofCat.Serialization/JsonProvider.cs b/src/HouseofCat.Serialization/JsonProvider.cs index 39132900..26475c68 100644 --- a/src/HouseofCat.Serialization/JsonProvider.cs +++ b/src/HouseofCat.Serialization/JsonProvider.cs @@ -38,13 +38,7 @@ public TOut Deserialize(Stream inputStream) if (inputStream.Position == inputStream.Length) { inputStream.Seek(0, SeekOrigin.Begin); } - var length = (int)inputStream.Length; - var buffer = new Span(new byte[length]); - var bytesRead = inputStream.Read(buffer); - if (bytesRead == 0) throw new InvalidDataException(); - - var utf8Reader = new Utf8JsonReader(buffer); - return JsonSerializer.Deserialize(ref utf8Reader, _options); + return JsonSerializer.Deserialize(inputStream, _options); } public async Task DeserializeAsync(Stream inputStream) @@ -63,7 +57,7 @@ public ReadOnlyMemory Serialize(TIn input) public void Serialize(Stream outputStream, TIn input) { - JsonSerializer.Serialize(new Utf8JsonWriter(outputStream), input, _options); + JsonSerializer.Serialize(outputStream, input, _options); outputStream.Seek(0, SeekOrigin.Begin); } From 375c92493fa3454227d0b4aad666f8135d4e6e8d Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Thu, 18 Apr 2024 23:50:50 -0500 Subject: [PATCH 44/47] Update JsonProvider.cs --- src/HouseofCat.Serialization/JsonProvider.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/HouseofCat.Serialization/JsonProvider.cs b/src/HouseofCat.Serialization/JsonProvider.cs index 26475c68..6a9c72a4 100644 --- a/src/HouseofCat.Serialization/JsonProvider.cs +++ b/src/HouseofCat.Serialization/JsonProvider.cs @@ -47,7 +47,9 @@ public async Task DeserializeAsync(Stream inputStream) if (inputStream.Position == inputStream.Length) { inputStream.Seek(0, SeekOrigin.Begin); } - return await JsonSerializer.DeserializeAsync(inputStream, _options).ConfigureAwait(false); + return await JsonSerializer + .DeserializeAsync(inputStream, _options) + .ConfigureAwait(false); } public ReadOnlyMemory Serialize(TIn input) @@ -63,7 +65,9 @@ public void Serialize(Stream outputStream, TIn input) public async Task SerializeAsync(Stream outputStream, TIn input) { - await JsonSerializer.SerializeAsync(outputStream, input, _options).ConfigureAwait(false); + await JsonSerializer + .SerializeAsync(outputStream, input, _options) + .ConfigureAwait(false); } private static readonly JsonSerializerOptions _prettyOptions = new JsonSerializerOptions From 8df58ae87296573fb16fc90821a128686b51b5d6 Mon Sep 17 00:00:00 2001 From: Tristan Hyams Date: Fri, 19 Apr 2024 00:02:49 -0500 Subject: [PATCH 45/47] Update common.props --- common.props | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/common.props b/common.props index 898db97e..62c01023 100644 --- a/common.props +++ b/common.props @@ -8,28 +8,28 @@ true true true - + HouseofCat.io RabbitMQ.Dataflows library helps use RabbitMQ as well as create powerful RabbitMQ powered workflows. Copyright © 2024 - HouseofCat.io - RabbitMQ.Workflows + HouseofCat.io - RabbitMQ.Dataflows houseofcat.png C# NET8 NetCore RabbitMQ HouseofCat HouseofCat.io TPL Workflows Dataflow Pipelines Major overhaul! - README.md - LICENSE + README.md + LICENSE https://github.com/houseofcat/RabbitMQ.Dataflows https://github.com/houseofcat/RabbitMQ.Dataflows git - + true true true snupkg $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb - + true true @@ -59,14 +59,14 @@ True - - True - - - - True - - + + True + + + + True + + From 6070992c1a9b8916d7a6f099882466419c0188bb Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Fri, 19 Apr 2024 00:03:48 -0500 Subject: [PATCH 46/47] Update RabbitServiceTests.cs --- tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs index bf3ffc4b..e0819d78 100644 --- a/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs +++ b/tests/RabbitMQ.Console.Tests/Tests/RabbitServiceTests.cs @@ -23,9 +23,10 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger exchange: Shared.ExchangeName, routingKey: Shared.RoutingKey, body: dataAsBytes, - payloadId: Guid.NewGuid().ToString()); - - message.ParentSpanContext = preProcessSpan.Context; + payloadId: Guid.NewGuid().ToString()) + { + ParentSpanContext = preProcessSpan.Context + }; await rabbitService.Publisher.QueueMessageAsync(message); preProcessSpan.End(); @@ -45,7 +46,7 @@ public static async Task RunRabbitServicePingPongTestAsync(ILoggerFactory logger } await rabbitService.DecomcryptAsync(receivedMessage.Message); - //await RequeueMessageAsync(rabbitService, receivedMessage); + await RequeueMessageAsync(rabbitService, receivedMessage); receivedMessage.AckMessage(); consumerSpan.End(); From 71a4b03042531ef3c88de0d474879720e9f028f1 Mon Sep 17 00:00:00 2001 From: "Tristan (HouseCat) Hyams" Date: Fri, 19 Apr 2024 00:04:12 -0500 Subject: [PATCH 47/47] Dependency update. --- .../HouseofCat.Compression.csproj | 2 +- src/HouseofCat.Data/HouseofCat.Data.csproj | 4 ++-- .../HouseofCat.Utilities.csproj | 10 +++++----- tests/UnitTests/UnitTests.csproj | 14 ++++++++++---- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/HouseofCat.Compression/HouseofCat.Compression.csproj b/src/HouseofCat.Compression/HouseofCat.Compression.csproj index 89a2a7ec..553587f2 100644 --- a/src/HouseofCat.Compression/HouseofCat.Compression.csproj +++ b/src/HouseofCat.Compression/HouseofCat.Compression.csproj @@ -2,7 +2,7 @@ - + diff --git a/src/HouseofCat.Data/HouseofCat.Data.csproj b/src/HouseofCat.Data/HouseofCat.Data.csproj index 91a60bc6..5dc3d51e 100644 --- a/src/HouseofCat.Data/HouseofCat.Data.csproj +++ b/src/HouseofCat.Data/HouseofCat.Data.csproj @@ -3,10 +3,10 @@ - + - + diff --git a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj index 676d00a3..2926d7be 100644 --- a/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj +++ b/src/HouseofCat.Utilities/HouseofCat.Utilities.csproj @@ -5,11 +5,11 @@ - - - - - + + + + + diff --git a/tests/UnitTests/UnitTests.csproj b/tests/UnitTests/UnitTests.csproj index d808aa0c..8b4edf2f 100644 --- a/tests/UnitTests/UnitTests.csproj +++ b/tests/UnitTests/UnitTests.csproj @@ -8,10 +8,16 @@ - - - - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive +