Merge branch 'main' into vibankwa/update-ilogger-event-attribute-names

.github/workflows/apicompatibility.yml

@@ -17,6 +17,9 @@ jobs:
       with:
         fetch-depth: 0 # fetching all
 
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v3
+
     - name: Install dependencies
       run: dotnet restore
 

.github/workflows/ci-aot.yml

@@ -24,6 +24,9 @@ jobs:
       with:
         fetch-depth: 0 # fetching all
 
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v3
+
     - name: publish AOT testApp, assert static analysis warning count, and run the app
       shell: pwsh
       run: .\build\test-aot-compatibility.ps1 ${{ matrix.version }}

.github/workflows/ci-instrumentation-libraries.yml

@@ -31,6 +31,9 @@ jobs:
       with:
         fetch-depth: 0 # fetching all
 
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v3
+
     - name: Install dependencies
       run: dotnet restore ./build/InstrumentationLibraries.proj
 

.github/workflows/ci.yml

@@ -27,6 +27,9 @@ jobs:
       with:
         fetch-depth: 0 # fetching all
 
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v3
+
     - name: Install dependencies
       run: dotnet restore
 

.github/workflows/code-coverage.yml

@@ -24,6 +24,9 @@ jobs:
       with:
         fetch-depth: 0 # fetching all
 
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v3
+
     - name: Install dependencies
       run: dotnet restore
 

.github/workflows/dotnet-format.yml

@@ -20,6 +20,9 @@ jobs:
     - name: check out code
       uses: actions/checkout@v3
 
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v3
+
     - name: Install format tool
       run: dotnet tool install -g dotnet-format
 

OpenTelemetry.sln

@@ -44,8 +44,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{7CB2F02E
 		build\process-codecoverage.ps1 = build\process-codecoverage.ps1
 		build\RELEASING.md = build\RELEASING.md
 		build\stylecop.json = build\stylecop.json
-		build\xunit.runner.json = build\xunit.runner.json
 		build\test-aot-compatibility.ps1 = build\test-aot-compatibility.ps1
+		build\xunit.runner.json = build\xunit.runner.json
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Exporter.Zipkin", "src\OpenTelemetry.Exporter.Zipkin\OpenTelemetry.Exporter.Zipkin.csproj", "{7EDAE7FA-B44E-42CA-80FA-7DF2FAA2C5DD}"
@@ -97,6 +97,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{
 	ProjectSection(SolutionItems) = preProject
 		.github\workflows\apicompatibility.yml = .github\workflows\apicompatibility.yml
 		.github\workflows\ci-aot.yml = .github\workflows\ci-aot.yml
+		.github\workflows\ci-instrumentation-libraries-md.yml = .github\workflows\ci-instrumentation-libraries-md.yml
+		.github\workflows\ci-instrumentation-libraries.yml = .github\workflows\ci-instrumentation-libraries.yml
 		.github\workflows\ci-md.yml = .github\workflows\ci-md.yml
 		.github\workflows\ci.yml = .github\workflows\ci.yml
 		.github\workflows\code-coverage.yml = .github\workflows\code-coverage.yml

build/Common.props

@@ -6,6 +6,7 @@
     <AssemblyOriginatorKeyFile>$(MSBuildThisFileDirectory)debug.snk</AssemblyOriginatorKeyFile>
     <DefineConstants>$(DefineConstants);SIGNED</DefineConstants>
     <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <!--temporarily disable. See 3958-->

docs/metrics/getting-started-aspnetcore/Program.cs

@@ -14,7 +14,6 @@
 // limitations under the License.
 // </copyright>
 
-using System.Diagnostics;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
 

docs/trace/getting-started-jaeger/Program.cs

@@ -14,12 +14,8 @@
 // limitations under the License.
 // </copyright>
 
-using System;
 using System.Diagnostics;
-using System.Net.Http;
-using System.Threading.Tasks;
 using OpenTelemetry;
-using OpenTelemetry.Exporter;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
 

global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
     "rollForward": "latestFeature",
-    "version": "7.0.101"
+    "version": "7.0.400"
   }
 }

src/OpenTelemetry/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## Unreleased
 
+* **Experimental Feature** Added an opt-in feature to aggregate any metric
+  measurements that were dropped due to reaching the [max MetricPoints
+  limit](https://github.com/open-telemetry/opentelemetry-dotnet/tree/core-1.6.0-alpha.1/docs/metrics/customizing-the-sdk).
+  When this feature is enabled, SDK would aggregate such measurements using a
+  reserved MetricPoint with a single tag with key as `otel.metric.overflow` and
+  value as `true`. The feature is turned-off by default. You can enable it by
+  setting the environment variable
+  `OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE` to `true` before
+  setting up the `MeterProvider`.
+  ([#4737](https://github.com/open-telemetry/opentelemetry-dotnet/pull/4737))
+
 ## 1.6.0-alpha.1
 
 Released 2023-Jul-12

src/OpenTelemetry/Metrics/AggregatorStore.cs

@@ -22,9 +22,11 @@ namespace OpenTelemetry.Metrics;
 
 internal sealed class AggregatorStore
 {
-    private static readonly string MetricPointCapHitFixMessage = "Modify instrumentation to reduce the number of unique key/value pair combinations. Or use Views to drop unwanted tags. Or use MeterProviderBuilder.SetMaxMetricPointsPerMetricStream to set higher limit.";
+    private static readonly string MetricPointCapHitFixMessage = "Consider opting in for the experimental SDK feature to emit all the throttled metrics under the overflow attribute by setting env variable OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE = true. You could also modify instrumentation to reduce the number of unique key/value pair combinations. Or use Views to drop unwanted tags. Or use MeterProviderBuilder.SetMaxMetricPointsPerMetricStream to set higher limit.";
     private static readonly Comparison<KeyValuePair<string, object>> DimensionComparisonDelegate = (x, y) => x.Key.CompareTo(y.Key);
+
     private readonly object lockZeroTags = new();
+    private readonly object lockOverflowTag = new();
     private readonly HashSet<string> tagKeysInteresting;
     private readonly int tagsKeysInterestingCount;
 
@@ -43,17 +45,21 @@ internal sealed class AggregatorStore
     private readonly UpdateLongDelegate updateLongCallback;
     private readonly UpdateDoubleDelegate updateDoubleCallback;
     private readonly int maxMetricPoints;
+    private readonly bool emitOverflowAttribute;
     private readonly ExemplarFilter exemplarFilter;
+
     private int metricPointIndex = 0;
     private int batchSize = 0;
     private int metricCapHitMessageLogged;
     private bool zeroTagMetricPointInitialized;
+    private bool overflowTagMetricPointInitialized;
 
     internal AggregatorStore(
         MetricStreamIdentity metricStreamIdentity,
         AggregationType aggType,
         AggregationTemporality temporality,
         int maxMetricPoints,
+        bool emitOverflowAttribute,
         ExemplarFilter exemplarFilter = null)
     {
         this.name = metricStreamIdentity.InstrumentName;
@@ -81,6 +87,15 @@ internal AggregatorStore(
             this.tagKeysInteresting = hs;
             this.tagsKeysInterestingCount = hs.Count;
         }
+
+        this.emitOverflowAttribute = emitOverflowAttribute;
+
+        if (emitOverflowAttribute)
+        {
+            // Setting metricPointIndex to 1 as we would reserve the metricPoints[1] for overflow attribute.
+            // Newer attributes should be added starting at the index: 2
+            this.metricPointIndex = 1;
+        }
     }
 
     private delegate void UpdateLongDelegate(long value, ReadOnlySpan<KeyValuePair<string, object>> tags);
@@ -197,6 +212,22 @@ private void InitializeZeroTagPointIfNotInitialized()
         }
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private void InitializeOverflowTagPointIfNotInitialized()
+    {
+        if (!this.overflowTagMetricPointInitialized)
+        {
+            lock (this.lockOverflowTag)
+            {
+                if (!this.overflowTagMetricPointInitialized)
+                {
+                    this.metricPoints[1] = new MetricPoint(this, this.aggType, new KeyValuePair<string, object>[] { new("otel.metric.overflow", true) }, this.histogramBounds, this.exponentialHistogramMaxSize, this.exponentialHistogramMaxScale);
+                    this.overflowTagMetricPointInitialized = true;
+                }
+            }
+        }
+    }
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     private int LookupAggregatorStore(KeyValuePair<string, object>[] tagKeysAndValues, int length)
     {
@@ -329,12 +360,21 @@ private void UpdateLong(long value, ReadOnlySpan<KeyValuePair<string, object>> t
             var index = this.FindMetricAggregatorsDefault(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (this.emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.
@@ -361,12 +401,21 @@ private void UpdateLongCustomTags(long value, ReadOnlySpan<KeyValuePair<string,
             var index = this.FindMetricAggregatorsCustomTag(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (this.emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.
@@ -393,12 +442,21 @@ private void UpdateDouble(double value, ReadOnlySpan<KeyValuePair<string, object
             var index = this.FindMetricAggregatorsDefault(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (this.emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.
@@ -425,12 +483,21 @@ private void UpdateDoubleCustomTags(double value, ReadOnlySpan<KeyValuePair<stri
             var index = this.FindMetricAggregatorsCustomTag(tags);
             if (index < 0)
             {
-                if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                if (this.emitOverflowAttribute)
                 {
-                    OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    this.InitializeOverflowTagPointIfNotInitialized();
+                    this.metricPoints[1].Update(value);
+                    return;
                 }
+                else
+                {
+                    if (Interlocked.CompareExchange(ref this.metricCapHitMessageLogged, 1, 0) == 0)
+                    {
+                        OpenTelemetrySdkEventSource.Log.MeasurementDropped(this.name, this.metricPointCapHitMessage, MetricPointCapHitFixMessage);
+                    }
 
-                return;
+                    return;
+                }
             }
 
             // TODO: can special case built-in filters to be bit faster.

src/OpenTelemetry/Metrics/MeterProviderSdk.cs

@@ -19,6 +19,7 @@
 using System.Diagnostics;
 using System.Diagnostics.Metrics;
 using System.Text;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using OpenTelemetry.Internal;
 using OpenTelemetry.Resources;
@@ -32,6 +33,8 @@ internal sealed class MeterProviderSdk : MeterProvider
     internal int ShutdownCount;
     internal bool Disposed;
 
+    private const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE";
+
     private readonly List<object> instrumentations = new();
     private readonly List<Func<Instrument, MetricStreamConfiguration?>> viewConfigs;
     private readonly object collectLock = new();
@@ -48,6 +51,9 @@ internal MeterProviderSdk(
         var state = serviceProvider!.GetRequiredService<MeterProviderBuilderSdk>();
         state.RegisterProvider(this);
 
+        var config = serviceProvider!.GetRequiredService<IConfiguration>();
+        _ = config.TryGetBoolValue(EmitOverFlowAttributeConfigKey, out bool isEmitOverflowAttributeKeySet);
+
         this.ServiceProvider = serviceProvider!;
 
         if (ownsServiceProvider)
@@ -79,7 +85,7 @@ internal MeterProviderSdk(
 
             reader.SetParentProvider(this);
             reader.SetMaxMetricStreams(state.MaxMetricStreams);
-            reader.SetMaxMetricPointsPerMetricStream(state.MaxMetricPointsPerMetricStream);
+            reader.SetMaxMetricPointsPerMetricStream(state.MaxMetricPointsPerMetricStream, isEmitOverflowAttributeKeySet);
             reader.SetExemplarFilter(state.ExemplarFilter);
 
             if (this.reader == null)

src/OpenTelemetry/Metrics/Metric.cs

@@ -35,6 +35,7 @@ internal Metric(
         MetricStreamIdentity instrumentIdentity,
         AggregationTemporality temporality,
         int maxMetricPointsPerMetricStream,
+        bool emitOverflowAttribute,
         ExemplarFilter exemplarFilter = null)
     {
         this.InstrumentIdentity = instrumentIdentity;
@@ -141,7 +142,7 @@ internal Metric(
             throw new NotSupportedException($"Unsupported Instrument Type: {instrumentIdentity.InstrumentType.FullName}");
         }
 
-        this.aggStore = new AggregatorStore(instrumentIdentity, aggType, temporality, maxMetricPointsPerMetricStream, exemplarFilter);
+        this.aggStore = new AggregatorStore(instrumentIdentity, aggType, temporality, maxMetricPointsPerMetricStream, emitOverflowAttribute, exemplarFilter);
         this.Temporality = temporality;
         this.InstrumentDisposed = false;
     }