From dca99b6e66ecb77557e78941a8f868ac303136b3 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Mon, 22 Jan 2024 15:28:26 -0800 Subject: [PATCH] Revise the metrics perf guidance (#5243) --- docs/metrics/README.md | 97 +++++---- test/Benchmarks/Metrics/MetricsBenchmarks.cs | 217 ++++++++++++++++--- 2 files changed, 239 insertions(+), 75 deletions(-) diff --git a/docs/metrics/README.md b/docs/metrics/README.md index 4499281cdd8..be261e377d1 100644 --- a/docs/metrics/README.md +++ b/docs/metrics/README.md @@ -14,7 +14,7 @@ into a dedicated class called [Instrumentation](../../examples/AspNetCore/Instrumentation.cs) which is then added as a `Singleton` service. -### Ordering of Tags +### Ordering of tags When emitting metrics with tags, DO NOT change the order in which you provide tags. Changing the order of tag keys would increase the time taken by the SDK to @@ -30,46 +30,53 @@ MyFruitCounter.Add(5, new("name", "apple"), new("color", "red"), new("taste", "s MyFruitCounter.Add(7, new("color", "red"), new("name", "apple"), new("taste", "sweet")); // <--- DON'T DO THIS ``` -### Use TagList where appropriate - -For the best performance, it is highly recommended to pass in tags in certain -ways so allocations are only happening on the stack rather than the heap, -which eliminates pressure on the GC (garbage collector): - -- When reporting measurements with 3 tags or less, - emit the tags individually. -- When reporting measurements with more than 3 tags, use - [`TagList`](https://learn.microsoft.com/dotnet/api/system.diagnostics.taglist?view=net-7.0#remarks) - for better performance. - -```csharp -var tags = new TagList -{ - { "DimName1", "DimValue1" }, - { "DimName2", "DimValue2" }, - { "DimName3", "DimValue3" }, - { "DimName4", "DimValue4" }, -}; - -// Uses a TagList as there are more than three tags -counter.Add(100, tags); // <--- DO THIS - -// Avoid the below mentioned approaches when there are more than three tags -var tag1 = new KeyValuePair("DimName1", "DimValue1"); -var tag2 = new KeyValuePair("DimName2", "DimValue2"); -var tag3 = new KeyValuePair("DimName3", "DimValue3"); -var tag4 = new KeyValuePair("DimName4", "DimValue4"); - -counter.Add(100, tag1, tag2, tag3, tag4); // <--- DON'T DO THIS - -var readOnlySpanOfTags = new KeyValuePair[4] { tag1, tag2, tag3, tag4}; -counter.Add(100, readOnlySpanOfTags); // <--- DON'T DO THIS -``` - -- When emitting metrics with more than eight tags, the SDK allocates memory on -the hot-path. You SHOULD try to keep the number of tags less than or equal to -eight. If you are exceeding this, check if you can model some of the tags as -Resource, as [shown here](#modeling-static-tags-as-resource). +### Use TagList accordingly + +There are two different ways of passing tags to an instrument API: + +* Pass the tags directly to the instrument API: + + ```csharp + counter.Add(100, ("Key1", "Value1"), ("Key2", "Value2")); + ``` + +* Use + [`TagList`](https://learn.microsoft.com/dotnet/api/system.diagnostics.taglist): + + ```csharp + var tags = new TagList + { + { "DimName1", "DimValue1" }, + { "DimName2", "DimValue2" }, + { "DimName3", "DimValue3" }, + { "DimName4", "DimValue4" }, + }; + + counter.Add(100, tags); + ``` + +Here is the rule of thumb: + +* When reporting measurements with 3 tags or less, pass the tags directly to the + instrument API. +* When reporting measurements with 4 to 8 tags (inclusive), use + [`TagList`](https://learn.microsoft.com/dotnet/api/system.diagnostics.taglist?#remarks) + to avoid heap allocation if avoiding GC pressure is a primary performance + goal. For high performance code which consider reducing CPU utilization more + important (e.g. to reduce latency, to save battery, etc.) than optimizing + memory allocations, use profiler and stress test to determine which approach + is better. + Here are some [metrics benchmark + results](../../test/Benchmarks/Metrics/MetricsBenchmarks.cs) for reference. +* When reporting measurements with more than 8 tags, the two approaches share + very similar CPU performance and heap allocation. `TagList` is recommended due + to its better readability and maintainability. + +> [!NOTE] +> When reporting measurements with more than 8 tags, the API allocates memory on +the hot-path. You SHOULD try to keep the number of tags less than or equal to 8. +If you are exceeding this, check if you can model some of the tags as Resource, +as [shown here](#modeling-static-tags-as-resource). ### Modeling static tags as Resource @@ -80,22 +87,22 @@ each metric measurement. Refer to this ## Common issues that lead to missing metrics -- The `Meter` used to create the instruments is not added to the +* The `Meter` used to create the instruments is not added to the `MeterProvider`. Use `AddMeter` method to enable the processing for the required metrics. -- Instrument name is invalid. When naming instruments, ensure that the name you +* Instrument name is invalid. When naming instruments, ensure that the name you choose meets the criteria defined in the [spec](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md#instrument-name-syntax). A few notable characters that are not allowed in the instrument name: `/` (forward slash), `\` (backward slash), any space character in the name. -- MetricPoint limit is reached. By default, the SDK limits the number of maximum +* MetricPoint limit is reached. By default, the SDK limits the number of maximum MetricPoints (unique combination of keys and values for a given Metric stream) to `2000`. This limit can be configured using `SetMaxMetricPointsPerMetricStream` method. Refer to this [doc](../../docs/metrics/customizing-the-sdk/README.md#changing-maximum-metricpoints-per-metricstream) for more information. The SDK would not process any newer unique key-value combination that it encounters, once this limit is reached. -- MeterProvider is disposed. You need to ensure that the `MeterProvider` +* MeterProvider is disposed. You need to ensure that the `MeterProvider` instance is kept active for metrics to be collected. In a typical application, a single MeterProvider is built at application startup, and is disposed of at application shutdown. For an ASP.NET Core application, use `AddOpenTelemetry` diff --git a/test/Benchmarks/Metrics/MetricsBenchmarks.cs b/test/Benchmarks/Metrics/MetricsBenchmarks.cs index e9ddd996903..e63aeb87576 100644 --- a/test/Benchmarks/Metrics/MetricsBenchmarks.cs +++ b/test/Benchmarks/Metrics/MetricsBenchmarks.cs @@ -9,27 +9,49 @@ using OpenTelemetry.Tests; /* -BenchmarkDotNet v0.13.10, Windows 11 (10.0.23424.1000) -Intel Core i7-9700 CPU 3.00GHz, 1 CPU, 8 logical and 8 physical cores -.NET SDK 8.0.100 - [Host] : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2 - DefaultJob : .NET 8.0.0 (8.0.23.53103), X64 RyuJIT AVX2 - - -| Method | AggregationTemporality | Mean | Error | StdDev | Allocated | -|-------------------------- |----------------------- |----------:|---------:|---------:|----------:| -| CounterHotPath | Cumulative | 15.37 ns | 0.072 ns | 0.064 ns | - | -| CounterWith1LabelsHotPath | Cumulative | 60.67 ns | 0.764 ns | 0.677 ns | - | -| CounterWith3LabelsHotPath | Cumulative | 112.98 ns | 0.843 ns | 0.704 ns | - | -| CounterWith5LabelsHotPath | Cumulative | 196.83 ns | 1.632 ns | 1.447 ns | - | -| CounterWith6LabelsHotPath | Cumulative | 225.96 ns | 2.676 ns | 2.503 ns | - | -| CounterWith7LabelsHotPath | Cumulative | 249.96 ns | 3.459 ns | 3.066 ns | - | -| CounterHotPath | Delta | 19.83 ns | 0.158 ns | 0.148 ns | - | -| CounterWith1LabelsHotPath | Delta | 59.88 ns | 0.251 ns | 0.235 ns | - | -| CounterWith3LabelsHotPath | Delta | 124.24 ns | 1.490 ns | 1.394 ns | - | -| CounterWith5LabelsHotPath | Delta | 203.75 ns | 3.755 ns | 5.504 ns | - | -| CounterWith6LabelsHotPath | Delta | 226.50 ns | 2.036 ns | 1.805 ns | - | -| CounterWith7LabelsHotPath | Delta | 253.83 ns | 1.247 ns | 0.973 ns | - | +BenchmarkDotNet v0.13.10, Windows 11 (10.0.22621.3007/22H2/2022Update/SunValley2) +11th Gen Intel Core i7-1185G7 3.00GHz, 1 CPU, 8 logical and 4 physical cores +.NET SDK 8.0.101 + [Host] : .NET 8.0.1 (8.0.123.58001), X64 RyuJIT AVX2 + DefaultJob : .NET 8.0.1 (8.0.123.58001), X64 RyuJIT AVX2 + + +| Method | AggregationTemporality | Mean | Error | StdDev | Gen0 | Allocated | +|-------------------------------------- |----------------------- |----------:|---------:|----------:|-------:|----------:| +| CounterHotPath | Cumulative | 11.27 ns | 0.173 ns | 0.145 ns | - | - | +| CounterWith1LabelsHotPath | Cumulative | 43.72 ns | 0.266 ns | 0.248 ns | - | - | +| CounterWith2LabelsHotPath | Cumulative | 65.90 ns | 1.334 ns | 1.248 ns | - | - | +| CounterWith3LabelsHotPath | Cumulative | 88.97 ns | 1.785 ns | 1.984 ns | - | - | +| CounterWith4LabelsHotPath | Cumulative | 122.00 ns | 2.131 ns | 1.994 ns | 0.0138 | 88 B | +| CounterWith5LabelsHotPath | Cumulative | 146.73 ns | 2.557 ns | 2.392 ns | 0.0165 | 104 B | +| CounterWith6LabelsHotPath | Cumulative | 163.22 ns | 2.136 ns | 1.783 ns | 0.0191 | 120 B | +| CounterWith7LabelsHotPath | Cumulative | 184.53 ns | 1.324 ns | 1.238 ns | 0.0215 | 136 B | +| CounterWith1LabelsHotPathUsingTagList | Cumulative | 58.27 ns | 1.157 ns | 1.504 ns | - | - | +| CounterWith2LabelsHotPathUsingTagList | Cumulative | 92.87 ns | 0.550 ns | 0.488 ns | - | - | +| CounterWith3LabelsHotPathUsingTagList | Cumulative | 120.31 ns | 0.739 ns | 0.617 ns | - | - | +| CounterWith4LabelsHotPathUsingTagList | Cumulative | 132.98 ns | 2.181 ns | 2.240 ns | - | - | +| CounterWith5LabelsHotPathUsingTagList | Cumulative | 154.56 ns | 2.685 ns | 2.380 ns | - | - | +| CounterWith6LabelsHotPathUsingTagList | Cumulative | 171.36 ns | 2.738 ns | 2.286 ns | - | - | +| CounterWith7LabelsHotPathUsingTagList | Cumulative | 194.81 ns | 1.894 ns | 1.582 ns | - | - | +| CounterWith8LabelsHotPathUsingTagList | Cumulative | 214.39 ns | 1.339 ns | 1.187 ns | - | - | +| CounterWith9LabelsHotPathUsingTagList | Cumulative | 300.38 ns | 3.945 ns | 3.690 ns | 0.0710 | 448 B | +| CounterHotPath | Delta | 14.11 ns | 0.257 ns | 0.228 ns | - | - | +| CounterWith1LabelsHotPath | Delta | 49.15 ns | 0.295 ns | 0.246 ns | - | - | +| CounterWith2LabelsHotPath | Delta | 68.99 ns | 0.477 ns | 0.398 ns | - | - | +| CounterWith3LabelsHotPath | Delta | 93.35 ns | 1.294 ns | 1.080 ns | - | - | +| CounterWith4LabelsHotPath | Delta | 141.40 ns | 2.846 ns | 6.539 ns | 0.0138 | 88 B | +| CounterWith5LabelsHotPath | Delta | 163.34 ns | 3.189 ns | 3.917 ns | 0.0165 | 104 B | +| CounterWith6LabelsHotPath | Delta | 181.62 ns | 3.582 ns | 4.125 ns | 0.0191 | 120 B | +| CounterWith7LabelsHotPath | Delta | 201.33 ns | 2.700 ns | 2.108 ns | 0.0215 | 136 B | +| CounterWith1LabelsHotPathUsingTagList | Delta | 75.56 ns | 1.457 ns | 1.496 ns | - | - | +| CounterWith2LabelsHotPathUsingTagList | Delta | 91.48 ns | 1.852 ns | 2.714 ns | - | - | +| CounterWith3LabelsHotPathUsingTagList | Delta | 129.23 ns | 2.608 ns | 3.298 ns | - | - | +| CounterWith4LabelsHotPathUsingTagList | Delta | 150.55 ns | 2.433 ns | 2.498 ns | - | - | +| CounterWith5LabelsHotPathUsingTagList | Delta | 191.60 ns | 3.119 ns | 2.918 ns | - | - | +| CounterWith6LabelsHotPathUsingTagList | Delta | 196.49 ns | 2.874 ns | 2.400 ns | - | - | +| CounterWith7LabelsHotPathUsingTagList | Delta | 224.42 ns | 4.482 ns | 8.196 ns | - | - | +| CounterWith8LabelsHotPathUsingTagList | Delta | 243.75 ns | 4.861 ns | 9.482 ns | - | - | +| CounterWith9LabelsHotPathUsingTagList | Delta | 331.22 ns | 6.493 ns | 11.373 ns | 0.0710 | 448 B | */ namespace Benchmarks.Metrics; @@ -79,10 +101,18 @@ public void CounterHotPath() [Benchmark] public void CounterWith1LabelsHotPath() { - var tag1 = new KeyValuePair("DimName1", this.dimensionValues[this.random.Next(0, 2)]); + var tag1 = new KeyValuePair("DimName1", this.dimensionValues[this.random.Next(0, 10)]); this.counter.Add(100, tag1); } + [Benchmark] + public void CounterWith2LabelsHotPath() + { + var tag1 = new KeyValuePair("DimName1", this.dimensionValues[this.random.Next(0, 10)]); + var tag2 = new KeyValuePair("DimName2", this.dimensionValues[this.random.Next(0, 10)]); + this.counter.Add(100, tag1, tag2); + } + [Benchmark] public void CounterWith3LabelsHotPath() { @@ -92,8 +122,100 @@ public void CounterWith3LabelsHotPath() this.counter.Add(100, tag1, tag2, tag3); } + [Benchmark] + public void CounterWith4LabelsHotPath() + { + var tag1 = new KeyValuePair("DimName1", this.dimensionValues[this.random.Next(0, 2)]); + var tag2 = new KeyValuePair("DimName2", this.dimensionValues[this.random.Next(0, 5)]); + var tag3 = new KeyValuePair("DimName3", this.dimensionValues[this.random.Next(0, 10)]); + var tag4 = new KeyValuePair("DimName4", this.dimensionValues[this.random.Next(0, 10)]); + this.counter.Add(100, tag1, tag2, tag3, tag4); + } + [Benchmark] public void CounterWith5LabelsHotPath() + { + var tag1 = new KeyValuePair("DimName1", this.dimensionValues[this.random.Next(0, 2)]); + var tag2 = new KeyValuePair("DimName2", this.dimensionValues[this.random.Next(0, 2)]); + var tag3 = new KeyValuePair("DimName3", this.dimensionValues[this.random.Next(0, 5)]); + var tag4 = new KeyValuePair("DimName4", this.dimensionValues[this.random.Next(0, 5)]); + var tag5 = new KeyValuePair("DimName5", this.dimensionValues[this.random.Next(0, 10)]); + this.counter.Add(100, tag1, tag2, tag3, tag4, tag5); + } + + [Benchmark] + public void CounterWith6LabelsHotPath() + { + var tag1 = new KeyValuePair("DimName1", this.dimensionValues[this.random.Next(0, 2)]); + var tag2 = new KeyValuePair("DimName2", this.dimensionValues[this.random.Next(0, 2)]); + var tag3 = new KeyValuePair("DimName3", this.dimensionValues[this.random.Next(0, 2)]); + var tag4 = new KeyValuePair("DimName4", this.dimensionValues[this.random.Next(0, 5)]); + var tag5 = new KeyValuePair("DimName5", this.dimensionValues[this.random.Next(0, 5)]); + var tag6 = new KeyValuePair("DimName6", this.dimensionValues[this.random.Next(0, 5)]); + this.counter.Add(100, tag1, tag2, tag3, tag4, tag5, tag6); + } + + [Benchmark] + public void CounterWith7LabelsHotPath() + { + var tag1 = new KeyValuePair("DimName1", this.dimensionValues[this.random.Next(0, 1)]); + var tag2 = new KeyValuePair("DimName2", this.dimensionValues[this.random.Next(0, 2)]); + var tag3 = new KeyValuePair("DimName3", this.dimensionValues[this.random.Next(0, 2)]); + var tag4 = new KeyValuePair("DimName4", this.dimensionValues[this.random.Next(0, 2)]); + var tag5 = new KeyValuePair("DimName5", this.dimensionValues[this.random.Next(0, 5)]); + var tag6 = new KeyValuePair("DimName6", this.dimensionValues[this.random.Next(0, 5)]); + var tag7 = new KeyValuePair("DimName7", this.dimensionValues[this.random.Next(0, 5)]); + this.counter.Add(100, tag1, tag2, tag3, tag4, tag5, tag6, tag7); + } + + [Benchmark] + public void CounterWith1LabelsHotPathUsingTagList() + { + var tags = new TagList + { + { "DimName1", this.dimensionValues[this.random.Next(0, 10)] }, + }; + this.counter.Add(100, tags); + } + + [Benchmark] + public void CounterWith2LabelsHotPathUsingTagList() + { + var tags = new TagList + { + { "DimName1", this.dimensionValues[this.random.Next(0, 10)] }, + { "DimName2", this.dimensionValues[this.random.Next(0, 10)] }, + }; + this.counter.Add(100, tags); + } + + [Benchmark] + public void CounterWith3LabelsHotPathUsingTagList() + { + var tags = new TagList + { + { "DimName1", this.dimensionValues[this.random.Next(0, 10)] }, + { "DimName2", this.dimensionValues[this.random.Next(0, 10)] }, + { "DimName3", this.dimensionValues[this.random.Next(0, 10)] }, + }; + this.counter.Add(100, tags); + } + + [Benchmark] + public void CounterWith4LabelsHotPathUsingTagList() + { + var tags = new TagList + { + { "DimName1", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName2", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName3", this.dimensionValues[this.random.Next(0, 10)] }, + { "DimName4", this.dimensionValues[this.random.Next(0, 10)] }, + }; + this.counter.Add(100, tags); + } + + [Benchmark] + public void CounterWith5LabelsHotPathUsingTagList() { var tags = new TagList { @@ -107,32 +229,67 @@ public void CounterWith5LabelsHotPath() } [Benchmark] - public void CounterWith6LabelsHotPath() + public void CounterWith6LabelsHotPathUsingTagList() { var tags = new TagList { { "DimName1", this.dimensionValues[this.random.Next(0, 2)] }, { "DimName2", this.dimensionValues[this.random.Next(0, 2)] }, - { "DimName3", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName3", this.dimensionValues[this.random.Next(0, 2)] }, { "DimName4", this.dimensionValues[this.random.Next(0, 5)] }, { "DimName5", this.dimensionValues[this.random.Next(0, 5)] }, - { "DimName6", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName6", this.dimensionValues[this.random.Next(0, 5)] }, }; this.counter.Add(100, tags); } [Benchmark] - public void CounterWith7LabelsHotPath() + public void CounterWith7LabelsHotPathUsingTagList() { var tags = new TagList { - { "DimName1", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName1", this.dimensionValues[this.random.Next(0, 1)] }, { "DimName2", this.dimensionValues[this.random.Next(0, 2)] }, - { "DimName3", this.dimensionValues[this.random.Next(0, 5)] }, - { "DimName4", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName3", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName4", this.dimensionValues[this.random.Next(0, 2)] }, { "DimName5", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName6", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName7", this.dimensionValues[this.random.Next(0, 5)] }, + }; + this.counter.Add(100, tags); + } + + [Benchmark] + public void CounterWith8LabelsHotPathUsingTagList() + { + var tags = new TagList + { + { "DimName1", this.dimensionValues[this.random.Next(0, 1)] }, + { "DimName2", this.dimensionValues[this.random.Next(0, 1)] }, + { "DimName3", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName4", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName5", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName6", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName7", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName8", this.dimensionValues[this.random.Next(0, 5)] }, + }; + this.counter.Add(100, tags); + } + + [Benchmark] + public void CounterWith9LabelsHotPathUsingTagList() + { + var tags = new TagList + { + { "DimName1", this.dimensionValues[this.random.Next(0, 1)] }, + { "DimName2", this.dimensionValues[this.random.Next(0, 1)] }, + { "DimName3", this.dimensionValues[this.random.Next(0, 1)] }, + { "DimName4", this.dimensionValues[this.random.Next(0, 2)] }, + { "DimName5", this.dimensionValues[this.random.Next(0, 2)] }, { "DimName6", this.dimensionValues[this.random.Next(0, 2)] }, - { "DimName7", this.dimensionValues[this.random.Next(0, 1)] }, + { "DimName7", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName8", this.dimensionValues[this.random.Next(0, 5)] }, + { "DimName9", this.dimensionValues[this.random.Next(0, 5)] }, }; this.counter.Add(100, tags); }