Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Do Not Merge] Add Meter Tags, Instrument Tags, and Scopes to System Diagnostics Metrics #237

Closed
wants to merge 7 commits into from
Closed

[Do Not Merge] Add Meter Tags, Instrument Tags, and Scopes to System Diagnostics Metrics #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b856bbb
End to end appears to be working, need to add testing and do cleanup.…
kkeirstead Oct 9, 2023
8f85ae4
Added prometheus test, still need json tests
kkeirstead Oct 11, 2023
8d3900e
Working on updated tests for JSON - not working yet.
kkeirstead Oct 11, 2023
5870813
Got live metrics test working
kkeirstead Oct 12, 2023
d661b1e
Small cleanup
kkeirstead Oct 13, 2023
28f5c37
Removed unintended change.
kkeirstead Oct 13, 2023
d27aca2
Small adjustments based on Noah's feedback.
kkeirstead Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/JsonCounterLogger.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ protected override async Task SerializeAsync(ICounterPayload counter)
}
else if (counter is CounterEndedPayload)
{
Logger.CounterEndedPayload(counter.Name);
Logger.CounterEndedPayload(counter.CounterInfo.CounterName);
return;
}
else if (!counter.IsValuePublishedEvent)
else if (!counter.EventType.IsValuePublishedEvent())
{
// Do we want to do anything with this payload?
return;
Expand All @@ -70,13 +70,16 @@ protected override async Task SerializeAsync(ICounterPayload counter)
Quantile quantile = aggregatePercentilePayload.Quantiles[i];

SerializeCounterValues(counter.Timestamp,
counter.Provider,
counter.Name,
counter.CounterInfo.ProviderName,
counter.CounterInfo.CounterName,
counter.DisplayName,
counter.Unit,
counter.CounterType.ToString(),
CounterUtilities.AppendPercentile(counter.Metadata, quantile.Percentage),
quantile.Value);
quantile.Value,
counter.CounterInfo.MeterTags,
counter.CounterInfo.InstrumentTags,
counter.CounterInfo.ScopeHash);

if (i < aggregatePercentilePayload.Quantiles.Length - 1)
{
Expand All @@ -90,13 +93,16 @@ protected override async Task SerializeAsync(ICounterPayload counter)
_bufferWriter.Clear();

SerializeCounterValues(counter.Timestamp,
counter.Provider,
counter.Name,
counter.CounterInfo.ProviderName,
counter.CounterInfo.CounterName,
counter.DisplayName,
counter.Unit,
counter.CounterType.ToString(),
counter.Metadata,
counter.Value);
counter.Value,
counter.CounterInfo.MeterTags,
counter.CounterInfo.InstrumentTags,
counter.CounterInfo.ScopeHash);
}
await _stream.WriteAsync(_bufferWriter.WrittenMemory);

Expand All @@ -111,7 +117,10 @@ private void SerializeCounterValues(
string unit,
string counterType,
string tags,
double value)
double value,
string meterTags,
string instrumentTags,
string scopeHash)
{
using var writer = new Utf8JsonWriter(_bufferWriter, new JsonWriterOptions { Indented = false });
writer.WriteStartObject();
Expand All @@ -127,6 +136,10 @@ private void SerializeCounterValues(
//Some versions of .Net return invalid metric numbers. See https://github.com/dotnet/runtime/pull/46938
writer.WriteNumber("value", double.IsNaN(value) ? 0.0 : value);

writer.WriteString("meterTags", meterTags);
writer.WriteString("instrumentTags", instrumentTags);
writer.WriteString("scopeHash", scopeHash);

writer.WriteEndObject();
}
}
Expand Down
18 changes: 9 additions & 9 deletions src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/MetricsStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ public MetricKey(ICounterPayload metric)
public override int GetHashCode()
{
HashCode code = new HashCode();
code.Add(_metric.Provider);
code.Add(_metric.Name);
code.Add(_metric.CounterInfo.ProviderName);
code.Add(_metric.CounterInfo.CounterName);
return code.ToHashCode();
}

Expand Down Expand Up @@ -73,9 +73,9 @@ public void AddMetric(ICounterPayload metric)
//Do not accept CounterEnded payloads.
if (metric is CounterEndedPayload counterEnded)
{
if (_observedEndedCounters.Add((counterEnded.Provider, counterEnded.Name)))
if (_observedEndedCounters.Add((counterEnded.CounterInfo.ProviderName, counterEnded.CounterInfo.CounterName)))
{
_logger.CounterEndedPayload(counterEnded.Name);
_logger.CounterEndedPayload(counterEnded.CounterInfo.CounterName);
}
return;
}
Expand All @@ -90,7 +90,7 @@ public void AddMetric(ICounterPayload metric)
}
return;
}
if (!metric.IsValuePublishedEvent)
if (!metric.EventType.IsValuePublishedEvent())
{
// Do we want to do anything with this payload?
return;
Expand Down Expand Up @@ -137,7 +137,7 @@ public async Task SnapshotMetrics(Stream outputStream, CancellationToken token)
{
ICounterPayload metricInfo = metricGroup.Value.First();

string metricName = PrometheusDataModel.GetPrometheusNormalizedName(metricInfo.Provider, metricInfo.Name, metricInfo.Unit);
string metricName = PrometheusDataModel.GetPrometheusNormalizedName(metricInfo.CounterInfo.ProviderName, metricInfo.CounterInfo.CounterName, metricInfo.Unit);

await WriteMetricHeader(metricInfo, writer, metricName);

Expand Down Expand Up @@ -165,10 +165,10 @@ public async Task SnapshotMetrics(Stream outputStream, CancellationToken token)

private static string GetMetricLabels(ICounterPayload metric, double? quantile)
{
string metadata = metric.Metadata;
string allMetadata = metric.CombineTags();

char separator = IsMeter(metric) ? '=' : ':';
var metadataValues = CounterUtilities.GetMetadata(metadata, separator);
var metadataValues = CounterUtilities.GetMetadata(allMetadata, separator);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So for prometheus export, we will effectively combine meter, instrument, and instance tags into one set correct?

if (quantile.HasValue)
{
metadataValues.Add("quantile", quantile.Value.ToString(CultureInfo.InvariantCulture));
Expand Down Expand Up @@ -243,7 +243,7 @@ private static async Task WriteMetricDetails(

private static bool CompareMetrics(ICounterPayload first, ICounterPayload second)
{
return string.Equals(first.Name, second.Name);
return string.Equals(first.CounterInfo.CounterName, second.CounterInfo.CounterName);
}

public void Clear()
Expand Down
6 changes: 6 additions & 0 deletions src/Tests/Microsoft.Diagnostics.Monitoring.TestCommon/CounterPayload.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,11 @@ internal class CounterPayload

[JsonPropertyName("tags")]
public string Metadata { get; set; }

[JsonPropertyName("meterTags")]
public string MeterTags { get; set; }

[JsonPropertyName("instrumentTags")]
public string InstrumentTags { get; set; }
}
}
5 changes: 5 additions & 0 deletions src/Tests/Microsoft.Diagnostics.Monitoring.TestCommon/LiveMetricsTestConstants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@ public static class LiveMetricsTestConstants
public const string HistogramName2 = "test-histogram-2";
public const string ProviderName1 = "P1";
public const string ProviderName2 = "P2";
public const string ProviderName3 = "P3";
public const string MetadataKey = "key1";
public const string MetadataValue = "value1";
public const string MeterMetadataKey = "mkey1";
public const string MeterMetadataValue = "mvalue1";
public const string InstrumentMetadataKey = "ikey1";
public const string InstrumentMetadataValue = "ivalue1";
}
}
17 changes: 17 additions & 0 deletions src/Tests/Microsoft.Diagnostics.Monitoring.TestCommon/LiveMetricsTestUtilities.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,23 @@ internal static async Task AggregateMetrics(IAsyncEnumerable<CounterPayload> act
}
}

internal static async Task AggregateMetrics(IAsyncEnumerable<CounterPayload> actualMetrics,
List<string> providers,
List<string> names,
List<string> metadata,
List<string> meterTags,
List<string> instrumentTags)
{
await foreach (CounterPayload counter in actualMetrics)
{
providers.Add(counter.Provider);
names.Add(counter.Name);
metadata.Add(counter.Metadata);
meterTags.Add(counter.MeterTags);
instrumentTags.Add(counter.InstrumentTags);
}
}

internal static async IAsyncEnumerable<CounterPayload> GetAllMetrics(Stream liveMetricsStream)
{
using var reader = new StreamReader(liveMetricsStream);
Expand Down
83 changes: 83 additions & 0 deletions src/Tests/Microsoft.Diagnostics.Monitoring.Tool.FunctionalTests/LiveMetricsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,5 +360,88 @@ await ScenarioRunner.SingleTarget(
});
});
}

#if NET8_0_OR_GREATER
[Fact]
public async Task TestSystemDiagnosticsMetrics_MeterInstrumentTags()
{
var instrumentNamesP3 = new[] { Constants.CounterName };

MetricProvider p3 = new MetricProvider()
{
ProviderName = Constants.ProviderName3
};

var providers = new List<MetricProvider>()
{
p3
};

await ScenarioRunner.SingleTarget(
_outputHelper,
_httpClientFactory,
DiagnosticPortConnectionMode.Connect,
TestAppScenarios.Metrics.Name,
appValidate: async (runner, client) =>
{
using ResponseStreamHolder holder = await client.CaptureMetricsAsync(await runner.ProcessIdTask,
durationSeconds: CommonTestTimeouts.LiveMetricsDurationSeconds,
metricsConfiguration: new EventMetricsConfiguration
{
IncludeDefaultProviders = false,
Meters = new[]
{
new EventMetricsMeter
{
MeterName = p3.ProviderName,
InstrumentNames = instrumentNamesP3,
}
}
});

await runner.SendCommandAsync(TestAppScenarios.Metrics.Commands.Continue);

var metrics = LiveMetricsTestUtilities.GetAllMetrics(holder.Stream);

List<string> actualMeterNames = new();
List<string> actualInstrumentNames = new();
List<string> actualMetadata = new();
List<string> actualMeterTags = new();
List<string> actualInstrumentTags = new();

await LiveMetricsTestUtilities.AggregateMetrics(metrics, actualMeterNames, actualInstrumentNames, actualMetadata, actualMeterTags, actualInstrumentTags);

LiveMetricsTestUtilities.ValidateMetrics(new[] { p3.ProviderName },
instrumentNamesP3,
actualMeterNames.ToHashSet(),
actualInstrumentNames.ToHashSet(),
strict: true);

string actualMeterTag = Assert.Single(actualMeterTags.Distinct());
string expectedMeterTag = Constants.MeterMetadataKey + "=" + Constants.MeterMetadataValue;
Assert.Equal(expectedMeterTag, actualMeterTag);

string actualInstrumentTag = Assert.Single(actualInstrumentTags.Distinct());
string expectedInstrumentTag = Constants.InstrumentMetadataKey + "=" + Constants.InstrumentMetadataValue;
Assert.Equal(expectedInstrumentTag, actualInstrumentTag);
},
configureTool: runner =>
{
runner.WriteKeyPerValueConfiguration(new RootOptions()
{
Metrics = new MetricsOptions()
{
Enabled = true,
IncludeDefaultProviders = false,
Providers = providers
},
GlobalCounter = new GlobalCounterOptions()
{
IntervalSeconds = 1
}
});
});
}
#endif
}
}
48 changes: 40 additions & 8 deletions src/Tests/Microsoft.Diagnostics.Monitoring.Tool.UnitTests/MetricsFormattingTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,16 @@ public async Task HistogramFormat_Test()
{
List<ICounterPayload> payload = new();

payload.Add(new AggregatePercentilePayload(MeterName, InstrumentName, "DisplayName", string.Empty, string.Empty,
CachedCounterInfo counterInfo = new CachedCounterInfo(MeterName, InstrumentName, null, null, null);

payload.Add(new AggregatePercentilePayload(counterInfo, "DisplayName", string.Empty, string.Empty,
new Quantile[] { new Quantile(0.5, Value1), new Quantile(0.95, Value2), new Quantile(0.99, Value3) },
Timestamp));

using MemoryStream stream = await GetMetrics(payload);
List<string> lines = ReadStream(stream);

string metricName = $"{MeterName.ToLowerInvariant()}_{payload[0].Name}";
string metricName = $"{MeterName.ToLowerInvariant()}_{payload[0].CounterInfo.CounterName}";

const string quantile_50 = "{quantile=\"0.5\"}";
const string quantile_95 = "{quantile=\"0.95\"}";
Expand All @@ -68,13 +70,15 @@ public async Task HistogramFormat_Test()
[Fact]
public async Task GaugeFormat_Test()
{
ICounterPayload payload = new GaugePayload(MeterName, InstrumentName, "DisplayName", "", null, Value1, Timestamp);
CachedCounterInfo counterInfo = new CachedCounterInfo(MeterName, InstrumentName, null, null, null);

ICounterPayload payload = new GaugePayload(counterInfo, "DisplayName", "", null, Value1, Timestamp);

MemoryStream stream = await GetMetrics(new() { payload });

List<string> lines = ReadStream(stream);

string metricName = $"{MeterName.ToLowerInvariant()}_{payload.Name}";
string metricName = $"{MeterName.ToLowerInvariant()}_{payload.CounterInfo.CounterName}";

Assert.Equal(3, lines.Count);
Assert.Equal(FormattableString.Invariant($"# HELP {metricName}{payload.Unit} {payload.DisplayName}"), lines[0]);
Expand All @@ -85,30 +89,58 @@ public async Task GaugeFormat_Test()
[Fact]
public async Task CounterFormat_Test()
{
ICounterPayload payload = new RatePayload(MeterName, InstrumentName, "DisplayName", "", null, Value1, IntervalSeconds, Timestamp);
CachedCounterInfo counterInfo = new CachedCounterInfo(MeterName, InstrumentName, null, null, null);

ICounterPayload payload = new RatePayload(counterInfo, "DisplayName", "", null, Value1, IntervalSeconds, Timestamp);

MemoryStream stream = await GetMetrics(new() { payload });

List<string> lines = ReadStream(stream);

string metricName = $"{MeterName.ToLowerInvariant()}_{payload.Name}";
string metricName = $"{MeterName.ToLowerInvariant()}_{payload.CounterInfo.CounterName}";

Assert.Equal(3, lines.Count);
Assert.Equal($"# HELP {metricName}{payload.Unit} {payload.DisplayName}", lines[0]);
Assert.Equal($"# TYPE {metricName} gauge", lines[1]);
Assert.Equal($"{metricName} {payload.Value} {new DateTimeOffset(payload.Timestamp).ToUnixTimeMilliseconds()}", lines[2]);
}

[Fact]
public async Task CounterFormat_Test_Tags()
{
string meterTags = "MeterTagKey=MeterTagValue,MeterTagKey2=MeterTagValue2";
string instrumentTags = "InstrumentTagKey=InstrumentTagValue,InstrumentTagKey2=InstrumentTagValue2";
string scopeHash = "123";

CachedCounterInfo counterInfo = new CachedCounterInfo(MeterName, InstrumentName, meterTags, instrumentTags, scopeHash);

ICounterPayload payload = new RatePayload(counterInfo, "DisplayName", "", null, Value1, IntervalSeconds, Timestamp);

MemoryStream stream = await GetMetrics(new() { payload });

List<string> lines = ReadStream(stream);

string metricName = $"{MeterName.ToLowerInvariant()}_{payload.CounterInfo.CounterName}";
string metricTags = "{MeterTagKey=\"MeterTagValue\", MeterTagKey2=\"MeterTagValue2\", InstrumentTagKey=\"InstrumentTagValue\", InstrumentTagKey2=\"InstrumentTagValue2\"}";

Assert.Equal(3, lines.Count);
Assert.Equal($"# HELP {metricName}{payload.Unit} {payload.DisplayName}", lines[0]);
Assert.Equal($"# TYPE {metricName} gauge", lines[1]);
Assert.Equal($"{metricName}{metricTags} {payload.Value} {new DateTimeOffset(payload.Timestamp).ToUnixTimeMilliseconds()}", lines[2]);
}

[Fact]
public async Task UpDownCounterFormat_Test()
{
ICounterPayload payload = new UpDownCounterPayload(MeterName, InstrumentName, "DisplayName", "", null, Value1, Timestamp);
CachedCounterInfo counterInfo = new CachedCounterInfo(MeterName, InstrumentName, null, null, null);

ICounterPayload payload = new UpDownCounterPayload(counterInfo, "DisplayName", "", null, Value1, Timestamp);

MemoryStream stream = await GetMetrics(new() { payload });

List<string> lines = ReadStream(stream);

string metricName = $"{MeterName.ToLowerInvariant()}_{payload.Name}";
string metricName = $"{MeterName.ToLowerInvariant()}_{payload.CounterInfo.CounterName}";

Assert.Equal(3, lines.Count);
Assert.Equal(FormattableString.Invariant($"# HELP {metricName}{payload.Unit} {payload.DisplayName}"), lines[0]);
Expand Down
Loading