diff --git a/build/Common.props b/build/Common.props index 3420b287d8..a35274d9ae 100644 --- a/build/Common.props +++ b/build/Common.props @@ -34,7 +34,7 @@ [3.3.3] [1.1.1,2.0) [1.4.0,2.0) - [1.4.0,2.0) + [1.5.0-alpha.1] [2.1.58,3.0) [1.2.0-beta.435,2.0) [4.3.4,) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index 531b8613e4..99ca6b4a83 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -5,10 +5,15 @@ * Changed the behavior of Unix domain socket connection at startup. Before this change, the exporter initialization would throw exception if the target Unix Domain Socket does not exist. After this change, the exporter initialization - would return success and the exporting background thread will try to - establish the connection. + would return success and the exporting background thread will try to establish + the connection. ([#935](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/935)) +* Update OTel SDK version to `1.5.0-alpha.1`. +* Update GenevaMetricExporter to use TLV format serialization. +* Add support for exporting exemplars. + ([#1069](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/1069)) + ## 1.4.0 Released 2023-Feb-27 diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs index 485982c91c..6f314f4249 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs @@ -35,8 +35,6 @@ public class GenevaMetricExporter : BaseExporter internal const int MaxDimensionValueSize = 1024; - private static readonly MetricData ulongZero = new MetricData { UInt64Value = 0 }; - private readonly ushort prepopulatedDimensionsCount; private readonly int fixedPayloadStartIndex; @@ -51,6 +49,8 @@ public class GenevaMetricExporter : BaseExporter private readonly List serializedPrepopulatedDimensionsValues; + private readonly byte[] buffer = new byte[BufferSize]; + private readonly byte[] bufferForNonHistogramMetrics = new byte[BufferSize]; private readonly byte[] bufferForHistogramMetrics = new byte[BufferSize]; @@ -126,19 +126,22 @@ public override ExportResult Export(in Batch batch) { try { + var exemplars = metricPoint.GetExemplars(); + switch (metric.MetricType) { case MetricType.LongSum: { var ulongSum = Convert.ToUInt64(metricPoint.GetSumLong()); var metricData = new MetricData { UInt64Value = ulongSum }; - var bodyLength = this.SerializeMetric( + var bodyLength = this.SerializeMetricWithTLV( MetricEventType.ULongMetric, metric.Name, metricPoint.EndTime.ToFileTime(), // Using the endTime here as the timestamp as Geneva Metrics only allows for one field for timestamp metricPoint.Tags, - metricData); - this.metricDataTransport.Send(MetricEventType.ULongMetric, this.bufferForNonHistogramMetrics, bodyLength); + metricData, + exemplars); + this.metricDataTransport.Send(MetricEventType.TLV, this.buffer, bodyLength); break; } @@ -149,13 +152,14 @@ public override ExportResult Export(in Batch batch) // see: https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/builtin-types/numeric-conversions#implicit-numeric-conversions var doubleSum = Convert.ToDouble(metricPoint.GetGaugeLastValueLong()); var metricData = new MetricData { DoubleValue = doubleSum }; - var bodyLength = this.SerializeMetric( + var bodyLength = this.SerializeMetricWithTLV( MetricEventType.DoubleMetric, metric.Name, metricPoint.EndTime.ToFileTime(), metricPoint.Tags, - metricData); - this.metricDataTransport.Send(MetricEventType.DoubleMetric, this.bufferForNonHistogramMetrics, bodyLength); + metricData, + exemplars); + this.metricDataTransport.Send(MetricEventType.TLV, this.buffer, bodyLength); break; } @@ -166,13 +170,14 @@ public override ExportResult Export(in Batch batch) // see: https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/builtin-types/numeric-conversions#implicit-numeric-conversions var doubleSum = Convert.ToDouble(metricPoint.GetSumLong()); var metricData = new MetricData { DoubleValue = doubleSum }; - var bodyLength = this.SerializeMetric( + var bodyLength = this.SerializeMetricWithTLV( MetricEventType.DoubleMetric, metric.Name, metricPoint.EndTime.ToFileTime(), metricPoint.Tags, - metricData); - this.metricDataTransport.Send(MetricEventType.DoubleMetric, this.bufferForNonHistogramMetrics, bodyLength); + metricData, + exemplars); + this.metricDataTransport.Send(MetricEventType.TLV, this.buffer, bodyLength); break; } @@ -181,13 +186,14 @@ public override ExportResult Export(in Batch batch) { var doubleSum = metricPoint.GetSumDouble(); var metricData = new MetricData { DoubleValue = doubleSum }; - var bodyLength = this.SerializeMetric( + var bodyLength = this.SerializeMetricWithTLV( MetricEventType.DoubleMetric, metric.Name, metricPoint.EndTime.ToFileTime(), metricPoint.Tags, - metricData); - this.metricDataTransport.Send(MetricEventType.DoubleMetric, this.bufferForNonHistogramMetrics, bodyLength); + metricData, + exemplars); + this.metricDataTransport.Send(MetricEventType.TLV, this.buffer, bodyLength); break; } @@ -195,29 +201,28 @@ public override ExportResult Export(in Batch batch) { var doubleSum = metricPoint.GetGaugeLastValueDouble(); var metricData = new MetricData { DoubleValue = doubleSum }; - var bodyLength = this.SerializeMetric( + var bodyLength = this.SerializeMetricWithTLV( MetricEventType.DoubleMetric, metric.Name, metricPoint.EndTime.ToFileTime(), metricPoint.Tags, - metricData); - this.metricDataTransport.Send(MetricEventType.DoubleMetric, this.bufferForNonHistogramMetrics, bodyLength); + metricData, + exemplars); + this.metricDataTransport.Send(MetricEventType.TLV, this.buffer, bodyLength); break; } case MetricType.Histogram: { - var sum = new MetricData { UInt64Value = Convert.ToUInt64(metricPoint.GetHistogramSum()) }; + var sum = Convert.ToUInt64(metricPoint.GetHistogramSum()); var count = Convert.ToUInt32(metricPoint.GetHistogramCount()); - MetricData min = ulongZero; - MetricData max = ulongZero; - if (metricPoint.TryGetHistogramMinMaxValues(out var minValue, out var maxValue)) + if (!metricPoint.TryGetHistogramMinMaxValues(out double min, out double max)) { - min = new MetricData { UInt64Value = Convert.ToUInt64(minValue) }; - max = new MetricData { UInt64Value = Convert.ToUInt64(maxValue) }; + min = 0; + max = 0; } - var bodyLength = this.SerializeHistogramMetric( + var bodyLength = this.SerializeHistogramMetricWithTLV( metric.Name, metricPoint.EndTime.ToFileTime(), metricPoint.Tags, @@ -225,8 +230,9 @@ public override ExportResult Export(in Batch batch) sum, count, min, - max); - this.metricDataTransport.Send(MetricEventType.ExternallyAggregatedULongDistributionMetric, this.bufferForHistogramMetrics, bodyLength); + max, + exemplars); + this.metricDataTransport.Send(MetricEventType.TLV, this.buffer, bodyLength); break; } } @@ -484,6 +490,384 @@ internal unsafe ushort SerializeHistogramMetric( return bodyLength; } + internal unsafe ushort SerializeMetricWithTLV( + MetricEventType eventType, + string metricName, + long timestamp, + in ReadOnlyTagCollection tags, + MetricData value, + Exemplar[] exemplars) + { + ushort bodyLength; + try + { + // The buffer format is as follows: + // -- BinaryHeader + // -- Sequence of payload types + + // Leave enough space for the header + var bufferIndex = sizeof(BinaryHeader); + + // Serialize metric name + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.MetricName); + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(metricName)); + + #region Serialize metric data + + var payloadType = eventType == MetricEventType.ULongMetric ? PayloadType.ULongMetric : PayloadType.DoubleMetric; + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)payloadType); + + // Get a placeholder to add the payloadType length + int payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, (ulong)timestamp); // timestamp + + if (payloadType == PayloadType.ULongMetric) + { + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, value.UInt64Value); + } + else + { + MetricSerializer.SerializeFloat64(this.buffer, ref bufferIndex, value.DoubleValue); + } + + var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + + #endregion + + #region Serialize metric dimensions + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Dimensions); + + // Get a placeholder to add the payloadType length + payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + // Get a placeholder to add dimensions count later + var bufferIndexForDimensionsCount = bufferIndex; + bufferIndex += 2; + + ushort dimensionsWritten = 0; + + // Serialize PrepopulatedDimensions keys + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsKeys[i]); + } + + if (this.prepopulatedDimensionsCount > 0) + { + dimensionsWritten += this.prepopulatedDimensionsCount; + } + + // Serialize MetricPoint Dimension keys + foreach (var tag in tags) + { + if (tag.Key.Length > MaxDimensionNameSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.buffer, ref bufferIndex, tag.Key); + } + + dimensionsWritten += (ushort)tags.Count; + + // Serialize PrepopulatedDimensions values + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsValues[i]); + } + + // Serialize MetricPoint Dimension values + foreach (var tag in tags) + { + var dimensionValue = Convert.ToString(tag.Value, CultureInfo.InvariantCulture); + if (dimensionValue.Length > MaxDimensionValueSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.buffer, ref bufferIndex, dimensionValue); + } + + // Backfill the number of dimensions written + MetricSerializer.SerializeUInt16(this.buffer, ref bufferIndexForDimensionsCount, dimensionsWritten); + + payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + + #endregion + + #region Serialize exemplars + + if (exemplars.Length > 0) + { + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Exemplars); + + // Get a placeholder to add the payloadType length + payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // version + + var exemplarsCount = 0; + foreach (var exemplar in exemplars) + { + if (exemplar.Timestamp != default) + { + exemplarsCount++; + } + } + + MetricSerializer.SerializeInt32AsBase128(this.buffer, ref bufferIndex, exemplarsCount); + + foreach (var exemplar in exemplars) + { + if (exemplar.Timestamp != default) + { + this.SerializeExemplar(exemplar, ref bufferIndex); + } + } + } + + payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + + #endregion + + // Serialize monitoring account + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.AccountName); + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.monitoringAccount)); + + // Serialize metric namespace + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Namespace); + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.metricNamespace)); + + // Write the final size of the payload + bodyLength = (ushort)(bufferIndex - this.fixedPayloadStartIndex); + + // Copy in the final structures to the front + fixed (byte* bufferBytes = this.buffer) + { + var ptr = (BinaryHeader*)bufferBytes; + ptr->EventId = (ushort)MetricEventType.TLV; + ptr->BodyLength = bodyLength; + } + } + finally + { + } + + return bodyLength; + } + + internal unsafe ushort SerializeHistogramMetricWithTLV( + string metricName, + long timestamp, + in ReadOnlyTagCollection tags, + HistogramBuckets buckets, + double sum, + uint count, + double min, + double max, + Exemplar[] exemplars) + { + ushort bodyLength; + try + { + // The buffer format is as follows: + // -- BinaryHeader + // -- Sequence of payload types + + // Leave enough space for the header + var bufferIndex = sizeof(BinaryHeader); + + // Serialize metric name + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.MetricName); + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(metricName)); + + #region Serialize histogram metric data + + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.ExternallyAggregatedULongDistributionMetric); + + // Get a placeholder to add the payloadType length + int payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + // Serialize sum, count, min, and max + MetricSerializer.SerializeUInt32(this.buffer, ref bufferIndex, count); // histogram count + MetricSerializer.SerializeUInt32(this.buffer, ref bufferIndex, 0); // padding + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, (ulong)timestamp); // timestamp + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(sum)); // histogram sum + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(min)); // histogram min + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(max)); // histogram max + + var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + + // Serialize histogram buckets as value-count pairs + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.HistogramULongValueCountPairs); + + // Get a placeholder to add the payloadType length + payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + // Get a placeholder to add the number of value-count pairs added + // with value being the bucket boundary and count being the respective count + + var itemsWrittenIndex = bufferIndex; + MetricSerializer.SerializeUInt16(this.buffer, ref bufferIndex, 0); + + // Bucket values + ushort bucketCount = 0; + double lastExplicitBound = default; + foreach (var bucket in buckets) + { + if (bucket.BucketCount > 0) + { + this.SerializeHistogramBucketWithTLV(bucket, ref bufferIndex, lastExplicitBound); + bucketCount++; + } + + lastExplicitBound = bucket.ExplicitBound; + } + + // Write the number of items in distribution emitted and reset back to end. + MetricSerializer.SerializeUInt16(this.buffer, ref itemsWrittenIndex, bucketCount); + + payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + + #endregion + + #region Serialize metric dimensions + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Dimensions); + + // Get a placeholder to add the payloadType length + payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + // Get a placeholder to add dimensions count later + var bufferIndexForDimensionsCount = bufferIndex; + bufferIndex += 2; + + ushort dimensionsWritten = 0; + + // Serialize PrepopulatedDimensions keys + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsKeys[i]); + } + + if (this.prepopulatedDimensionsCount > 0) + { + dimensionsWritten += this.prepopulatedDimensionsCount; + } + + // Serialize MetricPoint Dimension keys + foreach (var tag in tags) + { + if (tag.Key.Length > MaxDimensionNameSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.buffer, ref bufferIndex, tag.Key); + } + + dimensionsWritten += (ushort)tags.Count; + + // Serialize PrepopulatedDimensions values + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsValues[i]); + } + + // Serialize MetricPoint Dimension values + foreach (var tag in tags) + { + var dimensionValue = Convert.ToString(tag.Value, CultureInfo.InvariantCulture); + if (dimensionValue.Length > MaxDimensionValueSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.buffer, ref bufferIndex, dimensionValue); + } + + // Backfill the number of dimensions written + MetricSerializer.SerializeUInt16(this.buffer, ref bufferIndexForDimensionsCount, dimensionsWritten); + + payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + + #endregion + + #region Serialize exemplars + + if (exemplars.Length > 0) + { + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Exemplars); + + // Get a placeholder to add the payloadType length + payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // version + + var exemplarsCount = 0; + foreach (var exemplar in exemplars) + { + if (exemplar.Timestamp != default) + { + exemplarsCount++; + } + } + + MetricSerializer.SerializeInt32AsBase128(this.buffer, ref bufferIndex, exemplarsCount); + + foreach (var exemplar in exemplars) + { + if (exemplar.Timestamp != default) + { + this.SerializeExemplar(exemplar, ref bufferIndex); + } + } + } + + payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + + #endregion + + // Serialize monitoring account + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.AccountName); + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.monitoringAccount)); + + // Serialize metric namespace + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Namespace); + MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.metricNamespace)); + + // Write the final size of the payload + bodyLength = (ushort)(bufferIndex - this.fixedPayloadStartIndex); + + // Copy in the final structures to the front + fixed (byte* bufferBytes = this.buffer) + { + var ptr = (BinaryHeader*)bufferBytes; + ptr->EventId = (ushort)MetricEventType.TLV; + ptr->BodyLength = bodyLength; + } + } + finally + { + } + + return bodyLength; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void SerializeHistogramBucket(in HistogramBucket bucket, ref int bufferIndex, double lastExplicitBound) { @@ -500,6 +884,100 @@ private void SerializeHistogramBucket(in HistogramBucket bucket, ref int bufferI MetricSerializer.SerializeUInt32(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt32(bucket.BucketCount)); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SerializeHistogramBucketWithTLV(in HistogramBucket bucket, ref int bufferIndex, double lastExplicitBound) + { + if (bucket.ExplicitBound != double.PositiveInfinity) + { + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(bucket.ExplicitBound)); + } + else + { + // The bucket to catch the overflows is one greater than the last bound provided + MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(lastExplicitBound + 1)); + } + + MetricSerializer.SerializeUInt32(this.buffer, ref bufferIndex, Convert.ToUInt32(bucket.BucketCount)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SerializeExemplar(Exemplar exemplar, ref int bufferIndex) + { + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // version + + var bufferIndexForLength = bufferIndex; + bufferIndex++; + + var bufferIndexForFlags = bufferIndex; + bufferIndex++; + + var flags = ExemplarFlags.IsTimestampAvailable; // we only serialize exemplars with Timestamp != default + + // TODO: Update the code whenn Exemplars support long values + var value = exemplar.DoubleValue; + + // Check if the double value is actually a whole number that can be serialized as a long instead + var valueAsLong = (long)value; + bool isWholeNumber = valueAsLong == value; + if (isWholeNumber) + { + flags |= ExemplarFlags.IsMetricValueDoubleStoredAsLong; + MetricSerializer.SerializeInt64AsBase128(this.buffer, ref bufferIndex, valueAsLong); // serialize long value + } + else + { + MetricSerializer.SerializeFloat64(this.buffer, ref bufferIndex, value); // serialize double value + } + + var bufferIndexForNumberOfLabels = bufferIndex; + MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // serialize zero as the count of labels; this would be updated later if the exemplar has labels + byte numberOfLabels = 0; + + // Convert exemplar timestamp to unix nanoseconds + var unixNanoSeconds = DateTime.FromFileTimeUtc(exemplar.Timestamp.ToFileTime()) + .ToUniversalTime() + .Subtract(new DateTime(1970, 1, 1)) + .TotalMilliseconds * 1000000; + + MetricSerializer.SerializeInt64(this.buffer, ref bufferIndex, (long)unixNanoSeconds); // serialize timestamp + + if (exemplar.TraceId.HasValue) + { + Span traceIdBytes = stackalloc byte[16]; + exemplar.TraceId.Value.CopyTo(traceIdBytes); + MetricSerializer.SerializeSpanOfBytes(this.buffer, ref bufferIndex, traceIdBytes, traceIdBytes.Length); // serialize traceId + + flags |= ExemplarFlags.TraceIdExists; + } + + if (exemplar.SpanId.HasValue) + { + Span spanIdBytes = stackalloc byte[8]; + exemplar.SpanId.Value.CopyTo(spanIdBytes); + MetricSerializer.SerializeSpanOfBytes(this.buffer, ref bufferIndex, spanIdBytes, spanIdBytes.Length); // serialize spanId + + flags |= ExemplarFlags.SpanIdExists; + } + + bool hasLabels = exemplar.FilteredTags != null && exemplar.FilteredTags.Count > 0; + if (hasLabels) + { + foreach (var tag in exemplar.FilteredTags) + { + MetricSerializer.SerializeBase128String(this.buffer, ref bufferIndex, tag.Key); + MetricSerializer.SerializeBase128String(this.buffer, ref bufferIndex, Convert.ToString(tag.Value, CultureInfo.InvariantCulture)); + numberOfLabels++; + } + + MetricSerializer.SerializeByte(this.buffer, ref bufferIndexForNumberOfLabels, numberOfLabels); + } + + MetricSerializer.SerializeByte(this.buffer, ref bufferIndexForFlags, (byte)flags); + + var exemplarLength = bufferIndex - bufferIndexForLength + 1; + MetricSerializer.SerializeByte(this.buffer, ref bufferIndexForLength, (byte)exemplarLength); + } + private List SerializePrepopulatedDimensionsKeys(IEnumerable keys) { var serializedKeys = new List(this.prepopulatedDimensionsCount); diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/MetricSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/MetricSerializer.cs index fa1615c672..31322ec28a 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/MetricSerializer.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/MetricSerializer.cs @@ -193,6 +193,189 @@ public static void SerializeUInt64(byte[] buffer, ref int bufferIndex, ulong val buffer[bufferIndex + 7] = (byte)(value >> 0x38); bufferIndex += sizeof(ulong); } + + /// + /// Writes the long to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeInt64(byte[] buffer, ref int bufferIndex, long value) + { + if (bufferIndex + sizeof(long) >= buffer.Length) + { + } + + buffer[bufferIndex] = (byte)value; + buffer[bufferIndex + 1] = (byte)(value >> 8); + buffer[bufferIndex + 2] = (byte)(value >> 0x10); + buffer[bufferIndex + 3] = (byte)(value >> 0x18); + buffer[bufferIndex + 4] = (byte)(value >> 0x20); + buffer[bufferIndex + 5] = (byte)(value >> 0x28); + buffer[bufferIndex + 6] = (byte)(value >> 0x30); + buffer[bufferIndex + 7] = (byte)(value >> 0x38); + bufferIndex += sizeof(long); + } + + /// + /// Writes the double to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe void SerializeFloat64(byte[] buffer, ref int bufferIndex, double value) + { + if (bufferIndex + sizeof(double) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + + fixed (byte* bp = buffer) + { + *(double*)(bp + bufferIndex) = value; + } + + bufferIndex += sizeof(double); + } + + /// + /// Writes the base128 string to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeBase128String(byte[] buffer, ref int bufferIndex, string value) + { + if (!string.IsNullOrEmpty(value)) + { + if (bufferIndex + value.Length + sizeof(short) >= buffer.Length) + { + } + + var encodedValue = Encoding.UTF8.GetBytes(value); + SerializeUInt64AsBase128(buffer, ref bufferIndex, (ulong)encodedValue.Length); + Array.Copy(encodedValue, 0, buffer, bufferIndex, encodedValue.Length); + bufferIndex += encodedValue.Length; + } + else + { + SerializeInt16(buffer, ref bufferIndex, 0); + } + } + + /// + /// Writes unsigned int value Base-128 encoded. + /// + /// Buffer used for writing. + /// Offset to start with. Will be moved to the next byte after written. + /// Value to write. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeUInt32AsBase128(byte[] buffer, ref int offset, uint value) + { + SerializeUInt64AsBase128(buffer, ref offset, value); + } + + /// + /// Writes ulong value Base-128 encoded to the buffer starting from the specified offset. + /// + /// Buffer used for writing. + /// Offset to start with. Will be moved to the next byte after written. + /// Value to write. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeUInt64AsBase128(byte[] buffer, ref int offset, ulong value) + { + var t = value; + do + { + var b = (byte)(t & 0x7f); + t >>= 7; + if (t > 0) + { + b |= 0x80; + } + + buffer[offset++] = b; + } + while (t > 0); + } + + /// + /// Writes int value Base-128 encoded. + /// + /// Buffer used for writing. + /// Offset to start with. Will be moved to the next byte after written. + /// Value to write. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeInt32AsBase128(byte[] buffer, ref int offset, int value) + { + SerializeInt64AsBase128(buffer, ref offset, value); + } + + /// + /// Writes long value Base-128 encoded. + /// + /// Buffer used for writing. + /// Offset to start with. Will be moved to the next byte after written. + /// Value to write. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeInt64AsBase128(byte[] buffer, ref int offset, long value) + { + var negative = value < 0; + var t = negative ? -value : value; + var first = true; + do + { + byte b; + if (first) + { + b = (byte)(t & 0x3f); + t >>= 6; + if (negative) + { + b = (byte)(b | 0x40); + } + + first = false; + } + else + { + b = (byte)(t & 0x7f); + t >>= 7; + } + + if (t > 0) + { + b |= 0x80; + } + + buffer[offset++] = b; + } + while (t > 0); + } + + /// + /// Writes the encoded string to buffer. + /// + /// The buffer to write data into. + /// Index of the buffer. + /// Source data. + /// Number of bytes to copy. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeSpanOfBytes(byte[] buffer, ref int bufferIndex, Span data, int dataLength) + { + if (bufferIndex + dataLength + sizeof(short) >= buffer.Length) + { + } + + ReadOnlySpan source = data.Slice(0, dataLength); + var target = new Span(buffer, bufferIndex, dataLength); + + source.CopyTo(target); + bufferIndex += dataLength; + } } internal enum MetricEventType @@ -200,6 +383,31 @@ internal enum MetricEventType ULongMetric = 50, DoubleMetric = 55, ExternallyAggregatedULongDistributionMetric = 56, + TLV = 70, +} + +internal enum PayloadType +{ + AccountName = 1, + Namespace = 2, + MetricName = 3, + Dimensions = 4, + ULongMetric = 5, + DoubleMetric = 6, + ExternallyAggregatedULongDistributionMetric = 8, + HistogramULongValueCountPairs = 12, + Exemplars = 15, +} + +[Flags] +internal enum ExemplarFlags : byte +{ + None = 0x0, + IsMetricValueDoubleStoredAsLong = 0x1, + IsTimestampAvailable = 0x2, + SpanIdExists = 0x4, + TraceIdExists = 0x8, + SampleCountExists = 0x10, } /// diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricEtwDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricEtwDataTransport.cs index cee3871f86..9d825ce249 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricEtwDataTransport.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricEtwDataTransport.cs @@ -58,4 +58,9 @@ private void DoubleMetricEvent() private void ExternallyAggregatedDoubleDistributionMetric() { } + + [Event((int)MetricEventType.TLV)] + private void TLVMetricEvent() + { + } } diff --git a/src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj b/src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj index d095eef969..41c3ddcc7b 100644 --- a/src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj +++ b/src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj @@ -13,7 +13,7 @@ - + diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaMetricExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaMetricExporterTests.cs index 0f75779db8..a3b660bd58 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaMetricExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaMetricExporterTests.cs @@ -569,12 +569,15 @@ public void SuccessfulExportOnLinux() var metricPoint = metricPointsEnumerator.Current; var metricDataValue = Convert.ToUInt64(metricPoint.GetSumLong()); var metricData = new MetricData { UInt64Value = metricDataValue }; - var bodyLength = exporter.SerializeMetric( + + var exemplars = metricPoint.GetExemplars(); + var bodyLength = exporter.SerializeMetricWithTLV( MetricEventType.ULongMetric, metric.Name, metricPoint.EndTime.ToFileTime(), metricPoint.Tags, - metricData); + metricData, + exemplars); // Wait a little more than the ExportInterval for the exporter to export the data. Task.Delay(5500).Wait(); @@ -589,6 +592,9 @@ public void SuccessfulExportOnLinux() // BinaryHeader (fixed payload) + variable payload which starts with MetricPayload Assert.Equal(bodyLength + fixedPayloadLength, receivedDataSize); + // TODO: Update the unit test to test TLV based serialization + + /* var stream = new KaitaiStream(receivedData); var data = new MetricsContract(stream); @@ -611,6 +617,7 @@ public void SuccessfulExportOnLinux() Assert.Equal((ushort)MetricEventType.ULongMetric, data.EventId); Assert.Equal(bodyLength, data.LenBody); + */ } finally {