From 3768220ace7a2845efd2df97daa72658eb68f720 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Fri, 16 Dec 2022 11:22:07 -0800 Subject: [PATCH] [Geneva.Logs] Support prefix-based table name mapping (#796) * Added support for prefix based table mappings in geneva log exporter. * Benchmark tweak. * Patch CHANGELOG. * README updates. * Lint. * Code review. * Code review. * Test updates. * Code review. * Code review. * Code review. * Code review. * Warning cleanup. --- .../CHANGELOG.md | 3 + .../GenevaExporterOptions.cs | 22 +- .../GenevaLogExporter.cs | 2 +- .../GenevaTraceExporter.cs | 2 +- .../Internal/TableNameSerializer.cs | 328 ++++++++++++++++++ .../MsgPackExporter/MessagePackSerializer.cs | 25 +- .../MsgPackExporter/MsgPackExporter.cs | 2 +- .../MsgPackExporter/MsgPackLogExporter.cs | 143 +------- src/OpenTelemetry.Exporter.Geneva/README.md | 72 +++- .../Exporter/LogExporterBenchmarks.cs | 9 +- .../GenevaLogExporterTests.cs | 2 +- .../TableNameSerializerTests.cs | 158 +++++++++ 12 files changed, 616 insertions(+), 152 deletions(-) create mode 100644 src/OpenTelemetry.Exporter.Geneva/Internal/TableNameSerializer.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Tests/TableNameSerializerTests.cs diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index 893db5bfbb..6d1edafd77 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -17,6 +17,9 @@ Released 2022-Dec-09 * Fix EventSource logging ([#813](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/813)) +* Add support in logs for prefix-based table name mapping configuration. + [#796](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/796) + ## 1.4.0-beta.5 Released 2022-Nov-21 diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs index eee0865e0f..3af00bb09c 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs @@ -47,14 +47,32 @@ public IReadOnlyDictionary TableNameMappings foreach (var entry in value) { + if (string.IsNullOrWhiteSpace(entry.Key)) + { + throw new ArgumentException("A table name mapping key was null, empty, or consisted only of white-space characters.", nameof(this.TableNameMappings)); + } + if (string.IsNullOrWhiteSpace(entry.Value)) { - throw new ArgumentException($"A string-typed value provided for {nameof(this.TableNameMappings)} must not be null, empty, or consist only of white-space characters."); + throw new ArgumentException($"The table name mapping value provided for key '{entry.Key}' was null, empty, or consisted only of white-space characters.", nameof(this.TableNameMappings)); } if (Encoding.UTF8.GetByteCount(entry.Value) != entry.Value.Length) { - throw new ArgumentException($"A string-typed value provided for {nameof(this.TableNameMappings)} must not contain non-ASCII characters.", entry.Value); + throw new ArgumentException($"The table name mapping value '{entry.Value}' provided for key '{entry.Key}' contained non-ASCII characters.", nameof(this.TableNameMappings)); + } + + if (entry.Value != "*") + { + if (!TableNameSerializer.IsValidTableName(entry.Value)) + { + throw new ArgumentException($"The table name mapping value '{entry.Value}' provided for key '{entry.Key}' contained invalid characters or was too long.", nameof(this.TableNameMappings)); + } + + if (TableNameSerializer.IsReservedTableName(entry.Value)) + { + throw new ArgumentException($"The table name mapping value '{entry.Value}' provided for key '{entry.Key}' is reserved and cannot be specified.", nameof(this.TableNameMappings)); + } } copy[entry.Key] = entry.Value; diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs index b23bd32b10..2f5b6ce3da 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs @@ -45,7 +45,7 @@ public GenevaLogExporter(GenevaExporterOptions options) public override ExportResult Export(in Batch batch) { - return this.exportLogRecord(batch); + return this.exportLogRecord(in batch); } protected override void Dispose(bool disposing) diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs index 9c37b6db84..d732fe41cd 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs @@ -45,7 +45,7 @@ public GenevaTraceExporter(GenevaExporterOptions options) public override ExportResult Export(in Batch batch) { - return this.exportActivity(batch); + return this.exportActivity(in batch); } protected override void Dispose(bool disposing) diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/TableNameSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/TableNameSerializer.cs new file mode 100644 index 0000000000..0996827a48 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/TableNameSerializer.cs @@ -0,0 +1,328 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; + +namespace OpenTelemetry.Exporter.Geneva; + +internal sealed class TableNameSerializer +{ + public const int MaxSanitizedCategoryNameLength = 50; + public const int MaxSanitizedCategoryNameBytes = MaxSanitizedCategoryNameLength + 2; + public const int MaxCachedSanitizedTableNames = 1024; + private const StringComparison DictionaryKeyComparison = StringComparison.Ordinal; + +#pragma warning disable CA1825 // Avoid zero-length array allocations + /* Note: We don't use Array.Empty here because that is used to + indicate an invalid name. We need a different instance to trigger the + pass-through case. */ + private static readonly byte[] s_passthroughTableName = new byte[0]; +#pragma warning restore CA1825 // Avoid zero-length array allocations + private static readonly StringComparer s_dictionaryKeyComparer = StringComparer.Ordinal; + + private readonly byte[] m_defaultTableName; + private readonly Dictionary m_tableMappings; + private readonly bool m_shouldPassThruTableMappings; + private readonly object m_lockObject = new(); + private TableNameCacheDictionary m_tableNameCache = new(); + + public ITableNameCacheDictionary TableNameCache => this.m_tableNameCache; + + public TableNameSerializer(GenevaExporterOptions options, string defaultTableName) + { + Debug.Assert(options != null, "options were null"); + Debug.Assert(!string.IsNullOrWhiteSpace(defaultTableName), "defaultEventName was null or whitespace"); + Debug.Assert(IsValidTableName(defaultTableName), "defaultEventName was invalid"); + + this.m_defaultTableName = BuildStr8BufferForAsciiString(defaultTableName); + + if (options.TableNameMappings != null) + { + var tempTableMappings = new Dictionary(options.TableNameMappings.Count, s_dictionaryKeyComparer); + foreach (var kv in options.TableNameMappings) + { + if (kv.Key == "*") + { + if (kv.Value == "*") + { + this.m_shouldPassThruTableMappings = true; + } + else + { + this.m_defaultTableName = BuildStr8BufferForAsciiString(kv.Value); + } + } + else if (kv.Value == "*") + { + tempTableMappings[kv.Key] = s_passthroughTableName; + } + else + { + tempTableMappings[kv.Key] = BuildStr8BufferForAsciiString(kv.Value); + } + } + + this.m_tableMappings = tempTableMappings; + } + } + + public static bool IsReservedTableName(string tableName) + { + Debug.Assert(!string.IsNullOrWhiteSpace(tableName), "tableName was null or whitespace"); + + // TODO: Implement this if needed. + + return false; + } + + public static bool IsValidTableName(string tableName) + { + Debug.Assert(!string.IsNullOrWhiteSpace(tableName), "tableName was null or whitespace"); + + var length = tableName.Length; + if (length > MaxSanitizedCategoryNameLength) + { + return false; + } + + char firstChar = tableName[0]; + if (firstChar < 'A' || firstChar > 'Z') + { + return false; + } + + for (int i = 1; i < length; i++) + { + char cur = tableName[i]; + if ((cur >= 'a' && cur <= 'z') || (cur >= 'A' && cur <= 'Z') || (cur >= '0' && cur <= '9')) + { + continue; + } + + return false; + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int ResolveAndSerializeTableNameForCategoryName(byte[] destination, int offset, string categoryName, out ReadOnlySpan tableName) + { + byte[] mappedTableName = this.ResolveTableMappingForCategoryName(categoryName); + + if (mappedTableName == s_passthroughTableName) + { + // Pass-through mode with a full cache. + + int bytesWritten = WriteSanitizedCategoryNameToSpan(new Span(destination, offset, MaxSanitizedCategoryNameBytes), categoryName); + + tableName = new ReadOnlySpan(destination, offset, bytesWritten); + + return offset + bytesWritten; + } + + tableName = mappedTableName; + + return MessagePackSerializer.SerializeSpan(destination, offset, tableName); + } + + private static byte[] BuildStr8BufferForAsciiString(string value) + { + var length = value.Length; + + byte[] buffer = new byte[length + 2]; + + Encoding.ASCII.GetBytes(value, 0, length, buffer, 2); + + MessagePackSerializer.WriteStr8Header(buffer, 0, length); + + return buffer; + } + + // This method would map the logger category to a table name which only contains alphanumeric values with the following additions: + // Any character that is not allowed will be removed. + // If the resulting string is longer than 50 characters, only the first 50 characters will be taken. + // If the first character in the resulting string is a lower-case alphabet, it will be converted to the corresponding upper-case. + // If the resulting string still does not comply with Rule, the category name will not be serialized. + private static int WriteSanitizedCategoryNameToSpan(Span buffer, string categoryName) + { + // Reserve 2 bytes for storing LIMIT_MAX_STR8_LENGTH_IN_BYTES and (byte)validNameLength - + // these 2 bytes will be back filled after iterating through categoryName. + int cursor = 2; + int validNameLength = 0; + + // Special treatment for the first character. + var firstChar = categoryName[0]; + if (firstChar >= 'A' && firstChar <= 'Z') + { + buffer[cursor++] = (byte)firstChar; + ++validNameLength; + } + else if (firstChar >= 'a' && firstChar <= 'z') + { + // If the first character in the resulting string is a lower-case alphabet, + // it will be converted to the corresponding upper-case. + buffer[cursor++] = (byte)(firstChar - 32); + ++validNameLength; + } + else + { + // Not a valid name. + return 0; + } + + for (int i = 1; i < categoryName.Length; ++i) + { + if (validNameLength == MaxSanitizedCategoryNameLength) + { + break; + } + + var cur = categoryName[i]; + if ((cur >= 'a' && cur <= 'z') || (cur >= 'A' && cur <= 'Z') || (cur >= '0' && cur <= '9')) + { + buffer[cursor++] = (byte)cur; + ++validNameLength; + } + } + + // Backfilling MessagePack serialization protocol and valid category length to the startIdx of the categoryName byte array. + MessagePackSerializer.WriteStr8Header(buffer, 0, validNameLength); + + return cursor; + } + + private byte[] ResolveTableMappingForCategoryName(string categoryName) + { + var tableNameCache = this.m_tableNameCache; + + if (tableNameCache.TryGetValue(categoryName, out byte[] tableName)) + { + return tableName; + } + + return this.ResolveTableMappingForCategoryNameRare(categoryName); + } + + private byte[] ResolveTableMappingForCategoryNameRare(string categoryName) + { + byte[] mappedTableName = null; + + // If user configured table name mappings run resolution logic. + if (this.m_tableMappings != null + && !this.m_tableMappings.TryGetValue(categoryName, out mappedTableName)) + { + // Find best match if an exact match was not found. + + string currentKey = null; + + foreach (var mapping in this.m_tableMappings) + { + if (!categoryName.StartsWith(mapping.Key, DictionaryKeyComparison)) + { + continue; + } + + if (currentKey == null || mapping.Key.Length >= currentKey.Length) + { + currentKey = mapping.Key; + mappedTableName = mapping.Value; + } + } + } + + mappedTableName ??= !this.m_shouldPassThruTableMappings + ? this.m_defaultTableName + : s_passthroughTableName; + + Span sanitizedTableNameStorage = mappedTableName == s_passthroughTableName + ? stackalloc byte[MaxSanitizedCategoryNameBytes] + : Array.Empty(); + + if (sanitizedTableNameStorage.Length > 0) + { + // We resolved to a wildcard which is pass-through mode. + + int bytesWritten = WriteSanitizedCategoryNameToSpan(sanitizedTableNameStorage, categoryName); + if (bytesWritten > 0) + { + sanitizedTableNameStorage = sanitizedTableNameStorage.Slice(0, bytesWritten); + } + else + { + // Note: When the table name could not be sanitized we cache + // the empty array NOT s_passthroughTableName. + mappedTableName = Array.Empty(); + } + } + + lock (this.m_lockObject) + { + var tableNameCache = this.m_tableNameCache; + + // Check if another thread added the mapping while we waited on the + // lock. + if (tableNameCache.TryGetValue(categoryName, out byte[] tableName)) + { + return tableName; + } + + if (mappedTableName == s_passthroughTableName + && tableNameCache.CachedSanitizedTableNameCount < MaxCachedSanitizedTableNames) + { + mappedTableName = sanitizedTableNameStorage.ToArray(); + tableNameCache.CachedSanitizedTableNameCount++; + } + + // Note: This is using copy-on-write pattern to keep the happy + // path lockless once everything has spun up. + TableNameCacheDictionary newTableNameCache = new(tableNameCache) + { + [categoryName] = mappedTableName, + }; + + this.m_tableNameCache = newTableNameCache; + + return mappedTableName; + } + } + + // Note: This is used for tests. + public interface ITableNameCacheDictionary : IReadOnlyDictionary + { + int CachedSanitizedTableNameCount { get; } + } + + private sealed class TableNameCacheDictionary : Dictionary, ITableNameCacheDictionary + { + public TableNameCacheDictionary() + : base(0, s_dictionaryKeyComparer) + { + } + + public TableNameCacheDictionary(TableNameCacheDictionary sourceCache) + : base(sourceCache, s_dictionaryKeyComparer) + { + this.CachedSanitizedTableNameCount = sourceCache.CachedSanitizedTableNameCount; + } + + public int CachedSanitizedTableNameCount { get; set; } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MessagePackSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MessagePackSerializer.cs index 5236c386cf..6d8ed0bb9c 100644 --- a/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MessagePackSerializer.cs +++ b/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MessagePackSerializer.cs @@ -350,6 +350,13 @@ public static void WriteStr8Header(byte[] buffer, int nameStartIdx, int validNam buffer[nameStartIdx + 1] = unchecked((byte)validNameLength); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void WriteStr8Header(Span buffer, int nameStartIdx, int validNameLength) + { + buffer[nameStartIdx] = STR8; + buffer[nameStartIdx + 1] = unchecked((byte)validNameLength); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int SerializeAsciiString(byte[] buffer, int cursor, string value) { @@ -373,7 +380,7 @@ public static int SerializeAsciiString(byte[] buffer, int cursor, string value) } else { - throw new ArgumentException("The input string: \"{inputString}\" has non-ASCII characters in it.", value); + throw new ArgumentException($"The input string: \"{value}\" has non-ASCII characters in it.", nameof(value)); } } @@ -390,7 +397,7 @@ public static int SerializeAsciiString(byte[] buffer, int cursor, string value) } else { - throw new ArgumentException("The input string: \"{inputString}\" has non-ASCII characters in it.", value); + throw new ArgumentException($"The input string: \"{value}\" has non-ASCII characters in it.", nameof(value)); } } @@ -656,7 +663,7 @@ public static int Serialize(byte[] buffer, int cursor, object obj) #endif default: - string repr = null; + string repr; try { @@ -671,9 +678,17 @@ public static int Serialize(byte[] buffer, int cursor, object obj) } } - public static int SerializeSpan(byte[] buffer, int cursor, Span value) + public static int SerializeSpan(byte[] buffer, int cursor, ReadOnlySpan value) { + var length = value.Length; + + if (length == 0) + { + return SerializeNull(buffer, cursor); + } + value.CopyTo(buffer.AsSpan(cursor)); - return cursor + value.Length; + + return cursor + length; } } diff --git a/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackExporter.cs b/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackExporter.cs index f543b01689..f825c033a0 100644 --- a/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackExporter.cs @@ -58,7 +58,7 @@ protected static int AddPartAField(byte[] buffer, int cursor, string name, objec return cursor; } - protected static int AddPartAField(byte[] buffer, int cursor, string name, Span value) + protected static int AddPartAField(byte[] buffer, int cursor, string name, ReadOnlySpan value) { if (V40_PART_A_MAPPING.TryGetValue(name, out string replacementKey)) { diff --git a/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackLogExporter.cs index 17dde9799a..9719da091a 100644 --- a/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/MsgPackExporter/MsgPackLogExporter.cs @@ -28,56 +28,27 @@ namespace OpenTelemetry.Exporter.Geneva; internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable { private const int BUFFER_SIZE = 65360; // the maximum ETW payload (inclusive) - private const int MaxSanitizedEventNameLength = 50; - private readonly IReadOnlyDictionary m_customFields; - - private readonly ExceptionStackExportMode m_exportExceptionStack; - - private readonly string m_defaultEventName = "Log"; - private readonly IReadOnlyDictionary m_prepopulatedFields; - private readonly List m_prepopulatedFieldKeys; private static readonly ThreadLocal m_buffer = new ThreadLocal(() => null); - private readonly byte[] m_bufferEpilogue; private static readonly string[] logLevels = new string[7] { "Trace", "Debug", "Information", "Warning", "Error", "Critical", "None", }; + private readonly TableNameSerializer m_tableNameSerializer; + private readonly Dictionary m_customFields; + private readonly Dictionary m_prepopulatedFields; + private readonly ExceptionStackExportMode m_exportExceptionStack; + private readonly List m_prepopulatedFieldKeys; + private readonly byte[] m_bufferEpilogue; private readonly IDataTransport m_dataTransport; - private readonly bool shouldPassThruTableMappings; private bool isDisposed; public MsgPackLogExporter(GenevaExporterOptions options) { + this.m_tableNameSerializer = new(options, defaultTableName: "Log"); this.m_exportExceptionStack = options.ExceptionStackExportMode; - // TODO: Validate mappings for reserved tablenames etc. - if (options.TableNameMappings != null) - { - var tempTableMappings = new Dictionary(options.TableNameMappings.Count, StringComparer.Ordinal); - foreach (var kv in options.TableNameMappings) - { - if (kv.Key == "*") - { - if (kv.Value == "*") - { - this.shouldPassThruTableMappings = true; - } - else - { - this.m_defaultEventName = kv.Value; - } - } - else - { - tempTableMappings[kv.Key] = kv.Value; - } - } - - this.m_tableMappings = tempTableMappings; - } - var connectionStringBuilder = new ConnectionStringBuilder(options.ConnectionString); switch (connectionStringBuilder.Protocol) { @@ -133,8 +104,6 @@ public MsgPackLogExporter(GenevaExporterOptions options) Buffer.BlockCopy(buffer, 0, this.m_bufferEpilogue, 0, cursor - 0); } - private readonly IReadOnlyDictionary m_tableMappings; - public ExportResult Export(in Batch batch) { var result = ExportResult.Success; @@ -207,40 +176,8 @@ internal int SerializeLogRecord(LogRecord logRecord) cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 3); var categoryName = logRecord.CategoryName; - string eventName = null; - - Span sanitizedEventName = default; - - // If user configured explicit TableName, use it. - if (this.m_tableMappings != null && this.m_tableMappings.TryGetValue(categoryName, out eventName)) - { - cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, eventName); - } - else if (!this.shouldPassThruTableMappings) - { - eventName = this.m_defaultEventName; - cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, eventName); - } - else - { - int cursorStartIdx = cursor; - - if (categoryName.Length > 0) - { - cursor = SerializeSanitizedCategoryName(buffer, cursor, categoryName); - } - if (cursor == cursorStartIdx) - { - // Serializing null as categoryName could not be sanitized into a valid string. - cursor = MessagePackSerializer.SerializeNull(buffer, cursor); - } - else - { - // Sanitized category name has been serialized. - sanitizedEventName = buffer.AsSpan().Slice(cursorStartIdx, cursor - cursorStartIdx); - } - } + cursor = this.m_tableNameSerializer.ResolveAndSerializeTableNameForCategoryName(buffer, cursor, categoryName, out ReadOnlySpan eventName); cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 1); cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 2); @@ -261,14 +198,7 @@ internal int SerializeLogRecord(LogRecord logRecord) } // Part A - core envelope - if (sanitizedEventName.Length != 0) - { - cursor = AddPartAField(buffer, cursor, Schema.V40.PartA.Name, sanitizedEventName); - } - else - { - cursor = AddPartAField(buffer, cursor, Schema.V40.PartA.Name, eventName); - } + cursor = AddPartAField(buffer, cursor, Schema.V40.PartA.Name, eventName); cntFields += 1; @@ -493,61 +423,6 @@ private static byte GetSeverityNumber(LogLevel logLevel) } } - // This method would map the logger category to a table name which only contains alphanumeric values with the following additions: - // Any character that is not allowed will be removed. - // If the resulting string is longer than 50 characters, only the first 50 characters will be taken. - // If the first character in the resulting string is a lower-case alphabet, it will be converted to the corresponding upper-case. - // If the resulting string still does not comply with Rule, the category name will not be serialized. - private static int SerializeSanitizedCategoryName(byte[] buffer, int cursor, string categoryName) - { - int cursorStartIdx = cursor; - - // Reserve 2 bytes for storing LIMIT_MAX_STR8_LENGTH_IN_BYTES and (byte)validNameLength - - // these 2 bytes will be back filled after iterating through categoryName. - cursor += 2; - int validNameLength = 0; - - // Special treatment for the first character. - var firstChar = categoryName[0]; - if (firstChar >= 'A' && firstChar <= 'Z') - { - buffer[cursor++] = (byte)firstChar; - ++validNameLength; - } - else if (firstChar >= 'a' && firstChar <= 'z') - { - // If the first character in the resulting string is a lower-case alphabet, - // it will be converted to the corresponding upper-case. - buffer[cursor++] = (byte)(firstChar - 32); - ++validNameLength; - } - else - { - // Not a valid name. - return cursor -= 2; - } - - for (int i = 1; i < categoryName.Length; ++i) - { - if (validNameLength == MaxSanitizedEventNameLength) - { - break; - } - - var cur = categoryName[i]; - if ((cur >= 'a' && cur <= 'z') || (cur >= 'A' && cur <= 'Z') || (cur >= '0' && cur <= '9')) - { - buffer[cursor++] = (byte)cur; - ++validNameLength; - } - } - - // Backfilling MessagePack serialization protocol and valid category length to the startIdx of the categoryName byte array. - MessagePackSerializer.WriteStr8Header(buffer, cursorStartIdx, validNameLength); - - return cursor; - } - private static string ToInvariantString(Exception exception) { var originalUICulture = Thread.CurrentThread.CurrentUICulture; diff --git a/src/OpenTelemetry.Exporter.Geneva/README.md b/src/OpenTelemetry.Exporter.Geneva/README.md index 24b0ca1626..44d6a7ef25 100644 --- a/src/OpenTelemetry.Exporter.Geneva/README.md +++ b/src/OpenTelemetry.Exporter.Geneva/README.md @@ -104,14 +104,74 @@ sent through this exporter. This defines the mapping for the table name used to store Logs and Traces. -The default table name used for Traces is `Span`. For changing the table name -for Traces, add an entry with key as `Span`, and value as the custom table name. +##### Trace table name mappings -The default table name used for Logs is `Log`. Mappings can be specified for -each +The default table name used for Traces is `Span`. To change the table name for +Traces add an entry with the key `Span` and set the value to the desired custom +table name. + +**Note:** Only a single table name is supported for Traces. + +##### Log table name mappings + +The default table name used for Logs is `Log`. Mappings can be specified +universally or for individual log message [category](https://docs.microsoft.com/dotnet/core/extensions/logging#log-category) -of the log. For changing the default table name for Logs, add an entry with key -as `*`, and value as the custom table name. +values. + +* To change the default table name for Logs add an entry with the key `*` and + set the value to the desired custom table name. To enable "pass-through" + mapping set the value of the `*` key to `*`. For details on "pass-through" + mode see below. + +* To change the table name for a specific log + [category](https://docs.microsoft.com/dotnet/core/extensions/logging#log-category) + add an entry with the key set to the full "category" of the log messages or a + prefix that will match the starting portion of the log message "category". Set + the value of the key to either the desired custom table name or `*` to enable + "pass-through" mapping. For details on "pass-through" mode see below. + + For example, given the configuration... + + ```csharp + var options = new GenevaExporterOptions + { + TableNameMappings = new Dictionary() + { + ["*"] = "DefaultLogs", + ["MyCompany"] = "InternalLogs", + ["MyCompany.Product1"] = "InternalProduct1Logs", + ["MyCompany.Product2"] = "InternalProduct2Logs", + ["MyCompany.Product2.Security"] = "InternalSecurityLogs", + ["MyPartner"] = "*", + }, + }; + ``` + + ...log category mapping would be performed as such: + + * `ILogger`: This would go to "DefaultLogs" + * `ILogger`: This would go to "InternalLogs" + * `ILogger`: This would go to "InternalProduct1Logs" + * `ILogger`: This would go to "InternalProduct2Logs" + * `ILogger`: This would go to + "InternalSecurityLogs" + * `ILogger`: This is marked as pass-through ("*") so + it will be sanitized as "MyPartnerProductThing" table name + +##### Pass-through table name mapping rules + +When "pass-through" mapping is enabled for a given log message the runtime +[category](https://docs.microsoft.com/dotnet/core/extensions/logging#log-category) +value will be converted into a valid table name. + +* The first character MUST be an ASCII letter. If it is lower-case, it will be + converted into an upper-case letter. If the first character is invalid all log + messages for the "category" will be dropped. + +* Any non-ASCII letter or number will be removed. + +* Only the first 50 valid characters will be used. ### Enable Metrics diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs index 4c6c4bc122..8252c80b4e 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs @@ -87,12 +87,19 @@ public LogExporterBenchmarks() ["cloud.roleInstance"] = "CY1SCH030021417", ["cloud.roleVer"] = "9.0.15289.2", }; + exporterOptions.TableNameMappings = new Dictionary + { + ["*"] = "*", + ["TestCompany"] = "*", + ["TestCompany.TestNamespace"] = "*", + ["TestCompany.TestNamespace.TestLogger"] = "TestLoggerTable", + }; }); loggerOptions.IncludeFormattedMessage = this.IncludeFormattedMessage; })); - this.logger = this.loggerFactory.CreateLogger("TestLogger"); + this.logger = this.loggerFactory.CreateLogger("TestCompany.TestNamespace.TestLogger"); // For msgpack serialization + export this.logRecord = GenerateTestLogRecord(); diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index edd959d1e8..b317583b58 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -76,7 +76,7 @@ public void SpecialCharactersInTableNameMappings() TableNameMappings = new Dictionary { ["TestCategory"] = null }, }; }); - Assert.Contains("A string-typed value provided for TableNameMappings must not be null, empty, or consist only of white-space characters.", ex.Message); + Assert.Contains("The table name mapping value provided for key 'TestCategory' was null, empty, or consisted only of white-space characters.", ex.Message); // Throw when TableNameMappings is null Assert.Throws(() => diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/TableNameSerializerTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/TableNameSerializerTests.cs new file mode 100644 index 0000000000..399c49584e --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/TableNameSerializerTests.cs @@ -0,0 +1,158 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.Tests; + +public class TableNameSerializerTests +{ + [Theory] + [InlineData("Unknown", "DefaultLogs")] + [InlineData("Unknown", "LogTableName", "LogTableName")] + public void DefaultResolutionTests(string categoryName, string tableName, string defaultTableName = null) + { + var mappings = new Dictionary(); + + if (defaultTableName != null) + { + mappings["*"] = defaultTableName; + } + + var options = new GenevaExporterOptions + { + TableNameMappings = mappings, + }; + + RunTableNameSerializerTest(categoryName, tableName, options); + } + + [Theory] + [InlineData("Unknown", "Unknown")] + [InlineData("unknown.table", "Unknowntable")] + public void PassthroughResolutionTests(string categoryName, string tableName) + { + var options = new GenevaExporterOptions + { + TableNameMappings = new Dictionary() + { + ["*"] = "*", + }, + }; + + RunTableNameSerializerTest(categoryName, tableName, options); + } + + [Theory] + [InlineData("Unknown", "DefaultLogs")] + [InlineData("Prefix.Nonmatch", "PrefixNonmatch")] + [InlineData("Prefix.Sub.Nonmatch", "SubTableName")] + [InlineData("Prefix.Sub.Final", "FinalTableName")] + public void PrefixResolutionTests(string categoryName, string tableName) + { + var options = new GenevaExporterOptions + { + TableNameMappings = new Dictionary() + { + ["Prefix"] = "*", + ["Prefix.Sub"] = "SubTableName", + ["Prefix.Sub.Final"] = "FinalTableName", + }, + }; + + RunTableNameSerializerTest(categoryName, tableName, options); + } + + [Fact] + public void ResolvedTableNameCacheTest() + { + var options = new GenevaExporterOptions(); + + var buffer = new byte[1024]; + + var tableNameSerializer = new TableNameSerializer(options, "DefaultLogs"); + + Assert.Empty(tableNameSerializer.TableNameCache); + + tableNameSerializer.ResolveAndSerializeTableNameForCategoryName(buffer, 0, "MyCategory", out _); + + Assert.Single(tableNameSerializer.TableNameCache); + + tableNameSerializer.ResolveAndSerializeTableNameForCategoryName(buffer, 0, "MyCategory", out _); + + Assert.Single(tableNameSerializer.TableNameCache); + + tableNameSerializer.ResolveAndSerializeTableNameForCategoryName(buffer, 0, "MyCategory2", out _); + + Assert.Equal(2, tableNameSerializer.TableNameCache.Count); + + tableNameSerializer.ResolveAndSerializeTableNameForCategoryName(buffer, 0, "MyCategory2", out _); + + Assert.Equal(2, tableNameSerializer.TableNameCache.Count); + } + + [Fact] + public void TableNameCacheTest() + { + var options = new GenevaExporterOptions + { + TableNameMappings = new Dictionary() + { + ["*"] = "*", + }, + }; + + var buffer = new byte[1024]; + + var tableNameSerializer = new TableNameSerializer(options, "DefaultLogs"); + + var numberOfCategoryNames = TableNameSerializer.MaxCachedSanitizedTableNames * 2; + + for (int i = 0; i < numberOfCategoryNames; i++) + { + var categoryName = $"category.{i}-test"; + var sanitizedCategoryName = $"Category{i}test"; + + for (int c = 0; c < 10; c++) + { + var bytesWritten = tableNameSerializer.ResolveAndSerializeTableNameForCategoryName(buffer, 0, categoryName, out var tableName); + + Assert.Equal(sanitizedCategoryName.Length + 2, bytesWritten); + Assert.Equal(sanitizedCategoryName, Encoding.ASCII.GetString(tableName.ToArray(), 2, sanitizedCategoryName.Length)); + } + } + + var tableNameCache = tableNameSerializer.TableNameCache; + + Assert.NotNull(tableNameCache); + Assert.Equal(numberOfCategoryNames, tableNameCache.Count); + Assert.Equal(TableNameSerializer.MaxCachedSanitizedTableNames, tableNameCache.CachedSanitizedTableNameCount); + } + + private static void RunTableNameSerializerTest(string categoryName, string tableName, GenevaExporterOptions options) + { + var buffer = new byte[1024]; + + var tableNameSerializer = new TableNameSerializer(options, "DefaultLogs"); + + var bytesWritten = tableNameSerializer.ResolveAndSerializeTableNameForCategoryName(buffer, 0, categoryName, out var resolvedTableName); + + Assert.Equal(tableName.Length + 2, bytesWritten); + Assert.Equal(tableName, Encoding.ASCII.GetString(resolvedTableName.ToArray(), 2, tableName.Length)); + } +}