Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for custom event writer #388

Merged
merged 10 commits into from
May 29, 2024
Merged

Add support for custom event writer #388

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
cf046cb
Pass json serializer options through converters
sergiojrdotnet May 28, 2024
4026067
Update file generator
sergiojrdotnet May 28, 2024
dadd016
Add unit test
sergiojrdotnet May 28, 2024
1ffee03
Add integration tests
sergiojrdotnet May 28, 2024
0323bfe
Fix tests
sergiojrdotnet May 28, 2024
8167249
Fix test code style
sergiojrdotnet May 28, 2024
977ff61
Add type-specialized property converter write methods
sergiojrdotnet May 29, 2024
0186cdc
Use cached DateTimeOffset JsonConverter when default JsonSerializerOp…
sergiojrdotnet May 29, 2024
db56890
Use cached DateTimeOffset JsonConverter when default JsonSerializerOp…
sergiojrdotnet May 29, 2024
fb66742
Consolidate JsonConverter<DateTimeOffset> getter
sergiojrdotnet May 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 69 additions & 67 deletions src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ private static bool ReadProperties(
TBase ecsEvent,
ref DateTimeOffset? timestamp,
ref string loglevel,
ref string ecsVersion
ref string ecsVersion,
JsonSerializerOptions options
)
{
var propertyName = reader.GetString();
Expand All @@ -33,14 +34,14 @@ ref string ecsVersion
{
"log.level" => ReadString(ref reader, ref loglevel),
"ecs.version" => ReadString(ref reader, ref ecsVersion),
"metadata" => ReadProp<MetadataDictionary>(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v),
"@timestamp" => ReadDateTime(ref reader, ref @timestamp),
"message" => ReadProp<string>(ref reader, "message", ecsEvent, (b, v) => b.Message = v),
"tags" => ReadProp<string[]>(ref reader, "tags", ecsEvent, (b, v) => b.Tags = v),
"span.id" => ReadProp<string>(ref reader, "span.id", ecsEvent, (b, v) => b.SpanId = v),
"trace.id" => ReadProp<string>(ref reader, "trace.id", ecsEvent, (b, v) => b.TraceId = v),
"transaction.id" => ReadProp<string>(ref reader, "transaction.id", ecsEvent, (b, v) => b.TransactionId = v),
"labels" => ReadProp<Labels>(ref reader, "labels", ecsEvent, (b, v) => b.Labels = v),
"metadata" => ReadProp<MetadataDictionary>(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v, options),
"@timestamp" => ReadDateTime(ref reader, ref @timestamp, options),
"message" => ReadProp<string>(ref reader, "message", ecsEvent, (b, v) => b.Message = v, options),
"tags" => ReadProp<string[]>(ref reader, "tags", ecsEvent, (b, v) => b.Tags = v, options),
"span.id" => ReadProp<string>(ref reader, "span.id", ecsEvent, (b, v) => b.SpanId = v, options),
"trace.id" => ReadProp<string>(ref reader, "trace.id", ecsEvent, (b, v) => b.TraceId = v, options),
"transaction.id" => ReadProp<string>(ref reader, "transaction.id", ecsEvent, (b, v) => b.TransactionId = v, options),
"labels" => ReadProp<Labels>(ref reader, "labels", ecsEvent, (b, v) => b.Labels = v, options),
"agent" => ReadProp<Agent>(ref reader, "agent", EcsJsonContext.Default.Agent, ecsEvent, (b, v) => b.Agent = v),
"as" => ReadProp<As>(ref reader, "as", EcsJsonContext.Default.As, ecsEvent, (b, v) => b.As = v),
"client" => ReadProp<Client>(ref reader, "client", EcsJsonContext.Default.Client, ecsEvent, (b, v) => b.Client = v),
Expand Down Expand Up @@ -94,7 +95,7 @@ ref string ecsVersion
typeof(EcsDocument) == ecsEvent.GetType()
? false
: ecsEvent.TryRead(propertyName, out var t)
? ecsEvent.ReceiveProperty(propertyName, ReadPropDeserialize(ref reader, t))
? ecsEvent.ReceiveProperty(propertyName, ReadPropDeserialize(ref reader, t, options))
: false
};
}
Expand All @@ -109,71 +110,72 @@ public override void Write(Utf8JsonWriter writer, TBase value, JsonSerializerOpt
}
writer.WriteStartObject();

WriteTimestamp(writer, value);
WriteTimestamp(writer, value, options);
WriteLogLevel(writer, value);
WriteMessage(writer, value);
WriteEcsVersion(writer, value);
WriteLogEntity(writer, value.Log);
WriteEcsEntity(writer, value.Ecs);
WriteLogEntity(writer, value.Log, options);
WriteEcsEntity(writer, value.Ecs, options);

// Base fields
WriteProp(writer, "tags", value.Tags);
WriteProp(writer, "span.id", value.SpanId);
WriteProp(writer, "trace.id", value.TraceId);
WriteProp(writer, "transaction.id", value.TransactionId);
WriteProp(writer, "labels", value.Labels);
WriteProp(writer, "tags", value.Tags, options);
WriteProp(writer, "span.id", value.SpanId, options);
WriteProp(writer, "trace.id", value.TraceId, options);
WriteProp(writer, "transaction.id", value.TransactionId, options);
WriteProp(writer, "labels", value.Labels, options);

// Complex types
WriteProp(writer, "agent", value.Agent, EcsJsonContext.Default.Agent);
WriteProp(writer, "as", value.As, EcsJsonContext.Default.As);
WriteProp(writer, "client", value.Client, EcsJsonContext.Default.Client);
WriteProp(writer, "cloud", value.Cloud, EcsJsonContext.Default.Cloud);
WriteProp(writer, "code_signature", value.CodeSignature, EcsJsonContext.Default.CodeSignature);
WriteProp(writer, "container", value.Container, EcsJsonContext.Default.Container);
WriteProp(writer, "data_stream", value.DataStream, EcsJsonContext.Default.DataStream);
WriteProp(writer, "destination", value.Destination, EcsJsonContext.Default.Destination);
WriteProp(writer, "device", value.Device, EcsJsonContext.Default.Device);
WriteProp(writer, "dll", value.Dll, EcsJsonContext.Default.Dll);
WriteProp(writer, "dns", value.Dns, EcsJsonContext.Default.Dns);
WriteProp(writer, "elf", value.Elf, EcsJsonContext.Default.Elf);
WriteProp(writer, "email", value.Email, EcsJsonContext.Default.Email);
WriteProp(writer, "error", value.Error, EcsJsonContext.Default.Error);
WriteProp(writer, "event", value.Event, EcsJsonContext.Default.Event);
WriteProp(writer, "faas", value.Faas, EcsJsonContext.Default.Faas);
WriteProp(writer, "file", value.File, EcsJsonContext.Default.File);
WriteProp(writer, "geo", value.Geo, EcsJsonContext.Default.Geo);
WriteProp(writer, "group", value.Group, EcsJsonContext.Default.Group);
WriteProp(writer, "hash", value.Hash, EcsJsonContext.Default.Hash);
WriteProp(writer, "host", value.Host, EcsJsonContext.Default.Host);
WriteProp(writer, "http", value.Http, EcsJsonContext.Default.Http);
WriteProp(writer, "interface", value.Interface, EcsJsonContext.Default.Interface);
WriteProp(writer, "macho", value.Macho, EcsJsonContext.Default.Macho);
WriteProp(writer, "network", value.Network, EcsJsonContext.Default.Network);
WriteProp(writer, "observer", value.Observer, EcsJsonContext.Default.Observer);
WriteProp(writer, "orchestrator", value.Orchestrator, EcsJsonContext.Default.Orchestrator);
WriteProp(writer, "organization", value.Organization, EcsJsonContext.Default.Organization);
WriteProp(writer, "os", value.Os, EcsJsonContext.Default.Os);
WriteProp(writer, "package", value.Package, EcsJsonContext.Default.Package);
WriteProp(writer, "pe", value.Pe, EcsJsonContext.Default.Pe);
WriteProp(writer, "process", value.Process, EcsJsonContext.Default.Process);
WriteProp(writer, "registry", value.Registry, EcsJsonContext.Default.Registry);
WriteProp(writer, "related", value.Related, EcsJsonContext.Default.Related);
WriteProp(writer, "risk", value.Risk, EcsJsonContext.Default.Risk);
WriteProp(writer, "rule", value.Rule, EcsJsonContext.Default.Rule);
WriteProp(writer, "server", value.Server, EcsJsonContext.Default.Server);
WriteProp(writer, "service", value.Service, EcsJsonContext.Default.Service);
WriteProp(writer, "source", value.Source, EcsJsonContext.Default.Source);
WriteProp(writer, "threat", value.Threat, EcsJsonContext.Default.Threat);
WriteProp(writer, "tls", value.Tls, EcsJsonContext.Default.Tls);
WriteProp(writer, "url", value.Url, EcsJsonContext.Default.Url);
WriteProp(writer, "user", value.User, EcsJsonContext.Default.User);
WriteProp(writer, "user_agent", value.UserAgent, EcsJsonContext.Default.UserAgent);
WriteProp(writer, "vlan", value.Vlan, EcsJsonContext.Default.Vlan);
WriteProp(writer, "vulnerability", value.Vulnerability, EcsJsonContext.Default.Vulnerability);
WriteProp(writer, "x509", value.X509, EcsJsonContext.Default.X509);
WriteProp(writer, "metadata", value.Metadata);
WriteProp(writer, "agent", value.Agent, EcsJsonContext.Default.Agent, options);
WriteProp(writer, "as", value.As, EcsJsonContext.Default.As, options);
WriteProp(writer, "client", value.Client, EcsJsonContext.Default.Client, options);
WriteProp(writer, "cloud", value.Cloud, EcsJsonContext.Default.Cloud, options);
WriteProp(writer, "code_signature", value.CodeSignature, EcsJsonContext.Default.CodeSignature, options);
WriteProp(writer, "container", value.Container, EcsJsonContext.Default.Container, options);
WriteProp(writer, "data_stream", value.DataStream, EcsJsonContext.Default.DataStream, options);
WriteProp(writer, "destination", value.Destination, EcsJsonContext.Default.Destination, options);
WriteProp(writer, "device", value.Device, EcsJsonContext.Default.Device, options);
WriteProp(writer, "dll", value.Dll, EcsJsonContext.Default.Dll, options);
WriteProp(writer, "dns", value.Dns, EcsJsonContext.Default.Dns, options);
WriteProp(writer, "elf", value.Elf, EcsJsonContext.Default.Elf, options);
WriteProp(writer, "email", value.Email, EcsJsonContext.Default.Email, options);
WriteProp(writer, "error", value.Error, EcsJsonContext.Default.Error, options);
WriteProp(writer, "event", value.Event, EcsJsonContext.Default.Event, options);
WriteProp(writer, "faas", value.Faas, EcsJsonContext.Default.Faas, options);
WriteProp(writer, "file", value.File, EcsJsonContext.Default.File, options);
WriteProp(writer, "geo", value.Geo, EcsJsonContext.Default.Geo, options);
WriteProp(writer, "group", value.Group, EcsJsonContext.Default.Group, options);
WriteProp(writer, "hash", value.Hash, EcsJsonContext.Default.Hash, options);
WriteProp(writer, "host", value.Host, EcsJsonContext.Default.Host, options);
WriteProp(writer, "http", value.Http, EcsJsonContext.Default.Http, options);
WriteProp(writer, "interface", value.Interface, EcsJsonContext.Default.Interface, options);
WriteProp(writer, "macho", value.Macho, EcsJsonContext.Default.Macho, options);
WriteProp(writer, "network", value.Network, EcsJsonContext.Default.Network, options);
WriteProp(writer, "observer", value.Observer, EcsJsonContext.Default.Observer, options);
WriteProp(writer, "orchestrator", value.Orchestrator, EcsJsonContext.Default.Orchestrator, options);
WriteProp(writer, "organization", value.Organization, EcsJsonContext.Default.Organization, options);
WriteProp(writer, "os", value.Os, EcsJsonContext.Default.Os, options);
WriteProp(writer, "package", value.Package, EcsJsonContext.Default.Package, options);
WriteProp(writer, "pe", value.Pe, EcsJsonContext.Default.Pe, options);
WriteProp(writer, "process", value.Process, EcsJsonContext.Default.Process, options);
WriteProp(writer, "registry", value.Registry, EcsJsonContext.Default.Registry, options);
WriteProp(writer, "related", value.Related, EcsJsonContext.Default.Related, options);
WriteProp(writer, "risk", value.Risk, EcsJsonContext.Default.Risk, options);
WriteProp(writer, "rule", value.Rule, EcsJsonContext.Default.Rule, options);
WriteProp(writer, "server", value.Server, EcsJsonContext.Default.Server, options);
WriteProp(writer, "service", value.Service, EcsJsonContext.Default.Service, options);
WriteProp(writer, "source", value.Source, EcsJsonContext.Default.Source, options);
WriteProp(writer, "threat", value.Threat, EcsJsonContext.Default.Threat, options);
WriteProp(writer, "tls", value.Tls, EcsJsonContext.Default.Tls, options);
WriteProp(writer, "url", value.Url, EcsJsonContext.Default.Url, options);
WriteProp(writer, "user", value.User, EcsJsonContext.Default.User, options);
WriteProp(writer, "user_agent", value.UserAgent, EcsJsonContext.Default.UserAgent, options);
WriteProp(writer, "vlan", value.Vlan, EcsJsonContext.Default.Vlan, options);
WriteProp(writer, "vulnerability", value.Vulnerability, EcsJsonContext.Default.Vulnerability, options);
WriteProp(writer, "x509", value.X509, EcsJsonContext.Default.X509, options);
WriteProp(writer, "metadata", value.Metadata, options);

if (typeof(EcsDocument) != value.GetType())
value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v));
value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options));
writer.WriteEndObject();
}
}
Expand Down
15 changes: 8 additions & 7 deletions src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ namespace Elastic.CommonSchema.Serialization
if (reader.TokenType != JsonTokenType.PropertyName)
throw new JsonException();

var _ = ReadProperties(ref reader, ecsEvent, ref timestamp, ref loglevel, ref ecsVersion);
var _ = ReadProperties(ref reader, ecsEvent, ref timestamp, ref loglevel, ref ecsVersion, options);
}
if (!string.IsNullOrEmpty(loglevel))
{
Expand All @@ -65,11 +65,11 @@ private static void WriteMessage(Utf8JsonWriter writer, EcsDocument value)
writer.WriteString("message", value.Message);
}

private static void WriteLogEntity(Utf8JsonWriter writer, Log? value) {
private static void WriteLogEntity(Utf8JsonWriter writer, Log? value, JsonSerializerOptions options) {
if (value == null) return;
if (!value.ShouldSerialize) return;

WriteProp(writer, "log", value, EcsJsonContext.Default.Log);
WriteProp(writer, "log", value, EcsJsonContext.Default.Log, options);
}

private static void WriteLogLevel(Utf8JsonWriter writer, EcsDocument value)
Expand All @@ -78,23 +78,24 @@ private static void WriteLogLevel(Utf8JsonWriter writer, EcsDocument value)
writer.WriteString("log.level", value.Log?.Level);
}

private static void WriteEcsEntity(Utf8JsonWriter writer, Ecs? value)
private static void WriteEcsEntity(Utf8JsonWriter writer, Ecs? value, JsonSerializerOptions options)
{
if (value == null) return;
if (!value.ShouldSerialize) return;

WriteProp(writer, "ecs", value, EcsJsonContext.Default.Ecs);
WriteProp(writer, "ecs", value, EcsJsonContext.Default.Ecs, options);
}

private static void WriteEcsVersion(Utf8JsonWriter writer, EcsDocument value) =>
writer.WriteString("ecs.version", value.Ecs?.Version ?? EcsDocument.Version);

private static void WriteTimestamp(Utf8JsonWriter writer, BaseFieldSet value)
private static void WriteTimestamp(Utf8JsonWriter writer, BaseFieldSet value, JsonSerializerOptions options)
{
if (!value.Timestamp.HasValue) return;

writer.WritePropertyName("@timestamp");
EcsJsonConfiguration.DateTimeOffsetConverter.Write(writer, value.Timestamp.Value, EcsJsonConfiguration.SerializerOptions);
var converter = (JsonConverter<DateTimeOffset>)options.GetConverter(typeof(DateTimeOffset));
converter.Write(writer, value.Timestamp.Value, options);
}
}

Expand Down
25 changes: 11 additions & 14 deletions src/Elastic.CommonSchema/Serialization/EcsJsonConverterBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,16 @@ namespace Elastic.CommonSchema.Serialization
public abstract class EcsJsonConverterBase<T> : JsonConverter<T>
{
/// <summary></summary>
protected static bool ReadDateTime(ref Utf8JsonReader reader, ref DateTimeOffset? dateTime)
protected static bool ReadDateTime(ref Utf8JsonReader reader, ref DateTimeOffset? dateTime, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.Null)
{
dateTime = null;
return true;
}

dateTime = EcsJsonConfiguration.DateTimeOffsetConverter.Read(ref reader, typeof(DateTimeOffset), EcsJsonConfiguration.SerializerOptions);
var converter = (JsonConverter<DateTimeOffset>)options.GetConverter(typeof(DateTimeOffset));
sergiojrdotnet marked this conversation as resolved.
Show resolved Hide resolved
dateTime = converter.Read(ref reader, typeof(DateTimeOffset), options);
return true;
}

Expand All @@ -34,20 +35,19 @@ protected static bool ReadString(ref Utf8JsonReader reader, ref string? stringPr
}

/// <summary></summary>
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value)
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value, JsonSerializerOptions options)
{
if (value == null) return;

var options = EcsJsonConfiguration.SerializerOptions;

writer.WritePropertyName(key);
// Attempt to use existing converter first before re-entering through JsonSerializer.Serialize().
// The default converter for object does not support writing.
JsonSerializer.Serialize<TValue>(writer, value, options);
}

/// <summary></summary>
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value, JsonTypeInfo<TValue> typeInfo)
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value, JsonTypeInfo<TValue> typeInfo,
JsonSerializerOptions options)
{
if (value == null) return;

Expand All @@ -56,16 +56,14 @@ protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValu

//To support user supplied subtypes
if (type != typeof(TValue))
JsonSerializer.Serialize(writer, value, type, EcsJsonConfiguration.SerializerOptions);
JsonSerializer.Serialize(writer, value, type, options);
else JsonSerializer.Serialize(writer, value, typeInfo);
}

internal static object? ReadPropDeserialize(ref Utf8JsonReader reader, Type type)
internal static object? ReadPropDeserialize(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.Null) return null;

var options = EcsJsonConfiguration.SerializerOptions;

return JsonSerializer.Deserialize(ref reader, type, options);
}

Expand Down Expand Up @@ -100,19 +98,18 @@ protected static bool ReadPropString(ref Utf8JsonReader reader, string key, T b,

/// <summary></summary>
// ReSharper disable once UnusedParameter.Local (key is used for readability)
private static TValue? ReadProp<TValue>(ref Utf8JsonReader reader, string key) where TValue : class
private static TValue? ReadProp<TValue>(ref Utf8JsonReader reader, string key, JsonSerializerOptions options) where TValue : class
{
if (reader.TokenType == JsonTokenType.Null) return null;

var options = EcsJsonConfiguration.SerializerOptions;
return JsonSerializer.Deserialize<TValue>(ref reader, options);
}

/// <summary></summary>
protected static bool ReadProp<TValue>(ref Utf8JsonReader reader, string key, T b, Action<T, TValue?> set)
protected static bool ReadProp<TValue>(ref Utf8JsonReader reader, string key, T b, Action<T, TValue?> set, JsonSerializerOptions options)
where TValue : class
{
set(b, ReadProp<TValue>(ref reader, key));
set(b, ReadProp<TValue>(ref reader, key, options));
return true;
}

Expand Down
3 changes: 0 additions & 3 deletions src/Elastic.CommonSchema/Serialization/JsonConfiguration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,6 @@ public static class EcsJsonConfiguration
}
};

internal static readonly JsonConverter<DateTimeOffset> DateTimeOffsetConverter =
(JsonConverter<DateTimeOffset>)SerializerOptions.GetConverter(typeof(DateTimeOffset));

/// <summary>Default <see cref="JsonConverter{T}"/> for <see cref="EcsDocument"/></summary>
public static readonly EcsDocumentJsonConverter DefaultEcsDocumentJsonConverter = new();

Expand Down
Loading
Loading