diff --git a/src/documentation/articles/kafkadbcontext.md b/src/documentation/articles/kafkadbcontext.md index 4f02d3a5..22592f71 100644 --- a/src/documentation/articles/kafkadbcontext.md +++ b/src/documentation/articles/kafkadbcontext.md @@ -8,8 +8,8 @@ _description: Describe what is and how use KafkaDbContext class from Entity Fram `KafkaDbContext` is a special class which helps to define the `DbContext` and use [Entity Framework Core](https://learn.microsoft.com/it-it/ef/core/) provider for [Apache Kafka](https://kafka.apache.org/): - `KafkaDbContext` inherits from `DbContext`: to define the model, and/or creating the database, see [getting started](https://docs.microsoft.com/ef/core/get-started/) in the docs and [KEFCore usage](usage.md) - `KafkaDbContext` defines the following properties: - - **KeySerializationType**: the .NET type to be used to allocate an external serializer for Apache Kafka record key - - **ValueSerializationType**: the .NET type to be used to allocate an external serializer for Apache Kafka record value + - **KeySerDesSelectorType**: the .NET type to be used to allocate an external serializer for Apache Kafka record key + - **ValueSerDesSelectorType**: the .NET type to be used to allocate an external serializer for Apache Kafka record value - **ValueContainerType**: the .NET type to be used to allocate an external container class for Apache Kafka record value - **UseNameMatching**: set to **false** to avoid Entity matching based on Name - **BootstrapServers**: the server hosting the broker of Apache Kafka @@ -19,10 +19,11 @@ _description: Describe what is and how use KafkaDbContext class from Entity Fram - **DefaultReplicationFactor**: the replication factor to use when data are stored in Apache Kafka - **DefaultConsumerInstances**: the consumer instances to be allocated when UseCompactedReplicator is **true** - **UsePersistentStorage**: set to **true** to use a persistent storage between multiple application startup + - **UseEnumeratorWithPrefetch**: set to **true** to prefer enumerator instances able to do a prefetch on data speeding up execution, used if **UseKNetStreams** is **true** and **UseCompactedReplicator** is **false** + - **UseByteBufferDataTransfer**: set to **true** to prefer data exchange in serializer instances - **UseDeletePolicyForTopic**: set to **true** to enable [delete cleanup policy](https://kafka.apache.org/documentation/#topicconfigs_cleanup.policy) - **UseCompactedReplicator**: Use `KNetCompactedReplicator` instead of Apache Kafka Streams to manage data to or from topics - **UseKNetStreams**: Use KNet version of Apache Kafka Streams instead of standard Apache Kafka Streams, used if **UseCompactedReplicator** is **false** - - **UseEnumeratorWithPrefetch**: Setting this property to **true** the engine prefers to use enumerator instances able to do a prefetch on data speeding up execution, used if **UseKNetStreams** is **true** and **UseCompactedReplicator** is **false** - **ConsumerConfig**: parameters to use for Producer - **ProducerConfig**: parameters to use for Producer - **StreamsConfig**: parameters to use for Apche Kafka Streams application diff --git a/src/documentation/articles/serialization.md b/src/documentation/articles/serialization.md index e0aa1fb5..8dfd6a4e 100644 --- a/src/documentation/articles/serialization.md +++ b/src/documentation/articles/serialization.md @@ -510,7 +510,7 @@ The extension converted this schema into code to speedup the exection of seriali ### How to use Protobuf `KafkaDbContext` contains three properties can be used to override the default types: -- **KeySerializationType**: set this value to `ProtobufKEFCoreSerDes..Key.BinaryRaw<>` or use `ProtobufKEFCoreSerDes.DefaultKeySerialization`, the type automatically manages simple or complex Primary Key +- **KeySerializationType**: set this value to `ProtobufKEFCoreSerDes.Key.BinaryRaw<>` or use `ProtobufKEFCoreSerDes.DefaultKeySerialization`, the type automatically manages simple or complex Primary Key - **ValueSerializationType**: set this value to `ProtobufKEFCoreSerDes.ValueContainer.BinaryRaw<>` or use `ProtobufKEFCoreSerDes.DefaultValueContainerSerialization` - **ValueContainerType**: set this value to `ProtobufValueContainerRaw<>` or use `ProtobufKEFCoreSerDes.DefaultValueContainer` diff --git a/src/net/KEFCore.SerDes.Avro.Compiler/KEFCore.SerDes.Avro.Compiler.csproj b/src/net/KEFCore.SerDes.Avro.Compiler/KEFCore.SerDes.Avro.Compiler.csproj index f0549c95..274f9e4b 100644 --- a/src/net/KEFCore.SerDes.Avro.Compiler/KEFCore.SerDes.Avro.Compiler.csproj +++ b/src/net/KEFCore.SerDes.Avro.Compiler/KEFCore.SerDes.Avro.Compiler.csproj @@ -23,7 +23,7 @@ - + diff --git a/src/net/KEFCore.SerDes.Avro/AvroKEFCoreSerDes.cs b/src/net/KEFCore.SerDes.Avro/AvroKEFCoreSerDes.cs index 67393692..5b5660d9 100644 --- a/src/net/KEFCore.SerDes.Avro/AvroKEFCoreSerDes.cs +++ b/src/net/KEFCore.SerDes.Avro/AvroKEFCoreSerDes.cs @@ -29,558 +29,730 @@ namespace MASES.EntityFrameworkCore.KNet.Serialization.Avro; /// -/// Avro base class to define extensions of , for example +/// Avro base class to define extensions of , for example /// public static class AvroKEFCoreSerDes { /// /// Returns the default serializer for keys /// - public static readonly Type DefaultKeySerialization = typeof(Key.BinaryRaw<>); + public static readonly Type DefaultKeySerialization = typeof(Key.Binary<>); /// /// Returns the default serializer for value containers /// - public static readonly Type DefaultValueContainerSerialization = typeof(ValueContainer.BinaryRaw<>); + public static readonly Type DefaultValueContainerSerialization = typeof(ValueContainer.Binary<>); /// /// Returns the default for value containers /// public static readonly Type DefaultValueContainer = typeof(AvroValueContainer<>); /// - /// Base class to define key extensions of , for example + /// Base class to define key extensions of , for example /// public static class Key { /// - /// Avro Key Binary encoder extension of , for example based on array + /// Base class to define key extensions of , for example /// - /// - public class BinaryRaw : SerDesRaw + public class Binary : ISerDesSelector { - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(BinaryRaw<>).ToAssemblyQualified()); - readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); - readonly ISerDesRaw _defaultSerDes = default!; - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Returns a new instance of /// - public BinaryRaw() - { - if (KNetSerialization.IsInternalManaged()) + /// The of + public static ISerDesSelector NewInstance() => new Binary(); + /// + public static string SelectorTypeName => typeof(Binary<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(BinaryRaw); + /// + public static Type ByteBufferSerDes => typeof(BinaryBuffered); + /// + public static ISerDes NewSerDes() + { + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static ISerDesRaw NewByteArraySerDes() { return new BinaryRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new BinaryBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + + /// + /// Avro Key Binary encoder extension of , for example based on array + /// + /// + sealed class BinaryRaw : SerDesRaw + { + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly byte[] keySerDesName ; + readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); + readonly ISerDesRaw _defaultSerDes = default!; + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public BinaryRaw(string selectorName) { - _defaultSerDes = new SerDesRaw(); + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) + { + _defaultSerDes = new SerDesRaw(); + } + else if (!typeof(TData).IsArray) + { + throw new InvalidOperationException($"{typeof(BinaryRaw<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); + } } - else if (!typeof(T).IsArray) + + /// + public override byte[] Serialize(string topic, TData data) { - throw new InvalidOperationException($"{typeof(BinaryRaw<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + return SerializeWithHeaders(topic, null!, data); } - } - - /// - public override byte[] Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); - headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); + headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); - if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); + if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); - using MemoryStream memStream = new(); - BinaryEncoder encoder = new(memStream); - var container = new AvroKeyContainer(); - if (data is object[] dataArray) + using MemoryStream memStream = new(); + BinaryEncoder encoder = new(memStream); + var container = new AvroKeyContainer(); + if (data is object[] dataArray) + { + container.PrimaryKey = new List(dataArray); + } + else throw new InvalidDataException($"Cannot manage inputs different from object[], input is {data?.GetType()}"); + SpecificWriter.Write(container, encoder); + return memStream.ToArray(); + } + /// + public override TData Deserialize(string topic, byte[] data) { - container.PrimaryKey = new List(dataArray); + return DeserializeWithHeaders(topic, null!, data); } - else throw new InvalidDataException($"Cannot manage inputs different from object[], input is {data?.GetType()}"); - SpecificWriter.Write(container, encoder); - return memStream.ToArray(); - } - /// - public override T Deserialize(string topic, byte[] data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) - { - if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) + { + if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); - using MemoryStream memStream = new(data); - BinaryDecoder decoder = new(memStream); - AvroKeyContainer t = new AvroKeyContainer(); - t = SpecificReader.Read(t!, decoder); - return (T)(object)(t.PrimaryKey.ToArray()); + using MemoryStream memStream = new(data); + BinaryDecoder decoder = new(memStream); + AvroKeyContainer t = new AvroKeyContainer(); + t = SpecificReader.Read(t!, decoder); + return (TData)(object)(t.PrimaryKey.ToArray()); + } } - } - /// - /// Avro Key Binary encoder extension of , for example based on - /// - /// - public class BinaryBuffered : SerDesBuffered - { - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(BinaryBuffered<>).ToAssemblyQualified()); - readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); - readonly ISerDesBuffered _defaultSerDes = default!; - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Avro Key Binary encoder extension of , for example based on /// - public BinaryBuffered() - { - if (KNetSerialization.IsInternalManaged()) + /// + sealed class BinaryBuffered : SerDesBuffered + { + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly byte[] keySerDesName; + readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); + readonly ISerDesBuffered _defaultSerDes = default!; + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public BinaryBuffered(string selectorName) { - _defaultSerDes = new SerDesBuffered(); + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) + { + _defaultSerDes = new SerDesBuffered(); + } + else if (!typeof(TData).IsArray) + { + throw new InvalidOperationException($"{typeof(BinaryBuffered<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); + } } - else if (!typeof(T).IsArray) + + /// + public override ByteBuffer Serialize(string topic, TData data) { - throw new InvalidOperationException($"{typeof(BinaryBuffered<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + return SerializeWithHeaders(topic, null!, data); } - } - - /// - public override ByteBuffer Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); - headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); + headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); - if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); + if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); - MemoryStream memStream = new(); - BinaryEncoder encoder = new(memStream); - var container = new AvroKeyContainer(); - if (data is object[] dataArray) + MemoryStream memStream = new(); + BinaryEncoder encoder = new(memStream); + var container = new AvroKeyContainer(); + if (data is object[] dataArray) + { + container.PrimaryKey = new List(dataArray); + } + else throw new InvalidDataException($"Cannot manage inputs different from object[], input is {data?.GetType()}"); + SpecificWriter.Write(container, encoder); + return ByteBuffer.From(memStream); + } + /// + public override TData Deserialize(string topic, ByteBuffer data) { - container.PrimaryKey = new List(dataArray); + return DeserializeWithHeaders(topic, null!, data); } - else throw new InvalidDataException($"Cannot manage inputs different from object[], input is {data?.GetType()}"); - SpecificWriter.Write(container, encoder); - return ByteBuffer.From(memStream); - } - /// - public override T Deserialize(string topic, ByteBuffer data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) - { - if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + { + if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); - BinaryDecoder decoder = new(data); - AvroKeyContainer t = new AvroKeyContainer(); - t = SpecificReader.Read(t!, decoder); - return (T)(object)(t.PrimaryKey.ToArray()); + BinaryDecoder decoder = new(data); + AvroKeyContainer t = new AvroKeyContainer(); + t = SpecificReader.Read(t!, decoder); + return (TData)(object)(t.PrimaryKey.ToArray()); + } } } /// - /// Avro Key Json encoder extension of , for example based on array + /// Base class to define key extensions of , for example /// - /// - public class JsonRaw : SerDesRaw + public class Json : ISerDesSelector { - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(JsonRaw<>).ToAssemblyQualified()); - readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); - readonly ISerDesRaw _defaultSerDes = default!; - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Returns a new instance of /// - public JsonRaw() - { - if (KNetSerialization.IsInternalManaged()) + /// The of + public static ISerDesSelector NewInstance() => new Json(); + /// + public static string SelectorTypeName => typeof(Json<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(JsonRaw); + /// + public static Type ByteBufferSerDes => typeof(JsonBuffered); + /// + public static ISerDes NewSerDes() + { + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static ISerDesRaw NewByteArraySerDes() { return new JsonRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new JsonBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + + /// + /// Avro Key Json encoder extension of , for example based on array + /// + /// + sealed class JsonRaw : SerDesRaw + { + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly byte[] keySerDesName; + readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); + readonly ISerDesRaw _defaultSerDes = default!; + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public JsonRaw(string selectorName) { - _defaultSerDes = new SerDesRaw(); + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) + { + _defaultSerDes = new SerDesRaw(); + } + else if (!typeof(TData).IsArray) + { + throw new InvalidOperationException($"{typeof(JsonRaw<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); + } } - else if (!typeof(T).IsArray) + + /// + public override byte[] Serialize(string topic, TData data) { - throw new InvalidOperationException($"{typeof(JsonRaw<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + return SerializeWithHeaders(topic, null!, data); } - } - - /// - public override byte[] Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); - headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); + headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); - if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); + if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); - using MemoryStream memStream = new(); - JsonEncoder encoder = new(AvroKeyContainer._SCHEMA, memStream); - SpecificWriter.Write(data, encoder); - encoder.Flush(); - return memStream.ToArray(); - } - /// - public override T Deserialize(string topic, byte[] data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) - { - if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); + using MemoryStream memStream = new(); + JsonEncoder encoder = new(AvroKeyContainer._SCHEMA, memStream); + SpecificWriter.Write(data, encoder); + encoder.Flush(); + return memStream.ToArray(); + } + /// + public override TData Deserialize(string topic, byte[] data) + { + return DeserializeWithHeaders(topic, null!, data); + } + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) + { + if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); - using MemoryStream memStream = new(data); - JsonDecoder decoder = new(AvroKeyContainer._SCHEMA, memStream); - T t = (T)Activator.CreateInstance(typeof(T))!; - t = SpecificReader.Read(t!, decoder); - return t; + using MemoryStream memStream = new(data); + JsonDecoder decoder = new(AvroKeyContainer._SCHEMA, memStream); + TData t = (TData)Activator.CreateInstance(typeof(TData))!; + t = SpecificReader.Read(t!, decoder); + return t; + } } - } - /// - /// Avro Key Json encoder extension of , for example based on - /// - /// - public class JsonBuffered : SerDesBuffered - { - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(JsonBuffered<>).ToAssemblyQualified()); - readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); - readonly ISerDesBuffered _defaultSerDes = default!; - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Avro Key Json encoder extension of , for example based on /// - public JsonBuffered() - { - if (KNetSerialization.IsInternalManaged()) + /// + sealed class JsonBuffered : SerDesBuffered + { + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly byte[] keySerDesName; + readonly SpecificDefaultWriter SpecificWriter = new(AvroKeyContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroKeyContainer._SCHEMA, AvroKeyContainer._SCHEMA); + readonly ISerDesBuffered _defaultSerDes = default!; + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public JsonBuffered(string selectorName) { - _defaultSerDes = new SerDesBuffered(); + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) + { + _defaultSerDes = new SerDesBuffered(); + } + else if (!typeof(TData).IsArray) + { + throw new InvalidOperationException($"{typeof(JsonBuffered<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); + } } - else if (!typeof(T).IsArray) + + /// + public override ByteBuffer Serialize(string topic, TData data) { - throw new InvalidOperationException($"{typeof(JsonBuffered<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + return SerializeWithHeaders(topic, null!, data); } - } - - /// - public override ByteBuffer Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); - headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); + headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); - if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); + if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); - MemoryStream memStream = new(); - JsonEncoder encoder = new(AvroKeyContainer._SCHEMA, memStream); - SpecificWriter.Write(data, encoder); - encoder.Flush(); - return ByteBuffer.From(memStream); - } - /// - public override T Deserialize(string topic, ByteBuffer data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) - { - if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); + MemoryStream memStream = new(); + JsonEncoder encoder = new(AvroKeyContainer._SCHEMA, memStream); + SpecificWriter.Write(data, encoder); + encoder.Flush(); + return ByteBuffer.From(memStream); + } + /// + public override TData Deserialize(string topic, ByteBuffer data) + { + return DeserializeWithHeaders(topic, null!, data); + } + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + { + if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); - JsonDecoder decoder = new(AvroKeyContainer._SCHEMA, data); - T t = (T)Activator.CreateInstance(typeof(T))!; - t = SpecificReader.Read(t!, decoder); - return t; + JsonDecoder decoder = new(AvroKeyContainer._SCHEMA, data); + TData t = (TData)Activator.CreateInstance(typeof(TData))!; + t = SpecificReader.Read(t!, decoder); + return t; + } } } } /// - /// Base class to define ValueContainer extensions of , for example + /// Base class to define ValueContainer extensions of , for example /// public static class ValueContainer { /// - /// Avro ValueContainer Binary encoder extension of , for example based on array + /// Base class to define key extensions of , for example /// - /// - public class BinaryRaw : SerDesRaw + public class Binary : ISerDesSelector { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(BinaryRaw<>).ToAssemblyQualified()); - readonly byte[] valueContainerName = null!; - readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Returns a new instance of /// - public BinaryRaw() - { - var tt = typeof(T); - if (tt.IsGenericType) + /// The of + public static ISerDesSelector NewInstance() => new Binary(); + /// + public static string SelectorTypeName => typeof(Binary<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(BinaryRaw); + /// + public static Type ByteBufferSerDes => typeof(BinaryBuffered); + /// + public static ISerDes NewSerDes() + { + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static ISerDesRaw NewByteArraySerDes() { return new BinaryRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new BinaryBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + + /// + /// Avro ValueContainer Binary encoder extension of , for example based on array + /// + /// + sealed class BinaryRaw : SerDesRaw + { + readonly byte[] valueContainerSerDesName; + readonly byte[] valueContainerName = null!; + readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public BinaryRaw(string selectorName) { - var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } - var t = tt.GetGenericTypeDefinition(); - if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); + if (tt.IsGenericType) { - valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); - return; + var keyT = tt.GetGenericArguments(); + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + var t = tt.GetGenericTypeDefinition(); + if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + { + valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); + return; + } + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); - } - /// - public override byte[] Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); - headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); + /// + public override byte[] Serialize(string topic, TData data) + { + return SerializeWithHeaders(topic, null!, data); + } + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); + headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); - using MemoryStream memStream = new(); - BinaryEncoder encoder = new(memStream); - SpecificWriter.Write(data, encoder); - return memStream.ToArray(); - } - /// - public override T Deserialize(string topic, byte[] data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) - { - using MemoryStream memStream = new(data); - BinaryDecoder decoder = new(memStream); - T t = (T)Activator.CreateInstance(typeof(T))!; - t = SpecificReader.Read(t!, decoder); - return t; + using MemoryStream memStream = new(); + BinaryEncoder encoder = new(memStream); + SpecificWriter.Write(data, encoder); + return memStream.ToArray(); + } + /// + public override TData Deserialize(string topic, byte[] data) + { + return DeserializeWithHeaders(topic, null!, data); + } + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) + { + using MemoryStream memStream = new(data); + BinaryDecoder decoder = new(memStream); + TData t = (TData)Activator.CreateInstance(typeof(TData))!; + t = SpecificReader.Read(t!, decoder); + return t; + } } - } - /// - /// Avro ValueContainer Binary encoder extension of , for example based on - /// - /// - public class BinaryBuffered : SerDesBuffered - { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(BinaryBuffered<>).ToAssemblyQualified()); - readonly byte[] valueContainerName = null!; - readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Avro ValueContainer Binary encoder extension of , for example based on /// - public BinaryBuffered() - { - var tt = typeof(T); - if (tt.IsGenericType) + /// + sealed class BinaryBuffered : SerDesBuffered + { + readonly byte[] valueContainerSerDesName; + readonly byte[] valueContainerName = null!; + readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public BinaryBuffered(string selectorName) { - var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } - var t = tt.GetGenericTypeDefinition(); - if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); + if (tt.IsGenericType) { - valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); - return; + var keyT = tt.GetGenericArguments(); + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + var t = tt.GetGenericTypeDefinition(); + if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + { + valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); + return; + } + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); - } - /// - public override ByteBuffer Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); - headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); + /// + public override ByteBuffer Serialize(string topic, TData data) + { + return SerializeWithHeaders(topic, null!, data); + } + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); + headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); - MemoryStream memStream = new(); - BinaryEncoder encoder = new(memStream); - SpecificWriter.Write(data, encoder); - return ByteBuffer.From(memStream); - } - /// - public override T Deserialize(string topic, ByteBuffer data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) - { - BinaryDecoder decoder = new(data); - T t = (T)Activator.CreateInstance(typeof(T))!; - t = SpecificReader.Read(t!, decoder); - return t; + MemoryStream memStream = new(); + BinaryEncoder encoder = new(memStream); + SpecificWriter.Write(data, encoder); + return ByteBuffer.From(memStream); + } + /// + public override TData Deserialize(string topic, ByteBuffer data) + { + return DeserializeWithHeaders(topic, null!, data); + } + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + { + BinaryDecoder decoder = new(data); + TData t = (TData)Activator.CreateInstance(typeof(TData))!; + t = SpecificReader.Read(t!, decoder); + return t; + } } } /// - /// Avro ValueContainer Json encoder extension of , for example based on array + /// Base class to define key extensions of , for example /// - /// - public class JsonRaw : SerDesRaw + public class Json : ISerDesSelector { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(JsonRaw<>).ToAssemblyQualified()); - readonly byte[] valueContainerName = null!; - readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Returns a new instance of /// - public JsonRaw() - { - var tt = typeof(T); - if (tt.IsGenericType) + /// The of + public static ISerDesSelector NewInstance() => new Json(); + /// + public static string SelectorTypeName => typeof(Json<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(JsonRaw); + /// + public static Type ByteBufferSerDes => typeof(JsonBuffered); + /// + public static ISerDes NewSerDes() + { + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static ISerDesRaw NewByteArraySerDes() { return new JsonRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new JsonBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + + /// + /// Avro ValueContainer Json encoder extension of , for example based on array + /// + /// + sealed class JsonRaw : SerDesRaw + { + readonly byte[] valueContainerSerDesName; + readonly byte[] valueContainerName = null!; + readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public JsonRaw(string selectorName) { - var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } - var t = tt.GetGenericTypeDefinition(); - if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); + if (tt.IsGenericType) { - valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); - return; + var keyT = tt.GetGenericArguments(); + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + var t = tt.GetGenericTypeDefinition(); + if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + { + valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); + return; + } + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); - } - /// - public override byte[] Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); - headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); - - using MemoryStream memStream = new(); - JsonEncoder encoder = new(AvroValueContainer._SCHEMA, memStream); - SpecificWriter.Write(data, encoder); - encoder.Flush(); - return memStream.ToArray(); - } - /// - public override T Deserialize(string topic, byte[] data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) - { - using MemoryStream memStream = new(data); - JsonDecoder decoder = new(AvroValueContainer._SCHEMA, memStream); - T t = (T)Activator.CreateInstance(typeof(T))!; - t = SpecificReader.Read(t!, decoder); - return t; + /// + public override byte[] Serialize(string topic, TData data) + { + return SerializeWithHeaders(topic, null!, data); + } + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); + headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); + + using MemoryStream memStream = new(); + JsonEncoder encoder = new(AvroValueContainer._SCHEMA, memStream); + SpecificWriter.Write(data, encoder); + encoder.Flush(); + return memStream.ToArray(); + } + /// + public override TData Deserialize(string topic, byte[] data) + { + return DeserializeWithHeaders(topic, null!, data); + } + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) + { + using MemoryStream memStream = new(data); + JsonDecoder decoder = new(AvroValueContainer._SCHEMA, memStream); + TData t = (TData)Activator.CreateInstance(typeof(TData))!; + t = SpecificReader.Read(t!, decoder); + return t; + } } - } - /// - /// Avro ValueContainer Json encoder extension of , for example based on - /// - /// - public class JsonBuffered : SerDesBuffered - { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(JsonBuffered<>).ToAssemblyQualified()); - readonly byte[] valueContainerName = null!; - readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); - readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); - /// - public override bool UseHeaders => true; /// - /// Default initializer + /// Avro ValueContainer Json encoder extension of , for example based on /// - public JsonBuffered() - { - var tt = typeof(T); - if (tt.IsGenericType) + /// + sealed class JsonBuffered : SerDesBuffered + { + readonly byte[] valueContainerSerDesName; + readonly byte[] valueContainerName = null!; + readonly SpecificDefaultWriter SpecificWriter = new(AvroValueContainer._SCHEMA); + readonly SpecificDefaultReader SpecificReader = new(AvroValueContainer._SCHEMA, AvroValueContainer._SCHEMA); + /// + public override bool UseHeaders => true; + /// + /// Default initializer + /// + public JsonBuffered(string selectorName) { - var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } - var t = tt.GetGenericTypeDefinition(); - if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); + if (tt.IsGenericType) { - valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); - return; + var keyT = tt.GetGenericArguments(); + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + var t = tt.GetGenericTypeDefinition(); + if (t.GetInterface(typeof(IValueContainer<>).Name) != null) + { + valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); + return; + } + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); - } - /// - public override ByteBuffer Serialize(string topic, T data) - { - return SerializeWithHeaders(topic, null!, data); - } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) - { - headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); - headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); - - MemoryStream memStream = new(); - JsonEncoder encoder = new(AvroValueContainer._SCHEMA, memStream); - SpecificWriter.Write(data, encoder); - encoder.Flush(); - return ByteBuffer.From(memStream); - } - /// - public override T Deserialize(string topic, ByteBuffer data) - { - return DeserializeWithHeaders(topic, null!, data); - } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) - { - JsonDecoder decoder = new(AvroValueContainer._SCHEMA, data); - T t = (T)Activator.CreateInstance(typeof(T))!; - t = SpecificReader.Read(t!, decoder); - return t; + /// + public override ByteBuffer Serialize(string topic, TData data) + { + return SerializeWithHeaders(topic, null!, data); + } + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) + { + headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); + headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); + + MemoryStream memStream = new(); + JsonEncoder encoder = new(AvroValueContainer._SCHEMA, memStream); + SpecificWriter.Write(data, encoder); + encoder.Flush(); + return ByteBuffer.From(memStream); + } + /// + public override TData Deserialize(string topic, ByteBuffer data) + { + return DeserializeWithHeaders(topic, null!, data); + } + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + { + JsonDecoder decoder = new(AvroValueContainer._SCHEMA, data); + TData t = (TData)Activator.CreateInstance(typeof(TData))!; + t = SpecificReader.Read(t!, decoder); + return t; + } } } } diff --git a/src/net/KEFCore.SerDes.Avro/AvroValueContainer.cs b/src/net/KEFCore.SerDes.Avro/AvroValueContainer.cs index df43da7c..2fb6baed 100644 --- a/src/net/KEFCore.SerDes.Avro/AvroValueContainer.cs +++ b/src/net/KEFCore.SerDes.Avro/AvroValueContainer.cs @@ -21,6 +21,7 @@ #nullable enable using Avro; +using MASES.KNet.Serialization; namespace MASES.EntityFrameworkCore.KNet.Serialization.Avro.Storage; @@ -44,7 +45,7 @@ public partial class AvroValueContainer : AvroValueContainer, IValueContai public AvroValueContainer(IEntityType tName, object[] rData) { EntityName = tName.Name; - ClrType = tName.ClrType.FullName!; + ClrType = tName.ClrType?.ToAssemblyQualified()!; Data = new List(); foreach (var item in tName.GetProperties()) { @@ -53,7 +54,7 @@ public AvroValueContainer(IEntityType tName, object[] rData) { PropertyIndex = index, PropertyName = item.Name, - ClrType = item.ClrType?.FullName, + ClrType = item.ClrType?.ToAssemblyQualified(), Value = rData[index] }; Data.Add(pRecord); diff --git a/src/net/KEFCore.SerDes.Protobuf/Generated/GenericValue.cs b/src/net/KEFCore.SerDes.Protobuf/Generated/GenericValue.cs index eb526c6f..b8e99fd4 100644 --- a/src/net/KEFCore.SerDes.Protobuf/Generated/GenericValue.cs +++ b/src/net/KEFCore.SerDes.Protobuf/Generated/GenericValue.cs @@ -772,7 +772,11 @@ public void MergeFrom(pb::CodedInputStream input) { #else uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; @@ -846,7 +850,11 @@ public void MergeFrom(pb::CodedInputStream input) { void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; diff --git a/src/net/KEFCore.SerDes.Protobuf/Generated/KeyContainer.cs b/src/net/KEFCore.SerDes.Protobuf/Generated/KeyContainer.cs index 39fca702..2082af44 100644 --- a/src/net/KEFCore.SerDes.Protobuf/Generated/KeyContainer.cs +++ b/src/net/KEFCore.SerDes.Protobuf/Generated/KeyContainer.cs @@ -190,7 +190,11 @@ public void MergeFrom(pb::CodedInputStream input) { #else uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; @@ -209,7 +213,11 @@ public void MergeFrom(pb::CodedInputStream input) { void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; @@ -386,7 +394,11 @@ public void MergeFrom(pb::CodedInputStream input) { #else uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; @@ -408,7 +420,11 @@ public void MergeFrom(pb::CodedInputStream input) { void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; diff --git a/src/net/KEFCore.SerDes.Protobuf/Generated/ValueContainer.cs b/src/net/KEFCore.SerDes.Protobuf/Generated/ValueContainer.cs index 48698379..a4d70f89 100644 --- a/src/net/KEFCore.SerDes.Protobuf/Generated/ValueContainer.cs +++ b/src/net/KEFCore.SerDes.Protobuf/Generated/ValueContainer.cs @@ -294,7 +294,11 @@ public void MergeFrom(pb::CodedInputStream input) { #else uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; @@ -328,7 +332,11 @@ public void MergeFrom(pb::CodedInputStream input) { void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; @@ -564,7 +572,11 @@ public void MergeFrom(pb::CodedInputStream input) { #else uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; @@ -591,7 +603,11 @@ public void MergeFrom(pb::CodedInputStream input) { void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { uint tag; while ((tag = input.ReadTag()) != 0) { - switch(tag) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; diff --git a/src/net/KEFCore.SerDes.Protobuf/KEFCore.SerDes.Protobuf.csproj b/src/net/KEFCore.SerDes.Protobuf/KEFCore.SerDes.Protobuf.csproj index fcbfa0b1..dd17575a 100644 --- a/src/net/KEFCore.SerDes.Protobuf/KEFCore.SerDes.Protobuf.csproj +++ b/src/net/KEFCore.SerDes.Protobuf/KEFCore.SerDes.Protobuf.csproj @@ -42,8 +42,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/net/KEFCore.SerDes.Protobuf/ProtobufKEFCoreSerDes.cs b/src/net/KEFCore.SerDes.Protobuf/ProtobufKEFCoreSerDes.cs index 414c41fa..103dd90c 100644 --- a/src/net/KEFCore.SerDes.Protobuf/ProtobufKEFCoreSerDes.cs +++ b/src/net/KEFCore.SerDes.Protobuf/ProtobufKEFCoreSerDes.cs @@ -28,60 +28,95 @@ namespace MASES.EntityFrameworkCore.KNet.Serialization.Protobuf; /// -/// Protobuf base class to define extensions of , for example +/// Protobuf base class to define extensions of , for example /// public static class ProtobufKEFCoreSerDes { /// /// Returns the default serializer for keys /// - public static readonly Type DefaultKeySerialization = typeof(Key.BinaryRaw<>); + public static readonly Type DefaultKeySerialization = typeof(Key<>); /// /// Returns the default serializer for value containers /// - public static readonly Type DefaultValueContainerSerialization = typeof(ValueContainer.BinaryRaw<>); + public static readonly Type DefaultValueContainerSerialization = typeof(ValueContainer<>); /// /// Returns the default for value containers /// public static readonly Type DefaultValueContainer = typeof(ProtobufValueContainer<>); /// - /// Base class to define key extensions of , for example based on array + /// Base class to define key extensions of , for example /// - public static class Key + public class Key : ISerDesSelector where T : class, IMessage { /// - /// Protobuf Key Binary encoder extension of , for example + /// Returns a new instance of /// - /// - public class BinaryRaw : SerDesRaw + /// The of + public static ISerDesSelector NewInstance() => new Key(); + /// + public static string SelectorTypeName => typeof(Key<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(BinaryRaw); + /// + public static Type ByteBufferSerDes => typeof(BinaryBuffered); + /// + public static ISerDes NewSerDes() { - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(BinaryRaw<>).ToAssemblyQualified()); - readonly ISerDesRaw _defaultSerDes = default!; + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static ISerDesRaw NewByteArraySerDes() { return new BinaryRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new BinaryBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + /// + /// Protobuf Key Binary encoder extension of , for example + /// + /// + sealed class BinaryRaw : SerDesRaw + { + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly byte[] keySerDesName; + readonly ISerDesRaw _defaultSerDes = default!; /// public override bool UseHeaders => true; /// /// Default initializer /// - public BinaryRaw() + public BinaryRaw(string selectorName) { - if (KNetSerialization.IsInternalManaged()) + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) { - _defaultSerDes = new SerDesRaw(); + _defaultSerDes = new SerDesRaw(); } - else if (!typeof(T).IsArray) + else if (!typeof(TData).IsArray) { - throw new InvalidOperationException($"{typeof(BinaryRaw<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + throw new InvalidOperationException($"{typeof(BinaryRaw<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); } } - /// - public override byte[] Serialize(string topic, T data) + /// + public override byte[] Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); @@ -97,13 +132,13 @@ public override byte[] SerializeWithHeaders(string topic, Headers headers, T dat keyContainer.WriteTo(stream); return stream.ToArray(); } - /// - public override T Deserialize(string topic, byte[] data) + /// + public override TData Deserialize(string topic, byte[] data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) { if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); @@ -111,43 +146,44 @@ public override T DeserializeWithHeaders(string topic, Headers headers, byte[] d KeyContainer container = KeyContainer.Parser.ParseFrom(data); - return (T)container.GetContent(); + return (TData)container.GetContent(); } } /// - /// Protobuf Key Binary encoder extension of , for example based on + /// Protobuf Key Binary encoder extension of , for example based on /// - /// - public class BinaryBuffered : SerDesBuffered + /// + sealed class BinaryBuffered : SerDesBuffered { - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(BinaryBuffered<>).ToAssemblyQualified()); - readonly ISerDesBuffered _defaultSerDes = default!; + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly byte[] keySerDesName; + readonly ISerDesBuffered _defaultSerDes = default!; /// public override bool UseHeaders => true; /// /// Default initializer /// - public BinaryBuffered() + public BinaryBuffered(string selectorName) { - if (KNetSerialization.IsInternalManaged()) + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) { - _defaultSerDes = new SerDesBuffered(); + _defaultSerDes = new SerDesBuffered(); } - else if (!typeof(T).IsArray) + else if (!typeof(TData).IsArray) { - throw new InvalidOperationException($"{typeof(BinaryBuffered<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + throw new InvalidOperationException($"{typeof(BinaryBuffered<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); } } - /// - public override ByteBuffer Serialize(string topic, T data) + /// + public override ByteBuffer Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); @@ -163,13 +199,13 @@ public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T keyContainer.WriteTo(stream); return stream.ToArray(); } - /// - public override T Deserialize(string topic, ByteBuffer data) + /// + public override TData Deserialize(string topic, ByteBuffer data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) { if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); @@ -177,54 +213,90 @@ public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuff KeyContainer container = KeyContainer.Parser.ParseFrom(data.ToStream()); - return (T)container.GetContent(); + return (TData)container.GetContent(); } } } /// - /// Base class to define ValueContainer extensions of , for example + /// Base class to define ValueContainer extensions of , for example /// - public static class ValueContainer + public class ValueContainer : ISerDesSelector where T : class, IMessage { /// - /// Protobuf ValueContainer Binary encoder extension of , for example based on array + /// Returns a new instance of + /// + /// The of + public static ISerDesSelector NewInstance() => new ValueContainer(); + /// + public static string SelectorTypeName => typeof(ValueContainer<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(BinaryRaw); + /// + public static Type ByteBufferSerDes => typeof(BinaryBuffered); + /// + public static ISerDes NewSerDes() + { + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static ISerDesRaw NewByteArraySerDes() { return new BinaryRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new BinaryBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + + /// + /// Protobuf ValueContainer Binary encoder extension of , for example based on array /// - /// - public class BinaryRaw : SerDesRaw where T : class, IMessage + /// + sealed class BinaryRaw : SerDesRaw where TData : class, IMessage { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(BinaryRaw<>).ToAssemblyQualified()); + readonly byte[] valueContainerSerDesName; readonly byte[] valueContainerName = null!; /// public override bool UseHeaders => true; /// /// Default initializer /// - public BinaryRaw() + public BinaryRaw(string selectorName) { - var tt = typeof(T); + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); if (tt.IsGenericType) { var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } var t = tt.GetGenericTypeDefinition(); if (t.GetInterface(typeof(IValueContainer<>).Name) != null) { valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); return; } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - /// - public override byte[] Serialize(string topic, T data) + /// + public override byte[] Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); @@ -235,58 +307,59 @@ public override byte[] SerializeWithHeaders(string topic, Headers headers, T dat return stream.ToArray(); } } - /// - public override T Deserialize(string topic, byte[] data) + /// + public override TData Deserialize(string topic, byte[] data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) { if (data == null) return default!; var container = Storage.ValueContainer.Parser.ParseFrom(data); - return (Activator.CreateInstance(typeof(T), container) as T)!; + return (Activator.CreateInstance(typeof(TData), container) as TData)!; } } /// - /// Protobuf ValueContainer Binary encoder extension of , for example based on + /// Protobuf ValueContainer Binary encoder extension of , for example based on /// - /// - public class BinaryBuffered : SerDesBuffered where T : class, IMessage + /// + sealed class BinaryBuffered : SerDesBuffered where TData : class, IMessage { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(BinaryBuffered<>).ToAssemblyQualified()); + readonly byte[] valueContainerSerDesName; readonly byte[] valueContainerName = null!; /// public override bool UseHeaders => true; /// /// Default initializer /// - public BinaryBuffered() + public BinaryBuffered(string selectorName) { - var tt = typeof(T); + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); if (tt.IsGenericType) { var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } var t = tt.GetGenericTypeDefinition(); if (t.GetInterface(typeof(IValueContainer<>).Name) != null) { valueContainerName = Encoding.UTF8.GetBytes(t.ToAssemblyQualified()); return; } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - /// - public override ByteBuffer Serialize(string topic, T data) + /// + public override ByteBuffer Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); @@ -295,17 +368,17 @@ public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data.WriteTo(stream); return ByteBuffer.From(stream); } - /// - public override T Deserialize(string topic, ByteBuffer data) + /// + public override TData Deserialize(string topic, ByteBuffer data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) { if (data == null) return default!; var container = Storage.ValueContainer.Parser.ParseFrom(data.ToStream()); - return (Activator.CreateInstance(typeof(T), container) as T)!; + return (Activator.CreateInstance(typeof(TData), container) as TData)!; } } } diff --git a/src/net/KEFCore.SerDes.Protobuf/Storage/ProtobufValueContainer.cs b/src/net/KEFCore.SerDes.Protobuf/Storage/ProtobufValueContainer.cs index 342d566b..0191f6a8 100644 --- a/src/net/KEFCore.SerDes.Protobuf/Storage/ProtobufValueContainer.cs +++ b/src/net/KEFCore.SerDes.Protobuf/Storage/ProtobufValueContainer.cs @@ -20,6 +20,7 @@ using Google.Protobuf; using Google.Protobuf.Reflection; +using MASES.KNet.Serialization; namespace MASES.EntityFrameworkCore.KNet.Serialization.Protobuf.Storage; @@ -58,7 +59,7 @@ public ProtobufValueContainer(IEntityType tName, object[] rData) _innerMessage = new ValueContainer { EntityName = tName.Name, - ClrType = tName.ClrType.FullName! + ClrType = tName.ClrType?.ToAssemblyQualified()! }; _innerMessage.Data.Clear(); foreach (var item in tName.GetProperties()) @@ -68,7 +69,7 @@ public ProtobufValueContainer(IEntityType tName, object[] rData) { PropertyIndex = index, PropertyName = item.Name, - ClrType = item.ClrType?.FullName, + ClrType = item.ClrType?.ToAssemblyQualified(), Value = new GenericValue(rData[index]) }; _innerMessage.Data.Add(pRecord); diff --git a/src/net/KEFCore.SerDes/DefaultKEFCoreSerDes.cs b/src/net/KEFCore.SerDes/DefaultKEFCoreSerDes.cs index c70e2746..d5054792 100644 --- a/src/net/KEFCore.SerDes/DefaultKEFCoreSerDes.cs +++ b/src/net/KEFCore.SerDes/DefaultKEFCoreSerDes.cs @@ -27,51 +27,87 @@ namespace MASES.EntityFrameworkCore.KNet.Serialization.Json; /// -/// Default base class to define extensions of , for example +/// Default base class to define extensions of , for example /// public static class DefaultKEFCoreSerDes { /// /// Returns the default serializer for keys /// - public static readonly Type DefaultKeySerialization = typeof(Key.JsonRaw<>); + public static readonly Type DefaultKeySerialization = typeof(Key<>); /// /// Returns the default serializer for value containers /// - public static readonly Type DefaultValueContainerSerialization = typeof(ValueContainer.JsonRaw<>); + public static readonly Type DefaultValueContainerSerialization = typeof(ValueContainer<>); /// /// Returns the default for value containers /// public static readonly Type DefaultValueContainer = typeof(DefaultValueContainer<>); /// - /// Base class to define key extensions of , for example + /// Base class to define key extensions of , for example /// - public static class Key + public class Key : ISerDesSelector { /// - /// Json extension of , for example based on array + /// Returns a new instance of /// - /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like - public class JsonRaw : SerDesRaw + /// The of + public static ISerDesSelector NewInstance() => new Key(); + /// + public static string SelectorTypeName => typeof(Key<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(JsonRaw); + /// + public static Type ByteBufferSerDes => typeof(JsonBuffered); + /// + public static ISerDes NewSerDes() { - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(JsonRaw<>).ToAssemblyQualified()); - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly ISerDesRaw _defaultSerDes = default!; + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static ISerDesRaw NewByteArraySerDes() { return new JsonRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new JsonBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + + /// + /// Json extension of , for example based on array + /// + /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like + sealed class JsonRaw : SerDesRaw + { + readonly byte[] keySerDesName; + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly ISerDesRaw _defaultSerDes = default!; readonly JsonSerializerOptions? _options = null; /// public override bool UseHeaders => true; /// /// Default initializer /// - public JsonRaw() + public JsonRaw(string selectorName) { - if (KNetSerialization.IsInternalManaged()) + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) { - _defaultSerDes = new SerDesRaw(); + _defaultSerDes = new SerDesRaw(); } - else if (!typeof(T).IsArray) + else if (!typeof(TData).IsArray) { - throw new InvalidOperationException($"{typeof(JsonRaw<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + throw new InvalidOperationException($"{typeof(JsonRaw<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); } else { @@ -82,60 +118,61 @@ public JsonRaw() } } - /// - public override byte[] Serialize(string topic, T data) + /// + public override byte[] Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); - var jsonStr = System.Text.Json.JsonSerializer.Serialize(data); + var jsonStr = System.Text.Json.JsonSerializer.Serialize(data); return Encoding.UTF8.GetBytes(jsonStr); } - /// - public override T Deserialize(string topic, byte[] data) + /// + public override TData Deserialize(string topic, byte[] data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) { if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); if (data == null) return default!; - return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; + return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; } } /// - /// Json extension of , for example based on + /// Json extension of , for example based on /// - /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like - public class JsonBuffered : SerDesBuffered + /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like + sealed class JsonBuffered : SerDesBuffered { - readonly byte[] keySerDesName = Encoding.UTF8.GetBytes(typeof(JsonBuffered<>).ToAssemblyQualified()); - readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(T).FullName!); - readonly ISerDesBuffered _defaultSerDes = default!; + readonly byte[] keySerDesName; + readonly byte[] keyTypeName = Encoding.UTF8.GetBytes(typeof(TData).FullName!); + readonly ISerDesBuffered _defaultSerDes = default!; readonly JsonSerializerOptions? _options = null; /// public override bool UseHeaders => true; /// /// Default initializer /// - public JsonBuffered() + public JsonBuffered(string selectorName) { - if (KNetSerialization.IsInternalManaged()) + keySerDesName = Encoding.UTF8.GetBytes(selectorName); + if (KNetSerialization.IsInternalManaged()) { - _defaultSerDes = new SerDesBuffered(); + _defaultSerDes = new SerDesBuffered(); } - else if (!typeof(T).IsArray) + else if (!typeof(TData).IsArray) { - throw new InvalidOperationException($"{typeof(JsonBuffered<>).ToAssemblyQualified()} cannot manage {typeof(T).Name}, override or build a new serializaer"); + throw new InvalidOperationException($"{typeof(JsonBuffered<>).ToAssemblyQualified()} cannot manage {typeof(TData).Name}, override or build a new serializaer"); } else { @@ -146,13 +183,13 @@ public JsonBuffered() } } - /// - public override ByteBuffer Serialize(string topic, T data) + /// + public override ByteBuffer Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.KeyTypeIdentifier, keyTypeName); headers?.Add(KNetSerialization.KeySerializerIdentifier, keySerDesName); @@ -160,37 +197,72 @@ public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T if (_defaultSerDes != null) return _defaultSerDes.SerializeWithHeaders(topic, headers, data); var ms = new MemoryStream(); - System.Text.Json.JsonSerializer.Serialize(ms, data, _options); + System.Text.Json.JsonSerializer.Serialize(ms, data, _options); return ByteBuffer.From(ms); } - /// - public override T Deserialize(string topic, ByteBuffer data) + /// + public override TData Deserialize(string topic, ByteBuffer data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) { if (_defaultSerDes != null) return _defaultSerDes.DeserializeWithHeaders(topic, headers, data); if (data == null) return default!; - return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; + return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; } } - } + } /// - /// Base class to define ValueContainer extensions of , for example + /// Base class to define ValueContainer extensions of , for example /// - public static class ValueContainer + public class ValueContainer : ISerDesSelector { /// - /// Json extension of , for example based on array + /// Returns a new instance of + /// + /// The of + public static ISerDesSelector NewInstance() => new ValueContainer(); + /// + public static string SelectorTypeName => typeof(ValueContainer<>).ToAssemblyQualified(); + /// + public static Type ByteArraySerDes => typeof(JsonRaw); + /// + public static ISerDes NewSerDes() + { + if (typeof(TJVM) == typeof(Java.Nio.ByteBuffer)) return (ISerDes)NewByteBufferSerDes(); + return (ISerDes)NewByteArraySerDes(); + } + /// + public static Type ByteBufferSerDes => typeof(JsonBuffered); + /// + public static ISerDesRaw NewByteArraySerDes() { return new JsonRaw(SelectorTypeName); } + /// + public static ISerDesBuffered NewByteBufferSerDes() { return new JsonBuffered(SelectorTypeName); } + + /// + string ISerDesSelector.SelectorTypeName => SelectorTypeName; + /// + Type ISerDesSelector.ByteArraySerDes => ByteArraySerDes; + /// + Type ISerDesSelector.ByteBufferSerDes => ByteBufferSerDes; + /// + ISerDes ISerDesSelector.NewSerDes() => NewSerDes(); + /// + ISerDesRaw ISerDesSelector.NewByteArraySerDes() => NewByteArraySerDes(); + /// + ISerDesBuffered ISerDesSelector.NewByteBufferSerDes() => NewByteBufferSerDes(); + + /// + /// Json extension of , for example based on array /// - /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like - public class JsonRaw : SerDesRaw + /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like + sealed class JsonRaw : SerDesRaw { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(JsonRaw<>).ToAssemblyQualified()); + readonly byte[] valueContainerSerDesName; readonly byte[] valueContainerName = null!; readonly System.Text.Json.JsonSerializerOptions _options; /// @@ -198,13 +270,14 @@ public class JsonRaw : SerDesRaw /// /// Default initializer /// - public JsonRaw() + public JsonRaw(string selectorName) { - var tt = typeof(T); + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); if (tt.IsGenericType) { var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } var t = tt.GetGenericTypeDefinition(); if (t.GetInterface(typeof(IValueContainer<>).Name) != null) { @@ -215,45 +288,45 @@ public JsonRaw() }; return; } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - /// - public override byte[] Serialize(string topic, T data) + /// + public override byte[] Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override byte[] SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override byte[] SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); - var jsonStr = System.Text.Json.JsonSerializer.Serialize(data, _options); + var jsonStr = System.Text.Json.JsonSerializer.Serialize(data, _options); return Encoding.UTF8.GetBytes(jsonStr); } - /// - public override T Deserialize(string topic, byte[] data) + /// + public override TData Deserialize(string topic, byte[] data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, byte[] data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, byte[] data) { if (data == null) return default!; - return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; + return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; } } /// - /// Json extension of , for example based on + /// Json extension of , for example based on /// - /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like - public class JsonBuffered : SerDesBuffered + /// The type to be serialized or deserialized. It can be a Primary Key or a ValueContainer like + sealed class JsonBuffered : SerDesBuffered { - readonly byte[] valueContainerSerDesName = Encoding.UTF8.GetBytes(typeof(JsonBuffered<>).ToAssemblyQualified()); + readonly byte[] valueContainerSerDesName; readonly byte[] valueContainerName = null!; readonly System.Text.Json.JsonSerializerOptions _options; /// @@ -261,13 +334,14 @@ public class JsonBuffered : SerDesBuffered /// /// Default initializer /// - public JsonBuffered() + public JsonBuffered(string selectorName) { - var tt = typeof(T); + valueContainerSerDesName = Encoding.UTF8.GetBytes(selectorName); + var tt = typeof(TData); if (tt.IsGenericType) { var keyT = tt.GetGenericArguments(); - if (keyT.Length != 1) { throw new ArgumentException($"{typeof(T).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } + if (keyT.Length != 1) { throw new ArgumentException($"{typeof(TData).Name} does not contains a single generic argument and cannot be used because it is not a valid ValueContainer type"); } var t = tt.GetGenericTypeDefinition(); if (t.GetInterface(typeof(IValueContainer<>).Name) != null) { @@ -278,36 +352,36 @@ public JsonBuffered() }; return; } - else throw new ArgumentException($"{typeof(T).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); + else throw new ArgumentException($"{typeof(TData).Name} does not implement IValueContainer<> and cannot be used because it is not a valid ValueContainer type"); } - throw new ArgumentException($"{typeof(T).Name} is not a generic type and cannot be used as a valid ValueContainer type"); + throw new ArgumentException($"{typeof(TData).Name} is not a generic type and cannot be used as a valid ValueContainer type"); } - /// - public override ByteBuffer Serialize(string topic, T data) + /// + public override ByteBuffer Serialize(string topic, TData data) { return SerializeWithHeaders(topic, null!, data); } - /// - public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, T data) + /// + public override ByteBuffer SerializeWithHeaders(string topic, Headers headers, TData data) { headers?.Add(KNetSerialization.ValueSerializerIdentifier, valueContainerSerDesName); headers?.Add(KNetSerialization.ValueTypeIdentifier, valueContainerName); var ms = new MemoryStream(); - System.Text.Json.JsonSerializer.Serialize(ms, data, _options); + System.Text.Json.JsonSerializer.Serialize(ms, data, _options); return ByteBuffer.From(ms); } - /// - public override T Deserialize(string topic, ByteBuffer data) + /// + public override TData Deserialize(string topic, ByteBuffer data) { return DeserializeWithHeaders(topic, null!, data); } - /// - public override T DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) + /// + public override TData DeserializeWithHeaders(string topic, Headers headers, ByteBuffer data) { if (data == null) return default!; - return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; + return System.Text.Json.JsonSerializer.Deserialize(data, _options)!; } } } diff --git a/src/net/KEFCore.SerDes/DefaultValueContainer.cs b/src/net/KEFCore.SerDes/DefaultValueContainer.cs index 0e89d421..555b1311 100644 --- a/src/net/KEFCore.SerDes/DefaultValueContainer.cs +++ b/src/net/KEFCore.SerDes/DefaultValueContainer.cs @@ -20,6 +20,7 @@ #nullable enable +using MASES.KNet.Serialization; using System.Collections.Concurrent; using System.Text.Json; using System.Text.Json.Serialization; @@ -126,7 +127,7 @@ public PropertyData(IProperty property, object value) { if (!dict.TryGetValue(property.ClrType, out ManagedTypes _type)) _type = ManagedTypes.Undefined; ManagedType = _type; - ClrType = property.ClrType?.FullName; + ClrType = property.ClrType?.ToAssemblyQualified(); PropertyName = property.Name; Value = value; } @@ -141,7 +142,7 @@ public void OnDeserialized() { case JsonValueKind.String: Value = elem.GetString()!; - if (ClrType != typeof(string).FullName) + if (ClrType != typeof(string).ToAssemblyQualified()) { try { @@ -150,15 +151,15 @@ public void OnDeserialized() catch (InvalidCastException) { // failed conversion, try with other methods for known types - if (ClrType == typeof(Guid).FullName) + if (ClrType == typeof(Guid).ToAssemblyQualified()) { Value = elem.GetGuid(); } - else if (ClrType == typeof(DateTime).FullName) + else if (ClrType == typeof(DateTime).ToAssemblyQualified()) { Value = elem.GetDateTime(); } - else if (ClrType == typeof(DateTimeOffset).FullName) + else if (ClrType == typeof(DateTimeOffset).ToAssemblyQualified()) { Value = elem.GetDateTimeOffset(); } @@ -287,7 +288,7 @@ public class DefaultValueContainer : IValueContainer where TKey : no public DefaultValueContainer(IEntityType tName, object[] rData) { EntityName = tName.Name; - ClrType = tName.ClrType.FullName!; + ClrType = tName.ClrType?.ToAssemblyQualified()!; Data = new Dictionary(); foreach (var item in tName.GetProperties()) { diff --git a/src/net/KEFCore.SerDes/KEFCore.SerDes.csproj b/src/net/KEFCore.SerDes/KEFCore.SerDes.csproj index 2d3666a4..ac962da3 100644 --- a/src/net/KEFCore.SerDes/KEFCore.SerDes.csproj +++ b/src/net/KEFCore.SerDes/KEFCore.SerDes.csproj @@ -44,7 +44,7 @@ - + All None diff --git a/src/net/KEFCore.SerDes/KEFCoreSerDes.Helpers.cs b/src/net/KEFCore.SerDes/KEFCoreSerDes.Helpers.cs index 69a48d96..ac45251f 100644 --- a/src/net/KEFCore.SerDes/KEFCoreSerDes.Helpers.cs +++ b/src/net/KEFCore.SerDes/KEFCoreSerDes.Helpers.cs @@ -140,8 +140,8 @@ public static TEntity FromRecord(Org.Apache.Kafka.Clients.Consumer.Cons /// The extracted entity public static object FromRecord(Org.Apache.Kafka.Clients.Consumer.ConsumerRecord record, bool throwUnmatch = false) { - Type? keySerializerType = null; - Type? valueSerializerType = null; + Type? keySerializerSelectorType = null; + Type? valueSerializerSelectorType = null; Type? keyType = null; Type? valueType = null; @@ -159,7 +159,7 @@ public static object FromRecord(Org.Apache.Kafka.Clients.Consumer.ConsumerRecord if (key == KNetSerialization.KeySerializerIdentifier) { var strType = Encoding.UTF8.GetString(header.Value()); - keySerializerType = Type.GetType(strType, true)!; + keySerializerSelectorType = Type.GetType(strType, true)!; } if (key == KNetSerialization.ValueTypeIdentifier) { @@ -169,17 +169,17 @@ public static object FromRecord(Org.Apache.Kafka.Clients.Consumer.ConsumerRecord if (key == KNetSerialization.ValueSerializerIdentifier) { var strType = Encoding.UTF8.GetString(header.Value()); - valueSerializerType = Type.GetType(strType, true)!; + valueSerializerSelectorType = Type.GetType(strType, true)!; } } } - if (keyType == null || keySerializerType == null || valueType == null || valueSerializerType == null) + if (keyType == null || keySerializerSelectorType == null || valueType == null || valueSerializerSelectorType == null) { - throw new InvalidOperationException($"Missing one, or more, mandatory information in record: keyType: {keyType} - keySerializerType: {keySerializerType} - valueType: {valueType} - valueSerializerType: {valueSerializerType}"); + throw new InvalidOperationException($"Missing one, or more, mandatory information in record: keyType: {keyType} - keySerializerType: {keySerializerSelectorType} - valueType: {valueType} - valueSerializerType: {valueSerializerSelectorType}"); } - return FromRawValueData(keyType!, valueType!, keySerializerType!, valueSerializerType!, record.Topic(), record.Value(), record.Key(), throwUnmatch); + return FromRawValueData(keyType!, valueType!, keySerializerSelectorType!, valueSerializerSelectorType!, record.Topic(), record.Value(), record.Key(), throwUnmatch); } /// @@ -187,43 +187,24 @@ public static object FromRecord(Org.Apache.Kafka.Clients.Consumer.ConsumerRecord /// /// Expected key /// Expected ValueContainer - /// Key serializer to be used - /// ValueContainer serializer to be used + /// Key serializer to be used + /// ValueContainer serializer to be used /// The Apache Kafka topic the data is coming from /// The Apache Kafka record value containing the information /// The Apache Kafka record key containing the information /// Throws exceptions if there is unmatch in data retrieve, e.g. a property not available or a not settable /// The extracted entity - public static object FromRawValueData(Type keyType, Type valueContainer, Type keySerializer, Type valueContainerSerializer, string topic, byte[] recordValue, byte[] recordKey, bool throwUnmatch = false) + public static object FromRawValueData(Type keyType, Type valueContainer, Type keySerializerSelectorType, Type valueSerializerSelectorType, string topic, byte[] recordValue, byte[] recordKey, bool throwUnmatch = false) { - var fullKeySerializer = keySerializer.MakeGenericType(keyType); - Type jvmKeyType = null!; - foreach (var interfaceType in fullKeySerializer.GetInterfaces()) - { - if (interfaceType.IsGenericType && interfaceType.Name.StartsWith(typeof(ISerDes<,>).Name)) - { - jvmKeyType = interfaceType.GetGenericArguments()[1]; - } - } - if (jvmKeyType == null) throw new InvalidOperationException($"Cannot identity JVM type from {keySerializer}"); + var fullKeySerializer = keySerializerSelectorType.MakeGenericType(keyType); var fullValueContainer = valueContainer.MakeGenericType(keyType); - var fullValueContainerSerializer = valueContainerSerializer.MakeGenericType(fullValueContainer); - Type jvmKValueContainerType = null!; - foreach (var interfaceType in fullValueContainerSerializer.GetInterfaces()) - { - if (interfaceType.IsGenericType && interfaceType.Name.StartsWith(typeof(ISerDes<,>).Name)) - { - jvmKValueContainerType = interfaceType.GetGenericArguments()[1]; - } - } - - if (jvmKValueContainerType == null) throw new InvalidOperationException($"Cannot identity JVM type from {fullValueContainerSerializer}"); + var fullValueContainerSerializer = valueSerializerSelectorType.MakeGenericType(fullValueContainer); var ccType = typeof(LocalEntityExtractor<,,,,,>); - var extractorType = ccType.MakeGenericType(keyType, fullValueContainer, jvmKeyType, jvmKValueContainerType, fullKeySerializer, fullValueContainerSerializer); + var extractorType = ccType.MakeGenericType(keyType, fullValueContainer, typeof(byte[]), typeof(byte[]), fullKeySerializer, fullValueContainerSerializer); var methodInfo = extractorType.GetMethod("GetEntity"); var extractor = Activator.CreateInstance(extractorType); - return methodInfo?.Invoke(extractor, new object[] { topic, recordValue, recordKey, throwUnmatch })!; + return methodInfo?.Invoke(extractor, new object[] { topic, recordKey, recordValue, throwUnmatch })!; } } diff --git a/src/net/KEFCore.SerDes/LocalEntityExtractor.cs b/src/net/KEFCore.SerDes/LocalEntityExtractor.cs index 1da78e02..205b2c4c 100644 --- a/src/net/KEFCore.SerDes/LocalEntityExtractor.cs +++ b/src/net/KEFCore.SerDes/LocalEntityExtractor.cs @@ -19,6 +19,7 @@ #nullable enable using MASES.KNet.Serialization; +using System.Collections.Concurrent; namespace MASES.EntityFrameworkCore.KNet.Serialization; @@ -27,19 +28,23 @@ interface ILocalEntityExtractor object GetEntity(string topic, TJVMKey recordKey, TJVMValueContainer recordValue, bool throwUnmatch); } -class LocalEntityExtractor : ILocalEntityExtractor +class LocalEntityExtractor + : ILocalEntityExtractor where TKey : notnull where TValueContainer : class, IValueContainer - where TKeySerializer : class, new() - where TValueSerializer : class, new() + where TKeySerDesSelectorType : class, ISerDesSelector, new() + where TValueContainerSerDesSelectorType : class, ISerDesSelector, new() { + static ConcurrentDictionary<(Type, Type), ISerDes> _keySerdeses = new(); + static ConcurrentDictionary<(Type, Type), ISerDes> _valueSerdeses = new(); + private readonly ISerDes? _keySerdes; private readonly ISerDes? _valueSerdes; public LocalEntityExtractor() { - _keySerdes = new TKeySerializer() as ISerDes; - _valueSerdes = new TValueSerializer() as ISerDes; + _keySerdes = _keySerdeses.GetOrAdd((typeof(TKeySerDesSelectorType), typeof(TJVMKey)), (o) => { return new TKeySerDesSelectorType().NewSerDes(); }); + _valueSerdes = _valueSerdeses.GetOrAdd((typeof(TValueContainerSerDesSelectorType), typeof(TValueContainer)), (o) => { return new TValueContainerSerDesSelectorType().NewSerDes(); }); } public object GetEntity(string topic, TJVMKey recordKey, TJVMValueContainer recordValue, bool throwUnmatch) diff --git a/src/net/KEFCore/Extensions/KafkaDbContextOptionsExtensions.cs b/src/net/KEFCore/Extensions/KafkaDbContextOptionsExtensions.cs index 0a4a4953..2327e906 100644 --- a/src/net/KEFCore/Extensions/KafkaDbContextOptionsExtensions.cs +++ b/src/net/KEFCore/Extensions/KafkaDbContextOptionsExtensions.cs @@ -19,6 +19,7 @@ using MASES.EntityFrameworkCore.KNet.Diagnostics; using MASES.EntityFrameworkCore.KNet.Infrastructure; using MASES.EntityFrameworkCore.KNet.Infrastructure.Internal; +using MASES.KNet.Serialization; namespace MASES.EntityFrameworkCore.KNet; @@ -125,26 +126,17 @@ var coreOptionsExtension /// /// Creates a serializer for keys /// - public static Type SerializerTypeForKey(this IKafkaSingletonOptions options, IEntityType entityType) + public static Type KeyType(this IKafkaSingletonOptions options, IEntityType entityType) { var primaryKey = entityType.FindPrimaryKey()!.GetKeyType(); - return options.KeySerializationType?.MakeGenericType(primaryKey)!; - } - /// - /// Creates a serialzier for values - /// - public static Type SerializerTypeForValue(this IKafkaSingletonOptions options, IEntityType entityType) - { - var primaryKey = entityType.FindPrimaryKey()!.GetKeyType(); - return options.ValueSerializationType?.MakeGenericType(ValueContainerType(options, entityType))!; + return primaryKey; } /// /// Create the ValueContainer /// public static Type ValueContainerType(this IKafkaSingletonOptions options, IEntityType entityType) { - var primaryKey = entityType.FindPrimaryKey()!.GetKeyType(); - return options.ValueContainerType?.MakeGenericType(primaryKey)!; + return options.ValueContainerType?.MakeGenericType(KeyType(options, entityType))!; } /// /// Create the ValueContainer @@ -152,12 +144,38 @@ public static Type ValueContainerType(this IKafkaSingletonOptions options, IEnti public static Type JVMKeyType(this IKafkaSingletonOptions options, IEntityType entityType) { return typeof(byte[]); + + var keySerDesType = SerDesSelectorTypeForKey(options, entityType); + ISerDes serDes = Activator.CreateInstance(keySerDesType) as ISerDes; + if (serDes == null) throw new InvalidOperationException($"{keySerDesType} is not a valid {nameof(ISerDes)}"); + return serDes.JVMType; } /// /// Create the ValueContainer /// public static Type JVMValueContainerType(this IKafkaSingletonOptions options, IEntityType entityType) { + if (options.UseByteBufferDataTransfer) return typeof(Java.Nio.ByteBuffer); return typeof(byte[]); + + + var valueSerDesType = SerDesSelectorTypeForValue(options, entityType); + ISerDes serDes = Activator.CreateInstance(valueSerDesType) as ISerDes; + if (serDes == null) throw new InvalidOperationException($"{valueSerDesType} is not a valid {nameof(ISerDes)}"); + return serDes.JVMType; + } + /// + /// Creates a serializer for keys + /// + public static Type SerDesSelectorTypeForKey(this IKafkaSingletonOptions options, IEntityType entityType) + { + return options.KeySerDesSelectorType?.MakeGenericType(KeyType(options, entityType))!; + } + /// + /// Creates a serialzier for values + /// + public static Type SerDesSelectorTypeForValue(this IKafkaSingletonOptions options, IEntityType entityType) + { + return options.ValueSerDesSelectorType?.MakeGenericType(ValueContainerType(options, entityType))!; } } diff --git a/src/net/KEFCore/Infrastructure/Internal/IKafkaSingletonOptions.cs b/src/net/KEFCore/Infrastructure/Internal/IKafkaSingletonOptions.cs index 15dd40cf..3ac4983a 100644 --- a/src/net/KEFCore/Infrastructure/Internal/IKafkaSingletonOptions.cs +++ b/src/net/KEFCore/Infrastructure/Internal/IKafkaSingletonOptions.cs @@ -33,10 +33,10 @@ namespace MASES.EntityFrameworkCore.KNet.Infrastructure.Internal; /// public interface IKafkaSingletonOptions : ISingletonOptions { - /// - Type? KeySerializationType { get; } - /// - Type? ValueSerializationType { get; } + /// + Type? KeySerDesSelectorType { get; } + /// + Type? ValueSerDesSelectorType { get; } /// Type? ValueContainerType { get; } /// @@ -57,6 +57,8 @@ public interface IKafkaSingletonOptions : ISingletonOptions bool UsePersistentStorage { get; } /// bool UseEnumeratorWithPrefetch { get; } + /// + bool UseByteBufferDataTransfer { get; } /// int DefaultNumPartitions { get; } /// diff --git a/src/net/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs b/src/net/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs index 41b9b3c5..24ccb346 100644 --- a/src/net/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs +++ b/src/net/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs @@ -25,6 +25,7 @@ using MASES.KNet.Common; using MASES.KNet.Consumer; using MASES.KNet.Producer; +using MASES.KNet.Serialization; using MASES.KNet.Streams; using Org.Apache.Kafka.Streams.State; using System.Globalization; @@ -36,10 +37,10 @@ namespace MASES.EntityFrameworkCore.KNet.Infrastructure.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class KafkaOptionsExtension : IDbContextOptionsExtension +public class KafkaOptionsExtension : IDbContextOptionsExtension, IKafkaSingletonOptions { - private Type _keySerializationType = DefaultKEFCoreSerDes.DefaultKeySerialization; - private Type _valueSerializationType = DefaultKEFCoreSerDes.DefaultValueContainerSerialization; + private Type _keySerDesSelectorType = DefaultKEFCoreSerDes.DefaultKeySerialization; + private Type _valueSerDesSelectorType = DefaultKEFCoreSerDes.DefaultValueContainerSerialization; private Type _valueContainerType = DefaultKEFCoreSerDes.DefaultValueContainer; private bool _useNameMatching = true; private string? _databaseName; @@ -50,6 +51,7 @@ public class KafkaOptionsExtension : IDbContextOptionsExtension private bool _useKNetStreams = true; private bool _usePersistentStorage = false; private bool _useEnumeratorWithPrefetch = true; + private bool _useByteBufferDataTransfer = false; private int _defaultNumPartitions = 1; private int? _defaultConsumerInstances = null; private short _defaultReplicationFactor = 1; @@ -73,8 +75,8 @@ public KafkaOptionsExtension() /// protected KafkaOptionsExtension(KafkaOptionsExtension copyFrom) { - _keySerializationType = copyFrom._keySerializationType; - _valueSerializationType = copyFrom._valueSerializationType; + _keySerDesSelectorType = copyFrom._keySerDesSelectorType; + _valueSerDesSelectorType = copyFrom._valueSerDesSelectorType; _valueContainerType = copyFrom._valueContainerType; _useNameMatching = copyFrom._useNameMatching; _databaseName = copyFrom._databaseName; @@ -85,6 +87,7 @@ protected KafkaOptionsExtension(KafkaOptionsExtension copyFrom) _useKNetStreams = copyFrom._useKNetStreams; _usePersistentStorage = copyFrom._usePersistentStorage; _useEnumeratorWithPrefetch = copyFrom._useEnumeratorWithPrefetch; + _useByteBufferDataTransfer = copyFrom._useByteBufferDataTransfer; _defaultNumPartitions = copyFrom._defaultNumPartitions; _defaultConsumerInstances = copyFrom._defaultConsumerInstances; _defaultReplicationFactor = copyFrom._defaultReplicationFactor; @@ -102,10 +105,10 @@ protected KafkaOptionsExtension(KafkaOptionsExtension copyFrom) /// Internal property /// public virtual string ClusterId => _bootstrapServers!; - /// - public virtual Type KeySerializationType => _keySerializationType; - /// - public virtual Type ValueSerializationType => _valueSerializationType; + /// + public virtual Type KeySerDesSelectorType => _keySerDesSelectorType; + /// + public virtual Type ValueSerDesSelectorType => _valueSerDesSelectorType; /// public virtual Type ValueContainerType => _valueContainerType; /// @@ -124,6 +127,8 @@ protected KafkaOptionsExtension(KafkaOptionsExtension copyFrom) public virtual bool UseKNetStreams => _useKNetStreams; /// public virtual bool UsePersistentStorage => _usePersistentStorage; + /// + public virtual bool UseByteBufferDataTransfer => _useByteBufferDataTransfer; /// public virtual bool UseEnumeratorWithPrefetch => _useEnumeratorWithPrefetch; /// @@ -142,25 +147,28 @@ protected KafkaOptionsExtension(KafkaOptionsExtension copyFrom) public virtual TopicConfigBuilder TopicConfig => _topicConfigBuilder!; /// public virtual Action OnChangeEvent => _onChangeEvent!; - /// - public virtual KafkaOptionsExtension WithKeySerializationType(Type serializationType) + + int IKafkaSingletonOptions.DefaultReplicationFactor => throw new NotImplementedException(); + + /// + public virtual KafkaOptionsExtension WithKeySerDesSelectorType(Type serializationType) { if (!serializationType.IsGenericTypeDefinition) throw new InvalidOperationException($"{serializationType.Name} shall be a generic type and shall be defined using \"<>\""); var clone = Clone(); - clone._keySerializationType = serializationType; + clone._keySerDesSelectorType = serializationType; return clone; } - /// - public virtual KafkaOptionsExtension WithValueSerializationType(Type serializationType) + /// + public virtual KafkaOptionsExtension WithValueSerDesSelectorType(Type serializationType) { if (!serializationType.IsGenericTypeDefinition) throw new InvalidOperationException($"{serializationType.Name} shall be a generic type and shall be defined using \"<>\""); var clone = Clone(); - clone._valueSerializationType = serializationType; + clone._valueSerDesSelectorType = serializationType; return clone; } @@ -256,6 +264,15 @@ public virtual KafkaOptionsExtension WithUseEnumeratorWithPrefetch(bool useEnume return clone; } + /// + public virtual KafkaOptionsExtension WithUseByteBufferDataTransfer(bool useByteBufferDataTransfer = false) + { + var clone = Clone(); + + clone._useByteBufferDataTransfer = useByteBufferDataTransfer; + + return clone; + } /// public virtual KafkaOptionsExtension WithDefaultNumPartitions(int defaultNumPartitions = 1) { @@ -328,31 +345,28 @@ public virtual KafkaOptionsExtension WithOnChangeEvent(Action return clone; } - /// - /// Build for - /// - public virtual Properties StreamsOptions(IEntityType entityType) - { - return StreamsOptions(entityType.ApplicationIdForTable(this)); - } /// /// Build from options /// - public virtual StreamsConfigBuilder StreamsOptions() + public virtual StreamsConfigBuilder StreamsOptions(IEntityType entityType) { _streamsConfigBuilder ??= new(); StreamsConfigBuilder builder = StreamsConfigBuilder.CreateFrom(_streamsConfigBuilder); Properties props = builder; - builder.KNetKeySerDes = KeySerializationType; - builder.KNetValueSerDes = ValueSerializationType; + builder.KeySerDesSelector = KeySerDesSelectorType; + builder.ValueSerDesSelector = ValueSerDesSelectorType; builder.ApplicationId = ApplicationId; builder.BootstrapServers = BootstrapServers; string baSerdesName = Class.ClassNameOf(); - builder.DefaultKeySerdeClass = Class.ForName(baSerdesName, true, SystemClassLoader); - builder.DefaultValueSerdeClass = Class.ForName(baSerdesName, true, SystemClassLoader); + string bbSerdesName = Class.ClassNameOf(); + + builder.DefaultKeySerdeClass = this.JVMKeyType(entityType) == typeof(byte[]) ? Class.ForName(baSerdesName, true, SystemClassLoader) + : Class.ForName(bbSerdesName, true, SystemClassLoader); + builder.DefaultValueSerdeClass = this.JVMValueContainerType(entityType) == typeof(byte[]) ? Class.ForName(baSerdesName, true, SystemClassLoader) + : Class.ForName(bbSerdesName, true, SystemClassLoader); builder.DSLStoreSuppliersClass = UsePersistentStorage ? Class.ForName(Class.ClassNameOf(), true, SystemClassLoader) : Class.ForName(Class.ClassNameOf(), true, SystemClassLoader); @@ -481,6 +495,11 @@ public virtual void Validate(IDbContextOptions options) if (string.IsNullOrEmpty(kafkaOptions.BootstrapServers)) throw new ArgumentException("It is manadatory", "BootstrapServers"); } + public void Initialize(IDbContextOptions options) + { + throw new NotImplementedException(); + } + private sealed class ExtensionInfo : DbContextOptionsExtensionInfo { private string? _logFragment; diff --git a/src/net/KEFCore/Infrastructure/Internal/KafkaSingletonOptions.cs b/src/net/KEFCore/Infrastructure/Internal/KafkaSingletonOptions.cs index 8e354f9c..90b8df20 100644 --- a/src/net/KEFCore/Infrastructure/Internal/KafkaSingletonOptions.cs +++ b/src/net/KEFCore/Infrastructure/Internal/KafkaSingletonOptions.cs @@ -38,8 +38,8 @@ public virtual void Initialize(IDbContextOptions options) if (kafkaOptions != null) { - KeySerializationType = kafkaOptions.KeySerializationType; - ValueSerializationType = kafkaOptions.ValueSerializationType; + KeySerDesSelectorType = kafkaOptions.KeySerDesSelectorType; + ValueSerDesSelectorType = kafkaOptions.ValueSerDesSelectorType; ValueContainerType = kafkaOptions.ValueContainerType; UseNameMatching = kafkaOptions.UseNameMatching; DatabaseName = kafkaOptions.DatabaseName; @@ -50,6 +50,7 @@ public virtual void Initialize(IDbContextOptions options) UseKNetStreams = kafkaOptions.UseKNetStreams; UsePersistentStorage = kafkaOptions.UsePersistentStorage; UseEnumeratorWithPrefetch = kafkaOptions.UseEnumeratorWithPrefetch; + UseByteBufferDataTransfer = kafkaOptions.UseByteBufferDataTransfer; DefaultNumPartitions = kafkaOptions.DefaultNumPartitions; DefaultConsumerInstances = kafkaOptions.DefaultConsumerInstances; DefaultReplicationFactor = kafkaOptions.DefaultReplicationFactor; @@ -75,9 +76,9 @@ public virtual void Validate(IDbContextOptions options) } } /// - public virtual Type? KeySerializationType { get; private set; } + public virtual Type? KeySerDesSelectorType { get; private set; } /// - public virtual Type? ValueSerializationType { get; private set; } + public virtual Type? ValueSerDesSelectorType { get; private set; } /// public virtual Type? ValueContainerType { get; private set; } /// @@ -99,6 +100,8 @@ public virtual void Validate(IDbContextOptions options) /// public virtual bool UseEnumeratorWithPrefetch { get; private set; } /// + public virtual bool UseByteBufferDataTransfer { get; private set; } + /// public virtual int DefaultNumPartitions { get; private set; } /// public virtual int? DefaultConsumerInstances { get; private set; } diff --git a/src/net/KEFCore/Infrastructure/KafkaDbContext.cs b/src/net/KEFCore/Infrastructure/KafkaDbContext.cs index 7d3c9817..9956caf3 100644 --- a/src/net/KEFCore/Infrastructure/KafkaDbContext.cs +++ b/src/net/KEFCore/Infrastructure/KafkaDbContext.cs @@ -154,19 +154,19 @@ public KafkaDbContext(DbContextOptions options) : base(options) } /// - /// The optional to use for key serialization + /// The optional to use for key serialization selection /// - /// Default value is , any custom shall implement + /// Default value is , any custom shall implement /// /// - public virtual Type? KeySerializationType { get; set; } = null; + public virtual Type? KeySerDesSelectorType { get; set; } = null; /// - /// The optional to use for value serialization + /// The optional to use for value serialization selection /// - /// Default value is , any custom shall implement + /// Default value is , any custom shall implement /// /// - public virtual Type? ValueSerializationType { get; set; } = null; + public virtual Type? ValueSerDesSelectorType { get; set; } = null; /// /// The optional to use as value container /// @@ -207,6 +207,15 @@ public KafkaDbContext(DbContextOptions options) : base(options) /// public virtual bool UsePersistentStorage { get; set; } = false; /// + /// Setting this property to the engine prefers to use enumerator instances able to do a prefetch on data speeding up execution + /// + /// Used only if is and is , not available in EFCore 6. + public virtual bool UseEnumeratorWithPrefetch { get; set; } = false; + /// + /// Setting this property to the engine prefers to use data exchange in serializer instances + /// + public virtual bool UseByteBufferDataTransfer { get; set; } = false; + /// /// Use delete cleanup policy when a topic is created /// public bool UseDeletePolicyForTopic { get; set; } = false; @@ -218,11 +227,7 @@ public KafkaDbContext(DbContextOptions options) : base(options) /// Use KNet version of Apache Kafka Streams instead of standard Apache Kafka Streams /// public virtual bool UseKNetStreams { get; set; } = true; - /// - /// Setting this property to the engine prefers to use enumerator instances able to do a prefetch on data speeding up execution - /// - /// Used only if is and is , not available in EFCore 6. - public virtual bool UseEnumeratorWithPrefetch { get; set; } = false; + /// /// The optional used when is /// @@ -264,8 +269,8 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) o.WithCompactedReplicator(UseCompactedReplicator); o.WithUseKNetStreams(UseKNetStreams); o.WithDefaultReplicationFactor(DefaultReplicationFactor); - if (KeySerializationType != null) o.WithKeySerializationType(KeySerializationType); - if (ValueSerializationType != null) o.WithValueSerializationType(ValueSerializationType); + if (KeySerDesSelectorType != null) o.WithKeySerDesSelectorType(KeySerDesSelectorType); + if (ValueSerDesSelectorType != null) o.WithValueSerDesSelectorType(ValueSerDesSelectorType); if (ValueContainerType != null) o.WithValueContainerType(ValueContainerType); if (OnChangeEvent != null) o.WithOnChangeEvent(OnChangeEvent); }); diff --git a/src/net/KEFCore/Infrastructure/KafkaDbContextOptionsBuilder.cs b/src/net/KEFCore/Infrastructure/KafkaDbContextOptionsBuilder.cs index 8bf71919..32de9fc5 100644 --- a/src/net/KEFCore/Infrastructure/KafkaDbContextOptionsBuilder.cs +++ b/src/net/KEFCore/Infrastructure/KafkaDbContextOptionsBuilder.cs @@ -75,14 +75,14 @@ public KafkaDbContextOptionsBuilder(DbContextOptionsBuilder optionsBuilder) /// /// The implementing the serialization model. /// The same builder instance so that multiple calls can be chained. - public virtual KafkaDbContextOptionsBuilder WithKeySerializationType(Type serializationType) + public virtual KafkaDbContextOptionsBuilder WithKeySerDesSelectorType(Type serializationType) { if (!serializationType.IsGenericTypeDefinition) throw new InvalidOperationException($"{serializationType.Name} shall be a generic type and shall be defined using \"<>\""); var extension = OptionsBuilder.Options.FindExtension() ?? new KafkaOptionsExtension(); - extension = extension.WithKeySerializationType(serializationType); + extension = extension.WithKeySerDesSelectorType(serializationType); ((IDbContextOptionsBuilderInfrastructure)OptionsBuilder).AddOrUpdateExtension(extension); @@ -98,14 +98,14 @@ public virtual KafkaDbContextOptionsBuilder WithKeySerializationType(Type serial /// /// The implementing the serialization model. /// The same builder instance so that multiple calls can be chained. - public virtual KafkaDbContextOptionsBuilder WithValueSerializationType(Type serializationType) + public virtual KafkaDbContextOptionsBuilder WithValueSerDesSelectorType(Type serializationType) { if (!serializationType.IsGenericTypeDefinition) throw new InvalidOperationException($"{serializationType.Name} shall be a generic type and shall be defined using \"<>\""); var extension = OptionsBuilder.Options.FindExtension() ?? new KafkaOptionsExtension(); - extension = extension.WithValueSerializationType(serializationType); + extension = extension.WithValueSerDesSelectorType(serializationType); ((IDbContextOptionsBuilderInfrastructure)OptionsBuilder).AddOrUpdateExtension(extension); @@ -249,7 +249,7 @@ public virtual KafkaDbContextOptionsBuilder WithUsePersistentStorage(bool usePer /// See Using DbContextOptions, and /// The EF Core Kafka database provider for more information and examples. /// - /// If , persistent storage will be used. + /// If , prefetch in enumeration will be used. /// The same builder instance so that multiple calls can be chained. public virtual KafkaDbContextOptionsBuilder WithUseEnumeratorWithPrefetch(bool useEnumeratorWithPrefetch = true) { @@ -263,6 +263,27 @@ public virtual KafkaDbContextOptionsBuilder WithUseEnumeratorWithPrefetch(bool u return this; } + /// + /// Setting this property to the engine prefers to use data exchange in serializer instances + /// + /// + /// See Using DbContextOptions, and + /// The EF Core Kafka database provider for more information and examples. + /// + /// If , data exchange will be preferred. + /// The same builder instance so that multiple calls can be chained. + public virtual KafkaDbContextOptionsBuilder WithUseByteBufferDataTransfer(bool useByteBufferDataTransfer = true) + { + var extension = OptionsBuilder.Options.FindExtension() + ?? new KafkaOptionsExtension(); + + extension = extension.WithUseByteBufferDataTransfer(useByteBufferDataTransfer); + + ((IDbContextOptionsBuilderInfrastructure)OptionsBuilder).AddOrUpdateExtension(extension); + + return this; + } + /// /// Defines the default number of partitions to use when a new topic is created /// diff --git a/src/net/KEFCore/Storage/Internal/EntityTypeProducer.cs b/src/net/KEFCore/Storage/Internal/EntityTypeProducer.cs index 5d76c6a8..5edb98a3 100644 --- a/src/net/KEFCore/Storage/Internal/EntityTypeProducer.cs +++ b/src/net/KEFCore/Storage/Internal/EntityTypeProducer.cs @@ -35,11 +35,11 @@ namespace MASES.EntityFrameworkCore.KNet.Storage.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class EntityTypeProducer : IEntityTypeProducer +public class EntityTypeProducer : IEntityTypeProducer where TKey : notnull where TValueContainer : class, IValueContainer - where TKeySerializer : class, new() - where TValueSerializer : class, new() + where TKeySerDesSelectorType : class, ISerDesSelector, new() + where TValueContainerSerDesSelectorType : class, ISerDesSelector, new() { private readonly ConstructorInfo TValueContainerConstructor; private readonly bool _useCompactedReplicator; @@ -191,11 +191,11 @@ public EntityTypeProducer(IEntityType entityType, IKafkaCluster cluster) var tTValueContainer = typeof(TValueContainer); TValueContainerConstructor = tTValueContainer.GetConstructors().Single(ci => ci.GetParameters().Length == 2); - _keySerdes = new TKeySerializer() as ISerDes; - _valueSerdes = new TValueSerializer() as ISerDes; + _keySerdes = new TKeySerDesSelectorType().NewSerDes(); + _valueSerdes = new TValueContainerSerDesSelectorType().NewSerDes(); - if (_keySerdes == null) throw new InvalidOperationException($"{typeof(TKeySerializer)} is not a {typeof(ISerDes)}"); - if (_valueSerdes == null) throw new InvalidOperationException($"{typeof(TValueSerializer)} is not a {typeof(ISerDes)}"); + if (_keySerdes == null) throw new InvalidOperationException($"{typeof(TKeySerDesSelectorType)} does not returned a {typeof(ISerDes)}"); + if (_valueSerdes == null) throw new InvalidOperationException($"{typeof(TValueContainerSerDesSelectorType)} does not returned a {typeof(ISerDes)}"); if (_useCompactedReplicator) { diff --git a/src/net/KEFCore/Storage/Internal/EntityTypeProducers.cs b/src/net/KEFCore/Storage/Internal/EntityTypeProducers.cs index d4e66da7..940e9ba7 100644 --- a/src/net/KEFCore/Storage/Internal/EntityTypeProducers.cs +++ b/src/net/KEFCore/Storage/Internal/EntityTypeProducers.cs @@ -19,6 +19,7 @@ #nullable enable using MASES.EntityFrameworkCore.KNet.Serialization; +using MASES.KNet.Serialization; using System.Collections.Concurrent; namespace MASES.EntityFrameworkCore.KNet.Storage.Internal; @@ -34,13 +35,13 @@ public class EntityTypeProducers /// /// Allocates a new /// - public static IEntityTypeProducer Create(IEntityType entityType, IKafkaCluster cluster) + public static IEntityTypeProducer Create(IEntityType entityType, IKafkaCluster cluster) where TKey : notnull where TValueContainer : class, IValueContainer - where TKeySerializer : class, new() - where TValueSerializer : class, new() + where TKeySerDesSelectorType : class, ISerDesSelector, new() + where TValueContainerSerDesSelectorType : class, ISerDesSelector, new() { - return _producers.GetOrAdd(entityType, _ => CreateProducerLocal(entityType, cluster)); + return _producers.GetOrAdd(entityType, _ => CreateProducerLocal(entityType, cluster)); } /// /// Dispose a previously allocated @@ -54,10 +55,10 @@ public static void Dispose(IEntityTypeProducer producer) producer.Dispose(); } - static IEntityTypeProducer CreateProducerLocal(IEntityType entityType, IKafkaCluster cluster) + static IEntityTypeProducer CreateProducerLocal(IEntityType entityType, IKafkaCluster cluster) where TKey : notnull where TValueContainer : class, IValueContainer - where TKeySerializer : class, new() - where TValueSerializer : class, new() - => new EntityTypeProducer(entityType, cluster); + where TKeySerDesSelectorType : class, ISerDesSelector, new() + where TValueContainerSerDesSelectorType : class, ISerDesSelector, new() + => new EntityTypeProducer(entityType, cluster); } diff --git a/src/net/KEFCore/Storage/Internal/KNetStreamsRetriever.cs b/src/net/KEFCore/Storage/Internal/KNetStreamsRetriever.cs index 1d4a1c5b..dcb94a2a 100644 --- a/src/net/KEFCore/Storage/Internal/KNetStreamsRetriever.cs +++ b/src/net/KEFCore/Storage/Internal/KNetStreamsRetriever.cs @@ -98,7 +98,7 @@ public KNetStreamsRetriever(IKafkaCluster kafkaCluster, IEntityType entityType) { _kafkaCluster = kafkaCluster; _entityType = entityType; - _streamsConfig ??= _kafkaCluster.Options.StreamsOptions(); + _streamsConfig ??= _kafkaCluster.Options.StreamsOptions(_entityType); _builder ??= new StreamsBuilder(_streamsConfig); _topicName = _entityType.TopicName(kafkaCluster.Options); _usePersistentStorage = _kafkaCluster.Options.UsePersistentStorage; diff --git a/src/net/KEFCore/Storage/Internal/KafkaTable.cs b/src/net/KEFCore/Storage/Internal/KafkaTable.cs index 698a173d..71f6cd20 100644 --- a/src/net/KEFCore/Storage/Internal/KafkaTable.cs +++ b/src/net/KEFCore/Storage/Internal/KafkaTable.cs @@ -25,6 +25,7 @@ using Java.Util.Concurrent; using Org.Apache.Kafka.Clients.Producer; using MASES.EntityFrameworkCore.KNet.Serialization; +using MASES.KNet.Serialization; namespace MASES.EntityFrameworkCore.KNet.Storage.Internal; /// @@ -33,11 +34,11 @@ namespace MASES.EntityFrameworkCore.KNet.Storage.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class KafkaTable : IKafkaTable +public class KafkaTable : IKafkaTable where TKey : notnull where TValueContainer : class, IValueContainer - where TKeySerializer : class, new() - where TValueSerializer : class, new() + where TKeySerDesSelectorType : class, ISerDesSelector, new() + where TValueContainerSerDesSelectorType : class, ISerDesSelector, new() { private readonly IPrincipalKeyValueFactory _keyValueFactory; private readonly bool _sensitiveLoggingEnabled; @@ -62,7 +63,7 @@ public KafkaTable( Cluster = cluster; EntityType = entityType; _tableAssociatedTopicName = Cluster.CreateTable(entityType); - _producer = EntityTypeProducers.Create(entityType, Cluster); + _producer = EntityTypeProducers.Create(entityType, Cluster); _keyValueFactory = entityType.FindPrimaryKey()!.GetPrincipalKeyValueFactory(); _sensitiveLoggingEnabled = sensitiveLoggingEnabled; _rows = new Dictionary(_keyValueFactory.EqualityComparer); diff --git a/src/net/KEFCore/Storage/Internal/KafkaTableFactory.cs b/src/net/KEFCore/Storage/Internal/KafkaTableFactory.cs index 9b9eb347..492d6232 100644 --- a/src/net/KEFCore/Storage/Internal/KafkaTableFactory.cs +++ b/src/net/KEFCore/Storage/Internal/KafkaTableFactory.cs @@ -19,6 +19,7 @@ using System.Collections.Concurrent; using MASES.EntityFrameworkCore.KNet.Infrastructure.Internal; using MASES.EntityFrameworkCore.KNet.Serialization; +using MASES.KNet.Serialization; namespace MASES.EntityFrameworkCore.KNet.Storage.Internal; /// @@ -59,17 +60,17 @@ private Func CreateTable(IKafkaCluster cluster, IEntityType entityT _options.ValueContainerType(entityType), _options.JVMKeyType(entityType), _options.JVMValueContainerType(entityType), - _options.SerializerTypeForKey(entityType), - _options.SerializerTypeForValue(entityType)) + _options.SerDesSelectorTypeForKey(entityType), + _options.SerDesSelectorTypeForValue(entityType)) .Invoke(null, new object?[] { cluster, entityType, _sensitiveLoggingEnabled })!; - private static Func CreateFactory( + private static Func CreateFactory( IKafkaCluster cluster, IEntityType entityType, bool sensitiveLoggingEnabled) where TKey : notnull where TValueContainer : class, IValueContainer - where TKeySerializer : class, new() - where TValueSerializer : class, new() - => () => new KafkaTable(cluster, entityType, sensitiveLoggingEnabled); + where TKeySerDesSelectorType : class, ISerDesSelector, new() + where TValueContainerSerDesSelectorType : class, ISerDesSelector, new() + => () => new KafkaTable(cluster, entityType, sensitiveLoggingEnabled); } diff --git a/test/Common/KEFCore.Common.Test.csproj b/test/Common/KEFCore.Common.Test.csproj new file mode 100644 index 00000000..fbf538c1 --- /dev/null +++ b/test/Common/KEFCore.Common.Test.csproj @@ -0,0 +1,11 @@ + + + + MASES.EntityFrameworkCore.KNet.Test.Common + MASES.EntityFrameworkCore.KNet.Test.Common + EntityFrameworkCore KNet Test + EntityFrameworkCore KNet Test.Common + MASES.EntityFrameworkCore.KNet.Test.Common + ..\..\bin\ + + diff --git a/test/Common/ProgramConfig.cs b/test/Common/ProgramConfig.cs index 43383fcc..8901e0c0 100644 --- a/test/Common/ProgramConfig.cs +++ b/test/Common/ProgramConfig.cs @@ -22,19 +22,32 @@ * SOFTWARE. */ +using MASES.EntityFrameworkCore.KNet.Infrastructure; +using MASES.EntityFrameworkCore.KNet.Serialization.Avro; +using MASES.EntityFrameworkCore.KNet.Serialization.Avro.Storage; +using MASES.EntityFrameworkCore.KNet.Serialization.Json; +using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf; +using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf.Storage; +using MASES.KNet.Streams; +using System.Diagnostics; +using System; +using System.IO; +using System.Text.Json; + namespace MASES.EntityFrameworkCore.KNet.Test.Common { public class ProgramConfig { public bool UseProtobuf { get; set; } = false; public bool UseAvro { get; set; } = false; - public bool UseAvroBinary { get; set; } = true; + public bool UseAvroBinary { get; set; } = false; public bool EnableKEFCoreTracing { get; set; } = false; public bool UseInMemoryProvider { get; set; } = false; public bool UseModelBuilder { get; set; } = false; public bool UseCompactedReplicator { get; set; } = true; public bool UseKNetStreams { get; set; } = true; public bool UseEnumeratorWithPrefetch { get; set; } = true; + public bool UseByteBufferDataTransfer { get; set; } = true; public bool PreserveInformationAcrossContexts { get; set; } = true; public bool UsePersistentStorage { get; set; } = false; public string DatabaseName { get; set; } = "TestDB"; @@ -48,5 +61,69 @@ public class ProgramConfig public int NumberOfExecutions { get; set; } = 1; public int NumberOfExtraElements { get; set; } = 100; public bool WithEvents { get; set; } = false; + + public void ApplyOnContext(KafkaDbContext context) + { + var databaseName = UseModelBuilder ? DatabaseNameWithModel : DatabaseName; + + StreamsConfigBuilder streamConfig = null; + if (!UseInMemoryProvider) + { + streamConfig = StreamsConfigBuilder.Create(); + streamConfig = streamConfig.WithAcceptableRecoveryLag(100); + } + + context.DatabaseName = databaseName; + context.StreamsConfig = streamConfig; + context.BootstrapServers = BootstrapServers; + context.ApplicationId = ApplicationId; + context.UsePersistentStorage = UsePersistentStorage; + context.UseCompactedReplicator = UseCompactedReplicator; + context.UseKNetStreams = UseKNetStreams; + context.UseEnumeratorWithPrefetch = UseEnumeratorWithPrefetch; + context.UseByteBufferDataTransfer = UseByteBufferDataTransfer; + + if (UseProtobuf) + { + context.KeySerDesSelectorType = typeof(ProtobufKEFCoreSerDes.Key<>); + context.ValueContainerType = typeof(ProtobufValueContainer<>); + context.ValueSerDesSelectorType = typeof(ProtobufKEFCoreSerDes.ValueContainer<>); + } + else if (UseAvro) + { + context.KeySerDesSelectorType = UseAvroBinary ? typeof(AvroKEFCoreSerDes.Key.Binary<>) + : typeof(AvroKEFCoreSerDes.Key.Json<>); + context.ValueContainerType = typeof(AvroValueContainer<>); + context.ValueSerDesSelectorType = UseAvroBinary ? typeof(AvroKEFCoreSerDes.ValueContainer.Binary<>) + : typeof(AvroKEFCoreSerDes.ValueContainer.Json<>); + } + } + + public static ProgramConfig Config { get; private set; } + + public static void LoadConfig(string[] args) + { + if (args.Length > 0) + { + if (!File.Exists(args[0])) { throw new FileNotFoundException($"{args[0]} is not a configuration file.", args[0]); } + Config = JsonSerializer.Deserialize(File.ReadAllText(args[0])); + } + else Config = new(); + ReportString(JsonSerializer.Serialize(Config, new JsonSerializerOptions() { WriteIndented = true })); + + if (!KafkaDbContext.EnableKEFCoreTracing) KafkaDbContext.EnableKEFCoreTracing = Config.EnableKEFCoreTracing; + } + + public static void ReportString(string message) + { + if (Debugger.IsAttached) + { + Trace.WriteLine($"{DateTime.Now:HH::mm::ss:ffff} - {message}"); + } + else + { + Console.WriteLine($"{DateTime.Now:HH::mm::ss:ffff} - {message}"); + } + } } } diff --git a/test/KEFCore.Benchmark.Test/KEFCore.Benchmark.Test.csproj b/test/KEFCore.Benchmark.Test/KEFCore.Benchmark.Test.csproj index 3a966ee2..086884d5 100644 --- a/test/KEFCore.Benchmark.Test/KEFCore.Benchmark.Test.csproj +++ b/test/KEFCore.Benchmark.Test/KEFCore.Benchmark.Test.csproj @@ -10,8 +10,7 @@ ..\..\bin\ - - + diff --git a/test/KEFCore.Benchmark.Test/Program.cs b/test/KEFCore.Benchmark.Test/Program.cs index f9b77f47..9d9e6e01 100644 --- a/test/KEFCore.Benchmark.Test/Program.cs +++ b/test/KEFCore.Benchmark.Test/Program.cs @@ -23,27 +23,18 @@ */ using MASES.EntityFrameworkCore.KNet.Infrastructure; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro.Storage; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf.Storage; using MASES.EntityFrameworkCore.KNet.Test.Common; using MASES.EntityFrameworkCore.KNet.Test.Model; -using MASES.KNet.Streams; using Microsoft.EntityFrameworkCore; using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Text.Json; namespace MASES.EntityFrameworkCore.KNet.Test.Benchmark { partial class Program { - internal static ProgramConfig config = new(); - struct ExecutionData { public ExecutionData(int executionIndex, int maxTests) @@ -55,18 +46,6 @@ public ExecutionData(int executionIndex, int maxTests) public TimeSpan[] QueryTimes; } - static void ReportString(string message) - { - if (Debugger.IsAttached) - { - Trace.WriteLine($"{DateTime.Now:HH::mm::ss:ffff} - {message}"); - } - else - { - Console.WriteLine($"{DateTime.Now:HH::mm::ss:ffff} - {message}"); - } - } - static void Main(string[] args) { const int maxTests = 10; @@ -75,72 +54,40 @@ static void Main(string[] args) var testWatcher = new Stopwatch(); var globalWatcher = new Stopwatch(); - if (args.Length > 0) - { - if (!File.Exists(args[0])) { ReportString($"{args[0]} is not a configuration file."); return; } - config = JsonSerializer.Deserialize(File.ReadAllText(args[0])); - } + ProgramConfig.LoadConfig(args); - if (!KafkaDbContext.EnableKEFCoreTracing) KafkaDbContext.EnableKEFCoreTracing = config.EnableKEFCoreTracing; - - if (!config.UseInMemoryProvider) + if (!ProgramConfig.Config.UseInMemoryProvider) { KEFCore.CreateGlobalInstance(); - KEFCore.PreserveInformationAcrossContexts = config.PreserveInformationAcrossContexts; + KEFCore.PreserveInformationAcrossContexts = ProgramConfig.Config.PreserveInformationAcrossContexts; } - var databaseName = config.UseModelBuilder ? config.DatabaseNameWithModel : config.DatabaseName; - try { globalWatcher.Start(); - StreamsConfigBuilder streamConfig = null; - if (!config.UseInMemoryProvider) + using (context = new BloggingContext()) { - streamConfig = StreamsConfigBuilder.Create(); - streamConfig = streamConfig.WithAcceptableRecoveryLag(100); - } + ProgramConfig.Config.ApplyOnContext(context); - using (context = new BloggingContext() - { - BootstrapServers = config.BootstrapServers, - ApplicationId = config.ApplicationId, - DatabaseName = databaseName, - StreamsConfig = streamConfig, - }) - { - if (config.UseProtobuf) - { - context.KeySerializationType = typeof(ProtobufKEFCoreSerDes.Key.BinaryRaw<>); - context.ValueContainerType = typeof(ProtobufValueContainer<>); - context.ValueSerializationType = typeof(ProtobufKEFCoreSerDes.ValueContainer.BinaryRaw<>); - } - else if (config.UseAvro) - { - context.KeySerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.Key.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.Key.JsonRaw<>); - context.ValueContainerType = typeof(AvroValueContainer<>); - context.ValueSerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.ValueContainer.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.ValueContainer.JsonRaw<>); - } - - if (config.DeleteApplicationData) + if (ProgramConfig.Config.DeleteApplicationData) { context.Database.EnsureDeleted(); if (context.Database.EnsureCreated()) { - ReportString("EnsureCreated created database"); + ProgramConfig.ReportString("EnsureCreated created database"); } else { - ReportString("EnsureCreated does not created database"); + ProgramConfig.ReportString("EnsureCreated does not created database"); } } testWatcher.Start(); Stopwatch watch = new Stopwatch(); - if (config.LoadApplicationData) + if (ProgramConfig.Config.LoadApplicationData) { watch.Start(); - for (int i = 0; i < config.NumberOfElements; i++) + for (int i = 0; i < ProgramConfig.Config.NumberOfElements; i++) { context.Add(new Blog { @@ -157,64 +104,47 @@ static void Main(string[] args) }); } watch.Stop(); - ReportString($"Elapsed data load: {watch.Elapsed}"); + ProgramConfig.ReportString($"Elapsed data load: {watch.Elapsed}"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges: {watch.Elapsed}"); + ProgramConfig.ReportString($"Elapsed SaveChanges: {watch.Elapsed}"); } } - for (int execution = 0; execution < config.NumberOfExecutions; execution++) + for (int execution = 0; execution < ProgramConfig.Config.NumberOfExecutions; execution++) { _tests.Add(execution, new ExecutionData(execution, maxTests)); - ReportString($"Starting cycle number {execution}"); + ProgramConfig.ReportString($"Starting cycle number {execution}"); Stopwatch singleTestWatch = Stopwatch.StartNew(); - using (context = new BloggingContext() + using (context = new BloggingContext()) { - BootstrapServers = config.BootstrapServers, - ApplicationId = config.ApplicationId, - DatabaseName = databaseName, - StreamsConfig = streamConfig, - }) - { - if (config.UseProtobuf) - { - context.KeySerializationType = typeof(ProtobufKEFCoreSerDes.Key.BinaryRaw<>); - context.ValueContainerType = typeof(ProtobufValueContainer<>); - context.ValueSerializationType = typeof(ProtobufKEFCoreSerDes.ValueContainer.BinaryRaw<>); - } - else if (config.UseAvro) - { - context.KeySerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.Key.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.Key.JsonRaw<>); - context.ValueContainerType = typeof(AvroValueContainer<>); - context.ValueSerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.ValueContainer.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.ValueContainer.JsonRaw<>); - } + ProgramConfig.Config.ApplyOnContext(context); Stopwatch watch = new(); watch.Restart(); var post = context.Posts.Single(b => b.BlogId == 2); watch.Stop(); _tests[execution].QueryTimes[0] = watch.Elapsed; - ReportString($"First execution of context.Posts.Single(b => b.BlogId == 2) takes {watch.Elapsed}. Result is {post}"); + ProgramConfig.ReportString($"First execution of context.Posts.Single(b => b.BlogId == 2) takes {watch.Elapsed}. Result is {post}"); watch.Restart(); post = context.Posts.Single(b => b.BlogId == 2); watch.Stop(); _tests[execution].QueryTimes[1] = watch.Elapsed; - ReportString($"Second execution of context.Posts.Single(b => b.BlogId == 2) takes {watch.Elapsed}. Result is {post}"); + ProgramConfig.ReportString($"Second execution of context.Posts.Single(b => b.BlogId == 2) takes {watch.Elapsed}. Result is {post}"); watch.Restart(); - post = context.Posts.Single(b => b.BlogId == config.NumberOfElements - 1); + post = context.Posts.Single(b => b.BlogId == ProgramConfig.Config.NumberOfElements - 1); watch.Stop(); _tests[execution].QueryTimes[2] = watch.Elapsed; - ReportString($"Execution of context.Posts.Single(b => b.BlogId == {config.NumberOfElements - 1}) takes {watch.Elapsed}. Result is {post}"); + ProgramConfig.ReportString($"Execution of context.Posts.Single(b => b.BlogId == {ProgramConfig.Config.NumberOfElements - 1}) takes {watch.Elapsed}. Result is {post}"); watch.Restart(); var all = context.Posts.All((o) => true); watch.Stop(); _tests[execution].QueryTimes[3] = watch.Elapsed; - ReportString($"Execution of context.Posts.All((o) => true) takes {watch.Elapsed}. Result is {all}"); + ProgramConfig.ReportString($"Execution of context.Posts.All((o) => true) takes {watch.Elapsed}. Result is {all}"); Blog blog = null; watch.Restart(); @@ -222,13 +152,13 @@ static void Main(string[] args) watch.Stop(); _tests[execution].QueryTimes[4] = watch.Elapsed; - ReportString($"First execution of context.Blogs.Single(b => b.BlogId == 1) takes {watch.Elapsed}. Result is {blog}"); + ProgramConfig.ReportString($"First execution of context.Blogs.Single(b => b.BlogId == 1) takes {watch.Elapsed}. Result is {blog}"); watch.Restart(); blog = context.Blogs.Single(b => b.BlogId == 1); watch.Stop(); _tests[execution].QueryTimes[5] = watch.Elapsed; - ReportString($"Second execution of context.Blogs.Single(b => b.BlogId == 1) takes {watch.Elapsed}. Result is {blog}"); + ProgramConfig.ReportString($"Second execution of context.Blogs.Single(b => b.BlogId == 1) takes {watch.Elapsed}. Result is {blog}"); watch.Restart(); var selector = (from op in context.Blogs @@ -238,7 +168,7 @@ join pg in context.Posts on op.BlogId equals pg.BlogId watch.Stop(); _tests[execution].QueryTimes[6] = watch.Elapsed; var result = selector.ToList(); - ReportString($"Execution of first complex query takes {watch.Elapsed}. Result is {result.Count} element{(result.Count == 1 ? string.Empty : "s")}"); + ProgramConfig.ReportString($"Execution of first complex query takes {watch.Elapsed}. Result is {result.Count} element{(result.Count == 1 ? string.Empty : "s")}"); watch.Restart(); var selector2 = (from op in context.Blogs @@ -248,24 +178,24 @@ join pg in context.Posts on op.BlogId equals pg.BlogId watch.Stop(); _tests[execution].QueryTimes[7] = watch.Elapsed; var result2 = selector.ToList(); - ReportString($"Execution of second complex query takes {watch.Elapsed}. Result is {result2.Count} element{(result2.Count == 1 ? string.Empty : "s")}"); + ProgramConfig.ReportString($"Execution of second complex query takes {watch.Elapsed}. Result is {result2.Count} element{(result2.Count == 1 ? string.Empty : "s")}"); singleTestWatch.Stop(); _tests[execution].QueryTimes[8] = singleTestWatch.Elapsed; - ReportString($"Test {execution} takes {singleTestWatch.Elapsed}."); + ProgramConfig.ReportString($"Test {execution} takes {singleTestWatch.Elapsed}."); } } } catch (Exception ex) { - ReportString(ex.ToString()); + ProgramConfig.ReportString(ex.ToString()); } finally { testWatcher.Stop(); globalWatcher.Stop(); context?.Dispose(); - ReportString(string.Empty); - ReportString($"Full test completed in {globalWatcher.Elapsed}, only tests completed in {testWatcher.Elapsed}"); + ProgramConfig.ReportString(string.Empty); + ProgramConfig.ReportString($"Full test completed in {globalWatcher.Elapsed}, only tests completed in {testWatcher.Elapsed}"); TimeSpan[] max = new TimeSpan[maxTests]; for (int i = 0; i < max.Length; i++) { max[i] = TimeSpan.Zero; } @@ -273,7 +203,7 @@ join pg in context.Posts on op.BlogId equals pg.BlogId for (int i = 0; i < min.Length; i++) { min[i] = TimeSpan.MaxValue; } TimeSpan[] total = new TimeSpan[maxTests]; for (int i = 0; i < total.Length; i++) { total[i] = TimeSpan.Zero; } - for (int i = 0; i < config.NumberOfExecutions; i++) + for (int i = 0; i < ProgramConfig.Config.NumberOfExecutions; i++) { var item = _tests[i].QueryTimes; @@ -287,7 +217,7 @@ join pg in context.Posts on op.BlogId equals pg.BlogId for (int testId = 0; testId < maxTests; testId++) { - ReportString($"Test {testId} -> Max {max[testId]} Min {min[testId]} Mean {total[testId] / config.NumberOfExecutions}"); + ProgramConfig.ReportString($"Test {testId} -> Max {max[testId]} Min {min[testId]} Mean {total[testId] / ProgramConfig.Config.NumberOfExecutions}"); } } } @@ -295,19 +225,14 @@ join pg in context.Posts on op.BlogId equals pg.BlogId public class BloggingContext : KafkaDbContext { - public override bool UsePersistentStorage { get; set; } = Program.config.UsePersistentStorage; - public override bool UseCompactedReplicator { get; set; } = Program.config.UseCompactedReplicator; - public override bool UseKNetStreams { get; set; } = Program.config.UseKNetStreams; - public override bool UseEnumeratorWithPrefetch { get; set; } = Program.config.UseEnumeratorWithPrefetch; - public DbSet Blogs { get; set; } public DbSet Posts { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { - if (Program.config.UseInMemoryProvider) + if (ProgramConfig.Config.UseInMemoryProvider) { - optionsBuilder.UseInMemoryDatabase(Program.config.DatabaseName); + optionsBuilder.UseInMemoryDatabase(ProgramConfig.Config.DatabaseName); } else { @@ -317,7 +242,7 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder) { - if (!Program.config.UseModelBuilder) return; + if (!ProgramConfig.Config.UseModelBuilder) return; modelBuilder.Entity().HasKey(c => new { c.BlogId, c.Rating }); } diff --git a/test/KEFCore.Complex.Test/KEFCore.Complex.Test.csproj b/test/KEFCore.Complex.Test/KEFCore.Complex.Test.csproj index 2586002d..3d6bdf14 100644 --- a/test/KEFCore.Complex.Test/KEFCore.Complex.Test.csproj +++ b/test/KEFCore.Complex.Test/KEFCore.Complex.Test.csproj @@ -10,8 +10,7 @@ ..\..\bin\ - - + diff --git a/test/KEFCore.Complex.Test/Program.cs b/test/KEFCore.Complex.Test/Program.cs index dcfa57de..c43c0809 100644 --- a/test/KEFCore.Complex.Test/Program.cs +++ b/test/KEFCore.Complex.Test/Program.cs @@ -23,101 +23,43 @@ */ using MASES.EntityFrameworkCore.KNet.Infrastructure; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro.Storage; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf.Storage; using MASES.EntityFrameworkCore.KNet.Test.Common; using MASES.EntityFrameworkCore.KNet.Test.Model; -using MASES.KNet.Streams; using Microsoft.EntityFrameworkCore; using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Text.Json; namespace MASES.EntityFrameworkCore.KNet.Complex.Test { partial class Program { - internal static ProgramConfig config = new(); - - static void ReportString(string message) - { - if (Debugger.IsAttached) - { - Trace.WriteLine(message); - } - else - { - Console.WriteLine(message); - } - } - static void Main(string[] args) { BloggingContext context = null; var testWatcher = new Stopwatch(); var globalWatcher = new Stopwatch(); - if (args.Length > 0) - { - if (!File.Exists(args[0])) { ReportString($"{args[0]} is not a configuration file."); return; } - config = JsonSerializer.Deserialize(File.ReadAllText(args[0])); - } + ProgramConfig.LoadConfig(args); - if (!config.UseInMemoryProvider) + if (!ProgramConfig.Config.UseInMemoryProvider) { KEFCore.CreateGlobalInstance(); } - var databaseName = config.UseModelBuilder ? config.DatabaseNameWithModel : config.DatabaseName; - try { globalWatcher.Start(); - StreamsConfigBuilder streamConfig = null; - if (!config.UseInMemoryProvider) - { - streamConfig = StreamsConfigBuilder.Create(); - streamConfig = streamConfig.WithAcceptableRecoveryLag(100); - } - - context = new BloggingContext() - { - BootstrapServers = config.BootstrapServers, - ApplicationId = config.ApplicationId, - DatabaseName = databaseName, - StreamsConfig = streamConfig, - }; - - if (config.UseProtobuf) - { - context.KeySerializationType = typeof(ProtobufKEFCoreSerDes.Key.BinaryRaw<>); - context.ValueContainerType = typeof(ProtobufValueContainer<>); - context.ValueSerializationType = typeof(ProtobufKEFCoreSerDes.ValueContainer.BinaryRaw<>); - } - else if (config.UseAvro) - { - context.KeySerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.Key.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.Key.JsonRaw<>); - context.ValueContainerType = typeof(AvroValueContainer<>); - context.ValueSerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.ValueContainer.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.ValueContainer.JsonRaw<>); - } - - if (config.DeleteApplicationData) - { - context.Database.EnsureDeleted(); - context.Database.EnsureCreated(); - } + context = new BloggingContext(); + ProgramConfig.Config.ApplyOnContext(context); testWatcher.Start(); Stopwatch watch = new Stopwatch(); - if (config.LoadApplicationData) + if (ProgramConfig.Config.LoadApplicationData) { watch.Start(); - for (int i = 0; i < config.NumberOfElements; i++) + for (int i = 0; i < ProgramConfig.Config.NumberOfElements; i++) { context.Add(new BlogComplex { @@ -137,14 +79,14 @@ static void Main(string[] args) }); } watch.Stop(); - ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); } - if (config.UseModelBuilder) + if (ProgramConfig.Config.UseModelBuilder) { watch.Restart(); var selector = (from op in context.Blogs @@ -153,30 +95,30 @@ join pg in context.Posts on op.BlogId equals pg.BlogId select new { pg, op }); var pageObject = selector.SingleOrDefault(); watch.Stop(); - ReportString($"Elapsed UseModelBuilder {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed UseModelBuilder {watch.ElapsedMilliseconds} ms"); } watch.Restart(); var post = context.Posts.Single(b => b.BlogId == 2); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 2) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 2) {watch.ElapsedMilliseconds} ms. Result is {post}"); try { watch.Restart(); post = context.Posts.Single(b => b.BlogId == 1); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {post}"); } catch { - if (config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run + if (ProgramConfig.Config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run } watch.Restart(); var all = context.Posts.All((o) => true); watch.Stop(); - ReportString($"Elapsed context.Posts.All((o) => true) {watch.ElapsedMilliseconds} ms. Result is {all}"); + ProgramConfig.ReportString($"Elapsed context.Posts.All((o) => true) {watch.ElapsedMilliseconds} ms. Result is {all}"); Blog blog = null; try @@ -184,28 +126,28 @@ join pg in context.Posts on op.BlogId equals pg.BlogId watch.Restart(); blog = context.Blogs!.Single(b => b.BlogId == 1); watch.Stop(); - ReportString($"Elapsed context.Blogs!.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {blog}"); + ProgramConfig.ReportString($"Elapsed context.Blogs!.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {blog}"); } catch { - if (config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run + if (ProgramConfig.Config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run } - if (config.LoadApplicationData) + if (ProgramConfig.Config.LoadApplicationData) { watch.Restart(); context.Remove(post); context.Remove(blog); watch.Stop(); - ReportString($"Elapsed data remove {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data remove {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); watch.Restart(); - for (int i = config.NumberOfElements; i < config.NumberOfElements + config.NumberOfExtraElements; i++) + for (int i = ProgramConfig.Config.NumberOfElements; i < ProgramConfig.Config.NumberOfElements + ProgramConfig.Config.NumberOfExtraElements; i++) { context.Add(new BlogComplex { @@ -225,17 +167,17 @@ join pg in context.Posts on op.BlogId equals pg.BlogId }); } watch.Stop(); - ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); } watch.Restart(); - post = context.Posts.Single(b => b.BlogId == config.NumberOfElements + (config.NumberOfExtraElements != 0 ? 1 : 0)); + post = context.Posts.Single(b => b.BlogId == ProgramConfig.Config.NumberOfElements + (ProgramConfig.Config.NumberOfExtraElements != 0 ? 1 : 0)); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == config.NumberOfElements + (config.NumberOfExtraElements != 0 ? 1 : 0)) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == config.NumberOfElements + (config.NumberOfExtraElements != 0 ? 1 : 0)) {watch.ElapsedMilliseconds} ms. Result is {post}"); var value = context.Blogs.AsQueryable().ToQueryString(); } @@ -255,18 +197,14 @@ join pg in context.Posts on op.BlogId equals pg.BlogId public class BloggingContext : KafkaDbContext { - public override bool UsePersistentStorage { get; set; } = Program.config.UsePersistentStorage; - public override bool UseCompactedReplicator { get; set; } = Program.config.UseCompactedReplicator; - public override bool UseKNetStreams { get; set; } = Program.config.UseKNetStreams; - public DbSet Blogs { get; set; } public DbSet Posts { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { - if (Program.config.UseInMemoryProvider) + if (ProgramConfig.Config.UseInMemoryProvider) { - optionsBuilder.UseInMemoryDatabase(Program.config.DatabaseName); + optionsBuilder.UseInMemoryDatabase(ProgramConfig.Config.DatabaseName); } else { @@ -276,7 +214,7 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder) { - if (!Program.config.UseModelBuilder) return; + if (!ProgramConfig.Config.UseModelBuilder) return; modelBuilder.Entity().HasKey(c => new { c.BlogId, c.Rating }); } diff --git a/test/KEFCore.Extractor.Test/Extractor.Test.Blog.Avro.json b/test/KEFCore.Extractor.Test/Extractor.Test.Blog.Avro.json index 05c516b9..a36f91a2 100644 --- a/test/KEFCore.Extractor.Test/Extractor.Test.Blog.Avro.json +++ b/test/KEFCore.Extractor.Test/Extractor.Test.Blog.Avro.json @@ -1,4 +1,4 @@ { "BootstrapServers": "192.168.0.101:9092", - "TopicToSubscribe": "TestDBBenchmarkAvro.MASES.EntityFrameworkCore.KNet.Test.Blog" + "TopicToSubscribe": "TestDBBenchmarkAvro.MASES.EntityFrameworkCore.KNet.Test.Model.Blog" } diff --git a/test/KEFCore.Extractor.Test/Extractor.Test.Blog.json b/test/KEFCore.Extractor.Test/Extractor.Test.Blog.json index 63e94cfa..6fcc9a51 100644 --- a/test/KEFCore.Extractor.Test/Extractor.Test.Blog.json +++ b/test/KEFCore.Extractor.Test/Extractor.Test.Blog.json @@ -1,4 +1,4 @@ { "BootstrapServers": "192.168.0.101:9092", - "TopicToSubscribe": "TestDBBenchmark.MASES.EntityFrameworkCore.KNet.Test.Blog" + "TopicToSubscribe": "TestDBBenchmark.MASES.EntityFrameworkCore.KNet.Test.Model.Blog" } diff --git a/test/KEFCore.Extractor.Test/KEFCore.Extractor.Test.csproj b/test/KEFCore.Extractor.Test/KEFCore.Extractor.Test.csproj index b2af333a..1a254a83 100644 --- a/test/KEFCore.Extractor.Test/KEFCore.Extractor.Test.csproj +++ b/test/KEFCore.Extractor.Test/KEFCore.Extractor.Test.csproj @@ -10,8 +10,7 @@ ..\..\bin\ - - + diff --git a/test/KEFCore.Extractor.Test/Program.cs b/test/KEFCore.Extractor.Test/Program.cs index 5e4a4875..27c0807f 100644 --- a/test/KEFCore.Extractor.Test/Program.cs +++ b/test/KEFCore.Extractor.Test/Program.cs @@ -25,9 +25,6 @@ using MASES.EntityFrameworkCore.KNet.Serialization; using MASES.EntityFrameworkCore.KNet.Test.Common; using System; -using System.Diagnostics; -using System.IO; -using System.Text.Json; using System.Threading; namespace MASES.EntityFrameworkCore.KNet.Test.Extractor @@ -35,36 +32,19 @@ namespace MASES.EntityFrameworkCore.KNet.Test.Extractor partial class Program { internal static CancellationTokenSource runApplication = new CancellationTokenSource(); - internal static ProgramConfig config = new(); - - static void ReportString(string message) - { - if (Debugger.IsAttached) - { - Trace.WriteLine(message); - } - else - { - Console.WriteLine(message); - } - } static void Main(string[] args) { try { - if (args.Length > 0) - { - if (!File.Exists(args[0])) { ReportString($"{args[0]} is not a configuration file."); return; } - config = JsonSerializer.Deserialize(File.ReadAllText(args[0])); - } + ProgramConfig.LoadConfig(args); - if (string.IsNullOrWhiteSpace(config.BootstrapServers)) throw new ArgumentException("BootstrapServers must be set"); - if (string.IsNullOrWhiteSpace(config.TopicToSubscribe)) throw new ArgumentException("TopicToSubscribe must be set"); + if (string.IsNullOrWhiteSpace(ProgramConfig.Config.BootstrapServers)) throw new ArgumentException("BootstrapServers must be set"); + if (string.IsNullOrWhiteSpace(ProgramConfig.Config.TopicToSubscribe)) throw new ArgumentException("TopicToSubscribe must be set"); KEFCore.CreateGlobalInstance(); Console.CancelKeyPress += Console_CancelKeyPress; - EntityExtractor.FromTopic(config.BootstrapServers, config.TopicToSubscribe, ReportData, runApplication.Token); + EntityExtractor.FromTopic(ProgramConfig.Config.BootstrapServers, ProgramConfig.Config.TopicToSubscribe, ReportData, runApplication.Token); } catch (Exception ex) { diff --git a/test/KEFCore.StreamTest/KEFCore.StreamTest.csproj b/test/KEFCore.StreamTest/KEFCore.StreamTest.csproj index b150774a..867df086 100644 --- a/test/KEFCore.StreamTest/KEFCore.StreamTest.csproj +++ b/test/KEFCore.StreamTest/KEFCore.StreamTest.csproj @@ -10,8 +10,7 @@ ..\..\bin\ - - + diff --git a/test/KEFCore.StreamTest/Program.cs b/test/KEFCore.StreamTest/Program.cs index 9f8cda2d..7c856d60 100644 --- a/test/KEFCore.StreamTest/Program.cs +++ b/test/KEFCore.StreamTest/Program.cs @@ -23,89 +23,37 @@ */ using MASES.EntityFrameworkCore.KNet.Infrastructure; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro.Storage; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf.Storage; using MASES.EntityFrameworkCore.KNet.Test.Common; using MASES.EntityFrameworkCore.KNet.Test.Model; -using MASES.KNet.Streams; using Microsoft.EntityFrameworkCore; using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Text.Json; namespace MASES.EntityFrameworkCore.KNet.Test.Stream { partial class Program { - internal static ProgramConfig config = new(); - - static void ReportString(string message) - { - if (Debugger.IsAttached) - { - Trace.WriteLine(message); - } - else - { - Console.WriteLine(message); - } - } - static void Main(string[] args) { var testWatcher = new Stopwatch(); var globalWatcher = new Stopwatch(); - if (args.Length > 0) - { - if (!File.Exists(args[0])) { ReportString($"{args[0]} is not a configuration file."); return; } - config = JsonSerializer.Deserialize(File.ReadAllText(args[0])); - } + ProgramConfig.LoadConfig(args); - if (!config.UseInMemoryProvider) + if (!ProgramConfig.Config.UseInMemoryProvider) { KEFCore.CreateGlobalInstance(); } - var databaseName = config.UseModelBuilder ? config.DatabaseNameWithModel : config.DatabaseName; - try { globalWatcher.Start(); - StreamsConfigBuilder streamConfig = null; - if (!config.UseInMemoryProvider) - { - streamConfig = StreamsConfigBuilder.Create(); - streamConfig = streamConfig.WithAcceptableRecoveryLag(100); - } + using var context = new BloggingContext(); + ProgramConfig.Config.ApplyOnContext(context); - using var context = new BloggingContext() - { - BootstrapServers = config.BootstrapServers, - ApplicationId = config.ApplicationId, - DatabaseName = databaseName, - StreamsConfig = streamConfig, - }; - - if (config.UseProtobuf) - { - context.KeySerializationType = typeof(ProtobufKEFCoreSerDes.Key.BinaryRaw<>); - context.ValueContainerType = typeof(ProtobufValueContainer<>); - context.ValueSerializationType = typeof(ProtobufKEFCoreSerDes.ValueContainer.BinaryRaw<>); - } - else if (config.UseAvro) - { - context.KeySerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.Key.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.Key.JsonRaw<>); - context.ValueContainerType = typeof(AvroValueContainer<>); - context.ValueSerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.ValueContainer.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.ValueContainer.JsonRaw<>); - } - - if (config.DeleteApplicationData) + if (ProgramConfig.Config.DeleteApplicationData) { context.Database.EnsureDeleted(); context.Database.EnsureCreated(); @@ -113,10 +61,10 @@ static void Main(string[] args) testWatcher.Start(); Stopwatch watch = new Stopwatch(); - if (config.LoadApplicationData) + if (ProgramConfig.Config.LoadApplicationData) { watch.Start(); - for (int i = 0; i < config.NumberOfElements; i++) + for (int i = 0; i < ProgramConfig.Config.NumberOfElements; i++) { context.Add(new Blog { @@ -133,22 +81,16 @@ static void Main(string[] args) }); } watch.Stop(); - ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); } // ended data load - - - - - - - if (config.UseModelBuilder) + if (ProgramConfig.Config.UseModelBuilder) { watch.Restart(); var selector = (from op in context.Blogs @@ -157,30 +99,30 @@ join pg in context.Posts on op.BlogId equals pg.BlogId select new { pg, op }); var pageObject = selector.FirstOrDefault(); watch.Stop(); - ReportString($"Elapsed UseModelBuilder {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed UseModelBuilder {watch.ElapsedMilliseconds} ms"); } watch.Restart(); var post = context.Posts.Single(b => b.BlogId == 2); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 2) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 2) {watch.ElapsedMilliseconds} ms. Result is {post}"); try { watch.Restart(); post = context.Posts.Single(b => b.BlogId == 1); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {post}"); } catch { - if (config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run + if (ProgramConfig.Config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run } watch.Restart(); var all = context.Posts.All((o) => true); watch.Stop(); - ReportString($"Elapsed context.Posts.All((o) => true) {watch.ElapsedMilliseconds} ms. Result is {all}"); + ProgramConfig.ReportString($"Elapsed context.Posts.All((o) => true) {watch.ElapsedMilliseconds} ms. Result is {all}"); Blog blog = null; try @@ -188,28 +130,28 @@ join pg in context.Posts on op.BlogId equals pg.BlogId watch.Restart(); blog = context.Blogs!.Single(b => b.BlogId == 1); watch.Stop(); - ReportString($"Elapsed context.Blogs!.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {blog}"); + ProgramConfig.ReportString($"Elapsed context.Blogs!.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {blog}"); } catch { - if (config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run + if (ProgramConfig.Config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run } - if (config.LoadApplicationData) + if (ProgramConfig.Config.LoadApplicationData) { watch.Restart(); context.Remove(post); context.Remove(blog); watch.Stop(); - ReportString($"Elapsed data remove {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data remove {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); watch.Restart(); - for (int i = config.NumberOfElements; i < config.NumberOfElements + config.NumberOfExtraElements; i++) + for (int i = ProgramConfig.Config.NumberOfElements; i < ProgramConfig.Config.NumberOfElements + ProgramConfig.Config.NumberOfExtraElements; i++) { context.Add(new Blog { @@ -226,18 +168,18 @@ join pg in context.Posts on op.BlogId equals pg.BlogId }); } watch.Stop(); - ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); } - var postion = config.NumberOfElements + config.NumberOfExtraElements - 1; + var postion = ProgramConfig.Config.NumberOfElements + ProgramConfig.Config.NumberOfExtraElements - 1; watch.Restart(); post = context.Posts.Single(b => b.BlogId == postion); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == {postion}) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == {postion}) {watch.ElapsedMilliseconds} ms. Result is {post}"); var value = context.Blogs.AsQueryable().ToQueryString(); } @@ -257,18 +199,14 @@ join pg in context.Posts on op.BlogId equals pg.BlogId public class BloggingContext : KafkaDbContext { - public override bool UsePersistentStorage { get; set; } = Program.config.UsePersistentStorage; - public override bool UseCompactedReplicator { get; set; } = Program.config.UseCompactedReplicator; - public override bool UseKNetStreams { get; set; } = Program.config.UseKNetStreams; - public DbSet Blogs { get; set; } public DbSet Posts { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { - if (Program.config.UseInMemoryProvider) + if (ProgramConfig.Config.UseInMemoryProvider) { - optionsBuilder.UseInMemoryDatabase(Program.config.DatabaseName); + optionsBuilder.UseInMemoryDatabase(ProgramConfig.Config.DatabaseName); } else { @@ -278,7 +216,7 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder) { - if (!Program.config.UseModelBuilder) return; + if (!ProgramConfig.Config.UseModelBuilder) return; modelBuilder.Entity().HasKey(c => new { c.BlogId, c.Rating }); } diff --git a/test/KEFCore.Test.sln b/test/KEFCore.Test.sln index d542242d..3291e2bb 100644 --- a/test/KEFCore.Test.sln +++ b/test/KEFCore.Test.sln @@ -35,6 +35,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KEFCore.SerDes.Protobuf", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KEFCore.StreamTest", "KEFCore.StreamTest\KEFCore.StreamTest.csproj", "{BD4EC954-4C0A-40C4-8629-C5562582ED6A}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KEFCore.Common.Test", "Common\KEFCore.Common.Test.csproj", "{03B24253-0A2D-4FF0-9C9E-1962DDD189AC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -93,6 +95,10 @@ Global {BD4EC954-4C0A-40C4-8629-C5562582ED6A}.Debug|Any CPU.Build.0 = Debug|Any CPU {BD4EC954-4C0A-40C4-8629-C5562582ED6A}.Release|Any CPU.ActiveCfg = Release|Any CPU {BD4EC954-4C0A-40C4-8629-C5562582ED6A}.Release|Any CPU.Build.0 = Release|Any CPU + {03B24253-0A2D-4FF0-9C9E-1962DDD189AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {03B24253-0A2D-4FF0-9C9E-1962DDD189AC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03B24253-0A2D-4FF0-9C9E-1962DDD189AC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {03B24253-0A2D-4FF0-9C9E-1962DDD189AC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -111,6 +117,7 @@ Global {B18081C0-E85F-4228-8150-E7EDD4794BEB} = {B35B16BB-890F-4385-AB20-7AA4DD6E9C01} {1EA5DAE7-67D1-4E2B-ACAF-F107BB86EB66} = {B35B16BB-890F-4385-AB20-7AA4DD6E9C01} {BD4EC954-4C0A-40C4-8629-C5562582ED6A} = {4A0AD520-9BC4-4F92-893B-6F92BBC35BFA} + {03B24253-0A2D-4FF0-9C9E-1962DDD189AC} = {4A0AD520-9BC4-4F92-893B-6F92BBC35BFA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {36C294ED-9ECE-42AA-8273-31E008749AF3} diff --git a/test/KEFCore.Test/KEFCore.Test.csproj b/test/KEFCore.Test/KEFCore.Test.csproj index 5a5f27d1..2b6e5e9b 100644 --- a/test/KEFCore.Test/KEFCore.Test.csproj +++ b/test/KEFCore.Test/KEFCore.Test.csproj @@ -10,8 +10,7 @@ ..\..\bin\ - - + diff --git a/test/KEFCore.Test/Program.cs b/test/KEFCore.Test/Program.cs index cf59c7e0..95ca547b 100644 --- a/test/KEFCore.Test/Program.cs +++ b/test/KEFCore.Test/Program.cs @@ -23,92 +23,44 @@ */ using MASES.EntityFrameworkCore.KNet.Infrastructure; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro; -using MASES.EntityFrameworkCore.KNet.Serialization.Avro.Storage; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf; -using MASES.EntityFrameworkCore.KNet.Serialization.Protobuf.Storage; using MASES.EntityFrameworkCore.KNet.Storage; using MASES.EntityFrameworkCore.KNet.Test.Common; using MASES.EntityFrameworkCore.KNet.Test.Model; -using MASES.KNet.Streams; using Microsoft.EntityFrameworkCore; using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Text.Json; namespace MASES.EntityFrameworkCore.KNet.Test { partial class Program { static BloggingContext context = null; - internal static ProgramConfig config = new(); - - static void ReportString(string message) - { - if (Debugger.IsAttached) - { - Trace.WriteLine(message); - } - else - { - Console.WriteLine(message); - } - } static void Main(string[] args) { var testWatcher = new Stopwatch(); var globalWatcher = new Stopwatch(); - if (args.Length > 0) - { - if (!File.Exists(args[0])) { ReportString($"{args[0]} is not a configuration file."); return; } - config = JsonSerializer.Deserialize(File.ReadAllText(args[0])); - } + ProgramConfig.LoadConfig(args); - if (!config.UseInMemoryProvider) + if (!ProgramConfig.Config.UseInMemoryProvider) { KEFCore.CreateGlobalInstance(); } - var databaseName = config.UseModelBuilder ? config.DatabaseNameWithModel : config.DatabaseName; - try { globalWatcher.Start(); - StreamsConfigBuilder streamConfig = null; - if (!config.UseInMemoryProvider) - { - streamConfig = StreamsConfigBuilder.Create(); - streamConfig = streamConfig.WithAcceptableRecoveryLag(100); - } - context = new BloggingContext() { - BootstrapServers = config.BootstrapServers, - ApplicationId = config.ApplicationId, - DatabaseName = databaseName, - StreamsConfig = streamConfig, - OnChangeEvent = config.WithEvents ? OnEvent : null, + OnChangeEvent = ProgramConfig.Config.WithEvents ? OnEvent : null, }; - if (config.UseProtobuf) - { - context.KeySerializationType = typeof(ProtobufKEFCoreSerDes.Key.BinaryRaw<>); - context.ValueContainerType = typeof(ProtobufValueContainer<>); - context.ValueSerializationType = typeof(ProtobufKEFCoreSerDes.ValueContainer.BinaryRaw<>); - } - else if (config.UseAvro) - { - context.KeySerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.Key.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.Key.JsonRaw<>); - context.ValueContainerType = typeof(AvroValueContainer<>); - context.ValueSerializationType = config.UseAvroBinary ? typeof(AvroKEFCoreSerDes.ValueContainer.BinaryRaw<>) : typeof(AvroKEFCoreSerDes.ValueContainer.JsonRaw<>); - } - - if (config.DeleteApplicationData) + ProgramConfig.Config.ApplyOnContext(context); + + if (ProgramConfig.Config.DeleteApplicationData) { context.Database.EnsureDeleted(); context.Database.EnsureCreated(); @@ -116,10 +68,10 @@ static void Main(string[] args) testWatcher.Start(); Stopwatch watch = new Stopwatch(); - if (config.LoadApplicationData) + if (ProgramConfig.Config.LoadApplicationData) { watch.Start(); - for (int i = 0; i < config.NumberOfElements; i++) + for (int i = 0; i < ProgramConfig.Config.NumberOfElements; i++) { context.Add(new Blog { @@ -136,14 +88,14 @@ static void Main(string[] args) }); } watch.Stop(); - ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); } - if (config.UseModelBuilder) + if (ProgramConfig.Config.UseModelBuilder) { watch.Restart(); var selector = (from op in context.Blogs @@ -152,30 +104,30 @@ join pg in context.Posts on op.BlogId equals pg.BlogId select new { pg, op }); var pageObject = selector.FirstOrDefault(); watch.Stop(); - ReportString($"Elapsed UseModelBuilder {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed UseModelBuilder {watch.ElapsedMilliseconds} ms"); } watch.Restart(); var post = context.Posts.Single(b => b.BlogId == 2); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 2) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 2) {watch.ElapsedMilliseconds} ms. Result is {post}"); try { watch.Restart(); post = context.Posts.Single(b => b.BlogId == 1); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {post}"); } catch { - if (config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run + if (ProgramConfig.Config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run } watch.Restart(); var all = context.Posts.All((o) => true); watch.Stop(); - ReportString($"Elapsed context.Posts.All((o) => true) {watch.ElapsedMilliseconds} ms. Result is {all}"); + ProgramConfig.ReportString($"Elapsed context.Posts.All((o) => true) {watch.ElapsedMilliseconds} ms. Result is {all}"); Blog blog = null; try @@ -183,28 +135,28 @@ join pg in context.Posts on op.BlogId equals pg.BlogId watch.Restart(); blog = context.Blogs!.Single(b => b.BlogId == 1); watch.Stop(); - ReportString($"Elapsed context.Blogs!.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {blog}"); + ProgramConfig.ReportString($"Elapsed context.Blogs!.Single(b => b.BlogId == 1) {watch.ElapsedMilliseconds} ms. Result is {blog}"); } catch { - if (config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run + if (ProgramConfig.Config.LoadApplicationData) throw; // throw only if the test is loading data otherwise it was removed in a previous run } - if (config.LoadApplicationData) + if (ProgramConfig.Config.LoadApplicationData) { watch.Restart(); context.Remove(post); context.Remove(blog); watch.Stop(); - ReportString($"Elapsed data remove {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data remove {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); watch.Restart(); - for (int i = config.NumberOfElements; i < config.NumberOfElements + config.NumberOfExtraElements; i++) + for (int i = ProgramConfig.Config.NumberOfElements; i < ProgramConfig.Config.NumberOfElements + ProgramConfig.Config.NumberOfExtraElements; i++) { context.Add(new Blog { @@ -221,18 +173,18 @@ join pg in context.Posts on op.BlogId equals pg.BlogId }); } watch.Stop(); - ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed data load {watch.ElapsedMilliseconds} ms"); watch.Restart(); context.SaveChanges(); watch.Stop(); - ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); + ProgramConfig.ReportString($"Elapsed SaveChanges {watch.ElapsedMilliseconds} ms"); } - var postion = config.NumberOfElements + config.NumberOfExtraElements - 1; + var postion = ProgramConfig.Config.NumberOfElements + ProgramConfig.Config.NumberOfExtraElements - 1; watch.Restart(); post = context.Posts.Single(b => b.BlogId == postion); watch.Stop(); - ReportString($"Elapsed context.Posts.Single(b => b.BlogId == {postion}) {watch.ElapsedMilliseconds} ms. Result is {post}"); + ProgramConfig.ReportString($"Elapsed context.Posts.Single(b => b.BlogId == {postion}) {watch.ElapsedMilliseconds} ms. Result is {post}"); var value = context.Blogs.AsQueryable().ToQueryString(); } @@ -257,26 +209,22 @@ static void OnEvent(EntityTypeChanged change) value = context.Find(change.EntityType.ClrType, change.Key); } catch (ObjectDisposedException) { } - catch (InvalidOperationException ) { } + catch (InvalidOperationException) { } - ReportString($"{change.EntityType.Name} -> {(change.KeyRemoved ? "removed" : "updated/added")}: {change.Key} - {value}"); + ProgramConfig.ReportString($"{change.EntityType.Name} -> {(change.KeyRemoved ? "removed" : "updated/added")}: {change.Key} - {value}"); } } public class BloggingContext : KafkaDbContext { - public override bool UsePersistentStorage { get; set; } = Program.config.UsePersistentStorage; - public override bool UseCompactedReplicator { get; set; } = Program.config.UseCompactedReplicator; - public override bool UseKNetStreams { get; set; } = Program.config.UseKNetStreams; - public DbSet Blogs { get; set; } public DbSet Posts { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { - if (Program.config.UseInMemoryProvider) + if (ProgramConfig.Config.UseInMemoryProvider) { - optionsBuilder.UseInMemoryDatabase(Program.config.DatabaseName); + optionsBuilder.UseInMemoryDatabase(ProgramConfig.Config.DatabaseName); } else { @@ -286,7 +234,7 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder) { - if (!Program.config.UseModelBuilder) return; + if (!ProgramConfig.Config.UseModelBuilder) return; modelBuilder.Entity().HasKey(c => new { c.BlogId, c.Rating }); }