Skip to content

Commit

Permalink
Updates on documentation (#129)
Browse files Browse the repository at this point in the history
* Code documentation update

* Changed DbName into DatabaseName to have a common property name, plus fixed UseNameMatching in KafkaDbContext

* Removed other warning messages, plus an unused class

* Added first level cache
  • Loading branch information
masesdevelopers authored Oct 21, 2023
1 parent 04d4157 commit 5af07a0
Show file tree
Hide file tree
Showing 61 changed files with 515 additions and 390 deletions.
6 changes: 5 additions & 1 deletion src/documentation/articles/howitworks.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,11 @@ Using the [topic compaction](https://kafka.apache.org/documentation/#compaction)
- Update: a producer storing a new record with a previously stored unique key will discard the old records
- Delete: a producer storing a new record with a previously stored unique key, and value set to null, will delete all records with that unique key

All CRUD operations are helped, behind the scene, from [`KNetCompactedReplicator`](https://github.com/masesgroup/KNet/blob/master/src/net/KNet/Specific/Replicator/KNetCompactedReplicator.cs) and/or [`KNetProducer`](https://github.com/masesgroup/KNet/blob/master/src/net/KNet/Specific/Producer/KNetProducer.cs)/[Apache Kafka Streams](https://kafka.apache.org/documentation/streams/).
All CRUD operations are helped, behind the scene, from [`KNetCompactedReplicator`](https://github.com/masesgroup/KNet/blob/master/src/net/KNet/Specific/Replicator/KNetCompactedReplicator.cs) or [`KNetProducer`](https://github.com/masesgroup/KNet/blob/master/src/net/KNet/Specific/Producer/KNetProducer.cs)/[Apache Kafka Streams](https://kafka.apache.org/documentation/streams/).

### First-level cache

[`KNetCompactedReplicator`](https://github.com/masesgroup/KNet/blob/master/src/net/KNet/Specific/Replicator/KNetCompactedReplicator.cs) or [Apache Kafka Streams](https://kafka.apache.org/documentation/streams/) act as first-level cache of [Entity Framework Core](https://learn.microsoft.com/it-it/ef/core/): **data coming from the Apache Kafka cluster updates their content while the system is running without a specific request**.

### Data storage

Expand Down
3 changes: 2 additions & 1 deletion src/documentation/articles/kafkadbcontext.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
- **KeySerializationType**: the .NET type to be used to allocate an external serializer for Apache Kafka record key
- **ValueSerializationType**: the .NET type to be used to allocate an external serializer for Apache Kafka record value
- **ValueContainerType**: the .NET type to be used to allocate an external container class for Apache Kafka record value
- **UseNameMatching**: set to **false** to avoid Entity matching based on Name
- **BootstrapServers**: the server hosting the broker of Apache Kafka
- **ApplicationId**: the application identifier used to identify the context
- **DbName**: the user defined name which declares the database name, it is used to prepend every Topic which belongs to this database
- **DatabaseName**: the user defined name which declares the database name, it is used to prepend every Topic which belongs to this database
- **DefaultNumPartitions**: the default number of partitions used when topics are created for each entity
- **DefaultReplicationFactor**: the replication factor to use when data are stored in Apache Kafka
- **DefaultConsumerInstances**: the consumer instances to be allocated when UseCompactedReplicator is **true**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down Expand Up @@ -32,6 +29,10 @@ namespace MASES.EntityFrameworkCore.KNet.Design.Internal;
/// </summary>
public class KafkaCSharpRuntimeAnnotationCodeGenerator : CSharpRuntimeAnnotationCodeGenerator
{
/// <summary>
/// Default initializer
/// </summary>
/// <param name="dependencies"><see cref="CSharpRuntimeAnnotationCodeGeneratorDependencies"/></param>
public KafkaCSharpRuntimeAnnotationCodeGenerator(
CSharpRuntimeAnnotationCodeGeneratorDependencies dependencies)
: base(dependencies)
Expand Down
4 changes: 1 addition & 3 deletions src/net/KEFCore/Design/Internal/KafkaDesignTimeServices.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down Expand Up @@ -32,6 +29,7 @@ namespace MASES.EntityFrameworkCore.KNet.Design.Internal;
/// </summary>
public class KafkaDesignTimeServices : IDesignTimeServices
{
/// <inheritdoc/>
public virtual void ConfigureDesignTimeServices(IServiceCollection serviceCollection)
{
serviceCollection.AddEntityFrameworkKafkaDatabase();
Expand Down
3 changes: 0 additions & 3 deletions src/net/KEFCore/Diagnostics/Internal/KafkaLoggerExtensions.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down
3 changes: 0 additions & 3 deletions src/net/KEFCore/Diagnostics/KafkaEventId.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down
3 changes: 0 additions & 3 deletions src/net/KEFCore/Extensions/KafkaDatabaseFacadeExtensions.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down
21 changes: 12 additions & 9 deletions src/net/KEFCore/Extensions/KafkaDbContextOptionsExtensions.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down Expand Up @@ -125,22 +122,28 @@ var coreOptionsExtension

((IDbContextOptionsBuilderInfrastructure)optionsBuilder).AddOrUpdateExtension(coreOptionsExtension);
}

/// <summary>
/// Creates a serializer <see cref="Type"/> for keys
/// </summary>
public static Type SerializerTypeForKey(this IKafkaSingletonOptions options, IEntityType entityType)
{
var primaryKey = entityType.FindPrimaryKey()!.GetKeyType();
return options.KeySerializationType.MakeGenericType(primaryKey);
return options.KeySerializationType?.MakeGenericType(primaryKey)!;
}

/// <summary>
/// Creates a serialzier <see cref="Type"/> for values
/// </summary>
public static Type SerializerTypeForValue(this IKafkaSingletonOptions options, IEntityType entityType)
{
var primaryKey = entityType.FindPrimaryKey()!.GetKeyType();
return options.ValueSerializationType.MakeGenericType(ValueContainerType(options, entityType));
return options.ValueSerializationType?.MakeGenericType(ValueContainerType(options, entityType))!;
}

/// <summary>
/// Create the ValueContainer <see cref="Type"/>
/// </summary>
public static Type ValueContainerType(this IKafkaSingletonOptions options, IEntityType entityType)
{
var primaryKey = entityType.FindPrimaryKey()!.GetKeyType();
return options.ValueContainerType.MakeGenericType(primaryKey);
return options.ValueContainerType?.MakeGenericType(primaryKey)!;
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down
27 changes: 18 additions & 9 deletions src/net/KEFCore/Extensions/KafkaEntityTypeExtensions.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down Expand Up @@ -90,34 +87,46 @@ public static void SetKafkaQuery(
=> entityType.FindAnnotation(CoreAnnotationNames.DefiningQuery)?.GetConfigurationSource();
#pragma warning restore CS0612 // Il tipo o il membro è obsoleto
#pragma warning restore EF1001 // Internal EF Core API usage.

/// <summary>
/// Creates the topic name
/// </summary>
public static string TopicName(this IEntityType entityType, KafkaOptionsExtension options)
{
return $"{options.DatabaseName}.{entityType.Name}";
}

/// <summary>
/// Creates the storage id
/// </summary>
public static string StorageIdForTable(this IEntityType entityType, KafkaOptionsExtension options)
{
return $"Table_{entityType.TopicName(options)}";
}

/// <summary>
/// Creates the application id
/// </summary>
public static string ApplicationIdForTable(this IEntityType entityType, KafkaOptionsExtension options)
{
return $"{options.ApplicationId}_{entityType.Name}";
}

/// <summary>
/// Gets replication factor
/// </summary>
public static short ReplicationFactor(this IEntityType entityType, KafkaOptionsExtension options)
{
var replicationFactor = options.DefaultReplicationFactor;
return replicationFactor;
}

/// <summary>
/// Gets number of partitions
/// </summary>
public static int NumPartitions(this IEntityType entityType, KafkaOptionsExtension options)
{
var numPartitions = options.DefaultNumPartitions;
return numPartitions;
}

/// <summary>
/// Gets consumer instances
/// </summary>
public static int? ConsumerInstances(this IEntityType entityType, KafkaOptionsExtension options)
{
var consumerInstances = options.DefaultConsumerInstances;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
* Copyright 2023 MASES s.r.l.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*
/*
* Copyright 2023 MASES s.r.l.
*
* Licensed under the Apache License, Version 2.0 (the "License");
Expand Down
43 changes: 22 additions & 21 deletions src/net/KEFCore/Infrastructure/Internal/IKafkaSingletonOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,39 +32,40 @@ namespace MASES.EntityFrameworkCore.KNet.Infrastructure.Internal;
/// </summary>
public interface IKafkaSingletonOptions : ISingletonOptions
{
Type KeySerializationType { get; }

Type ValueSerializationType { get; }

Type ValueContainerType { get; }

/// <inheritdoc cref="KafkaDbContext.KeySerializationType"/>
Type? KeySerializationType { get; }
/// <inheritdoc cref="KafkaDbContext.ValueSerializationType"/>
Type? ValueSerializationType { get; }
/// <inheritdoc cref="KafkaDbContext.ValueContainerType"/>
Type? ValueContainerType { get; }
/// <inheritdoc cref="KafkaDbContext.UseNameMatching"/>
bool UseNameMatching { get; }

/// <inheritdoc cref="KafkaDbContext.DatabaseName"/>
string? DatabaseName { get; }

/// <inheritdoc cref="KafkaDbContext.ApplicationId"/>
string? ApplicationId { get; }

/// <inheritdoc cref="KafkaDbContext.BootstrapServers"/>
string? BootstrapServers { get; }

/// <inheritdoc cref="KafkaDbContext.UseDeletePolicyForTopic"/>
bool UseDeletePolicyForTopic { get; }

/// <inheritdoc cref="KafkaDbContext.UseCompactedReplicator"/>
bool UseCompactedReplicator { get; }

/// <inheritdoc cref="KafkaDbContext.UsePersistentStorage"/>
bool UsePersistentStorage { get; }

/// <inheritdoc cref="KafkaDbContext.DefaultNumPartitions"/>
int DefaultNumPartitions { get; }

/// <inheritdoc cref="KafkaDbContext.DefaultConsumerInstances"/>
int? DefaultConsumerInstances { get; }

/// <inheritdoc cref="KafkaDbContext.DefaultReplicationFactor"/>
int DefaultReplicationFactor { get; }

/// <inheritdoc cref="KafkaDbContext.ConsumerConfig"/>
ConsumerConfigBuilder? ConsumerConfig { get; }

/// <inheritdoc cref="KafkaDbContext.ProducerConfig"/>
ProducerConfigBuilder? ProducerConfig { get; }

/// <inheritdoc cref="KafkaDbContext.StreamsConfig"/>
StreamsConfigBuilder? StreamsConfig { get; }

/// <inheritdoc cref="KafkaDbContext.TopicConfig"/>
TopicConfigBuilder? TopicConfig { get; }

Action<IEntityType, bool, object> OnChangeEvent { get; }
/// <inheritdoc cref="KafkaDbContext.OnChangeEvent"/>
Action<IEntityType, bool, object>? OnChangeEvent { get; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,14 @@ namespace MASES.EntityFrameworkCore.KNet.Infrastructure.Internal;
/// </summary>
public class KafkaModelValidator : ModelValidator
{
/// <summary>
/// Initializer
/// </summary>
public KafkaModelValidator(ModelValidatorDependencies dependencies)
: base(dependencies)
{
}

/// <inheritdoc/>
public override void Validate(IModel model, IDiagnosticsLogger<DbLoggerCategory.Model.Validation> logger)
{
base.Validate(model, logger);
Expand Down
Loading

0 comments on commit 5af07a0

Please sign in to comment.