From 768ba7644dabb77e1d9638dbcf09ae63bfa3ea8e Mon Sep 17 00:00:00 2001 From: George Drak Date: Tue, 3 Oct 2023 19:59:37 +0500 Subject: [PATCH 01/32] feat(tasks): tasks modules prototype --- Sitko.Core.sln | 60 ++++++ src/Directory.Packages.props | 11 +- src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj | 13 ++ .../ApplicationExtensions.cs | 15 ++ .../KafkaTasksModule.cs | 114 +++++++++++ .../Sitko.Core.Tasks.Kafka.csproj | 6 + src/Sitko.Core.Tasks/BaseTask.cs | 20 ++ src/Sitko.Core.Tasks/BaseTaskConfig.cs | 5 + src/Sitko.Core.Tasks/BaseTaskResult.cs | 8 + .../Execution/BaseTaskExecutor.cs | 158 ++++++++++++++++ .../Execution/ITaskExecutor.cs | 13 ++ .../Execution/JobFailedException.cs | 8 + .../Execution/TaskExecutorAttribute.cs | 18 ++ .../Execution/TaskExecutorHelper.cs | 22 +++ src/Sitko.Core.Tasks/IBaseTask.cs | 29 +++ .../IBaseTaskWithConfigAndResult.cs | 7 + .../Scheduling/BaseTaskScheduler.cs | 63 ++++++ .../Scheduling/ITaskScheduler.cs | 6 + .../Scheduling/ScheduleLocks.cs | 9 + .../Scheduling/TaskSchedulingOptions.cs | 6 + src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj | 10 + src/Sitko.Core.Tasks/TaskAttribute.cs | 8 + src/Sitko.Core.Tasks/TaskStatus.cs | 10 + src/Sitko.Core.Tasks/TasksModule.cs | 179 ++++++++++++++++++ .../BaseKafkaTasksTest.cs | 28 +++ .../Data/BaseTestTask.cs | 13 ++ .../Data/TestDbContext.cs | 10 + .../Data/TestTask.cs | 15 ++ .../Data/TestTaskExecutor.cs | 21 ++ .../Data/TestTaskScheduler.cs | 20 ++ .../Sitko.Core.Tasks.Kafka.Tests.csproj | 5 + 31 files changed, 909 insertions(+), 1 deletion(-) create mode 100644 src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj create mode 100644 src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs create mode 100644 src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs create mode 100644 src/Sitko.Core.Tasks.Kafka/Sitko.Core.Tasks.Kafka.csproj create mode 100644 src/Sitko.Core.Tasks/BaseTask.cs create mode 100644 src/Sitko.Core.Tasks/BaseTaskConfig.cs create mode 100644 src/Sitko.Core.Tasks/BaseTaskResult.cs create mode 100644 src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs create mode 100644 src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs create mode 100644 src/Sitko.Core.Tasks/Execution/JobFailedException.cs create mode 100644 src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs create mode 100644 src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs create mode 100644 src/Sitko.Core.Tasks/IBaseTask.cs create mode 100644 src/Sitko.Core.Tasks/IBaseTaskWithConfigAndResult.cs create mode 100644 src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs create mode 100644 src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs create mode 100644 src/Sitko.Core.Tasks/Scheduling/ScheduleLocks.cs create mode 100644 src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs create mode 100644 src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj create mode 100644 src/Sitko.Core.Tasks/TaskAttribute.cs create mode 100644 src/Sitko.Core.Tasks/TaskStatus.cs create mode 100644 src/Sitko.Core.Tasks/TasksModule.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Sitko.Core.Tasks.Kafka.Tests.csproj diff --git a/Sitko.Core.sln b/Sitko.Core.sln index 203ac7ab5..255923b7c 100644 --- a/Sitko.Core.sln +++ b/Sitko.Core.sln @@ -248,6 +248,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Repository.Grpc. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Sentry", "src\Sitko.Core.Sentry\Sitko.Core.Sentry.csproj", "{5D438687-8576-425A-A8A9-A80B893DCE0C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Tasks", "src\Sitko.Core.Tasks\Sitko.Core.Tasks.csproj", "{76EA307F-182B-4285-B0AD-2A8330198525}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Kafka", "src\Sitko.Core.Kafka\Sitko.Core.Kafka.csproj", "{80293122-8ABD-4CE8-945B-B88AABE87241}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Tasks.Kafka", "src\Sitko.Core.Tasks.Kafka\Sitko.Core.Tasks.Kafka.csproj", "{E1F076AD-2741-4DCC-BA44-9015ED73C875}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Tasks.Kafka.Tests", "tests\Sitko.Core.Tasks.Kafka.Tests\Sitko.Core.Tasks.Kafka.Tests.csproj", "{53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1497,6 +1505,54 @@ Global {5D438687-8576-425A-A8A9-A80B893DCE0C}.Release|x64.Build.0 = Release|Any CPU {5D438687-8576-425A-A8A9-A80B893DCE0C}.Release|x86.ActiveCfg = Release|Any CPU {5D438687-8576-425A-A8A9-A80B893DCE0C}.Release|x86.Build.0 = Release|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Debug|Any CPU.Build.0 = Debug|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Debug|x64.ActiveCfg = Debug|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Debug|x64.Build.0 = Debug|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Debug|x86.ActiveCfg = Debug|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Debug|x86.Build.0 = Debug|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Release|Any CPU.ActiveCfg = Release|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Release|Any CPU.Build.0 = Release|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Release|x64.ActiveCfg = Release|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Release|x64.Build.0 = Release|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Release|x86.ActiveCfg = Release|Any CPU + {76EA307F-182B-4285-B0AD-2A8330198525}.Release|x86.Build.0 = Release|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Debug|x64.ActiveCfg = Debug|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Debug|x64.Build.0 = Debug|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Debug|x86.ActiveCfg = Debug|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Debug|x86.Build.0 = Debug|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Release|Any CPU.ActiveCfg = Release|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Release|Any CPU.Build.0 = Release|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Release|x64.ActiveCfg = Release|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Release|x64.Build.0 = Release|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Release|x86.ActiveCfg = Release|Any CPU + {80293122-8ABD-4CE8-945B-B88AABE87241}.Release|x86.Build.0 = Release|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Debug|x64.ActiveCfg = Debug|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Debug|x64.Build.0 = Debug|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Debug|x86.ActiveCfg = Debug|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Debug|x86.Build.0 = Debug|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Release|Any CPU.Build.0 = Release|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Release|x64.ActiveCfg = Release|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Release|x64.Build.0 = Release|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Release|x86.ActiveCfg = Release|Any CPU + {E1F076AD-2741-4DCC-BA44-9015ED73C875}.Release|x86.Build.0 = Release|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Debug|x64.ActiveCfg = Debug|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Debug|x64.Build.0 = Debug|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Debug|x86.ActiveCfg = Debug|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Debug|x86.Build.0 = Debug|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Release|Any CPU.Build.0 = Release|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Release|x64.ActiveCfg = Release|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Release|x64.Build.0 = Release|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Release|x86.ActiveCfg = Release|Any CPU + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {6677865F-C349-4D25-9E19-3DEC43E92DD5} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} @@ -1603,5 +1659,9 @@ Global {910855FF-013F-4343-B4D1-27721CB65822} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} {5D54C212-07F4-46B6-B86D-EC9D7641ED19} = {10C46F0D-1B13-447E-AD22-F01DDB5A79FD} {5D438687-8576-425A-A8A9-A80B893DCE0C} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} + {76EA307F-182B-4285-B0AD-2A8330198525} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} + {80293122-8ABD-4CE8-945B-B88AABE87241} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} + {E1F076AD-2741-4DCC-BA44-9015ED73C875} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} + {53AC3FA6-64D0-4D26-9F47-EC44A83B0DF5} = {10C46F0D-1B13-447E-AD22-F01DDB5A79FD} EndGlobalSection EndGlobal diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 5446327df..39f733377 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -90,6 +90,15 @@ + + + + + + + + + @@ -157,4 +166,4 @@ - + \ No newline at end of file diff --git a/src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj b/src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj new file mode 100644 index 000000000..0f555d63a --- /dev/null +++ b/src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs new file mode 100644 index 000000000..cba397be9 --- /dev/null +++ b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs @@ -0,0 +1,15 @@ +using Sitko.Core.App; + +namespace Sitko.Core.Tasks.Kafka; + +public static class ApplicationExtensions +{ + public static Application AddKafkaTasks(this Application application, + Action> configure) where TBaseTask : BaseTask + where TDbContext : TasksDbContext + { + application.AddModule, KafkaTasksModuleOptions>( + configure); + return application; + } +} diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs new file mode 100644 index 000000000..aa2e1b3da --- /dev/null +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -0,0 +1,114 @@ +using Confluent.Kafka; +using FluentValidation; +using KafkaFlow; +using KafkaFlow.Consumers.DistributionStrategies; +using KafkaFlow.Serializer; +using KafkaFlow.TypedHandler; +using Microsoft.Extensions.DependencyInjection; +using Sitko.Core.App; +using Sitko.Core.Tasks.Execution; +using AutoOffsetReset = Confluent.Kafka.AutoOffsetReset; + +namespace Sitko.Core.Tasks.Kafka; + +public class + KafkaTasksModule : TasksModule> where TBaseTask : BaseTask + where TDbContext : TasksDbContext +{ + public override string OptionsKey => $"Kafka:Tasks:{typeof(TBaseTask).Name}"; + + protected override void ConfigureServicesInternal(IApplicationContext applicationContext, + IServiceCollection services, + KafkaTasksModuleOptions startupOptions, List executors) + { + var kafkaTopicPrefix = startupOptions.AddTopicPrefix + ? (string.IsNullOrEmpty(startupOptions.TopicPrefix) + ? $"{applicationContext.Name}_{applicationContext.Environment}" + : startupOptions.TopicPrefix) + : ""; + var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}"; + services.AddKafka(builder => + { + builder + .UseMicrosoftLog() + .AddCluster(clusterBuilder => + { + clusterBuilder + .WithName($"Tasks_{typeof(TBaseTask).Name}") + .WithBrokers(startupOptions.Brokers) + .AddProducer("default", producerBuilder => + { + producerBuilder + .DefaultTopic(kafkaTopic) + .AddMiddlewares(configurationBuilder => + configurationBuilder.AddSerializer()); + }); + // регистрируем консьюмеры на каждую группу экзекьюторов + foreach (var groupConsumers in executors.GroupBy(r => r.GroupId)) + { + var commonRegistration = groupConsumers.First(); + var name = + $"{applicationContext.Name}/{applicationContext.Id}/{commonRegistration.GroupId}"; + + var parallelThreadCount = groupConsumers.Max(r => r.ParallelThreadCount); + var bufferSize = groupConsumers.Max(r => r.BufferSize); + + clusterBuilder.AddConsumer( + consumerBuilder => + { + consumerBuilder.Topic(kafkaTopic); + consumerBuilder.WithName(name); + consumerBuilder.WithGroupId(commonRegistration.GroupId); + consumerBuilder.WithWorkersCount(parallelThreadCount); + consumerBuilder.WithBufferSize(bufferSize); + // для гарантии порядка событий + consumerBuilder + .WithWorkDistributionStrategy(); + var consumerConfig = new ConsumerConfig + { + AutoOffsetReset = AutoOffsetReset.Latest, + ClientId = name, + GroupInstanceId = name, + PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky + }; + consumerBuilder.WithConsumerConfig(consumerConfig); + consumerBuilder.AddMiddlewares( + middlewares => + { + middlewares + .AddSerializer(); + middlewares.AddTypedHandlers(handlers => + handlers.AddHandlers(groupConsumers.Select(r => r.ExecutorType))); + } + ); + } + ); + } + }); + }); + } +} + +public class KafkaTasksModuleOptions : TasksModuleOptions + where TBaseTask : BaseTask + where TDbContext : TasksDbContext +{ + public override Type GetValidatorType() => typeof(KafkaModuleOptionsValidator); + public string[] Brokers { get; set; } = Array.Empty(); + public string TasksTopic { get; set; } = ""; + public bool AddTopicPrefix { get; set; } = true; + public string TopicPrefix { get; set; } = ""; +} + +public class + KafkaModuleOptionsValidator : TasksModuleOptionsValidator> where TBaseTask : BaseTask + where TDbContext : TasksDbContext +{ + public KafkaModuleOptionsValidator() + { + RuleFor(options => options.Brokers).NotEmpty().WithMessage("Specify Kafka brokers"); + RuleFor(options => options.TasksTopic).NotEmpty().WithMessage("Specify Kafka topic"); + } +} diff --git a/src/Sitko.Core.Tasks.Kafka/Sitko.Core.Tasks.Kafka.csproj b/src/Sitko.Core.Tasks.Kafka/Sitko.Core.Tasks.Kafka.csproj new file mode 100644 index 000000000..474d1ae7b --- /dev/null +++ b/src/Sitko.Core.Tasks.Kafka/Sitko.Core.Tasks.Kafka.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/Sitko.Core.Tasks/BaseTask.cs b/src/Sitko.Core.Tasks/BaseTask.cs new file mode 100644 index 000000000..cf737ff6b --- /dev/null +++ b/src/Sitko.Core.Tasks/BaseTask.cs @@ -0,0 +1,20 @@ +using System.ComponentModel.DataAnnotations.Schema; +using Sitko.Core.Repository; + +namespace Sitko.Core.Tasks; + +public record BaseTask : EntityRecord, IBaseTask +{ + public string Queue { get; set; } = ""; + public DateTimeOffset DateAdded { get; set; } = DateTimeOffset.UtcNow; + public DateTimeOffset DateUpdated { get; set; } = DateTimeOffset.UtcNow; + public TaskStatus TaskStatus { get; set; } = TaskStatus.Wait; + public DateTimeOffset? ExecuteDateStart { get; set; } + public DateTimeOffset? ExecuteDateEnd { get; set; } +#pragma warning disable CS8618 + public string Type { get; set; } +#pragma warning restore CS8618 + public Guid? ParentId { get; set; } + public string? UserId { get; set; } + public DateTimeOffset? LastActivityDate { get; set; } +} diff --git a/src/Sitko.Core.Tasks/BaseTaskConfig.cs b/src/Sitko.Core.Tasks/BaseTaskConfig.cs new file mode 100644 index 000000000..fc9b19aba --- /dev/null +++ b/src/Sitko.Core.Tasks/BaseTaskConfig.cs @@ -0,0 +1,5 @@ +namespace Sitko.Core.Tasks; + +public record BaseTaskConfig +{ +} \ No newline at end of file diff --git a/src/Sitko.Core.Tasks/BaseTaskResult.cs b/src/Sitko.Core.Tasks/BaseTaskResult.cs new file mode 100644 index 000000000..0cd72f38d --- /dev/null +++ b/src/Sitko.Core.Tasks/BaseTaskResult.cs @@ -0,0 +1,8 @@ +namespace Sitko.Core.Tasks; + +public record BaseTaskResult +{ + public bool IsSuccess { get; set; } = true; + public bool HasWarnings { get; set; } + public string? ErrorMessage { get; set; } +} \ No newline at end of file diff --git a/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs new file mode 100644 index 000000000..040c48d41 --- /dev/null +++ b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs @@ -0,0 +1,158 @@ +using Elastic.Apm.Api; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Serilog.Context; +using Sitko.Core.Repository; + +namespace Sitko.Core.Tasks.Execution; + +public abstract class BaseTaskExecutor : ITaskExecutor + where TTask : class, IBaseTask + where TConfig : BaseTaskConfig, new() + where TResult : BaseTaskResult, new() +{ + private readonly ITracer? tracer; + private readonly ILogger> logger; + private readonly CancellationTokenSource activityTaskCts = new(); + private readonly IRepository repository; + private readonly IServiceScopeFactory serviceScopeFactory; + + protected BaseTaskExecutor(ILogger> logger, ITracer? tracer, + IServiceScopeFactory serviceScopeFactory, IRepository repository) + { + this.logger = logger; + this.tracer = tracer; + this.serviceScopeFactory = serviceScopeFactory; + this.repository = repository; + } + + public async Task ExecuteAsync(Guid id, CancellationToken cancellationToken) + { + using (LogContext.PushProperty("TaskId", id.ToString())) + using (LogContext.PushProperty("TaskType", typeof(TTask).Name)) + { + try + { + if (tracer is not null) + { + await tracer.CaptureTransaction($"Tasks/{typeof(TTask)}", "Task", async transaction => + { + transaction.SetLabel("jobId", id.ToString()); + var task = await ExecuteTaskAsync(id, cancellationToken); + if (task.TaskStatus == TaskStatus.Fails) + { + throw new JobFailedException(id, typeof(TTask).Name, + task.Result?.ErrorMessage ?? "Unknown error"); + } + }); + } + else + { + await ExecuteTaskAsync(id, cancellationToken); + } + } + catch (JobFailedException) + { + logger.LogDebug("Mark transaction as failed"); + } + catch (Exception ex) + { + logger.LogError(ex, "Task {JobId} ( {JobType} ) failed: {ErrorText}", id, typeof(TTask).Name, + ex.ToString()); + } + } + } + + private async Task ExecuteTaskAsync(Guid id, CancellationToken cancellationToken) + { + logger.LogInformation("Start job {JobId}", id); + var task = await repository.GetByIdAsync(id, cancellationToken); + if (task is null) + { + throw new InvalidOperationException($"Task {id} not found"); + } + + if (task.TaskStatus != TaskStatus.Wait) + { + logger.LogInformation("Skip retry task {JobId}", id); + return task; + } + + logger.LogInformation("Set job {JobId} status to in progress", id); + task.ExecuteDateStart = DateTimeOffset.UtcNow; + task.TaskStatus = TaskStatus.InProgress; + task.LastActivityDate = DateTimeOffset.UtcNow; + await repository.UpdateAsync(task, cancellationToken); + + TResult result; + TaskStatus status; + var activityTask = Task.Run(async () => + { + while (!activityTaskCts.IsCancellationRequested) + { + await using var scope = serviceScopeFactory.CreateAsyncScope(); + var scopedRepository = scope.ServiceProvider.GetRequiredService>(); + var scopedTask = await scopedRepository.GetByIdAsync(id, CancellationToken.None); + if (scopedTask is not null) + { + scopedTask.LastActivityDate = DateTimeOffset.UtcNow; + await scopedRepository.UpdateAsync(scopedTask, CancellationToken.None); + } + + await Task.Delay(TimeSpan.FromSeconds(5), activityTaskCts.Token); + } + }, activityTaskCts.Token); + try + { + logger.LogInformation("Try to execute job {JobId}", id); + result = await ExecuteAsync(task, cancellationToken); + if (result.IsSuccess) + { + logger.LogInformation("Job {JobId} executed successfully", id); + status = result.HasWarnings + ? TaskStatus.SuccessWithWarnings + : TaskStatus.Success; + } + else + { + logger.LogInformation("Job {JobId} execution failed", id); + status = TaskStatus.Fails; + } + } + catch (Exception ex) + { + logger.LogInformation(ex, "Job {JobId} execution failed with error: {ErrorText}", id, ex.ToString()); + status = TaskStatus.Fails; + result = new TResult { IsSuccess = false, ErrorMessage = ex.Message }; + } + + activityTaskCts.Cancel(); + try + { + await activityTask; + } + catch (OperationCanceledException) + { + // do nothing + } + catch (Exception ex) + { + logger.LogInformation(ex, "Activity task error: {ErrorText}", ex.ToString()); + } + + logger.LogInformation("Set job {JobId} result and save", id); + task = await repository.RefreshAsync(task, cancellationToken); + task.Result = result; + task.TaskStatus = status; + task.ExecuteDateEnd = DateTimeOffset.UtcNow; + + await repository.UpdateAsync(task, cancellationToken); + logger.LogInformation("Job {JobId} finished", id); + return task; + } + + protected abstract Task ExecuteAsync(TTask task, CancellationToken cancellationToken); +} + +public record ExecutorRegistration(Type ExecutorType, Type EventType, string GroupId, int ParallelThreadCount, + int BufferSize); diff --git a/src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs b/src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs new file mode 100644 index 000000000..e4a0c1d30 --- /dev/null +++ b/src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs @@ -0,0 +1,13 @@ +namespace Sitko.Core.Tasks.Execution; + +public interface ITaskExecutor +{ + Task ExecuteAsync(Guid id, CancellationToken cancellationToken); +} + +public interface ITaskExecutor : ITaskExecutor + where TTask : class, IBaseTask + where TConfig : BaseTaskConfig, new() + where TResult : BaseTaskResult, new() +{ +} diff --git a/src/Sitko.Core.Tasks/Execution/JobFailedException.cs b/src/Sitko.Core.Tasks/Execution/JobFailedException.cs new file mode 100644 index 000000000..ba286284b --- /dev/null +++ b/src/Sitko.Core.Tasks/Execution/JobFailedException.cs @@ -0,0 +1,8 @@ +namespace Sitko.Core.Tasks.Execution; + +public class JobFailedException : Exception +{ + public JobFailedException(Guid id, string type, string message) : base($"Task {id} ( {type} ) failed: {message}") + { + } +} \ No newline at end of file diff --git a/src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs b/src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs new file mode 100644 index 000000000..2172276f2 --- /dev/null +++ b/src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs @@ -0,0 +1,18 @@ +namespace Sitko.Core.Tasks.Execution; + +[AttributeUsage(AttributeTargets.Class)] +public class TaskExecutorAttribute : Attribute +{ + public TaskExecutorAttribute(string groupId, int parallelThreadCount, int bufferSize = 10) + { + GroupId = groupId; + ParallelThreadCount = Math.Max(parallelThreadCount, 1); + BufferSize = Math.Max(bufferSize, 1); + } + + public string GroupId { get; } + + public int ParallelThreadCount { get; } + + public int BufferSize { get; } +} diff --git a/src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs b/src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs new file mode 100644 index 000000000..d2d031f32 --- /dev/null +++ b/src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs @@ -0,0 +1,22 @@ +using System.Reflection; + +namespace Sitko.Core.Tasks.Execution; + +internal static class TaskExecutorHelper +{ + public static (string GroupId, int ParallelThreadCount, int BufferSize)? GetGroupInfo(Type eventProcessorType) + { + var attribute = eventProcessorType.FindAttribute(); + if (attribute is null) + { + return null; + } + + var groupId = attribute.GroupId; + return (groupId, attribute.ParallelThreadCount, attribute.BufferSize); + } + + public static TResult? FindAttribute(this ICustomAttributeProvider provider, bool withInherit = true) + where TResult : Attribute => + provider.GetCustomAttributes(typeof(TResult), withInherit).Cast().FirstOrDefault(); +} diff --git a/src/Sitko.Core.Tasks/IBaseTask.cs b/src/Sitko.Core.Tasks/IBaseTask.cs new file mode 100644 index 000000000..2c15f9d0a --- /dev/null +++ b/src/Sitko.Core.Tasks/IBaseTask.cs @@ -0,0 +1,29 @@ +using System.ComponentModel.DataAnnotations.Schema; +using Sitko.Core.Repository; + +namespace Sitko.Core.Tasks; + +public interface IBaseTask : IEntity +{ + TaskStatus TaskStatus { get; set; } + DateTimeOffset? ExecuteDateStart { get; set; } + DateTimeOffset? ExecuteDateEnd { get; set; } + string Type { get; set; } + Guid? ParentId { get; set; } + string? UserId { get; set; } + DateTimeOffset? LastActivityDate { get; set; } +} + +public interface IBaseTask : IBaseTask where TConfig : BaseTaskConfig, new() +{ + [Column(TypeName = "jsonb")] TConfig Config { get; set; } +} + +public interface IBaseTask : IBaseTask, IBaseTaskWithConfigAndResult + where TConfig : BaseTaskConfig, new() where TResult : BaseTaskResult +{ + [Column(TypeName = "jsonb")] TResult? Result { get; set; } + + Type IBaseTaskWithConfigAndResult.ConfigType => typeof(TConfig); + Type IBaseTaskWithConfigAndResult.ResultType => typeof(TResult); +} diff --git a/src/Sitko.Core.Tasks/IBaseTaskWithConfigAndResult.cs b/src/Sitko.Core.Tasks/IBaseTaskWithConfigAndResult.cs new file mode 100644 index 000000000..18576a5ba --- /dev/null +++ b/src/Sitko.Core.Tasks/IBaseTaskWithConfigAndResult.cs @@ -0,0 +1,7 @@ +namespace Sitko.Core.Tasks; + +public interface IBaseTaskWithConfigAndResult +{ + Type ConfigType { get; } + Type ResultType { get; } +} \ No newline at end of file diff --git a/src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs b/src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs new file mode 100644 index 000000000..d712a312d --- /dev/null +++ b/src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs @@ -0,0 +1,63 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Nito.AsyncEx; + +namespace Sitko.Core.Tasks.Scheduling; + +public abstract class BaseTaskScheduler : BackgroundService where TTask : IBaseTask +{ + private readonly IOptions> taskOptions; + private readonly IOptionsMonitor optionsMonitor; + private readonly ITaskScheduler taskScheduler; + protected ILogger> Logger { get; } + + protected BaseTaskScheduler(IOptions> taskOptions, + IOptionsMonitor optionsMonitor, + ITaskScheduler taskScheduler, + ILogger> logger) + { + this.taskOptions = taskOptions; + this.optionsMonitor = optionsMonitor; + this.taskScheduler = taskScheduler; + Logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + await ScheduleAsync(stoppingToken); + try + { + await Task.Delay(taskOptions.Value.Interval, stoppingToken); + } + catch (TaskCanceledException) + { + // do nothing + } + } + } + + private async Task ScheduleAsync(CancellationToken cancellationToken) + { + if (optionsMonitor.CurrentValue.IsAllTasksDisabled || + optionsMonitor.CurrentValue.DisabledTasks.Contains(typeof(TTask).Name)) + { + return; + } + + var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)); + var scheduleLock = ScheduleLocks.Locks.GetOrAdd(GetType().Name, _ => new AsyncLock()); + using (await scheduleLock.LockAsync(cts.Token)) + { + var tasks = await GetTasksAsync(cancellationToken); + foreach (var task in tasks) + { + await taskScheduler.ScheduleAsync(task); + } + } + } + + protected abstract Task GetTasksAsync(CancellationToken cancellationToken); +} diff --git a/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs b/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs new file mode 100644 index 000000000..b258894ee --- /dev/null +++ b/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs @@ -0,0 +1,6 @@ +namespace Sitko.Core.Tasks.Scheduling; + +public interface ITaskScheduler +{ + Task ScheduleAsync(IBaseTask task); +} diff --git a/src/Sitko.Core.Tasks/Scheduling/ScheduleLocks.cs b/src/Sitko.Core.Tasks/Scheduling/ScheduleLocks.cs new file mode 100644 index 000000000..ddd542c3d --- /dev/null +++ b/src/Sitko.Core.Tasks/Scheduling/ScheduleLocks.cs @@ -0,0 +1,9 @@ +using System.Collections.Concurrent; +using Nito.AsyncEx; + +namespace Sitko.Core.Tasks.Scheduling; + +public static class ScheduleLocks +{ + public static ConcurrentDictionary Locks { get; } = new(); +} diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs new file mode 100644 index 000000000..932ef800f --- /dev/null +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs @@ -0,0 +1,6 @@ +namespace Sitko.Core.Tasks.Scheduling; + +public class TaskSchedulingOptions where TTask : IBaseTask +{ + public TimeSpan Interval { get; set; } +} diff --git a/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj b/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj new file mode 100644 index 000000000..41a1bf675 --- /dev/null +++ b/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/Sitko.Core.Tasks/TaskAttribute.cs b/src/Sitko.Core.Tasks/TaskAttribute.cs new file mode 100644 index 000000000..d728720fd --- /dev/null +++ b/src/Sitko.Core.Tasks/TaskAttribute.cs @@ -0,0 +1,8 @@ +namespace Sitko.Core.Tasks; + +[AttributeUsage(AttributeTargets.Class)] +public class TaskAttribute : Attribute +{ + public TaskAttribute(string key) => Key = key; + public string Key { get; } +} \ No newline at end of file diff --git a/src/Sitko.Core.Tasks/TaskStatus.cs b/src/Sitko.Core.Tasks/TaskStatus.cs new file mode 100644 index 000000000..ed99bd170 --- /dev/null +++ b/src/Sitko.Core.Tasks/TaskStatus.cs @@ -0,0 +1,10 @@ +namespace Sitko.Core.Tasks; + +public enum TaskStatus +{ + Wait = 0, + InProgress = 1, + SuccessWithWarnings = 2, + Success = 3, + Fails = 4 +} \ No newline at end of file diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs new file mode 100644 index 000000000..b25fd22c8 --- /dev/null +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -0,0 +1,179 @@ +using System.Reflection; +using FluentValidation; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.Extensions.DependencyInjection; +using Sitko.Core.App; +using Sitko.Core.Db.Postgres; +using Sitko.Core.Tasks.Execution; + +namespace Sitko.Core.Tasks; + +public abstract class + TasksModule : BaseApplicationModule + where TOptions : TasksModuleOptions, new() + where TDbContext : TasksDbContext + where TBaseTask : BaseTask +{ + public override string OptionsKey => $"Tasks:{typeof(TBaseTask).Name}"; + + public override void ConfigureServices(IApplicationContext applicationContext, IServiceCollection services, + TOptions startupOptions) + { + base.ConfigureServices(applicationContext, services, startupOptions); + + var types = new List(); + if (startupOptions.Assemblies.Count > 0) + { + foreach (var assembly in startupOptions.Assemblies) + { + types.AddRange(assembly.ExportedTypes.Where(type => typeof(ITaskExecutor).IsAssignableFrom(type))); + } + } + + var executors = new List(); + foreach (var executorType in types) + { + var groupInfo = TaskExecutorHelper.GetGroupInfo(executorType); + if (groupInfo is null) + { + throw new InvalidOperationException( + $"Consumer {executorType} must have attribute TaskExecutorAttribute"); + } + + var eventType = executorType.GetInterfaces() + .First(i => i.IsGenericType && typeof(ITaskExecutor).IsAssignableFrom(i)).GenericTypeArguments + .First(); + var registration = new ExecutorRegistration(executorType, eventType, groupInfo.Value.GroupId, + groupInfo.Value.ParallelThreadCount, groupInfo.Value.BufferSize); + executors.Add(registration); + } + + ConfigureServicesInternal(applicationContext, services, startupOptions, executors); + } + + protected abstract void ConfigureServicesInternal(IApplicationContext applicationContext, + IServiceCollection services, TOptions startupOptions, + List executors); +} + +public class TasksModuleOptions +{ + public bool IsAllTasksDisabled { get; set; } + public string[] DisabledTasks { get; set; } = Array.Empty(); + + public int? AllTasksRetentionDays { get; set; } + public Dictionary RetentionDays { get; set; } = new(); +} + +public abstract class TasksModuleOptions : BaseModuleOptions, IModuleOptionsWithValidation + where TDbContext : TasksDbContext where TBaseTask : BaseTask +{ + public List Assemblies { get; } = new(); + + public TasksModuleOptions AddExecutorsFromAssemblyOf() + { + Assemblies.Add(typeof(TAssembly).Assembly); + return this; + } + + internal bool HasJobs => false; + + public TasksModuleOptions AddTask(TimeSpan interval) + where TTask : class, IBaseTask + where TConfig : BaseTaskConfig, new() + where TResult : BaseTaskResult, new() + { + return this; + } + + public abstract Type GetValidatorType(); +} + +public abstract class TasksDbContext : BaseDbContext where TBaseTask : BaseTask +{ + private readonly DbContextOptions options; + protected TasksDbContext(DbContextOptions options) : base(options) => this.options = options; + + private DbSet Tasks => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + var discriminatorBuilder = modelBuilder.Entity().HasDiscriminator(nameof(BaseTask.Type)); + + var tasksExtension = options.FindExtension>(); + if (tasksExtension is not null) + { + tasksExtension.Configure(modelBuilder, discriminatorBuilder); + } + + modelBuilder.Entity().Property(task => task.Queue).HasDefaultValue("default"); + } +} + +internal class TasksDbContextOptionsExtension : IDbContextOptionsExtension where TBaseTask : BaseTask +{ + private readonly List>> discriminatorConfigurations = new(); + public void ApplyServices(IServiceCollection services) { } + + public void Validate(IDbContextOptions options) { } + + public DbContextOptionsExtensionInfo Info => new TasksDbContextOptionsExtensionInfo(this); + + public void Register() where TTask : class, IBaseTask + where TConfig : BaseTaskConfig, new() + where TResult : BaseTaskResult => + discriminatorConfigurations.Add((modelBuilder, discriminatorBuilder) => + { + modelBuilder.Entity().Property(task => task.Config).HasColumnType("jsonb") + .HasColumnName(nameof(IBaseTask.Config)); + modelBuilder.Entity().Property(task => task.Result).HasColumnType("jsonb") + .HasColumnName(nameof(IBaseTask.Result)); + var attr = typeof(TTask).GetCustomAttributes(typeof(TaskAttribute), true).Cast() + .FirstOrDefault(); + discriminatorBuilder.HasValue(attr is null ? typeof(TTask).Name : attr.Key); + }); + + public void Configure(ModelBuilder modelBuilder, DiscriminatorBuilder discriminatorBuilder) + { + foreach (var discriminatorConfiguration in discriminatorConfigurations) + { + discriminatorConfiguration(modelBuilder, discriminatorBuilder); + } + } +} + +internal class TasksDbContextOptionsExtensionInfo : DbContextOptionsExtensionInfo where TBaseTask : BaseTask +{ + public TasksDbContextOptionsExtensionInfo(IDbContextOptionsExtension extension) : base(extension) + { + } + +#if NET6_0_OR_GREATER + public override int GetServiceProviderHashCode() => Extension.GetHashCode(); +#else + public override long GetServiceProviderHashCode() => Extension.GetHashCode(); +#endif + +#if NET6_0_OR_GREATER + public override bool ShouldUseSameServiceProvider(DbContextOptionsExtensionInfo other) => true; +#endif + public override void PopulateDebugInfo(IDictionary debugInfo) + { + } + + public override bool IsDatabaseProvider => false; + public override string LogFragment => nameof(TasksDbContextOptionsExtension); +} + +public abstract class + TasksModuleOptionsValidator : AbstractValidator + where TOptions : TasksModuleOptions + where TBaseTask : BaseTask + where TDbContext : TasksDbContext +{ + protected TasksModuleOptionsValidator() => RuleFor(o => o.HasJobs).Equal(true) + .WithMessage("Необходимо сконфигурировать хотя бы одну задачу"); +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs new file mode 100644 index 000000000..b809ec618 --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs @@ -0,0 +1,28 @@ +using Sitko.Core.Tasks.Kafka.Tests.Data; +using Sitko.Core.Xunit; +using Xunit.Abstractions; + +namespace Sitko.Core.Tasks.Kafka.Tests; + +public abstract class BaseKafkaTasksTest : BaseTest +{ + protected BaseKafkaTasksTest(ITestOutputHelper testOutputHelper) : base(testOutputHelper) + { + } +} + +public class BaseKafkaTasksTestScope : BaseTestScope +{ + protected override TestApplication ConfigureApplication(TestApplication application, string name) + { + base + .ConfigureApplication(application, name) + .AddKafkaTasks(options => + { + options + .AddTask(TimeSpan.FromMinutes(1)) + .AddExecutorsFromAssemblyOf(); + }); + return application; + } +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs new file mode 100644 index 000000000..5752adbe1 --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs @@ -0,0 +1,13 @@ +namespace Sitko.Core.Tasks.Kafka.Tests.Data; + +public abstract record BaseTestTask : BaseTask +{ + public int FooId { get; set; } +} + +public abstract record BaseTestTask : BaseTestTask, IBaseTask + where TConfig : BaseTaskConfig, new() where TResult : BaseTaskResult +{ + public TConfig Config { get; set; } = default!; + public TResult? Result { get; set; } +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs new file mode 100644 index 000000000..4d490c726 --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs @@ -0,0 +1,10 @@ +using Microsoft.EntityFrameworkCore; + +namespace Sitko.Core.Tasks.Kafka.Tests.Data; + +public class TestDbContext : TasksDbContext +{ + public TestDbContext(DbContextOptions options) : base(options) + { + } +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs new file mode 100644 index 000000000..34e02625c --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs @@ -0,0 +1,15 @@ +namespace Sitko.Core.Tasks.Kafka.Tests.Data; + +public record TestTask : BaseTestTask +{ +} + +public record TestTaskResult : BaseTaskResult +{ + public int Foo { get; init; } + public Guid Id { get; init; } +} + +public record TestTaskConfig : BaseTaskConfig +{ +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs new file mode 100644 index 000000000..6d515b4d4 --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs @@ -0,0 +1,21 @@ +using Elastic.Apm.Api; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Sitko.Core.Repository; +using Sitko.Core.Tasks.Execution; + +namespace Sitko.Core.Tasks.Kafka.Tests.Data; + +public class TestTaskExecutor : BaseTaskExecutor +{ + public TestTaskExecutor(ILogger logger, ITracer? tracer, IServiceScopeFactory serviceScopeFactory, + IRepository repository) : base(logger, tracer, serviceScopeFactory, repository) + { + } + + protected override Task ExecuteAsync(TestTask task, CancellationToken cancellationToken) + { + var result = new TestTaskResult { Id = Guid.NewGuid(), Foo = task.FooId }; + return Task.FromResult(result); + } +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs new file mode 100644 index 000000000..9a5566f19 --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs @@ -0,0 +1,20 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Sitko.Core.Tasks.Scheduling; + +namespace Sitko.Core.Tasks.Kafka.Tests.Data; + +public class TestTaskScheduler : BaseTaskScheduler +{ + public TestTaskScheduler(IOptions> taskOptions, + IOptionsMonitor optionsMonitor, ITaskScheduler taskScheduler, + ILogger logger) : base(taskOptions, optionsMonitor, taskScheduler, logger) + { + } + + protected override Task GetTasksAsync(CancellationToken cancellationToken) + { + var tasks = new[] { new TestTask { Id = Guid.NewGuid(), FooId = 1 } }; + return Task.FromResult(tasks); + } +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Sitko.Core.Tasks.Kafka.Tests.csproj b/tests/Sitko.Core.Tasks.Kafka.Tests/Sitko.Core.Tasks.Kafka.Tests.csproj new file mode 100644 index 000000000..8776bd8ba --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Sitko.Core.Tasks.Kafka.Tests.csproj @@ -0,0 +1,5 @@ + + + + + From 77384a3723e09b06e55cafd6624cac63b8f202c7 Mon Sep 17 00:00:00 2001 From: nadia Date: Thu, 5 Oct 2023 14:44:29 +0500 Subject: [PATCH 02/32] feat(tasks): refactors and improvements tasks module --- src/Directory.Packages.props | 1 + src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj | 1 + .../ApplicationExtensions.cs | 8 +- .../Execution/KafkaExecutor.cs | 16 +++ .../KafkaTasksModule.cs | 22 ++- .../KafkaTasksModuleOptions.cs | 15 ++ .../Scheduling/KafkaTaskScheduler.cs | 15 ++ src/Sitko.Core.Tasks/ApplicationExtensions.cs | 30 ++++ src/Sitko.Core.Tasks/BaseTaskConfig.cs | 5 - .../Components/TasksManager.cs | 76 +++++++++++ .../{ => Data/Entities}/BaseTask.cs | 6 +- .../Data/Entities/BaseTaskConfig.cs | 3 + .../{ => Data/Entities}/BaseTaskResult.cs | 2 +- .../{ => Data/Entities}/IBaseTask.cs | 3 +- .../Entities}/IBaseTaskWithConfigAndResult.cs | 2 +- .../{ => Data/Entities}/TaskStatus.cs | 4 +- .../Data/Repository/BaseTaskRepository.cs | 23 ++++ .../Data/Repository/ITaskRepository.cs | 10 ++ src/Sitko.Core.Tasks/Data/TasksDbContext.cs | 24 ++++ .../Data/TasksDbContextOptionsExtension.cs | 43 ++++++ .../TasksDbContextOptionsExtensionInfo.cs | 27 ++++ .../Execution/BaseTaskExecutor.cs | 6 +- .../Execution/ITaskExecutor.cs | 10 +- .../Scheduling/BaseTaskScheduler.cs | 63 --------- .../Scheduling/IBaseTaskFactory.cs | 8 ++ .../Scheduling/ITaskScheduler.cs | 6 +- .../Scheduling/TaskSchedulingOptions.cs | 2 + .../Scheduling/TaskSchedulingService.cs | 75 ++++++++++ src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj | 1 + src/Sitko.Core.Tasks/TasksModule.cs | 128 ++---------------- src/Sitko.Core.Tasks/TasksModuleOptions.cs | 85 ++++++++++++ .../Data/BaseTestTask.cs | 2 + .../Data/TestDbContext.cs | 1 + .../Data/TestTask.cs | 4 +- .../Data/TestTaskExecutor.cs | 2 +- .../Data/TestTaskFactory.cs | 12 ++ .../Data/TestTaskScheduler.cs | 20 --- 37 files changed, 526 insertions(+), 235 deletions(-) create mode 100644 src/Sitko.Core.Tasks.Kafka/Execution/KafkaExecutor.cs create mode 100644 src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs create mode 100644 src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs create mode 100644 src/Sitko.Core.Tasks/ApplicationExtensions.cs delete mode 100644 src/Sitko.Core.Tasks/BaseTaskConfig.cs create mode 100644 src/Sitko.Core.Tasks/Components/TasksManager.cs rename src/Sitko.Core.Tasks/{ => Data/Entities}/BaseTask.cs (85%) create mode 100644 src/Sitko.Core.Tasks/Data/Entities/BaseTaskConfig.cs rename src/Sitko.Core.Tasks/{ => Data/Entities}/BaseTaskResult.cs (80%) rename src/Sitko.Core.Tasks/{ => Data/Entities}/IBaseTask.cs (93%) rename src/Sitko.Core.Tasks/{ => Data/Entities}/IBaseTaskWithConfigAndResult.cs (72%) rename src/Sitko.Core.Tasks/{ => Data/Entities}/TaskStatus.cs (73%) create mode 100644 src/Sitko.Core.Tasks/Data/Repository/BaseTaskRepository.cs create mode 100644 src/Sitko.Core.Tasks/Data/Repository/ITaskRepository.cs create mode 100644 src/Sitko.Core.Tasks/Data/TasksDbContext.cs create mode 100644 src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtension.cs create mode 100644 src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtensionInfo.cs delete mode 100644 src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs create mode 100644 src/Sitko.Core.Tasks/Scheduling/IBaseTaskFactory.cs create mode 100644 src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs create mode 100644 src/Sitko.Core.Tasks/TasksModuleOptions.cs create mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskFactory.cs delete mode 100644 tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 39f733377..e5c592cce 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -98,6 +98,7 @@ + diff --git a/src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj b/src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj index 0f555d63a..f0ebb48d7 100644 --- a/src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj +++ b/src/Sitko.Core.Kafka/Sitko.Core.Kafka.csproj @@ -1,6 +1,7 @@ + diff --git a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs index cba397be9..39ad33fa9 100644 --- a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs +++ b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs @@ -1,15 +1,21 @@ using Sitko.Core.App; +using Sitko.Core.Db.Postgres; +using Sitko.Core.Tasks.Data; +using Sitko.Core.Tasks.Data.Entities; namespace Sitko.Core.Tasks.Kafka; public static class ApplicationExtensions { public static Application AddKafkaTasks(this Application application, - Action> configure) where TBaseTask : BaseTask + Action> configure, bool configurePostgres = false, + Action>? configurePostgresAction = null) where TBaseTask : BaseTask where TDbContext : TasksDbContext { + application.AddTasks(configurePostgres, configurePostgresAction); application.AddModule, KafkaTasksModuleOptions>( configure); + return application; } } diff --git a/src/Sitko.Core.Tasks.Kafka/Execution/KafkaExecutor.cs b/src/Sitko.Core.Tasks.Kafka/Execution/KafkaExecutor.cs new file mode 100644 index 000000000..a8e818c39 --- /dev/null +++ b/src/Sitko.Core.Tasks.Kafka/Execution/KafkaExecutor.cs @@ -0,0 +1,16 @@ +using KafkaFlow; +using KafkaFlow.TypedHandler; +using Sitko.Core.Tasks.Data.Entities; +using Sitko.Core.Tasks.Execution; + +namespace Sitko.Core.Tasks.Kafka.Execution; + +public class KafkaExecutor : IMessageHandler where TTask : class, IBaseTask where TExecutor : ITaskExecutor +{ + private readonly TExecutor taskExecutor; + + public KafkaExecutor(TExecutor taskExecutor) => this.taskExecutor = taskExecutor; + + public async Task Handle(IMessageContext context, TTask message) => + await taskExecutor.ExecuteAsync(message.Id, CancellationToken.None); +} diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index aa2e1b3da..fb12b356f 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -6,7 +6,12 @@ using KafkaFlow.TypedHandler; using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App; +using Sitko.Core.Tasks.Data; +using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Execution; +using Sitko.Core.Tasks.Kafka.Execution; +using Sitko.Core.Tasks.Kafka.Scheduling; +using Sitko.Core.Tasks.Scheduling; using AutoOffsetReset = Confluent.Kafka.AutoOffsetReset; namespace Sitko.Core.Tasks.Kafka; @@ -28,7 +33,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio : startupOptions.TopicPrefix) : ""; var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}"; - services.AddKafka(builder => + services.AddKafkaFlowHostedService(builder => { builder .UseMicrosoftLog() @@ -45,6 +50,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio configurationBuilder.AddSerializer()); }); // регистрируем консьюмеры на каждую группу экзекьюторов + var executorType = typeof(KafkaExecutor<,>); foreach (var groupConsumers in executors.GroupBy(r => r.GroupId)) { var commonRegistration = groupConsumers.First(); @@ -79,7 +85,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio middlewares .AddSerializer(); middlewares.AddTypedHandlers(handlers => - handlers.AddHandlers(groupConsumers.Select(r => r.ExecutorType))); + handlers.AddHandlers(groupConsumers.Select(r => executorType.MakeGenericType(r.EventType, r.ExecutorType))).WithHandlerLifetime(InstanceLifetime.Scoped)); } ); } @@ -87,20 +93,10 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio } }); }); + services.AddSingleton(typeof(ITaskScheduler<>), typeof(KafkaTaskScheduler<>)); } } -public class KafkaTasksModuleOptions : TasksModuleOptions - where TBaseTask : BaseTask - where TDbContext : TasksDbContext -{ - public override Type GetValidatorType() => typeof(KafkaModuleOptionsValidator); - public string[] Brokers { get; set; } = Array.Empty(); - public string TasksTopic { get; set; } = ""; - public bool AddTopicPrefix { get; set; } = true; - public string TopicPrefix { get; set; } = ""; -} - public class KafkaModuleOptionsValidator : TasksModuleOptionsValidator> where TBaseTask : BaseTask diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs new file mode 100644 index 000000000..193bf3549 --- /dev/null +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs @@ -0,0 +1,15 @@ +using Sitko.Core.Tasks.Data; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Kafka; + +public class KafkaTasksModuleOptions : TasksModuleOptions + where TBaseTask : BaseTask + where TDbContext : TasksDbContext +{ + public override Type GetValidatorType() => typeof(KafkaModuleOptionsValidator); + public string[] Brokers { get; set; } = Array.Empty(); + public string TasksTopic { get; set; } = ""; + public bool AddTopicPrefix { get; set; } = true; + public string TopicPrefix { get; set; } = ""; +} diff --git a/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs b/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs new file mode 100644 index 000000000..a5a97c652 --- /dev/null +++ b/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs @@ -0,0 +1,15 @@ +using KafkaFlow.Producers; +using Sitko.Core.Tasks.Data.Entities; +using Sitko.Core.Tasks.Scheduling; + +namespace Sitko.Core.Tasks.Kafka.Scheduling; + +public class KafkaTaskScheduler : ITaskScheduler where TTask : IBaseTask +{ + private readonly IProducerAccessor producerAccessor; + + public KafkaTaskScheduler(IProducerAccessor producerAccessor) => this.producerAccessor = producerAccessor; + + public async Task ScheduleAsync(TTask task) => + await producerAccessor.GetProducer("default").ProduceAsync(task.GetKey(), task); +} diff --git a/src/Sitko.Core.Tasks/ApplicationExtensions.cs b/src/Sitko.Core.Tasks/ApplicationExtensions.cs new file mode 100644 index 000000000..8b7ca3020 --- /dev/null +++ b/src/Sitko.Core.Tasks/ApplicationExtensions.cs @@ -0,0 +1,30 @@ +using Microsoft.Extensions.DependencyInjection; +using Sitko.Core.App; +using Sitko.Core.Db; +using Sitko.Core.Db.Postgres; +using Sitko.Core.Tasks.Data; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks; + +public static class ApplicationExtensions +{ + public static Application AddTasks(this Application application, bool configurePostgres = false, + Action>? configurePostgresAction = null) where TBaseTask : BaseTask + where TDbContext : TasksDbContext + { + if (configurePostgres) + { + application.AddPostgresDatabase(options => + { + configurePostgresAction?.Invoke(options); + options.ConfigureDbContextOptions = (builder, provider, _) => + { + builder.AddExtension(provider.GetRequiredService>()); + }; + }); + } + + return application; + } +} diff --git a/src/Sitko.Core.Tasks/BaseTaskConfig.cs b/src/Sitko.Core.Tasks/BaseTaskConfig.cs deleted file mode 100644 index fc9b19aba..000000000 --- a/src/Sitko.Core.Tasks/BaseTaskConfig.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace Sitko.Core.Tasks; - -public record BaseTaskConfig -{ -} \ No newline at end of file diff --git a/src/Sitko.Core.Tasks/Components/TasksManager.cs b/src/Sitko.Core.Tasks/Components/TasksManager.cs new file mode 100644 index 000000000..5dbad6cd7 --- /dev/null +++ b/src/Sitko.Core.Tasks/Components/TasksManager.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.DependencyInjection; +using Sitko.Core.App.Results; +using Sitko.Core.Repository; +using Sitko.Core.Tasks.Data.Entities; +using Sitko.Core.Tasks.Data.Repository; +using Sitko.Core.Tasks.Scheduling; + +namespace Sitko.Core.Tasks.Components; + +public class TasksManager +{ + private readonly IEnumerable repositories; + private readonly IServiceProvider serviceProvider; + + public TasksManager(IEnumerable repositories, IServiceProvider serviceProvider) + { + this.repositories = repositories; + this.serviceProvider = serviceProvider; + } + + private ITaskRepository GetRepository() where TTask : class, IBaseTask + { + foreach (var repository in repositories) + { + if (repository is ITaskRepository taskRepository) + { + return taskRepository; + } + } + + throw new InvalidOperationException("Repository not found"); + } + + public Task GetByIdAsync(Guid id, CancellationToken cancellationToken = default) + where TTask : class, IBaseTask => GetRepository().GetByIdAsync(id, cancellationToken); + + public async Task GetParentAsync(Guid id, CancellationToken cancellationToken = default) + where TTask : class, IBaseTask + { + var task = await GetByIdAsync(id, cancellationToken); + if (task is { ParentId: { } }) + { + return await GetByIdAsync(task.ParentId.Value, cancellationToken); + } + + return null; + } + + public async Task GetChildrenAsync(Guid id, CancellationToken cancellationToken = default) + where TTask : class, IBaseTask + { + var (childTasks, _) = + await GetRepository().GetAllAsync(q => q.Where(t => t.ParentId == id), cancellationToken); + return childTasks; + } + + public async Task> RunAsync(TTask newTask, Guid? parentId = null, + string? userId = null, + CancellationToken cancellationToken = default) + where TTask : class, IBaseTask + { + var repository = GetRepository(); + newTask.ParentId = parentId; + newTask.UserId = userId; + var addResponse = await repository.AddAsync(newTask, cancellationToken); + if (!addResponse.IsSuccess) + { + return new OperationResult(addResponse.ErrorsString); + } + + var taskScheduler = serviceProvider.GetRequiredService>(); + await taskScheduler.ScheduleAsync(newTask); + + return new OperationResult(addResponse.Entity); + } +} diff --git a/src/Sitko.Core.Tasks/BaseTask.cs b/src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs similarity index 85% rename from src/Sitko.Core.Tasks/BaseTask.cs rename to src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs index cf737ff6b..1866d508d 100644 --- a/src/Sitko.Core.Tasks/BaseTask.cs +++ b/src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs @@ -1,7 +1,6 @@ -using System.ComponentModel.DataAnnotations.Schema; -using Sitko.Core.Repository; +using Sitko.Core.Repository; -namespace Sitko.Core.Tasks; +namespace Sitko.Core.Tasks.Data.Entities; public record BaseTask : EntityRecord, IBaseTask { @@ -17,4 +16,5 @@ public record BaseTask : EntityRecord, IBaseTask public Guid? ParentId { get; set; } public string? UserId { get; set; } public DateTimeOffset? LastActivityDate { get; set; } + public string GetKey() => Type; } diff --git a/src/Sitko.Core.Tasks/Data/Entities/BaseTaskConfig.cs b/src/Sitko.Core.Tasks/Data/Entities/BaseTaskConfig.cs new file mode 100644 index 000000000..418bfc8b7 --- /dev/null +++ b/src/Sitko.Core.Tasks/Data/Entities/BaseTaskConfig.cs @@ -0,0 +1,3 @@ +namespace Sitko.Core.Tasks.Data.Entities; + +public record BaseTaskConfig; diff --git a/src/Sitko.Core.Tasks/BaseTaskResult.cs b/src/Sitko.Core.Tasks/Data/Entities/BaseTaskResult.cs similarity index 80% rename from src/Sitko.Core.Tasks/BaseTaskResult.cs rename to src/Sitko.Core.Tasks/Data/Entities/BaseTaskResult.cs index 0cd72f38d..1e59b623c 100644 --- a/src/Sitko.Core.Tasks/BaseTaskResult.cs +++ b/src/Sitko.Core.Tasks/Data/Entities/BaseTaskResult.cs @@ -1,4 +1,4 @@ -namespace Sitko.Core.Tasks; +namespace Sitko.Core.Tasks.Data.Entities; public record BaseTaskResult { diff --git a/src/Sitko.Core.Tasks/IBaseTask.cs b/src/Sitko.Core.Tasks/Data/Entities/IBaseTask.cs similarity index 93% rename from src/Sitko.Core.Tasks/IBaseTask.cs rename to src/Sitko.Core.Tasks/Data/Entities/IBaseTask.cs index 2c15f9d0a..a973a9005 100644 --- a/src/Sitko.Core.Tasks/IBaseTask.cs +++ b/src/Sitko.Core.Tasks/Data/Entities/IBaseTask.cs @@ -1,7 +1,7 @@ using System.ComponentModel.DataAnnotations.Schema; using Sitko.Core.Repository; -namespace Sitko.Core.Tasks; +namespace Sitko.Core.Tasks.Data.Entities; public interface IBaseTask : IEntity { @@ -12,6 +12,7 @@ public interface IBaseTask : IEntity Guid? ParentId { get; set; } string? UserId { get; set; } DateTimeOffset? LastActivityDate { get; set; } + string GetKey(); } public interface IBaseTask : IBaseTask where TConfig : BaseTaskConfig, new() diff --git a/src/Sitko.Core.Tasks/IBaseTaskWithConfigAndResult.cs b/src/Sitko.Core.Tasks/Data/Entities/IBaseTaskWithConfigAndResult.cs similarity index 72% rename from src/Sitko.Core.Tasks/IBaseTaskWithConfigAndResult.cs rename to src/Sitko.Core.Tasks/Data/Entities/IBaseTaskWithConfigAndResult.cs index 18576a5ba..debf942b2 100644 --- a/src/Sitko.Core.Tasks/IBaseTaskWithConfigAndResult.cs +++ b/src/Sitko.Core.Tasks/Data/Entities/IBaseTaskWithConfigAndResult.cs @@ -1,4 +1,4 @@ -namespace Sitko.Core.Tasks; +namespace Sitko.Core.Tasks.Data.Entities; public interface IBaseTaskWithConfigAndResult { diff --git a/src/Sitko.Core.Tasks/TaskStatus.cs b/src/Sitko.Core.Tasks/Data/Entities/TaskStatus.cs similarity index 73% rename from src/Sitko.Core.Tasks/TaskStatus.cs rename to src/Sitko.Core.Tasks/Data/Entities/TaskStatus.cs index ed99bd170..30103d8fb 100644 --- a/src/Sitko.Core.Tasks/TaskStatus.cs +++ b/src/Sitko.Core.Tasks/Data/Entities/TaskStatus.cs @@ -1,4 +1,4 @@ -namespace Sitko.Core.Tasks; +namespace Sitko.Core.Tasks.Data.Entities; public enum TaskStatus { @@ -7,4 +7,4 @@ public enum TaskStatus SuccessWithWarnings = 2, Success = 3, Fails = 4 -} \ No newline at end of file +} diff --git a/src/Sitko.Core.Tasks/Data/Repository/BaseTaskRepository.cs b/src/Sitko.Core.Tasks/Data/Repository/BaseTaskRepository.cs new file mode 100644 index 000000000..c8df1eea7 --- /dev/null +++ b/src/Sitko.Core.Tasks/Data/Repository/BaseTaskRepository.cs @@ -0,0 +1,23 @@ +using Sitko.Core.Repository.EntityFrameworkCore; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Data.Repository; + +public class BaseTaskRepository : EFRepository, + ITaskRepository + where TTask : class, IBaseTask where TDbContext : TasksDbContext where TBaseTask : BaseTask +{ + public BaseTaskRepository(EFRepositoryContext repositoryContext) : base(repositoryContext) + { + } +} + +public class TasksRepository : EFRepository, + ITaskRepository + where TDbContext : TasksDbContext where TBaseTask : BaseTask +{ + public TasksRepository(EFRepositoryContext repositoryContext) : base(repositoryContext) + { + } +} + diff --git a/src/Sitko.Core.Tasks/Data/Repository/ITaskRepository.cs b/src/Sitko.Core.Tasks/Data/Repository/ITaskRepository.cs new file mode 100644 index 000000000..81c764c80 --- /dev/null +++ b/src/Sitko.Core.Tasks/Data/Repository/ITaskRepository.cs @@ -0,0 +1,10 @@ +using Sitko.Core.Repository; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Data.Repository; + +public interface ITaskRepository : IRepository + where TEntity : class, IBaseTask +{ +} + diff --git a/src/Sitko.Core.Tasks/Data/TasksDbContext.cs b/src/Sitko.Core.Tasks/Data/TasksDbContext.cs new file mode 100644 index 000000000..44c7ef87d --- /dev/null +++ b/src/Sitko.Core.Tasks/Data/TasksDbContext.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore; +using Sitko.Core.Db.Postgres; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Data; + +public abstract class TasksDbContext : BaseDbContext where TBaseTask : BaseTask +{ + private readonly DbContextOptions options; + protected TasksDbContext(DbContextOptions options) : base(options) => this.options = options; + + private DbSet Tasks => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + var discriminatorBuilder = modelBuilder.Entity().HasDiscriminator(nameof(BaseTask.Type)); + + var tasksExtension = options.FindExtension>(); + tasksExtension?.Configure(modelBuilder, discriminatorBuilder); + + modelBuilder.Entity().Property(task => task.Queue).HasDefaultValue("default"); + } +} diff --git a/src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtension.cs b/src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtension.cs new file mode 100644 index 000000000..c12650f08 --- /dev/null +++ b/src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtension.cs @@ -0,0 +1,43 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.Extensions.DependencyInjection; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Data; + +internal class TasksDbContextOptionsExtension : IDbContextOptionsExtension where TBaseTask : BaseTask +{ + public string TableName { get; init; } = ""; + + private readonly List>> discriminatorConfigurations = new(); + public void ApplyServices(IServiceCollection services) { } + + public void Validate(IDbContextOptions options) { } + + public DbContextOptionsExtensionInfo Info => new TasksDbContextOptionsExtensionInfo(this); + + public void Register() where TTask : class, IBaseTask + where TConfig : BaseTaskConfig, new() + where TResult : BaseTaskResult => + discriminatorConfigurations.Add((modelBuilder, discriminatorBuilder) => + { + modelBuilder.Entity().ToTable(TableName); + modelBuilder.Entity().Property(task => task.Config).HasColumnType("jsonb") + .HasColumnName(nameof(IBaseTask.Config)); + modelBuilder.Entity().Property(task => task.Result).HasColumnType("jsonb") + .HasColumnName(nameof(IBaseTask.Result)); + var attr = typeof(TTask).GetCustomAttributes(typeof(TaskAttribute), true).Cast() + .FirstOrDefault(); + discriminatorBuilder.HasValue(attr is null ? typeof(TTask).Name : attr.Key); + }); + + public void Configure(ModelBuilder modelBuilder, DiscriminatorBuilder discriminatorBuilder) + { + modelBuilder.Entity().ToTable(TableName); + foreach (var discriminatorConfiguration in discriminatorConfigurations) + { + discriminatorConfiguration(modelBuilder, discriminatorBuilder); + } + } +} diff --git a/src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtensionInfo.cs b/src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtensionInfo.cs new file mode 100644 index 000000000..e7b707c96 --- /dev/null +++ b/src/Sitko.Core.Tasks/Data/TasksDbContextOptionsExtensionInfo.cs @@ -0,0 +1,27 @@ +using Microsoft.EntityFrameworkCore.Infrastructure; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Data; + +internal class TasksDbContextOptionsExtensionInfo : DbContextOptionsExtensionInfo where TBaseTask : BaseTask +{ + public TasksDbContextOptionsExtensionInfo(IDbContextOptionsExtension extension) : base(extension) + { + } + +#if NET6_0_OR_GREATER + public override int GetServiceProviderHashCode() => Extension.GetHashCode(); +#else + public override long GetServiceProviderHashCode() => Extension.GetHashCode(); +#endif + +#if NET6_0_OR_GREATER + public override bool ShouldUseSameServiceProvider(DbContextOptionsExtensionInfo other) => true; +#endif + public override void PopulateDebugInfo(IDictionary debugInfo) + { + } + + public override bool IsDatabaseProvider => false; + public override string LogFragment => nameof(TasksDbContextOptionsExtension); +} diff --git a/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs index 040c48d41..4f0e5428c 100644 --- a/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs +++ b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs @@ -3,6 +3,8 @@ using Microsoft.Extensions.Logging; using Serilog.Context; using Sitko.Core.Repository; +using Sitko.Core.Tasks.Data.Entities; +using TaskStatus = Sitko.Core.Tasks.Data.Entities.TaskStatus; namespace Sitko.Core.Tasks.Execution; @@ -17,8 +19,8 @@ public abstract class BaseTaskExecutor : ITaskExecutor< private readonly IRepository repository; private readonly IServiceScopeFactory serviceScopeFactory; - protected BaseTaskExecutor(ILogger> logger, ITracer? tracer, - IServiceScopeFactory serviceScopeFactory, IRepository repository) + protected BaseTaskExecutor(ILogger> logger, + IServiceScopeFactory serviceScopeFactory, IRepository repository, ITracer? tracer = null) { this.logger = logger; this.tracer = tracer; diff --git a/src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs b/src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs index e4a0c1d30..e6abc9156 100644 --- a/src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs +++ b/src/Sitko.Core.Tasks/Execution/ITaskExecutor.cs @@ -1,11 +1,17 @@ -namespace Sitko.Core.Tasks.Execution; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Execution; public interface ITaskExecutor { Task ExecuteAsync(Guid id, CancellationToken cancellationToken); } -public interface ITaskExecutor : ITaskExecutor +public interface ITaskExecutor: ITaskExecutor where TTask : class, IBaseTask +{ +} + +public interface ITaskExecutor : ITaskExecutor where TTask : class, IBaseTask where TConfig : BaseTaskConfig, new() where TResult : BaseTaskResult, new() diff --git a/src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs b/src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs deleted file mode 100644 index d712a312d..000000000 --- a/src/Sitko.Core.Tasks/Scheduling/BaseTaskScheduler.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Nito.AsyncEx; - -namespace Sitko.Core.Tasks.Scheduling; - -public abstract class BaseTaskScheduler : BackgroundService where TTask : IBaseTask -{ - private readonly IOptions> taskOptions; - private readonly IOptionsMonitor optionsMonitor; - private readonly ITaskScheduler taskScheduler; - protected ILogger> Logger { get; } - - protected BaseTaskScheduler(IOptions> taskOptions, - IOptionsMonitor optionsMonitor, - ITaskScheduler taskScheduler, - ILogger> logger) - { - this.taskOptions = taskOptions; - this.optionsMonitor = optionsMonitor; - this.taskScheduler = taskScheduler; - Logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - while (!stoppingToken.IsCancellationRequested) - { - await ScheduleAsync(stoppingToken); - try - { - await Task.Delay(taskOptions.Value.Interval, stoppingToken); - } - catch (TaskCanceledException) - { - // do nothing - } - } - } - - private async Task ScheduleAsync(CancellationToken cancellationToken) - { - if (optionsMonitor.CurrentValue.IsAllTasksDisabled || - optionsMonitor.CurrentValue.DisabledTasks.Contains(typeof(TTask).Name)) - { - return; - } - - var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)); - var scheduleLock = ScheduleLocks.Locks.GetOrAdd(GetType().Name, _ => new AsyncLock()); - using (await scheduleLock.LockAsync(cts.Token)) - { - var tasks = await GetTasksAsync(cancellationToken); - foreach (var task in tasks) - { - await taskScheduler.ScheduleAsync(task); - } - } - } - - protected abstract Task GetTasksAsync(CancellationToken cancellationToken); -} diff --git a/src/Sitko.Core.Tasks/Scheduling/IBaseTaskFactory.cs b/src/Sitko.Core.Tasks/Scheduling/IBaseTaskFactory.cs new file mode 100644 index 000000000..1ca7054b7 --- /dev/null +++ b/src/Sitko.Core.Tasks/Scheduling/IBaseTaskFactory.cs @@ -0,0 +1,8 @@ +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Scheduling; + +public interface IBaseTaskFactory where TTask : IBaseTask +{ + public Task GetTasksAsync(CancellationToken cancellationToken); +} diff --git a/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs b/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs index b258894ee..f705a8ccb 100644 --- a/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs +++ b/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs @@ -1,6 +1,8 @@ +using Sitko.Core.Tasks.Data.Entities; + namespace Sitko.Core.Tasks.Scheduling; -public interface ITaskScheduler +public interface ITaskScheduler where TTask : IBaseTask { - Task ScheduleAsync(IBaseTask task); + Task ScheduleAsync(TTask task); } diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs index 932ef800f..e4577ded6 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs @@ -1,3 +1,5 @@ +using Sitko.Core.Tasks.Data.Entities; + namespace Sitko.Core.Tasks.Scheduling; public class TaskSchedulingOptions where TTask : IBaseTask diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs new file mode 100644 index 000000000..a164bbcad --- /dev/null +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs @@ -0,0 +1,75 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Nito.AsyncEx; +using Sitko.Core.Tasks.Components; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Scheduling; + +public class TaskSchedulingService : BackgroundService where TTask : class, IBaseTask +{ + private readonly IServiceScopeFactory serviceScopeFactory; + private readonly IOptions> taskOptions; + private readonly IOptionsMonitor optionsMonitor; + private readonly ILogger> logger; + + public TaskSchedulingService(IServiceScopeFactory serviceScopeFactory, IOptions> taskOptions, IOptionsMonitor optionsMonitor, ILogger> logger) + { + this.serviceScopeFactory = serviceScopeFactory; + this.taskOptions = taskOptions; + this.optionsMonitor = optionsMonitor; + this.logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + await using var scope = serviceScopeFactory.CreateAsyncScope(); + try + { + var scheduler = scope.ServiceProvider.GetRequiredService>(); + if (optionsMonitor.CurrentValue.IsAllTasksDisabled || + optionsMonitor.CurrentValue.DisabledTasks.Contains(typeof(TTask).Name)) + { + return; + } + + var tasksManager = scope.ServiceProvider.GetRequiredService(); + var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)); + var scheduleLock = ScheduleLocks.Locks.GetOrAdd(GetType().Name, _ => new AsyncLock()); + using (await scheduleLock.LockAsync(cts.Token)) + { + var tasks = await scheduler.GetTasksAsync(stoppingToken); + foreach (var task in tasks) + { + try + { + var runResult = await tasksManager.RunAsync(task, cancellationToken: cts.Token); + if (!runResult.IsSuccess) + { + throw new Exception(runResult.ErrorMessage); + } + } + catch (Exception ex) + { + logger.LogError("Error running task {Type}: {Ex}", typeof(TTask), ex); + } + } + } + + await Task.Delay(taskOptions.Value.Interval, stoppingToken); + } + catch (TaskCanceledException) + { + // do nothing + } + catch (Exception ex) + { + logger.LogError("Error schedule task {Type}: {Error}", typeof(TTask), ex); + } + } + } +} diff --git a/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj b/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj index 41a1bf675..cd1e95ac1 100644 --- a/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj +++ b/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj @@ -3,6 +3,7 @@ + diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs index b25fd22c8..407c4c88c 100644 --- a/src/Sitko.Core.Tasks/TasksModule.cs +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -1,11 +1,9 @@ -using System.Reflection; -using FluentValidation; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Metadata.Builders; +using FluentValidation; using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App; -using Sitko.Core.Db.Postgres; +using Sitko.Core.Tasks.Components; +using Sitko.Core.Tasks.Data; +using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Execution; namespace Sitko.Core.Tasks; @@ -39,7 +37,7 @@ public override void ConfigureServices(IApplicationContext applicationContext, I if (groupInfo is null) { throw new InvalidOperationException( - $"Consumer {executorType} must have attribute TaskExecutorAttribute"); + $"Executor {executorType} must have attribute TaskExecutorAttribute"); } var eventType = executorType.GetInterfaces() @@ -50,7 +48,13 @@ public override void ConfigureServices(IApplicationContext applicationContext, I executors.Add(registration); } + services.Scan(selector => selector.FromTypes(executors.Select(e => e.ExecutorType)).AsSelfWithInterfaces() + .WithScopedLifetime()); + + services.AddScoped(); + ConfigureServicesInternal(applicationContext, services, startupOptions, executors); + startupOptions.ConfigureServices(services); } protected abstract void ConfigureServicesInternal(IApplicationContext applicationContext, @@ -58,116 +62,6 @@ protected abstract void ConfigureServicesInternal(IApplicationContext applicatio List executors); } -public class TasksModuleOptions -{ - public bool IsAllTasksDisabled { get; set; } - public string[] DisabledTasks { get; set; } = Array.Empty(); - - public int? AllTasksRetentionDays { get; set; } - public Dictionary RetentionDays { get; set; } = new(); -} - -public abstract class TasksModuleOptions : BaseModuleOptions, IModuleOptionsWithValidation - where TDbContext : TasksDbContext where TBaseTask : BaseTask -{ - public List Assemblies { get; } = new(); - - public TasksModuleOptions AddExecutorsFromAssemblyOf() - { - Assemblies.Add(typeof(TAssembly).Assembly); - return this; - } - - internal bool HasJobs => false; - - public TasksModuleOptions AddTask(TimeSpan interval) - where TTask : class, IBaseTask - where TConfig : BaseTaskConfig, new() - where TResult : BaseTaskResult, new() - { - return this; - } - - public abstract Type GetValidatorType(); -} - -public abstract class TasksDbContext : BaseDbContext where TBaseTask : BaseTask -{ - private readonly DbContextOptions options; - protected TasksDbContext(DbContextOptions options) : base(options) => this.options = options; - - private DbSet Tasks => Set(); - - protected override void OnModelCreating(ModelBuilder modelBuilder) - { - base.OnModelCreating(modelBuilder); - var discriminatorBuilder = modelBuilder.Entity().HasDiscriminator(nameof(BaseTask.Type)); - - var tasksExtension = options.FindExtension>(); - if (tasksExtension is not null) - { - tasksExtension.Configure(modelBuilder, discriminatorBuilder); - } - - modelBuilder.Entity().Property(task => task.Queue).HasDefaultValue("default"); - } -} - -internal class TasksDbContextOptionsExtension : IDbContextOptionsExtension where TBaseTask : BaseTask -{ - private readonly List>> discriminatorConfigurations = new(); - public void ApplyServices(IServiceCollection services) { } - - public void Validate(IDbContextOptions options) { } - - public DbContextOptionsExtensionInfo Info => new TasksDbContextOptionsExtensionInfo(this); - - public void Register() where TTask : class, IBaseTask - where TConfig : BaseTaskConfig, new() - where TResult : BaseTaskResult => - discriminatorConfigurations.Add((modelBuilder, discriminatorBuilder) => - { - modelBuilder.Entity().Property(task => task.Config).HasColumnType("jsonb") - .HasColumnName(nameof(IBaseTask.Config)); - modelBuilder.Entity().Property(task => task.Result).HasColumnType("jsonb") - .HasColumnName(nameof(IBaseTask.Result)); - var attr = typeof(TTask).GetCustomAttributes(typeof(TaskAttribute), true).Cast() - .FirstOrDefault(); - discriminatorBuilder.HasValue(attr is null ? typeof(TTask).Name : attr.Key); - }); - - public void Configure(ModelBuilder modelBuilder, DiscriminatorBuilder discriminatorBuilder) - { - foreach (var discriminatorConfiguration in discriminatorConfigurations) - { - discriminatorConfiguration(modelBuilder, discriminatorBuilder); - } - } -} - -internal class TasksDbContextOptionsExtensionInfo : DbContextOptionsExtensionInfo where TBaseTask : BaseTask -{ - public TasksDbContextOptionsExtensionInfo(IDbContextOptionsExtension extension) : base(extension) - { - } - -#if NET6_0_OR_GREATER - public override int GetServiceProviderHashCode() => Extension.GetHashCode(); -#else - public override long GetServiceProviderHashCode() => Extension.GetHashCode(); -#endif - -#if NET6_0_OR_GREATER - public override bool ShouldUseSameServiceProvider(DbContextOptionsExtensionInfo other) => true; -#endif - public override void PopulateDebugInfo(IDictionary debugInfo) - { - } - - public override bool IsDatabaseProvider => false; - public override string LogFragment => nameof(TasksDbContextOptionsExtension); -} - public abstract class TasksModuleOptionsValidator : AbstractValidator where TOptions : TasksModuleOptions diff --git a/src/Sitko.Core.Tasks/TasksModuleOptions.cs b/src/Sitko.Core.Tasks/TasksModuleOptions.cs new file mode 100644 index 000000000..e86b3713d --- /dev/null +++ b/src/Sitko.Core.Tasks/TasksModuleOptions.cs @@ -0,0 +1,85 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Sitko.Core.App; +using Sitko.Core.Repository; +using Sitko.Core.Tasks.Data; +using Sitko.Core.Tasks.Data.Entities; +using Sitko.Core.Tasks.Data.Repository; +using Sitko.Core.Tasks.Scheduling; + +namespace Sitko.Core.Tasks; + +public class TasksModuleOptions +{ + public bool IsAllTasksDisabled { get; set; } + public string[] DisabledTasks { get; set; } = Array.Empty(); + + public int? AllTasksRetentionDays { get; set; } + public Dictionary RetentionDays { get; set; } = new(); +} + + +public abstract class TasksModuleOptions : BaseModuleOptions, IModuleOptionsWithValidation + where TDbContext : TasksDbContext where TBaseTask : BaseTask +{ + public List Assemblies { get; } = new(); + private readonly List> jobServiceConfigurations = new(); + public string TableName { get; set; } = "Tasks"; + + private readonly List>> + tasksDbContextOptionsExtensionConfigurations = new(); + + internal void ConfigureServices(IServiceCollection serviceCollection) + { + foreach (var jobServiceConfiguration in jobServiceConfigurations) + { + jobServiceConfiguration(serviceCollection); + } + + var extension = new TasksDbContextOptionsExtension + { + TableName = TableName + }; + foreach (var configuration in tasksDbContextOptionsExtensionConfigurations) + { + configuration(extension); + } + + serviceCollection.AddSingleton(extension); + } + + public TasksModuleOptions AddExecutorsFromAssemblyOf() + { + Assemblies.Add(typeof(TAssembly).Assembly); + return this; + } + + internal bool HasJobs => jobServiceConfigurations.Any(); + + public TasksModuleOptions AddTask(TimeSpan interval) + where TTask : class, IBaseTask + where TConfig : BaseTaskConfig, new() + where TResult : BaseTaskResult, new() + { + tasksDbContextOptionsExtensionConfigurations.Add(extension => + { + extension.Register(); + }); + jobServiceConfigurations.Add(services => + { + services.Configure>(options => options.Interval = interval); + var schedulerType = typeof(IBaseTaskFactory); + services.Scan(selector => + selector.FromAssemblyOf() + .AddClasses(filter => filter.AssignableToAny(schedulerType)) + .As>().WithScopedLifetime()); + services.AddHostedService>(); + services.Scan(selector => selector.FromTypes(typeof(BaseTaskRepository)) + .AsSelf().As().As>().As>() + .WithTransientLifetime()); + }); + return this; + } + + public abstract Type GetValidatorType(); +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs index 5752adbe1..bc64eb4d9 100644 --- a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/BaseTestTask.cs @@ -1,3 +1,5 @@ +using Sitko.Core.Tasks.Data.Entities; + namespace Sitko.Core.Tasks.Kafka.Tests.Data; public abstract record BaseTestTask : BaseTask diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs index 4d490c726..0e2012da1 100644 --- a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestDbContext.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Sitko.Core.Tasks.Data; namespace Sitko.Core.Tasks.Kafka.Tests.Data; diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs index 34e02625c..9220de45c 100644 --- a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTask.cs @@ -1,4 +1,6 @@ -namespace Sitko.Core.Tasks.Kafka.Tests.Data; +using Sitko.Core.Tasks.Data.Entities; + +namespace Sitko.Core.Tasks.Kafka.Tests.Data; public record TestTask : BaseTestTask { diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs index 6d515b4d4..79e06e924 100644 --- a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskExecutor.cs @@ -9,7 +9,7 @@ namespace Sitko.Core.Tasks.Kafka.Tests.Data; public class TestTaskExecutor : BaseTaskExecutor { public TestTaskExecutor(ILogger logger, ITracer? tracer, IServiceScopeFactory serviceScopeFactory, - IRepository repository) : base(logger, tracer, serviceScopeFactory, repository) + IRepository repository) : base(logger, serviceScopeFactory, repository, tracer) { } diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskFactory.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskFactory.cs new file mode 100644 index 000000000..4e28d1e47 --- /dev/null +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskFactory.cs @@ -0,0 +1,12 @@ +using Sitko.Core.Tasks.Scheduling; + +namespace Sitko.Core.Tasks.Kafka.Tests.Data; + +public class TestTaskFactory : IBaseTaskFactory +{ + public Task GetTasksAsync(CancellationToken cancellationToken) + { + var tasks = new[] { new TestTask { Id = Guid.NewGuid(), FooId = 1 } }; + return Task.FromResult(tasks); + } +} diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs deleted file mode 100644 index 9a5566f19..000000000 --- a/tests/Sitko.Core.Tasks.Kafka.Tests/Data/TestTaskScheduler.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Sitko.Core.Tasks.Scheduling; - -namespace Sitko.Core.Tasks.Kafka.Tests.Data; - -public class TestTaskScheduler : BaseTaskScheduler -{ - public TestTaskScheduler(IOptions> taskOptions, - IOptionsMonitor optionsMonitor, ITaskScheduler taskScheduler, - ILogger logger) : base(taskOptions, optionsMonitor, taskScheduler, logger) - { - } - - protected override Task GetTasksAsync(CancellationToken cancellationToken) - { - var tasks = new[] { new TestTask { Id = Guid.NewGuid(), FooId = 1 } }; - return Task.FromResult(tasks); - } -} From 16b03e96be27bf5222d019bf1e8d4097a4246c8e Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 5 Oct 2023 15:10:33 +0500 Subject: [PATCH 03/32] fix(tasks): use non-generic scheduler --- src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs | 7 ++++--- .../Scheduling/KafkaTaskScheduler.cs | 4 ++-- src/Sitko.Core.Tasks/Components/TasksManager.cs | 8 +++----- src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs | 4 ++-- src/Sitko.Core.Tasks/TasksModule.cs | 5 ++++- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index fb12b356f..5796f139e 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -17,7 +17,7 @@ namespace Sitko.Core.Tasks.Kafka; public class - KafkaTasksModule : TasksModule : TasksModule> where TBaseTask : BaseTask where TDbContext : TasksDbContext { @@ -85,7 +85,9 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio middlewares .AddSerializer(); middlewares.AddTypedHandlers(handlers => - handlers.AddHandlers(groupConsumers.Select(r => executorType.MakeGenericType(r.EventType, r.ExecutorType))).WithHandlerLifetime(InstanceLifetime.Scoped)); + handlers.AddHandlers(groupConsumers.Select(r => + executorType.MakeGenericType(r.EventType, r.ExecutorType))) + .WithHandlerLifetime(InstanceLifetime.Scoped)); } ); } @@ -93,7 +95,6 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio } }); }); - services.AddSingleton(typeof(ITaskScheduler<>), typeof(KafkaTaskScheduler<>)); } } diff --git a/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs b/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs index a5a97c652..4d8ffa432 100644 --- a/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs +++ b/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs @@ -4,12 +4,12 @@ namespace Sitko.Core.Tasks.Kafka.Scheduling; -public class KafkaTaskScheduler : ITaskScheduler where TTask : IBaseTask +public class KafkaTaskScheduler : ITaskScheduler { private readonly IProducerAccessor producerAccessor; public KafkaTaskScheduler(IProducerAccessor producerAccessor) => this.producerAccessor = producerAccessor; - public async Task ScheduleAsync(TTask task) => + public async Task ScheduleAsync(IBaseTask task) => await producerAccessor.GetProducer("default").ProduceAsync(task.GetKey(), task); } diff --git a/src/Sitko.Core.Tasks/Components/TasksManager.cs b/src/Sitko.Core.Tasks/Components/TasksManager.cs index 5dbad6cd7..59b1e37cf 100644 --- a/src/Sitko.Core.Tasks/Components/TasksManager.cs +++ b/src/Sitko.Core.Tasks/Components/TasksManager.cs @@ -1,4 +1,3 @@ -using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App.Results; using Sitko.Core.Repository; using Sitko.Core.Tasks.Data.Entities; @@ -10,12 +9,12 @@ namespace Sitko.Core.Tasks.Components; public class TasksManager { private readonly IEnumerable repositories; - private readonly IServiceProvider serviceProvider; + private readonly ITaskScheduler taskScheduler; - public TasksManager(IEnumerable repositories, IServiceProvider serviceProvider) + public TasksManager(IEnumerable repositories, ITaskScheduler taskScheduler) { this.repositories = repositories; - this.serviceProvider = serviceProvider; + this.taskScheduler = taskScheduler; } private ITaskRepository GetRepository() where TTask : class, IBaseTask @@ -68,7 +67,6 @@ public async Task> RunAsync(TTask newTask, Guid? p return new OperationResult(addResponse.ErrorsString); } - var taskScheduler = serviceProvider.GetRequiredService>(); await taskScheduler.ScheduleAsync(newTask); return new OperationResult(addResponse.Entity); diff --git a/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs b/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs index f705a8ccb..3a7c25a89 100644 --- a/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs +++ b/src/Sitko.Core.Tasks/Scheduling/ITaskScheduler.cs @@ -2,7 +2,7 @@ namespace Sitko.Core.Tasks.Scheduling; -public interface ITaskScheduler where TTask : IBaseTask +public interface ITaskScheduler { - Task ScheduleAsync(TTask task); + Task ScheduleAsync(IBaseTask task); } diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs index 407c4c88c..f45c64acd 100644 --- a/src/Sitko.Core.Tasks/TasksModule.cs +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -5,13 +5,15 @@ using Sitko.Core.Tasks.Data; using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Execution; +using Sitko.Core.Tasks.Scheduling; namespace Sitko.Core.Tasks; public abstract class - TasksModule : BaseApplicationModule + TasksModule : BaseApplicationModule where TOptions : TasksModuleOptions, new() where TDbContext : TasksDbContext + where TTaskScheduler : class, ITaskScheduler where TBaseTask : BaseTask { public override string OptionsKey => $"Tasks:{typeof(TBaseTask).Name}"; @@ -50,6 +52,7 @@ public override void ConfigureServices(IApplicationContext applicationContext, I services.Scan(selector => selector.FromTypes(executors.Select(e => e.ExecutorType)).AsSelfWithInterfaces() .WithScopedLifetime()); + services.AddScoped(); services.AddScoped(); From d0b5eb14e8d0e98e8f5792ee6ec226bae6431693 Mon Sep 17 00:00:00 2001 From: nadia Date: Fri, 6 Oct 2023 10:13:57 +0500 Subject: [PATCH 04/32] feat(tasks): remove queue from db --- src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs | 1 - src/Sitko.Core.Tasks/Data/TasksDbContext.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs b/src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs index 1866d508d..8dfec5fc6 100644 --- a/src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs +++ b/src/Sitko.Core.Tasks/Data/Entities/BaseTask.cs @@ -4,7 +4,6 @@ namespace Sitko.Core.Tasks.Data.Entities; public record BaseTask : EntityRecord, IBaseTask { - public string Queue { get; set; } = ""; public DateTimeOffset DateAdded { get; set; } = DateTimeOffset.UtcNow; public DateTimeOffset DateUpdated { get; set; } = DateTimeOffset.UtcNow; public TaskStatus TaskStatus { get; set; } = TaskStatus.Wait; diff --git a/src/Sitko.Core.Tasks/Data/TasksDbContext.cs b/src/Sitko.Core.Tasks/Data/TasksDbContext.cs index 44c7ef87d..223c1bf6c 100644 --- a/src/Sitko.Core.Tasks/Data/TasksDbContext.cs +++ b/src/Sitko.Core.Tasks/Data/TasksDbContext.cs @@ -18,7 +18,5 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) var tasksExtension = options.FindExtension>(); tasksExtension?.Configure(modelBuilder, discriminatorBuilder); - - modelBuilder.Entity().Property(task => task.Queue).HasDefaultValue("default"); } } From ec84abcce0c62a2ee3e1794183b8b1e14eaf81fa Mon Sep 17 00:00:00 2001 From: nadia Date: Fri, 6 Oct 2023 12:00:07 +0500 Subject: [PATCH 05/32] feat(tasks): use cronos for get next scheduling date --- src/Directory.Packages.props | 5 +++-- src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs | 1 - .../Scheduling/TaskSchedulingOptions.cs | 2 +- .../Scheduling/TaskSchedulingService.cs | 11 ++++++++++- src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj | 1 + src/Sitko.Core.Tasks/TasksModuleOptions.cs | 2 +- .../BaseKafkaTasksTest.cs | 2 +- 7 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index e5c592cce..e408ffef7 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -3,6 +3,7 @@ true + @@ -98,7 +99,7 @@ - + @@ -167,4 +168,4 @@ - \ No newline at end of file + diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 5796f139e..2aa835fec 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -11,7 +11,6 @@ using Sitko.Core.Tasks.Execution; using Sitko.Core.Tasks.Kafka.Execution; using Sitko.Core.Tasks.Kafka.Scheduling; -using Sitko.Core.Tasks.Scheduling; using AutoOffsetReset = Confluent.Kafka.AutoOffsetReset; namespace Sitko.Core.Tasks.Kafka; diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs index e4577ded6..4ccabd0bb 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs @@ -4,5 +4,5 @@ namespace Sitko.Core.Tasks.Scheduling; public class TaskSchedulingOptions where TTask : IBaseTask { - public TimeSpan Interval { get; set; } + public string Interval { get; set; } = ""; } diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs index a164bbcad..3ab17931d 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs @@ -1,3 +1,4 @@ +using Cronos; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -60,7 +61,15 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - await Task.Delay(taskOptions.Value.Interval, stoppingToken); + if (!string.IsNullOrEmpty(taskOptions.Value.Interval)) + { + var now = DateTime.UtcNow; + var nextDate = CronExpression.Parse(taskOptions.Value.Interval).GetNextOccurrence(now); + if (nextDate != null) + { + await Task.Delay(TimeSpan.FromSeconds((nextDate - now).Value.TotalSeconds), stoppingToken); + } + } } catch (TaskCanceledException) { diff --git a/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj b/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj index cd1e95ac1..db6a9aa37 100644 --- a/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj +++ b/src/Sitko.Core.Tasks/Sitko.Core.Tasks.csproj @@ -6,6 +6,7 @@ + diff --git a/src/Sitko.Core.Tasks/TasksModuleOptions.cs b/src/Sitko.Core.Tasks/TasksModuleOptions.cs index e86b3713d..047ec4555 100644 --- a/src/Sitko.Core.Tasks/TasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks/TasksModuleOptions.cs @@ -56,7 +56,7 @@ public TasksModuleOptions AddExecutorsFromAssemblyOf jobServiceConfigurations.Any(); - public TasksModuleOptions AddTask(TimeSpan interval) + public TasksModuleOptions AddTask(string interval) where TTask : class, IBaseTask where TConfig : BaseTaskConfig, new() where TResult : BaseTaskResult, new() diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs index b809ec618..f8e12fa64 100644 --- a/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs @@ -20,7 +20,7 @@ protected override TestApplication ConfigureApplication(TestApplication applicat .AddKafkaTasks(options => { options - .AddTask(TimeSpan.FromMinutes(1)) + .AddTask("* * * * *") .AddExecutorsFromAssemblyOf(); }); return application; From 424d13271e7b2ef92a3cad0248cf4907000a9331 Mon Sep 17 00:00:00 2001 From: nadia Date: Fri, 6 Oct 2023 12:00:31 +0500 Subject: [PATCH 06/32] feat: configure options, repository, scheduler --- src/Sitko.Core.Tasks/TasksModule.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs index f45c64acd..ff0d59fce 100644 --- a/src/Sitko.Core.Tasks/TasksModule.cs +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -1,9 +1,11 @@ using FluentValidation; using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App; +using Sitko.Core.Repository; using Sitko.Core.Tasks.Components; using Sitko.Core.Tasks.Data; using Sitko.Core.Tasks.Data.Entities; +using Sitko.Core.Tasks.Data.Repository; using Sitko.Core.Tasks.Execution; using Sitko.Core.Tasks.Scheduling; @@ -52,9 +54,13 @@ public override void ConfigureServices(IApplicationContext applicationContext, I services.Scan(selector => selector.FromTypes(executors.Select(e => e.ExecutorType)).AsSelfWithInterfaces() .WithScopedLifetime()); - services.AddScoped(); + + services.AddScoped(); services.AddScoped(); + services.Configure(applicationContext.Configuration.GetSection(OptionsKey)); + services.AddTransient, TasksRepository>(); + services.AddTransient, TasksRepository>(); ConfigureServicesInternal(applicationContext, services, startupOptions, executors); startupOptions.ConfigureServices(services); From 3c9627279ac10af7d6ac665a0ecf96a210d2ccc2 Mon Sep 17 00:00:00 2001 From: nadia Date: Fri, 6 Oct 2023 13:45:51 +0500 Subject: [PATCH 07/32] feat(tasks): allow retry --- src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs | 9 +++++++-- src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs | 5 ++++- src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs index 4f0e5428c..8613d6f05 100644 --- a/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs +++ b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs @@ -76,8 +76,13 @@ private async Task ExecuteTaskAsync(Guid id, CancellationToken cancellati if (task.TaskStatus != TaskStatus.Wait) { - logger.LogInformation("Skip retry task {JobId}", id); - return task; + var groupInfo = TaskExecutorHelper.GetGroupInfo(GetType()); + if (groupInfo is not { allowRetry: true }) + { + logger.LogInformation("Skip retry task {JobId}", id); + return task; + } + logger.LogInformation("Retry task {JobId}", id); } logger.LogInformation("Set job {JobId} status to in progress", id); diff --git a/src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs b/src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs index 2172276f2..5990da93e 100644 --- a/src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs +++ b/src/Sitko.Core.Tasks/Execution/TaskExecutorAttribute.cs @@ -3,11 +3,12 @@ namespace Sitko.Core.Tasks.Execution; [AttributeUsage(AttributeTargets.Class)] public class TaskExecutorAttribute : Attribute { - public TaskExecutorAttribute(string groupId, int parallelThreadCount, int bufferSize = 10) + public TaskExecutorAttribute(string groupId, int parallelThreadCount, int bufferSize = 10, bool allowRetry = false) { GroupId = groupId; ParallelThreadCount = Math.Max(parallelThreadCount, 1); BufferSize = Math.Max(bufferSize, 1); + AllowRetry = allowRetry; } public string GroupId { get; } @@ -15,4 +16,6 @@ public TaskExecutorAttribute(string groupId, int parallelThreadCount, int buffer public int ParallelThreadCount { get; } public int BufferSize { get; } + + public bool AllowRetry { get; } } diff --git a/src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs b/src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs index d2d031f32..ccaec37ff 100644 --- a/src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs +++ b/src/Sitko.Core.Tasks/Execution/TaskExecutorHelper.cs @@ -4,7 +4,7 @@ namespace Sitko.Core.Tasks.Execution; internal static class TaskExecutorHelper { - public static (string GroupId, int ParallelThreadCount, int BufferSize)? GetGroupInfo(Type eventProcessorType) + public static (string GroupId, int ParallelThreadCount, int BufferSize, bool allowRetry)? GetGroupInfo(Type eventProcessorType) { var attribute = eventProcessorType.FindAttribute(); if (attribute is null) @@ -13,7 +13,7 @@ public static (string GroupId, int ParallelThreadCount, int BufferSize)? GetGrou } var groupId = attribute.GroupId; - return (groupId, attribute.ParallelThreadCount, attribute.BufferSize); + return (groupId, attribute.ParallelThreadCount, attribute.BufferSize, attribute.AllowRetry); } public static TResult? FindAttribute(this ICustomAttributeProvider provider, bool withInherit = true) From b4e69eb75db37c74db5b1486b7df90a403afd7ac Mon Sep 17 00:00:00 2001 From: nadia Date: Mon, 9 Oct 2023 10:37:10 +0500 Subject: [PATCH 08/32] feat(tasks): add maintenance and cleaner tasks --- src/Sitko.Core.Tasks/Tasks/CleanerTask.cs | 87 +++++++++++++++++++ src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs | 70 +++++++++++++++ src/Sitko.Core.Tasks/TasksExtensions.cs | 30 +++++++ src/Sitko.Core.Tasks/TasksModule.cs | 3 + src/Sitko.Core.Tasks/TasksModuleOptions.cs | 11 ++- 5 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 src/Sitko.Core.Tasks/Tasks/CleanerTask.cs create mode 100644 src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs create mode 100644 src/Sitko.Core.Tasks/TasksExtensions.cs diff --git a/src/Sitko.Core.Tasks/Tasks/CleanerTask.cs b/src/Sitko.Core.Tasks/Tasks/CleanerTask.cs new file mode 100644 index 000000000..6e09bf8e7 --- /dev/null +++ b/src/Sitko.Core.Tasks/Tasks/CleanerTask.cs @@ -0,0 +1,87 @@ +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Sitko.Core.Repository.EntityFrameworkCore; +using Sitko.Core.Tasks.Data.Entities; +using Sitko.Core.Tasks.Data.Repository; + +namespace Sitko.Core.Tasks.Tasks; + +public class CleanerTask : BackgroundService + where TBaseTask : BaseTask +{ + private readonly IServiceScopeFactory serviceScopeFactory; + private readonly IOptions options; + private readonly ILogger> logger; + private ITaskRepository tasksRepository; + + public CleanerTask(IOptions options, ILogger> logger, + IServiceScopeFactory serviceScopeFactory) + { + this.options = options; + this.logger = logger; + this.serviceScopeFactory = serviceScopeFactory; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + await using var scope = serviceScopeFactory.CreateAsyncScope(); + tasksRepository = scope.ServiceProvider.GetRequiredService>(); + if (options.Value.AllTasksRetentionDays is > 0) + { + var taskTypes = options.Value.RetentionDays.Select(r => r.Key).ToArray(); + await RemoveTasks(options.Value.AllTasksRetentionDays.Value, false, taskTypes); + } + + foreach (var (taskType, retentionDays) in options.Value.RetentionDays) + { + if (retentionDays > 0) + { + await RemoveTasks(retentionDays, true, new[] { taskType }); + } + } + + await Task.Delay(TimeSpan.FromDays(1), stoppingToken); + } + } + + private async Task RemoveTasks(int retentionDays, bool include, string[] types) + { + var date = DateTimeOffset.UtcNow.AddDays(retentionDays * -1); + logger.LogInformation("Deleting tasks from {Date}. Types: {Types}, include: {Include}", date, types, include); + if (tasksRepository is IEFRepository efRepository) + { + var condition = $"\"{nameof(BaseTask.DateAdded)}\" < '{date.ToString("O", CultureInfo.InvariantCulture)}'"; + if (types.Length > 0) + { + condition += + $" AND \"{nameof(BaseTask.Type)}\" {(include ? "IN" : "NOT IN")} ({string.Join(",", types.Select(type => $"'{type}'"))})"; + } + + var cnt = await efRepository.DeleteAllRawAsync(condition); + logger.LogInformation("Deleted {Count} tasks", cnt); + } + else + { + var (tasks, tasksCount) = + await tasksRepository.GetAllAsync(query => query.Where(task => task.DateAdded < date && + (types.Length == 0 || + (include + ? types.Contains(task.Type) + : !types.Contains(task.Type))))); + if (tasksCount > 0) + { + foreach (var task in tasks) + { + await tasksRepository.DeleteAsync(task); + } + + logger.LogInformation("Deleted {Count} tasks", tasksCount); + } + } + } +} diff --git a/src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs b/src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs new file mode 100644 index 000000000..9dedd04cf --- /dev/null +++ b/src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs @@ -0,0 +1,70 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Sitko.Core.Tasks.Data; +using Sitko.Core.Tasks.Data.Entities; +using Sitko.Core.Tasks.Data.Repository; +using Sitko.Core.Tasks.Execution; +using TaskStatus = Sitko.Core.Tasks.Data.Entities.TaskStatus; + +namespace Sitko.Core.Tasks.Tasks; + +public class MaintenanceTask : BackgroundService + where TBaseTask : BaseTask where TDbContext : TasksDbContext +{ + private readonly IOptions options; + private readonly IServiceScopeFactory serviceScopeFactory; + + public MaintenanceTask(IOptions options, + IServiceScopeFactory serviceScopeFactory) + { + this.options = options; + this.serviceScopeFactory = serviceScopeFactory; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + await using var scope = serviceScopeFactory.CreateAsyncScope(); + var tasksRepository = scope.ServiceProvider.GetRequiredService>(); + var inactivityDate = DateTimeOffset.UtcNow - options.Value.TasksInactivityTimeout; + var waitDate = DateTimeOffset.UtcNow - options.Value.TasksWaitTimeout; + var stuckTasks = await tasksRepository.GetAllAsync(query => + query.Where(task => + (task.TaskStatus == TaskStatus.InProgress && task.LastActivityDate < inactivityDate) || + (task.TaskStatus == TaskStatus.Wait && task.DateAdded < waitDate)), stoppingToken); + if (stuckTasks.items.Length > 0) + { + switch (options.Value.StuckTasksProcessMode) + { + case StuckTasksProcessMode.Fail: + await tasksRepository.BeginBatchAsync(stoppingToken); + foreach (var task in stuckTasks.items) + { + var error = task.TaskStatus == TaskStatus.InProgress + ? $"Task inactive since {task.LastActivityDate}" + : $"Task in queue since {task.DateAdded}"; + TasksExtensions.SetTaskErrorResult((IBaseTaskWithConfigAndResult)task, error); + await tasksRepository.UpdateAsync(task, stoppingToken); + } + + await tasksRepository.CommitBatchAsync(stoppingToken); + break; + case StuckTasksProcessMode.Restart: + var taskExecutor = scope.ServiceProvider.GetRequiredService(); + foreach (var stuckTask in stuckTasks.items) + { + await taskExecutor.ExecuteAsync(stuckTask.Id, stoppingToken); + } + + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + + await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken); + } + } +} diff --git a/src/Sitko.Core.Tasks/TasksExtensions.cs b/src/Sitko.Core.Tasks/TasksExtensions.cs new file mode 100644 index 000000000..f1f1b9fc8 --- /dev/null +++ b/src/Sitko.Core.Tasks/TasksExtensions.cs @@ -0,0 +1,30 @@ +using System.Reflection; +using Sitko.Core.Tasks.Data.Entities; +using TaskStatus = Sitko.Core.Tasks.Data.Entities.TaskStatus; + +namespace Sitko.Core.Tasks; + +internal static class TasksExtensions +{ + private static readonly MethodInfo? SetErrorResultMethodInfo = + typeof(TasksExtensions).GetMethod(nameof(SetErrorResult), BindingFlags.Static | BindingFlags.Public); + + public static void SetErrorResult(TTask task, string error) + where TTask : IBaseTask + where TConfig : BaseTaskConfig, new() + where TResult : BaseTaskResult, new() + { + var result = new TResult { ErrorMessage = error, IsSuccess = false }; + task.TaskStatus = TaskStatus.Fails; + task.Result = result; + task.ExecuteDateEnd = DateTimeOffset.UtcNow; + } + + public static void SetTaskErrorResult(IBaseTaskWithConfigAndResult task, string error) + { + var scheduleMethod = SetErrorResultMethodInfo!.MakeGenericMethod(task.GetType(), + task.ConfigType, task.ResultType); + scheduleMethod.Invoke(null, new object?[] { task, error }); + } +} + diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs index ff0d59fce..78403a92f 100644 --- a/src/Sitko.Core.Tasks/TasksModule.cs +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -8,6 +8,7 @@ using Sitko.Core.Tasks.Data.Repository; using Sitko.Core.Tasks.Execution; using Sitko.Core.Tasks.Scheduling; +using Sitko.Core.Tasks.Tasks; namespace Sitko.Core.Tasks; @@ -61,6 +62,8 @@ public override void ConfigureServices(IApplicationContext applicationContext, I services.Configure(applicationContext.Configuration.GetSection(OptionsKey)); services.AddTransient, TasksRepository>(); services.AddTransient, TasksRepository>(); + services.AddHostedService>(); + services.AddHostedService>(); ConfigureServicesInternal(applicationContext, services, startupOptions, executors); startupOptions.ConfigureServices(services); diff --git a/src/Sitko.Core.Tasks/TasksModuleOptions.cs b/src/Sitko.Core.Tasks/TasksModuleOptions.cs index 047ec4555..bbf3ef2e3 100644 --- a/src/Sitko.Core.Tasks/TasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks/TasksModuleOptions.cs @@ -13,9 +13,11 @@ public class TasksModuleOptions { public bool IsAllTasksDisabled { get; set; } public string[] DisabledTasks { get; set; } = Array.Empty(); - public int? AllTasksRetentionDays { get; set; } public Dictionary RetentionDays { get; set; } = new(); + public TimeSpan TasksInactivityTimeout { get; set; } = TimeSpan.FromMinutes(30); + public TimeSpan TasksWaitTimeout { get; set; } = TimeSpan.FromMinutes(60); + public StuckTasksProcessMode StuckTasksProcessMode { get; set; } = StuckTasksProcessMode.Fail; } @@ -24,6 +26,7 @@ public abstract class TasksModuleOptions : BaseModuleOpti { public List Assemblies { get; } = new(); private readonly List> jobServiceConfigurations = new(); + public string TableName { get; set; } = "Tasks"; private readonly List>> @@ -83,3 +86,9 @@ public TasksModuleOptions AddTask Date: Mon, 9 Oct 2023 12:50:06 +0500 Subject: [PATCH 09/32] refactor(tasks): change log message --- src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs index 3ab17931d..5d3c2a40f 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs @@ -77,7 +77,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } catch (Exception ex) { - logger.LogError("Error schedule task {Type}: {Error}", typeof(TTask), ex); + logger.LogError("Error schedule tasks {Type}: {Error}", typeof(TTask), ex); } } } From a43385f739105bae90c73a6ff6d49bd4a984a9d6 Mon Sep 17 00:00:00 2001 From: nadia Date: Mon, 9 Oct 2023 12:54:41 +0500 Subject: [PATCH 10/32] refactor(tasks): cron expression instead of interval in options --- .../Scheduling/TaskSchedulingOptions.cs | 3 ++- .../Scheduling/TaskSchedulingService.cs | 16 +++++++--------- src/Sitko.Core.Tasks/TasksModuleOptions.cs | 5 +++-- .../BaseKafkaTasksTest.cs | 5 +++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs index 4ccabd0bb..a26ac17a1 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingOptions.cs @@ -1,8 +1,9 @@ +using Cronos; using Sitko.Core.Tasks.Data.Entities; namespace Sitko.Core.Tasks.Scheduling; public class TaskSchedulingOptions where TTask : IBaseTask { - public string Interval { get; set; } = ""; + public CronExpression CronExpression { get; set; } = null!; } diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs index 5d3c2a40f..425419002 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs @@ -1,4 +1,3 @@ -using Cronos; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -16,7 +15,9 @@ public class TaskSchedulingService : BackgroundService where TTask : clas private readonly IOptionsMonitor optionsMonitor; private readonly ILogger> logger; - public TaskSchedulingService(IServiceScopeFactory serviceScopeFactory, IOptions> taskOptions, IOptionsMonitor optionsMonitor, ILogger> logger) + public TaskSchedulingService(IServiceScopeFactory serviceScopeFactory, + IOptions> taskOptions, IOptionsMonitor optionsMonitor, + ILogger> logger) { this.serviceScopeFactory = serviceScopeFactory; this.taskOptions = taskOptions; @@ -61,14 +62,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - if (!string.IsNullOrEmpty(taskOptions.Value.Interval)) + var now = DateTime.UtcNow; + var nextDate = taskOptions.Value.CronExpression.GetNextOccurrence(now); + if (nextDate != null) { - var now = DateTime.UtcNow; - var nextDate = CronExpression.Parse(taskOptions.Value.Interval).GetNextOccurrence(now); - if (nextDate != null) - { - await Task.Delay(TimeSpan.FromSeconds((nextDate - now).Value.TotalSeconds), stoppingToken); - } + await Task.Delay(TimeSpan.FromSeconds((nextDate - now).Value.TotalSeconds), stoppingToken); } } catch (TaskCanceledException) diff --git a/src/Sitko.Core.Tasks/TasksModuleOptions.cs b/src/Sitko.Core.Tasks/TasksModuleOptions.cs index bbf3ef2e3..04c73d1f9 100644 --- a/src/Sitko.Core.Tasks/TasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks/TasksModuleOptions.cs @@ -1,4 +1,5 @@ using System.Reflection; +using Cronos; using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App; using Sitko.Core.Repository; @@ -59,7 +60,7 @@ public TasksModuleOptions AddExecutorsFromAssemblyOf jobServiceConfigurations.Any(); - public TasksModuleOptions AddTask(string interval) + public TasksModuleOptions AddTask(CronExpression cronExpression) where TTask : class, IBaseTask where TConfig : BaseTaskConfig, new() where TResult : BaseTaskResult, new() @@ -70,7 +71,7 @@ public TasksModuleOptions AddTask { - services.Configure>(options => options.Interval = interval); + services.Configure>(options => options.CronExpression = cronExpression); var schedulerType = typeof(IBaseTaskFactory); services.Scan(selector => selector.FromAssemblyOf() diff --git a/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs b/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs index f8e12fa64..b00ff4f34 100644 --- a/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs +++ b/tests/Sitko.Core.Tasks.Kafka.Tests/BaseKafkaTasksTest.cs @@ -1,4 +1,5 @@ -using Sitko.Core.Tasks.Kafka.Tests.Data; +using Cronos; +using Sitko.Core.Tasks.Kafka.Tests.Data; using Sitko.Core.Xunit; using Xunit.Abstractions; @@ -20,7 +21,7 @@ protected override TestApplication ConfigureApplication(TestApplication applicat .AddKafkaTasks(options => { options - .AddTask("* * * * *") + .AddTask(CronExpression.Parse("* * * * *")) .AddExecutorsFromAssemblyOf(); }); return application; From 082b87864e381589d3b2abca1138ed625f865bcb Mon Sep 17 00:00:00 2001 From: nadia Date: Mon, 9 Oct 2023 12:57:55 +0500 Subject: [PATCH 11/32] refactor(tasks): rename system tasks --- .../TasksCleaner.cs} | 12 ++++++------ .../TasksMaintenance.cs} | 9 ++++----- src/Sitko.Core.Tasks/TasksModule.cs | 6 +++--- 3 files changed, 13 insertions(+), 14 deletions(-) rename src/Sitko.Core.Tasks/{Tasks/CleanerTask.cs => BackgroundServices/TasksCleaner.cs} (88%) rename src/Sitko.Core.Tasks/{Tasks/MaintenanceTask.cs => BackgroundServices/TasksMaintenance.cs} (91%) diff --git a/src/Sitko.Core.Tasks/Tasks/CleanerTask.cs b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs similarity index 88% rename from src/Sitko.Core.Tasks/Tasks/CleanerTask.cs rename to src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs index 6e09bf8e7..ae951be8c 100644 --- a/src/Sitko.Core.Tasks/Tasks/CleanerTask.cs +++ b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs @@ -7,17 +7,17 @@ using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Data.Repository; -namespace Sitko.Core.Tasks.Tasks; +namespace Sitko.Core.Tasks.BackgroundServices; -public class CleanerTask : BackgroundService +public class TasksCleaner : BackgroundService where TBaseTask : BaseTask { private readonly IServiceScopeFactory serviceScopeFactory; private readonly IOptions options; - private readonly ILogger> logger; + private readonly ILogger> logger; private ITaskRepository tasksRepository; - public CleanerTask(IOptions options, ILogger> logger, + public TasksCleaner(IOptions options, ILogger> logger, IServiceScopeFactory serviceScopeFactory) { this.options = options; @@ -62,8 +62,8 @@ private async Task RemoveTasks(int retentionDays, bool include, string[] types) $" AND \"{nameof(BaseTask.Type)}\" {(include ? "IN" : "NOT IN")} ({string.Join(",", types.Select(type => $"'{type}'"))})"; } - var cnt = await efRepository.DeleteAllRawAsync(condition); - logger.LogInformation("Deleted {Count} tasks", cnt); + var deletedCount = await efRepository.DeleteAllRawAsync(condition); + logger.LogInformation("Deleted {Count} tasks", deletedCount); } else { diff --git a/src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs b/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs similarity index 91% rename from src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs rename to src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs index 9dedd04cf..0c083a2ae 100644 --- a/src/Sitko.Core.Tasks/Tasks/MaintenanceTask.cs +++ b/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs @@ -1,21 +1,20 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; -using Sitko.Core.Tasks.Data; using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Data.Repository; using Sitko.Core.Tasks.Execution; using TaskStatus = Sitko.Core.Tasks.Data.Entities.TaskStatus; -namespace Sitko.Core.Tasks.Tasks; +namespace Sitko.Core.Tasks.BackgroundServices; -public class MaintenanceTask : BackgroundService - where TBaseTask : BaseTask where TDbContext : TasksDbContext +public class TasksMaintenance : BackgroundService + where TBaseTask : BaseTask { private readonly IOptions options; private readonly IServiceScopeFactory serviceScopeFactory; - public MaintenanceTask(IOptions options, + public TasksMaintenance(IOptions options, IServiceScopeFactory serviceScopeFactory) { this.options = options; diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs index 78403a92f..06482420b 100644 --- a/src/Sitko.Core.Tasks/TasksModule.cs +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -2,13 +2,13 @@ using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App; using Sitko.Core.Repository; +using Sitko.Core.Tasks.BackgroundServices; using Sitko.Core.Tasks.Components; using Sitko.Core.Tasks.Data; using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Data.Repository; using Sitko.Core.Tasks.Execution; using Sitko.Core.Tasks.Scheduling; -using Sitko.Core.Tasks.Tasks; namespace Sitko.Core.Tasks; @@ -62,8 +62,8 @@ public override void ConfigureServices(IApplicationContext applicationContext, I services.Configure(applicationContext.Configuration.GetSection(OptionsKey)); services.AddTransient, TasksRepository>(); services.AddTransient, TasksRepository>(); - services.AddHostedService>(); - services.AddHostedService>(); + services.AddHostedService>(); + services.AddHostedService>(); ConfigureServicesInternal(applicationContext, services, startupOptions, executors); startupOptions.ConfigureServices(services); From a0412728b24e07462b1b4ef2b9295c4d56c68546 Mon Sep 17 00:00:00 2001 From: nadia Date: Mon, 9 Oct 2023 13:15:43 +0500 Subject: [PATCH 12/32] refactor(tasks): create base hosted service; add delay to first run --- .../BackgroundServices/BaseService.cs | 40 +++++++++ .../BackgroundServices/TasksCleaner.cs | 48 +++++------ .../BackgroundServices/TasksMaintenance.cs | 85 +++++++++---------- 3 files changed, 100 insertions(+), 73 deletions(-) create mode 100644 src/Sitko.Core.Tasks/BackgroundServices/BaseService.cs diff --git a/src/Sitko.Core.Tasks/BackgroundServices/BaseService.cs b/src/Sitko.Core.Tasks/BackgroundServices/BaseService.cs new file mode 100644 index 000000000..d457ce540 --- /dev/null +++ b/src/Sitko.Core.Tasks/BackgroundServices/BaseService.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Sitko.Core.Tasks.BackgroundServices; + +public abstract class BaseService : BackgroundService +{ + private readonly IServiceScopeFactory serviceScopeFactory; + private readonly ILogger logger; + protected abstract TimeSpan InitDelay { get; } + protected abstract TimeSpan RunDelay { get; } + + protected BaseService(IServiceScopeFactory serviceScopeFactory, ILogger logger) + { + this.serviceScopeFactory = serviceScopeFactory; + this.logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.Delay(InitDelay, stoppingToken); + while (!stoppingToken.IsCancellationRequested) + { + await using var scope = serviceScopeFactory.CreateAsyncScope(); + try + { + await ExecuteAsync(scope, stoppingToken); + } + catch (Exception ex) + { + logger.LogError("Error execute service {Service}: {Error}", GetType(), ex); + } + + await Task.Delay(RunDelay, stoppingToken); + } + } + + protected abstract Task ExecuteAsync(AsyncServiceScope scope, CancellationToken stoppingToken); +} diff --git a/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs index ae951be8c..f6063f0ac 100644 --- a/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs +++ b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs @@ -1,6 +1,5 @@ using System.Globalization; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Sitko.Core.Repository.EntityFrameworkCore; @@ -9,47 +8,42 @@ namespace Sitko.Core.Tasks.BackgroundServices; -public class TasksCleaner : BackgroundService +public class TasksCleaner : BaseService where TBaseTask : BaseTask { - private readonly IServiceScopeFactory serviceScopeFactory; private readonly IOptions options; private readonly ILogger> logger; private ITaskRepository tasksRepository; + protected override TimeSpan InitDelay => TimeSpan.FromMinutes(2); + protected override TimeSpan RunDelay => TimeSpan.FromDays(1); - public TasksCleaner(IOptions options, ILogger> logger, - IServiceScopeFactory serviceScopeFactory) + public TasksCleaner(IServiceScopeFactory serviceScopeFactory, ILogger logger, + IOptions options, ILogger> logger1) : base(serviceScopeFactory, + logger) { this.options = options; - this.logger = logger; - this.serviceScopeFactory = serviceScopeFactory; + this.logger = logger1; } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) + protected override async Task ExecuteAsync(AsyncServiceScope scope, CancellationToken stoppingToken) { - while (!stoppingToken.IsCancellationRequested) + tasksRepository = scope.ServiceProvider.GetRequiredService>(); + if (options.Value.AllTasksRetentionDays is > 0) { - await using var scope = serviceScopeFactory.CreateAsyncScope(); - tasksRepository = scope.ServiceProvider.GetRequiredService>(); - if (options.Value.AllTasksRetentionDays is > 0) - { - var taskTypes = options.Value.RetentionDays.Select(r => r.Key).ToArray(); - await RemoveTasks(options.Value.AllTasksRetentionDays.Value, false, taskTypes); - } + var taskTypes = options.Value.RetentionDays.Select(r => r.Key).ToArray(); + await RemoveTasksAsync(options.Value.AllTasksRetentionDays.Value, false, taskTypes, stoppingToken); + } - foreach (var (taskType, retentionDays) in options.Value.RetentionDays) + foreach (var (taskType, retentionDays) in options.Value.RetentionDays) + { + if (retentionDays > 0) { - if (retentionDays > 0) - { - await RemoveTasks(retentionDays, true, new[] { taskType }); - } + await RemoveTasksAsync(retentionDays, true, new[] { taskType }, stoppingToken); } - - await Task.Delay(TimeSpan.FromDays(1), stoppingToken); } } - private async Task RemoveTasks(int retentionDays, bool include, string[] types) + private async Task RemoveTasksAsync(int retentionDays, bool include, string[] types, CancellationToken stoppingToken) { var date = DateTimeOffset.UtcNow.AddDays(retentionDays * -1); logger.LogInformation("Deleting tasks from {Date}. Types: {Types}, include: {Include}", date, types, include); @@ -62,7 +56,7 @@ private async Task RemoveTasks(int retentionDays, bool include, string[] types) $" AND \"{nameof(BaseTask.Type)}\" {(include ? "IN" : "NOT IN")} ({string.Join(",", types.Select(type => $"'{type}'"))})"; } - var deletedCount = await efRepository.DeleteAllRawAsync(condition); + var deletedCount = await efRepository.DeleteAllRawAsync(condition, stoppingToken); logger.LogInformation("Deleted {Count} tasks", deletedCount); } else @@ -72,12 +66,12 @@ await tasksRepository.GetAllAsync(query => query.Where(task => task.DateAdded < (types.Length == 0 || (include ? types.Contains(task.Type) - : !types.Contains(task.Type))))); + : !types.Contains(task.Type)))), stoppingToken); if (tasksCount > 0) { foreach (var task in tasks) { - await tasksRepository.DeleteAsync(task); + await tasksRepository.DeleteAsync(task, stoppingToken); } logger.LogInformation("Deleted {Count} tasks", tasksCount); diff --git a/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs b/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs index 0c083a2ae..01379c33d 100644 --- a/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs +++ b/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs @@ -1,5 +1,5 @@ using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Data.Repository; @@ -8,62 +8,55 @@ namespace Sitko.Core.Tasks.BackgroundServices; -public class TasksMaintenance : BackgroundService +public class TasksMaintenance : BaseService where TBaseTask : BaseTask { private readonly IOptions options; - private readonly IServiceScopeFactory serviceScopeFactory; - public TasksMaintenance(IOptions options, - IServiceScopeFactory serviceScopeFactory) - { - this.options = options; - this.serviceScopeFactory = serviceScopeFactory; - } + public TasksMaintenance(IServiceScopeFactory serviceScopeFactory, ILogger logger, + IOptions options) : base(serviceScopeFactory, logger) => this.options = options; + - protected override async Task ExecuteAsync(CancellationToken stoppingToken) + protected override TimeSpan InitDelay => TimeSpan.FromMinutes(5); + protected override TimeSpan RunDelay => TimeSpan.FromMinutes(5); + + protected override async Task ExecuteAsync(AsyncServiceScope scope, CancellationToken stoppingToken) { - while (!stoppingToken.IsCancellationRequested) + var tasksRepository = scope.ServiceProvider.GetRequiredService>(); + var inactivityDate = DateTimeOffset.UtcNow - options.Value.TasksInactivityTimeout; + var waitDate = DateTimeOffset.UtcNow - options.Value.TasksWaitTimeout; + var stuckTasks = await tasksRepository.GetAllAsync(query => + query.Where(task => + (task.TaskStatus == TaskStatus.InProgress && task.LastActivityDate < inactivityDate) || + (task.TaskStatus == TaskStatus.Wait && task.DateAdded < waitDate)), stoppingToken); + if (stuckTasks.items.Length > 0) { - await using var scope = serviceScopeFactory.CreateAsyncScope(); - var tasksRepository = scope.ServiceProvider.GetRequiredService>(); - var inactivityDate = DateTimeOffset.UtcNow - options.Value.TasksInactivityTimeout; - var waitDate = DateTimeOffset.UtcNow - options.Value.TasksWaitTimeout; - var stuckTasks = await tasksRepository.GetAllAsync(query => - query.Where(task => - (task.TaskStatus == TaskStatus.InProgress && task.LastActivityDate < inactivityDate) || - (task.TaskStatus == TaskStatus.Wait && task.DateAdded < waitDate)), stoppingToken); - if (stuckTasks.items.Length > 0) + switch (options.Value.StuckTasksProcessMode) { - switch (options.Value.StuckTasksProcessMode) - { - case StuckTasksProcessMode.Fail: - await tasksRepository.BeginBatchAsync(stoppingToken); - foreach (var task in stuckTasks.items) - { - var error = task.TaskStatus == TaskStatus.InProgress - ? $"Task inactive since {task.LastActivityDate}" - : $"Task in queue since {task.DateAdded}"; - TasksExtensions.SetTaskErrorResult((IBaseTaskWithConfigAndResult)task, error); - await tasksRepository.UpdateAsync(task, stoppingToken); - } + case StuckTasksProcessMode.Fail: + await tasksRepository.BeginBatchAsync(stoppingToken); + foreach (var task in stuckTasks.items) + { + var error = task.TaskStatus == TaskStatus.InProgress + ? $"Task inactive since {task.LastActivityDate}" + : $"Task in queue since {task.DateAdded}"; + TasksExtensions.SetTaskErrorResult((IBaseTaskWithConfigAndResult)task, error); + await tasksRepository.UpdateAsync(task, stoppingToken); + } - await tasksRepository.CommitBatchAsync(stoppingToken); - break; - case StuckTasksProcessMode.Restart: - var taskExecutor = scope.ServiceProvider.GetRequiredService(); - foreach (var stuckTask in stuckTasks.items) - { - await taskExecutor.ExecuteAsync(stuckTask.Id, stoppingToken); - } + await tasksRepository.CommitBatchAsync(stoppingToken); + break; + case StuckTasksProcessMode.Restart: + var taskExecutor = scope.ServiceProvider.GetRequiredService(); + foreach (var stuckTask in stuckTasks.items) + { + await taskExecutor.ExecuteAsync(stuckTask.Id, stoppingToken); + } - break; - default: - throw new ArgumentOutOfRangeException(); - } + break; + default: + throw new ArgumentOutOfRangeException(); } - - await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken); } } } From e28eb25ddf2a61e2babc043a3239eb2c15a36901 Mon Sep 17 00:00:00 2001 From: nadia Date: Mon, 9 Oct 2023 14:10:40 +0500 Subject: [PATCH 13/32] refactor(tasks): use base tasks options --- .../BackgroundServices/TasksCleaner.cs | 9 +++++---- .../BackgroundServices/TasksMaintenance.cs | 7 ++++--- .../Scheduling/TaskSchedulingService.cs | 10 +++++----- src/Sitko.Core.Tasks/TasksModule.cs | 5 ++--- src/Sitko.Core.Tasks/TasksModuleOptions.cs | 11 ++++++----- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs index f6063f0ac..64ca0fec5 100644 --- a/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs +++ b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs @@ -8,17 +8,18 @@ namespace Sitko.Core.Tasks.BackgroundServices; -public class TasksCleaner : BaseService +public class TasksCleaner : BaseService where TBaseTask : BaseTask + where TOptions : TasksModuleOptions { - private readonly IOptions options; - private readonly ILogger> logger; + private readonly IOptions options; + private readonly ILogger> logger; private ITaskRepository tasksRepository; protected override TimeSpan InitDelay => TimeSpan.FromMinutes(2); protected override TimeSpan RunDelay => TimeSpan.FromDays(1); public TasksCleaner(IServiceScopeFactory serviceScopeFactory, ILogger logger, - IOptions options, ILogger> logger1) : base(serviceScopeFactory, + IOptions options, ILogger> logger1) : base(serviceScopeFactory, logger) { this.options = options; diff --git a/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs b/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs index 01379c33d..de12ff3a8 100644 --- a/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs +++ b/src/Sitko.Core.Tasks/BackgroundServices/TasksMaintenance.cs @@ -8,13 +8,14 @@ namespace Sitko.Core.Tasks.BackgroundServices; -public class TasksMaintenance : BaseService +public class TasksMaintenance : BaseService where TBaseTask : BaseTask + where TOptions : TasksModuleOptions { - private readonly IOptions options; + private readonly IOptions options; public TasksMaintenance(IServiceScopeFactory serviceScopeFactory, ILogger logger, - IOptions options) : base(serviceScopeFactory, logger) => this.options = options; + IOptions options) : base(serviceScopeFactory, logger) => this.options = options; protected override TimeSpan InitDelay => TimeSpan.FromMinutes(5); diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs index 425419002..6216d02da 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs @@ -8,16 +8,16 @@ namespace Sitko.Core.Tasks.Scheduling; -public class TaskSchedulingService : BackgroundService where TTask : class, IBaseTask +public class TaskSchedulingService : BackgroundService where TTask : class, IBaseTask where TOptions : TasksModuleOptions { private readonly IServiceScopeFactory serviceScopeFactory; private readonly IOptions> taskOptions; - private readonly IOptionsMonitor optionsMonitor; - private readonly ILogger> logger; + private readonly IOptionsMonitor optionsMonitor; + private readonly ILogger> logger; public TaskSchedulingService(IServiceScopeFactory serviceScopeFactory, - IOptions> taskOptions, IOptionsMonitor optionsMonitor, - ILogger> logger) + IOptions> taskOptions, IOptionsMonitor optionsMonitor, + ILogger> logger) { this.serviceScopeFactory = serviceScopeFactory; this.taskOptions = taskOptions; diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs index 06482420b..c55da0015 100644 --- a/src/Sitko.Core.Tasks/TasksModule.cs +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -59,11 +59,10 @@ public override void ConfigureServices(IApplicationContext applicationContext, I services.AddScoped(); services.AddScoped(); - services.Configure(applicationContext.Configuration.GetSection(OptionsKey)); services.AddTransient, TasksRepository>(); services.AddTransient, TasksRepository>(); - services.AddHostedService>(); - services.AddHostedService>(); + services.AddHostedService>(); + services.AddHostedService>(); ConfigureServicesInternal(applicationContext, services, startupOptions, executors); startupOptions.ConfigureServices(services); diff --git a/src/Sitko.Core.Tasks/TasksModuleOptions.cs b/src/Sitko.Core.Tasks/TasksModuleOptions.cs index 04c73d1f9..75f560537 100644 --- a/src/Sitko.Core.Tasks/TasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks/TasksModuleOptions.cs @@ -1,6 +1,7 @@ using System.Reflection; using Cronos; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Sitko.Core.App; using Sitko.Core.Repository; using Sitko.Core.Tasks.Data; @@ -10,7 +11,7 @@ namespace Sitko.Core.Tasks; -public class TasksModuleOptions +public abstract class TasksModuleOptions : BaseModuleOptions, IModuleOptionsWithValidation { public bool IsAllTasksDisabled { get; set; } public string[] DisabledTasks { get; set; } = Array.Empty(); @@ -19,10 +20,12 @@ public class TasksModuleOptions public TimeSpan TasksInactivityTimeout { get; set; } = TimeSpan.FromMinutes(30); public TimeSpan TasksWaitTimeout { get; set; } = TimeSpan.FromMinutes(60); public StuckTasksProcessMode StuckTasksProcessMode { get; set; } = StuckTasksProcessMode.Fail; + + public abstract Type GetValidatorType(); } -public abstract class TasksModuleOptions : BaseModuleOptions, IModuleOptionsWithValidation +public abstract class TasksModuleOptions : TasksModuleOptions where TDbContext : TasksDbContext where TBaseTask : BaseTask { public List Assemblies { get; } = new(); @@ -77,15 +80,13 @@ public TasksModuleOptions AddTask() .AddClasses(filter => filter.AssignableToAny(schedulerType)) .As>().WithScopedLifetime()); - services.AddHostedService>(); + services.AddSingleton(typeof(IHostedService), typeof(TaskSchedulingService<,>).MakeGenericType(typeof(TTask), GetType())); services.Scan(selector => selector.FromTypes(typeof(BaseTaskRepository)) .AsSelf().As().As>().As>() .WithTransientLifetime()); }); return this; } - - public abstract Type GetValidatorType(); } public enum StuckTasksProcessMode From d5a2ea81cc83cb5effdb8bc84f9df926f986c741 Mon Sep 17 00:00:00 2001 From: nadia Date: Mon, 9 Oct 2023 14:57:31 +0500 Subject: [PATCH 14/32] refactor(tasks): use DeleteAll method --- .../BackgroundServices/TasksCleaner.cs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs index 64ca0fec5..b8da1f85a 100644 --- a/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs +++ b/src/Sitko.Core.Tasks/BackgroundServices/TasksCleaner.cs @@ -1,4 +1,3 @@ -using System.Globalization; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -48,16 +47,11 @@ private async Task RemoveTasksAsync(int retentionDays, bool include, string[] ty { var date = DateTimeOffset.UtcNow.AddDays(retentionDays * -1); logger.LogInformation("Deleting tasks from {Date}. Types: {Types}, include: {Include}", date, types, include); - if (tasksRepository is IEFRepository efRepository) + if (tasksRepository is IEFRepository efRepository) { - var condition = $"\"{nameof(BaseTask.DateAdded)}\" < '{date.ToString("O", CultureInfo.InvariantCulture)}'"; - if (types.Length > 0) - { - condition += - $" AND \"{nameof(BaseTask.Type)}\" {(include ? "IN" : "NOT IN")} ({string.Join(",", types.Select(type => $"'{type}'"))})"; - } - - var deletedCount = await efRepository.DeleteAllRawAsync(condition, stoppingToken); + var deletedCount = await efRepository.DeleteAllAsync(task => task.DateAdded < date && (types.Length == 0 || (include + ? types.Contains(task.Type) + : !types.Contains(task.Type))), stoppingToken); logger.LogInformation("Deleted {Count} tasks", deletedCount); } else From f73104e050b53cca1cc1f40f0c2e26dd2fcf9749 Mon Sep 17 00:00:00 2001 From: nadia Date: Thu, 12 Oct 2023 15:28:05 +0500 Subject: [PATCH 15/32] fix(tasks): fix get executor types --- src/Sitko.Core.Tasks/TasksModule.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Sitko.Core.Tasks/TasksModule.cs b/src/Sitko.Core.Tasks/TasksModule.cs index c55da0015..379e54270 100644 --- a/src/Sitko.Core.Tasks/TasksModule.cs +++ b/src/Sitko.Core.Tasks/TasksModule.cs @@ -1,5 +1,6 @@ using FluentValidation; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Sitko.Core.App; using Sitko.Core.Repository; using Sitko.Core.Tasks.BackgroundServices; @@ -31,7 +32,13 @@ public override void ConfigureServices(IApplicationContext applicationContext, I { foreach (var assembly in startupOptions.Assemblies) { - types.AddRange(assembly.ExportedTypes.Where(type => typeof(ITaskExecutor).IsAssignableFrom(type))); + types.AddRange(assembly.ExportedTypes.Where(type => typeof(ITaskExecutor).IsAssignableFrom(type) && + !type.IsAbstract && + typeof(TBaseTask).IsAssignableFrom(type + .GetInterfaces() + .First(i => i.IsGenericType && + typeof(ITaskExecutor).IsAssignableFrom(i)) + .GenericTypeArguments.First()))); } } @@ -56,9 +63,9 @@ public override void ConfigureServices(IApplicationContext applicationContext, I services.Scan(selector => selector.FromTypes(executors.Select(e => e.ExecutorType)).AsSelfWithInterfaces() .WithScopedLifetime()); - services.AddScoped(); + services.TryAddScoped(); - services.AddScoped(); + services.TryAddScoped(); services.AddTransient, TasksRepository>(); services.AddTransient, TasksRepository>(); services.AddHostedService>(); From 1315bf2481a69b4fb5222c7210f461c6fbda09ad Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:13:07 +0500 Subject: [PATCH 16/32] fix: single cluster for all kafka tasks modules --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 59 ++++++++++ .../KafkaTasksModule.cs | 101 ++++++++---------- 2 files changed, 102 insertions(+), 58 deletions(-) create mode 100644 src/Sitko.Core.Kafka/KafkaConfigurator.cs diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs new file mode 100644 index 000000000..916dd1c25 --- /dev/null +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -0,0 +1,59 @@ +using KafkaFlow; +using KafkaFlow.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace Sitko.Core.Kafka; + +public class KafkaConfigurator +{ + private static readonly Dictionary Configurators = new(); + private readonly string[] brokers; + + private KafkaConfigurator(string[] brokers) => this.brokers = brokers; + + private readonly List> consumerActions = new(); + private readonly Dictionary> producerActions = new(); + + public KafkaConfigurator AddProducer(string name, Action configure) + { + producerActions[name] = configure; + return this; + } + + public KafkaConfigurator AddConsumer(Action configure) + { + consumerActions.Add(configure); + return this; + } + + public IServiceCollection Build(string name, IServiceCollection serviceCollection) + { + serviceCollection.AddKafkaFlowHostedService(builder => + { + builder + .UseMicrosoftLog() + .AddCluster(clusterBuilder => + { + clusterBuilder + .WithName(name) + .WithBrokers(brokers); + foreach (var (producerName, configure) in producerActions) + { + clusterBuilder.AddProducer(producerName, configurationBuilder => + { + configure(configurationBuilder); + }); + } + + foreach (var consumerAction in consumerActions) + { + clusterBuilder.AddConsumer(consumerAction); + } + }); + }); + return serviceCollection; + } + + public static KafkaConfigurator Create(string[] brokers) => + Configurators.SafeGetOrAdd(string.Join("|", brokers.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)), _ => new KafkaConfigurator(brokers)); +} diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 2aa835fec..fae212e6c 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -6,6 +6,7 @@ using KafkaFlow.TypedHandler; using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App; +using Sitko.Core.Kafka; using Sitko.Core.Tasks.Data; using Sitko.Core.Tasks.Data.Entities; using Sitko.Core.Tasks.Execution; @@ -32,68 +33,52 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio : startupOptions.TopicPrefix) : ""; var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}"; - services.AddKafkaFlowHostedService(builder => + + var kafkaConfigurator = KafkaConfigurator.Create(startupOptions.Brokers); + kafkaConfigurator + .AddProducer("default", builder => + { + builder.DefaultTopic(kafkaTopic); + builder.AddMiddlewares(middlewareBuilder => + middlewareBuilder.AddSerializer()); + }); + var executorType = typeof(KafkaExecutor<,>); + foreach (var groupConsumers in executors.GroupBy(r => r.GroupId)) { - builder - .UseMicrosoftLog() - .AddCluster(clusterBuilder => + var commonRegistration = groupConsumers.First(); + var name = $"{applicationContext.Name}/{applicationContext.Id}/{commonRegistration.GroupId}"; + var parallelThreadCount = groupConsumers.Max(r => r.ParallelThreadCount); + var bufferSize = groupConsumers.Max(r => r.BufferSize); + kafkaConfigurator.AddConsumer(consumerBuilder => + { + consumerBuilder.Topic(kafkaTopic); + consumerBuilder.WithName(name); + consumerBuilder.WithGroupId(commonRegistration.GroupId); + consumerBuilder.WithWorkersCount(parallelThreadCount); + consumerBuilder.WithBufferSize(bufferSize); + // для гарантии порядка событий + consumerBuilder + .WithWorkDistributionStrategy(); + var consumerConfig = new ConsumerConfig { - clusterBuilder - .WithName($"Tasks_{typeof(TBaseTask).Name}") - .WithBrokers(startupOptions.Brokers) - .AddProducer("default", producerBuilder => - { - producerBuilder - .DefaultTopic(kafkaTopic) - .AddMiddlewares(configurationBuilder => - configurationBuilder.AddSerializer()); - }); - // регистрируем консьюмеры на каждую группу экзекьюторов - var executorType = typeof(KafkaExecutor<,>); - foreach (var groupConsumers in executors.GroupBy(r => r.GroupId)) + AutoOffsetReset = AutoOffsetReset.Latest, ClientId = name, GroupInstanceId = name, PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky + }; + consumerBuilder.WithConsumerConfig(consumerConfig); + consumerBuilder.AddMiddlewares( + middlewares => { - var commonRegistration = groupConsumers.First(); - var name = - $"{applicationContext.Name}/{applicationContext.Id}/{commonRegistration.GroupId}"; - - var parallelThreadCount = groupConsumers.Max(r => r.ParallelThreadCount); - var bufferSize = groupConsumers.Max(r => r.BufferSize); - - clusterBuilder.AddConsumer( - consumerBuilder => - { - consumerBuilder.Topic(kafkaTopic); - consumerBuilder.WithName(name); - consumerBuilder.WithGroupId(commonRegistration.GroupId); - consumerBuilder.WithWorkersCount(parallelThreadCount); - consumerBuilder.WithBufferSize(bufferSize); - // для гарантии порядка событий - consumerBuilder - .WithWorkDistributionStrategy(); - var consumerConfig = new ConsumerConfig - { - AutoOffsetReset = AutoOffsetReset.Latest, - ClientId = name, - GroupInstanceId = name, - PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky - }; - consumerBuilder.WithConsumerConfig(consumerConfig); - consumerBuilder.AddMiddlewares( - middlewares => - { - middlewares - .AddSerializer(); - middlewares.AddTypedHandlers(handlers => - handlers.AddHandlers(groupConsumers.Select(r => - executorType.MakeGenericType(r.EventType, r.ExecutorType))) - .WithHandlerLifetime(InstanceLifetime.Scoped)); - } - ); - } - ); + middlewares + .AddSerializer(); + middlewares.AddTypedHandlers(handlers => + handlers.AddHandlers(groupConsumers.Select(r => + executorType.MakeGenericType(r.EventType, r.ExecutorType))) + .WithHandlerLifetime(InstanceLifetime.Scoped)); } - }); - }); + ); + }); + } + + kafkaConfigurator.Build($"Kafka_Tasks_Cluster", services); } } From 3506def69898968c43ff3a4dbaf0b6956f31836e Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:33:24 +0500 Subject: [PATCH 17/32] feat: add di post-configuration hook --- src/Sitko.Core.App/Application.cs | 5 +++++ src/Sitko.Core.App/ApplicationModuleRegistration.cs | 11 +++++++++++ src/Sitko.Core.App/BaseApplicationModule.cs | 5 +++++ src/Sitko.Core.App/IApplicationModule.cs | 3 +++ 4 files changed, 24 insertions(+) diff --git a/src/Sitko.Core.App/Application.cs b/src/Sitko.Core.App/Application.cs index 28a6e40b5..1f9dd19e3 100644 --- a/src/Sitko.Core.App/Application.cs +++ b/src/Sitko.Core.App/Application.cs @@ -105,6 +105,11 @@ protected void RegisterApplicationServices(IApplicationCont { servicesConfigurationAction(applicationContext, services); } + + foreach (var moduleRegistration in GetEnabledModuleRegistrations(applicationContext)) + { + moduleRegistration.PostConfigureServices(applicationContext, services); + } } protected void ConfigureConfiguration(IApplicationContext appContext, IConfigurationBuilder builder) diff --git a/src/Sitko.Core.App/ApplicationModuleRegistration.cs b/src/Sitko.Core.App/ApplicationModuleRegistration.cs index d25e90f25..b5a232526 100644 --- a/src/Sitko.Core.App/ApplicationModuleRegistration.cs +++ b/src/Sitko.Core.App/ApplicationModuleRegistration.cs @@ -194,6 +194,14 @@ public override ApplicationModuleRegistration ConfigureServices( return this; } + public override ApplicationModuleRegistration PostConfigureServices(IApplicationContext context, + IServiceCollection services) + { + var options = CreateOptions(context, true); + instance.PostConfigureServices(context, services, options); + return this; + } + public override Task ApplicationStopped(IApplicationContext applicationContext, IServiceProvider serviceProvider) => instance.ApplicationStopped(applicationContext, serviceProvider); @@ -226,6 +234,9 @@ public abstract LoggerConfiguration ConfigureLogging(IApplicationContext context public abstract ApplicationModuleRegistration ConfigureServices(IApplicationContext context, IServiceCollection services); + public abstract ApplicationModuleRegistration PostConfigureServices(IApplicationContext context, + IServiceCollection services); + public abstract Task ApplicationStopped(IApplicationContext applicationContext, IServiceProvider serviceProvider); diff --git a/src/Sitko.Core.App/BaseApplicationModule.cs b/src/Sitko.Core.App/BaseApplicationModule.cs index 77b4b252d..e22f6d3f8 100644 --- a/src/Sitko.Core.App/BaseApplicationModule.cs +++ b/src/Sitko.Core.App/BaseApplicationModule.cs @@ -19,6 +19,11 @@ public virtual void ConfigureServices(IApplicationContext applicationContext, IS { } + public virtual void PostConfigureServices(IApplicationContext applicationContext, IServiceCollection services, + TModuleOptions startupOptions) + { + } + public virtual Task InitAsync(IApplicationContext applicationContext, IServiceProvider serviceProvider) => Task.CompletedTask; diff --git a/src/Sitko.Core.App/IApplicationModule.cs b/src/Sitko.Core.App/IApplicationModule.cs index 71e9a57ba..bebd1e74f 100644 --- a/src/Sitko.Core.App/IApplicationModule.cs +++ b/src/Sitko.Core.App/IApplicationModule.cs @@ -13,6 +13,9 @@ namespace Sitko.Core.App; void ConfigureServices(IApplicationContext applicationContext, IServiceCollection services, TModuleOptions startupOptions); + void PostConfigureServices(IApplicationContext applicationContext, IServiceCollection services, + TModuleOptions startupOptions); + IEnumerable GetRequiredModules(IApplicationContext applicationContext, TModuleOptions options); } From 256ca3c2b2f3633734a4cdaa8fbfd226ab7f97d9 Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:33:39 +0500 Subject: [PATCH 18/32] feat: configure kafka cluster once after all modules added --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 13 ++++++---- src/Sitko.Core.Kafka/KafkaModule.cs | 26 +++++++++++++++++++ .../ApplicationExtensions.cs | 6 ++++- .../KafkaTasksModule.cs | 9 ++++--- 4 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 src/Sitko.Core.Kafka/KafkaModule.cs diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index 916dd1c25..aedc85d47 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -6,10 +6,14 @@ namespace Sitko.Core.Kafka; public class KafkaConfigurator { - private static readonly Dictionary Configurators = new(); + private readonly string name; private readonly string[] brokers; - private KafkaConfigurator(string[] brokers) => this.brokers = brokers; + internal KafkaConfigurator(string name, string[] brokers) + { + this.name = name; + this.brokers = brokers; + } private readonly List> consumerActions = new(); private readonly Dictionary> producerActions = new(); @@ -26,7 +30,7 @@ public KafkaConfigurator AddConsumer(Action confi return this; } - public IServiceCollection Build(string name, IServiceCollection serviceCollection) + public IServiceCollection Build(IServiceCollection serviceCollection) { serviceCollection.AddKafkaFlowHostedService(builder => { @@ -54,6 +58,5 @@ public IServiceCollection Build(string name, IServiceCollection serviceCollectio return serviceCollection; } - public static KafkaConfigurator Create(string[] brokers) => - Configurators.SafeGetOrAdd(string.Join("|", brokers.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)), _ => new KafkaConfigurator(brokers)); + } diff --git a/src/Sitko.Core.Kafka/KafkaModule.cs b/src/Sitko.Core.Kafka/KafkaModule.cs new file mode 100644 index 000000000..f88547b94 --- /dev/null +++ b/src/Sitko.Core.Kafka/KafkaModule.cs @@ -0,0 +1,26 @@ +using KafkaFlow; +using Microsoft.Extensions.DependencyInjection; +using Sitko.Core.App; + +namespace Sitko.Core.Kafka; + +public class KafkaModule : BaseApplicationModule +{ + private static readonly Dictionary Configurators = new(); + public override bool AllowMultiple => false; + + public static KafkaConfigurator CreateConfigurator(string name, string[] brokers) => + Configurators.SafeGetOrAdd(name, _ => new KafkaConfigurator(name, brokers)); + + public override string OptionsKey => "Kafka"; + + public override void ConfigureServices(IApplicationContext applicationContext, IServiceCollection services, + BaseApplicationModuleOptions startupOptions) + { + base.ConfigureServices(applicationContext, services, startupOptions); + foreach (var configurator in Configurators) + { + configurator.Value.Build(services); + } + } +} diff --git a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs index 39ad33fa9..d678e2de0 100644 --- a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs +++ b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs @@ -1,5 +1,6 @@ using Sitko.Core.App; using Sitko.Core.Db.Postgres; +using Sitko.Core.Kafka; using Sitko.Core.Tasks.Data; using Sitko.Core.Tasks.Data.Entities; @@ -15,7 +16,10 @@ public static Application AddKafkaTasks(this Application application.AddTasks(configurePostgres, configurePostgresAction); application.AddModule, KafkaTasksModuleOptions>( configure); - + if (!application.HasModule()) + { + application.AddModule(); + } return application; } } diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index fae212e6c..19281902a 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -34,7 +34,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio : ""; var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}"; - var kafkaConfigurator = KafkaConfigurator.Create(startupOptions.Brokers); + var kafkaConfigurator = KafkaModule.CreateConfigurator($"Kafka_Tasks_Cluster", startupOptions.Brokers); kafkaConfigurator .AddProducer("default", builder => { @@ -61,7 +61,10 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio .WithWorkDistributionStrategy(); var consumerConfig = new ConsumerConfig { - AutoOffsetReset = AutoOffsetReset.Latest, ClientId = name, GroupInstanceId = name, PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky + AutoOffsetReset = AutoOffsetReset.Latest, + ClientId = name, + GroupInstanceId = name, + PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky }; consumerBuilder.WithConsumerConfig(consumerConfig); consumerBuilder.AddMiddlewares( @@ -77,8 +80,6 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio ); }); } - - kafkaConfigurator.Build($"Kafka_Tasks_Cluster", services); } } From 897ea5baf9024975b3550cd2e1ac72af4624bf72 Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:40:22 +0500 Subject: [PATCH 19/32] fix: use post di hook --- src/Sitko.Core.Kafka/KafkaModule.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sitko.Core.Kafka/KafkaModule.cs b/src/Sitko.Core.Kafka/KafkaModule.cs index f88547b94..abc2e46b1 100644 --- a/src/Sitko.Core.Kafka/KafkaModule.cs +++ b/src/Sitko.Core.Kafka/KafkaModule.cs @@ -14,7 +14,7 @@ public static KafkaConfigurator CreateConfigurator(string name, string[] brokers public override string OptionsKey => "Kafka"; - public override void ConfigureServices(IApplicationContext applicationContext, IServiceCollection services, + public override void PostConfigureServices(IApplicationContext applicationContext, IServiceCollection services, BaseApplicationModuleOptions startupOptions) { base.ConfigureServices(applicationContext, services, startupOptions); From 03dedc8ef877fb9bedcd45facf1c6b53aa035b02 Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:48:36 +0500 Subject: [PATCH 20/32] fix: single kafkaflow builder per app --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 44 ++++++++++------------- src/Sitko.Core.Kafka/KafkaModule.cs | 9 +++-- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index aedc85d47..0640c14e3 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -30,33 +30,25 @@ public KafkaConfigurator AddConsumer(Action confi return this; } - public IServiceCollection Build(IServiceCollection serviceCollection) - { - serviceCollection.AddKafkaFlowHostedService(builder => - { - builder - .UseMicrosoftLog() - .AddCluster(clusterBuilder => + public void Build(IKafkaConfigurationBuilder builder) => + builder + .UseMicrosoftLog() + .AddCluster(clusterBuilder => + { + clusterBuilder + .WithName(name) + .WithBrokers(brokers); + foreach (var (producerName, configure) in producerActions) { - clusterBuilder - .WithName(name) - .WithBrokers(brokers); - foreach (var (producerName, configure) in producerActions) - { - clusterBuilder.AddProducer(producerName, configurationBuilder => - { - configure(configurationBuilder); - }); - } - - foreach (var consumerAction in consumerActions) + clusterBuilder.AddProducer(producerName, configurationBuilder => { - clusterBuilder.AddConsumer(consumerAction); - } - }); - }); - return serviceCollection; - } - + configure(configurationBuilder); + }); + } + foreach (var consumerAction in consumerActions) + { + clusterBuilder.AddConsumer(consumerAction); + } + }); } diff --git a/src/Sitko.Core.Kafka/KafkaModule.cs b/src/Sitko.Core.Kafka/KafkaModule.cs index abc2e46b1..ca75dbcfe 100644 --- a/src/Sitko.Core.Kafka/KafkaModule.cs +++ b/src/Sitko.Core.Kafka/KafkaModule.cs @@ -18,9 +18,12 @@ public override void PostConfigureServices(IApplicationContext applicationContex BaseApplicationModuleOptions startupOptions) { base.ConfigureServices(applicationContext, services, startupOptions); - foreach (var configurator in Configurators) + services.AddKafkaFlowHostedService(builder => { - configurator.Value.Build(services); - } + foreach (var (_, configurator) in Configurators) + { + configurator.Build(builder); + } + }); } } From 5a06140a0892401e9e1835a4abfa36f629e8b90e Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:57:00 +0500 Subject: [PATCH 21/32] feat: create topic with correct settings --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 11 +++++++++++ src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs | 1 + src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs | 2 ++ 3 files changed, 14 insertions(+) diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index 0640c14e3..007f91b32 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -17,6 +17,7 @@ internal KafkaConfigurator(string name, string[] brokers) private readonly List> consumerActions = new(); private readonly Dictionary> producerActions = new(); + private readonly Dictionary topics = new(); public KafkaConfigurator AddProducer(string name, Action configure) { @@ -30,6 +31,12 @@ public KafkaConfigurator AddConsumer(Action confi return this; } + public KafkaConfigurator AutoCreateTopic(string topic, int partitions, short replicationFactor) + { + topics[topic] = (partitions, replicationFactor); + return this; + } + public void Build(IKafkaConfigurationBuilder builder) => builder .UseMicrosoftLog() @@ -38,6 +45,10 @@ public void Build(IKafkaConfigurationBuilder builder) => clusterBuilder .WithName(name) .WithBrokers(brokers); + foreach (var (topic, config) in topics) + { + clusterBuilder.CreateTopicIfNotExists(topic, config.Partitions, config.ReplicationFactor); + } foreach (var (producerName, configure) in producerActions) { clusterBuilder.AddProducer(producerName, configurationBuilder => diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 19281902a..1c1960fde 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -36,6 +36,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio var kafkaConfigurator = KafkaModule.CreateConfigurator($"Kafka_Tasks_Cluster", startupOptions.Brokers); kafkaConfigurator + .AutoCreateTopic(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) .AddProducer("default", builder => { builder.DefaultTopic(kafkaTopic); diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs index 193bf3549..0a225fec2 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs @@ -12,4 +12,6 @@ public class KafkaTasksModuleOptions : TasksModuleOptions public string TasksTopic { get; set; } = ""; public bool AddTopicPrefix { get; set; } = true; public string TopicPrefix { get; set; } = ""; + public int TopicPartitions { get; set; } = 24; + public short TopicReplicationFactor { get; set; } = 1; } From f885f178ff9107130d8de17554b6c10b1a7cf7ac Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:57:07 +0500 Subject: [PATCH 22/32] refactor: naming --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index 007f91b32..d0fcadf33 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -19,9 +19,9 @@ internal KafkaConfigurator(string name, string[] brokers) private readonly Dictionary> producerActions = new(); private readonly Dictionary topics = new(); - public KafkaConfigurator AddProducer(string name, Action configure) + public KafkaConfigurator AddProducer(string producerName, Action configure) { - producerActions[name] = configure; + producerActions[producerName] = configure; return this; } From 2bbcef7aaf343cf92d0ac7eb5f7ff3e9b2095c7e Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 16:57:19 +0500 Subject: [PATCH 23/32] fix: more unique names --- src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 1c1960fde..8b222424f 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -32,7 +32,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio ? $"{applicationContext.Name}_{applicationContext.Environment}" : startupOptions.TopicPrefix) : ""; - var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}"; + var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}".Replace(".", "_"); var kafkaConfigurator = KafkaModule.CreateConfigurator($"Kafka_Tasks_Cluster", startupOptions.Brokers); kafkaConfigurator @@ -47,7 +47,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio foreach (var groupConsumers in executors.GroupBy(r => r.GroupId)) { var commonRegistration = groupConsumers.First(); - var name = $"{applicationContext.Name}/{applicationContext.Id}/{commonRegistration.GroupId}"; + var name = $"{applicationContext.Name}/{applicationContext.Id}/{nameof(TBaseTask)}/{commonRegistration.GroupId}"; var parallelThreadCount = groupConsumers.Max(r => r.ParallelThreadCount); var bufferSize = groupConsumers.Max(r => r.BufferSize); kafkaConfigurator.AddConsumer(consumerBuilder => From 4375a0c4f0a80afacf492400c63104cd55adcc42 Mon Sep 17 00:00:00 2001 From: nadia Date: Thu, 12 Oct 2023 17:33:55 +0500 Subject: [PATCH 24/32] feat(tasks): move get next date to run execute --- .../Scheduling/TaskSchedulingService.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs index 6216d02da..746c4c6ac 100644 --- a/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs +++ b/src/Sitko.Core.Tasks/Scheduling/TaskSchedulingService.cs @@ -29,9 +29,16 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { - await using var scope = serviceScopeFactory.CreateAsyncScope(); try { + var now = DateTime.UtcNow; + var nextDate = taskOptions.Value.CronExpression.GetNextOccurrence(now); + if (nextDate != null) + { + await Task.Delay(TimeSpan.FromSeconds((nextDate - now).Value.TotalSeconds), stoppingToken); + } + + await using var scope = serviceScopeFactory.CreateAsyncScope(); var scheduler = scope.ServiceProvider.GetRequiredService>(); if (optionsMonitor.CurrentValue.IsAllTasksDisabled || optionsMonitor.CurrentValue.DisabledTasks.Contains(typeof(TTask).Name)) @@ -61,13 +68,6 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } } - - var now = DateTime.UtcNow; - var nextDate = taskOptions.Value.CronExpression.GetNextOccurrence(now); - if (nextDate != null) - { - await Task.Delay(TimeSpan.FromSeconds((nextDate - now).Value.TotalSeconds), stoppingToken); - } } catch (TaskCanceledException) { From e88e8a6b18adcb81619e670f8705049dc3f96167 Mon Sep 17 00:00:00 2001 From: nadia Date: Thu, 12 Oct 2023 17:34:24 +0500 Subject: [PATCH 25/32] feat(tasks): unique producer name for base tasks --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 1 - src/Sitko.Core.Tasks.Kafka/EventsRegistry.cs | 12 ++++++++++++ src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs | 9 +++++++-- .../Scheduling/KafkaTaskScheduler.cs | 2 +- 4 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 src/Sitko.Core.Tasks.Kafka/EventsRegistry.cs diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index d0fcadf33..e44460600 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -1,6 +1,5 @@ using KafkaFlow; using KafkaFlow.Configuration; -using Microsoft.Extensions.DependencyInjection; namespace Sitko.Core.Kafka; diff --git a/src/Sitko.Core.Tasks.Kafka/EventsRegistry.cs b/src/Sitko.Core.Tasks.Kafka/EventsRegistry.cs new file mode 100644 index 000000000..2ce1e6720 --- /dev/null +++ b/src/Sitko.Core.Tasks.Kafka/EventsRegistry.cs @@ -0,0 +1,12 @@ +namespace Sitko.Core.Tasks.Kafka; + +internal static class EventsRegistry +{ + private static readonly Dictionary events = new(); + + public static void Register(Type eventType, string topic, string producerName) => events[eventType] = (topic, producerName); + + public static string GetProducerName(Type eventType) => events.TryGetValue(eventType, out var eventData) + ? eventData.ProducerName + : throw new InvalidOperationException($"Can't find producer for event {eventType}"); +} diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 8b222424f..ea171283d 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -34,10 +34,15 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio : ""; var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}".Replace(".", "_"); + var producerName = $"Tasks_{typeof(TBaseTask).Name}"; + foreach (var executor in executors) + { + EventsRegistry.Register(executor.EventType, kafkaTopic, producerName); + } var kafkaConfigurator = KafkaModule.CreateConfigurator($"Kafka_Tasks_Cluster", startupOptions.Brokers); kafkaConfigurator .AutoCreateTopic(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) - .AddProducer("default", builder => + .AddProducer(producerName, builder => { builder.DefaultTopic(kafkaTopic); builder.AddMiddlewares(middlewareBuilder => @@ -47,7 +52,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio foreach (var groupConsumers in executors.GroupBy(r => r.GroupId)) { var commonRegistration = groupConsumers.First(); - var name = $"{applicationContext.Name}/{applicationContext.Id}/{nameof(TBaseTask)}/{commonRegistration.GroupId}"; + var name = $"{applicationContext.Name}/{applicationContext.Id}/{typeof(TBaseTask).Name}/{commonRegistration.GroupId}"; var parallelThreadCount = groupConsumers.Max(r => r.ParallelThreadCount); var bufferSize = groupConsumers.Max(r => r.BufferSize); kafkaConfigurator.AddConsumer(consumerBuilder => diff --git a/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs b/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs index 4d8ffa432..110bf5cb5 100644 --- a/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs +++ b/src/Sitko.Core.Tasks.Kafka/Scheduling/KafkaTaskScheduler.cs @@ -11,5 +11,5 @@ public class KafkaTaskScheduler : ITaskScheduler public KafkaTaskScheduler(IProducerAccessor producerAccessor) => this.producerAccessor = producerAccessor; public async Task ScheduleAsync(IBaseTask task) => - await producerAccessor.GetProducer("default").ProduceAsync(task.GetKey(), task); + await producerAccessor.GetProducer(EventsRegistry.GetProducerName(task.GetType())).ProduceAsync(task.GetKey(), task); } From cb351aeb19bc4b29ae64f01c77fea832bbd9dabe Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 17:54:58 +0500 Subject: [PATCH 26/32] feat: ensure consumer offsets --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 18 +- .../KafkaConsumerOffsetsEnsurer.cs | 162 ++++++++++++++++++ src/Sitko.Core.Kafka/KafkaModule.cs | 2 + .../KafkaTasksModule.cs | 1 + 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index e44460600..e6c0465c3 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -17,6 +17,7 @@ internal KafkaConfigurator(string name, string[] brokers) private readonly List> consumerActions = new(); private readonly Dictionary> producerActions = new(); private readonly Dictionary topics = new(); + private bool ensureOffsets; public KafkaConfigurator AddProducer(string producerName, Action configure) { @@ -36,6 +37,12 @@ public KafkaConfigurator AutoCreateTopic(string topic, int partitions, short rep return this; } + public KafkaConfigurator EnsureOffsets(bool enable = true) + { + ensureOffsets = enable; + return this; + } + public void Build(IKafkaConfigurationBuilder builder) => builder .UseMicrosoftLog() @@ -58,7 +65,16 @@ public void Build(IKafkaConfigurationBuilder builder) => foreach (var consumerAction in consumerActions) { - clusterBuilder.AddConsumer(consumerAction); + clusterBuilder.AddConsumer(consumerBuilder => + { + consumerAction(consumerBuilder); + if (ensureOffsets) + { + consumerBuilder.WithPartitionsAssignedHandler((resolver, list) => + resolver.Resolve() + .EnsureOffsets(brokers, name, list)); + } + }); } }); } diff --git a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs new file mode 100644 index 000000000..6976f6f41 --- /dev/null +++ b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs @@ -0,0 +1,162 @@ +using System.Collections.Concurrent; +using System.Reflection; +using Confluent.Kafka; +using KafkaFlow; +using KafkaFlow.Consumers; + +namespace Sitko.Core.Kafka; + +internal class KafkaConsumerOffsetsEnsurer +{ + private static FieldInfo? consumerManagerField; + private static FieldInfo? directConsumerField; + private static PropertyInfo? consumerProperty; + private static readonly HashSet ProcessedPartitions = new(); + + private static readonly + ConcurrentDictionary confluentConsumer + )> Consumers = new(); + + private readonly IConsumerAccessor consumerAccessor; + private readonly ILogHandler logHandler; + private readonly ConcurrentDictionary tasks = new(); + private IAdminClient? adminClient; + + public KafkaConsumerOffsetsEnsurer(IConsumerAccessor consumerAccessor, ILogHandler logHandler) + { + this.consumerAccessor = consumerAccessor; + this.logHandler = logHandler; + } + + private IAdminClient GetAdminClient(string[] brokers) + { + if (adminClient is null) + { + var adminClientConfig = new AdminClientConfig + { + BootstrapServers = string.Join(",", brokers), ClientId = "AdminClient" + }; + adminClient = new AdminClientBuilder(adminClientConfig) + .SetLogHandler((_, m) => logHandler.Info(m.Message, m)) + .SetErrorHandler((_, error) => logHandler.Error("Kafka Consumer Error", null, new { Error = error })) + .Build(); + } + + return adminClient; + } + + public void EnsureOffsets( + string[] brokers, + string name, + List list + ) + { + foreach (var partition in list) + { + var key = $"{name}/{partition.Partition.Value}"; + if (ProcessedPartitions.Contains(key)) + { + continue; + } + + tasks.GetOrAdd( + key, _ => { return Task.Run(async () => await ProcessPartition(brokers, name, partition)); } + ); + ProcessedPartitions.Add(key); + } + } + + private async Task ProcessPartition(string[] brokers, string name, TopicPartition partition) + { + var messageConsumer = consumerAccessor.GetConsumer(name); + messageConsumer.Pause(new[] { partition }); + try + { + var (kafkaFlowConsumer, confluentConsumer) = GetConsumers(messageConsumer); + + var commited = await GetAdminClient(brokers).ListConsumerGroupOffsetsAsync(new[] + { + new ConsumerGroupTopicPartitions(messageConsumer.GroupId, new List { partition }) + }); + if (!commited.Any()) + { + logHandler.Warning( + $"Не получилось найти оффсеты для назначенных партиций консьюмера {messageConsumer.ConsumerName}", + null); + return; + } + + var currentOffset = commited.First().Partitions.FirstOrDefault( + partitionOffset => + partitionOffset.TopicPartition == partition + ); + + if (currentOffset is null || currentOffset.Offset == Offset.Unset) + { + var partitionOffset = confluentConsumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(30)); + var newOffset = new TopicPartitionOffset(partition, partitionOffset.High); + logHandler.Warning( + $"Сохраняем отсутствующий оффсет для партиции {partition} консьюмера {name}: {newOffset.Offset}", + null); + kafkaFlowConsumer.Commit(new[] { newOffset }); + } + } + finally + { + messageConsumer.Resume(new[] { partition }); + } + } + + private static (IConsumer kafkaFlowConsumer, IConsumer confluentConsumer) GetConsumers( + IMessageConsumer consumer) => + Consumers.GetOrAdd( + consumer, messageConsumer => + { + consumerManagerField ??= messageConsumer.GetType().GetField( + "consumerManager", + BindingFlags.Instance | + BindingFlags.NonPublic + ) ?? + throw new InvalidOperationException( + "Can't find field consumerManager" + ); + var consumerManager = + consumerManagerField.GetValue(messageConsumer) ?? + throw new InvalidOperationException( + "Can't get consumerManager" + ); + consumerProperty ??= consumerManager.GetType() + .GetProperty( + "Consumer", + BindingFlags.Instance | + BindingFlags.Public + ) ?? + throw new InvalidOperationException( + "Can't find field consumer" + ); + var flowConsumer = + consumerProperty.GetValue(consumerManager) as IConsumer ?? + throw new InvalidOperationException( + "Can't get flowConsumer" + ); + + directConsumerField ??= flowConsumer.GetType() + .GetField( + "consumer", + BindingFlags.Instance | + BindingFlags.NonPublic + ) ?? + throw new InvalidOperationException( + "Can't find field directConsumer" + ); + var confluentConsumer = + directConsumerField.GetValue(flowConsumer) as + IConsumer ?? + throw new InvalidOperationException( + "Can't getdirectConsumer" + ); + + return (flowConsumer, confluentConsumer); + } + ); +} diff --git a/src/Sitko.Core.Kafka/KafkaModule.cs b/src/Sitko.Core.Kafka/KafkaModule.cs index ca75dbcfe..bf1b01f48 100644 --- a/src/Sitko.Core.Kafka/KafkaModule.cs +++ b/src/Sitko.Core.Kafka/KafkaModule.cs @@ -1,5 +1,6 @@ using KafkaFlow; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Sitko.Core.App; namespace Sitko.Core.Kafka; @@ -18,6 +19,7 @@ public override void PostConfigureServices(IApplicationContext applicationContex BaseApplicationModuleOptions startupOptions) { base.ConfigureServices(applicationContext, services, startupOptions); + services.TryAddSingleton(); services.AddKafkaFlowHostedService(builder => { foreach (var (_, configurator) in Configurators) diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index ea171283d..b808b9a83 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -42,6 +42,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio var kafkaConfigurator = KafkaModule.CreateConfigurator($"Kafka_Tasks_Cluster", startupOptions.Brokers); kafkaConfigurator .AutoCreateTopic(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) + .EnsureOffsets() .AddProducer(producerName, builder => { builder.DefaultTopic(kafkaTopic); From 270cbdf290439b75cca6c219e89d4032f986bd0e Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 17:58:09 +0500 Subject: [PATCH 27/32] feat: prefix consumer groups --- src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs | 10 +++++++++- src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index b808b9a83..f90c8e0b7 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -33,12 +33,18 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio : startupOptions.TopicPrefix) : ""; var kafkaTopic = $"{kafkaTopicPrefix}_{startupOptions.TasksTopic}".Replace(".", "_"); + var kafkaGroupPrefix = startupOptions.AddConsumerGroupPrefix + ? (string.IsNullOrEmpty(startupOptions.ConsumerGroupPrefix) + ? $"{applicationContext.Name}_{applicationContext.Environment}" + : startupOptions.ConsumerGroupPrefix) + : ""; var producerName = $"Tasks_{typeof(TBaseTask).Name}"; foreach (var executor in executors) { EventsRegistry.Register(executor.EventType, kafkaTopic, producerName); } + var kafkaConfigurator = KafkaModule.CreateConfigurator($"Kafka_Tasks_Cluster", startupOptions.Brokers); kafkaConfigurator .AutoCreateTopic(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) @@ -53,11 +59,13 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio foreach (var groupConsumers in executors.GroupBy(r => r.GroupId)) { var commonRegistration = groupConsumers.First(); - var name = $"{applicationContext.Name}/{applicationContext.Id}/{typeof(TBaseTask).Name}/{commonRegistration.GroupId}"; + var name = + $"{applicationContext.Name}/{applicationContext.Id}/{typeof(TBaseTask).Name}/{commonRegistration.GroupId}"; var parallelThreadCount = groupConsumers.Max(r => r.ParallelThreadCount); var bufferSize = groupConsumers.Max(r => r.BufferSize); kafkaConfigurator.AddConsumer(consumerBuilder => { + var groupName = $"{kafkaGroupPrefix}_{commonRegistration.GroupId}".Replace(".", "_"); consumerBuilder.Topic(kafkaTopic); consumerBuilder.WithName(name); consumerBuilder.WithGroupId(commonRegistration.GroupId); diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs index 0a225fec2..1eb278e02 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs @@ -14,4 +14,6 @@ public class KafkaTasksModuleOptions : TasksModuleOptions public string TopicPrefix { get; set; } = ""; public int TopicPartitions { get; set; } = 24; public short TopicReplicationFactor { get; set; } = 1; + public bool AddConsumerGroupPrefix { get; set; } = true; + public string ConsumerGroupPrefix { get; set; } = ""; } From 1f1d8dc6f7f5e7bbc88e1a457d90c3f8561908b2 Mon Sep 17 00:00:00 2001 From: nadia Date: Thu, 12 Oct 2023 18:22:36 +0500 Subject: [PATCH 28/32] fix(tasks): fixes --- .../KafkaConsumerOffsetsEnsurer.cs | 29 +++++++++++-------- .../KafkaTasksModule.cs | 4 +-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs index 6976f6f41..030d6754a 100644 --- a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs +++ b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs @@ -1,8 +1,8 @@ using System.Collections.Concurrent; using System.Reflection; using Confluent.Kafka; -using KafkaFlow; using KafkaFlow.Consumers; +using Microsoft.Extensions.Logging; namespace Sitko.Core.Kafka; @@ -18,14 +18,14 @@ private static readonly )> Consumers = new(); private readonly IConsumerAccessor consumerAccessor; - private readonly ILogHandler logHandler; private readonly ConcurrentDictionary tasks = new(); private IAdminClient? adminClient; + private ILogger logger; - public KafkaConsumerOffsetsEnsurer(IConsumerAccessor consumerAccessor, ILogHandler logHandler) + public KafkaConsumerOffsetsEnsurer(IConsumerAccessor consumerAccessor, ILogger logger) { this.consumerAccessor = consumerAccessor; - this.logHandler = logHandler; + this.logger = logger; } private IAdminClient GetAdminClient(string[] brokers) @@ -37,8 +37,8 @@ private IAdminClient GetAdminClient(string[] brokers) BootstrapServers = string.Join(",", brokers), ClientId = "AdminClient" }; adminClient = new AdminClientBuilder(adminClientConfig) - .SetLogHandler((_, m) => logHandler.Info(m.Message, m)) - .SetErrorHandler((_, error) => logHandler.Error("Kafka Consumer Error", null, new { Error = error })) + .SetLogHandler((_, m) => logger.LogInformation("{Message}", m.Message)) + .SetErrorHandler((_, error) => logger.LogError("Kafka Consumer Error: {Error}", error)) .Build(); } @@ -80,9 +80,9 @@ private async Task ProcessPartition(string[] brokers, string name, TopicPartitio }); if (!commited.Any()) { - logHandler.Warning( - $"Не получилось найти оффсеты для назначенных партиций консьюмера {messageConsumer.ConsumerName}", - null); + logger.LogWarning( + "Не получилось найти оффсеты для назначенных партиций консьюмера {Consumer}", + messageConsumer.ConsumerName); return; } @@ -95,12 +95,17 @@ private async Task ProcessPartition(string[] brokers, string name, TopicPartitio { var partitionOffset = confluentConsumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(30)); var newOffset = new TopicPartitionOffset(partition, partitionOffset.High); - logHandler.Warning( - $"Сохраняем отсутствующий оффсет для партиции {partition} консьюмера {name}: {newOffset.Offset}", - null); + logger.LogWarning( + "Сохраняем отсутствующий оффсет для партиции {Partition} консьюмера {Consumer}: {Offset}", + partition, name, newOffset.Offset); kafkaFlowConsumer.Commit(new[] { newOffset }); } } + catch (Exception ex) + { + logger.LogError(ex, "Error process partition {Partition}: {Error}", partition, ex); + throw; + } finally { messageConsumer.Resume(new[] { partition }); diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index f90c8e0b7..5de4f84a0 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -65,10 +65,10 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio var bufferSize = groupConsumers.Max(r => r.BufferSize); kafkaConfigurator.AddConsumer(consumerBuilder => { - var groupName = $"{kafkaGroupPrefix}_{commonRegistration.GroupId}".Replace(".", "_"); + var groupId = $"{kafkaGroupPrefix}_{commonRegistration.GroupId}".Replace(".", "_"); consumerBuilder.Topic(kafkaTopic); consumerBuilder.WithName(name); - consumerBuilder.WithGroupId(commonRegistration.GroupId); + consumerBuilder.WithGroupId(groupId); consumerBuilder.WithWorkersCount(parallelThreadCount); consumerBuilder.WithBufferSize(bufferSize); // для гарантии порядка событий From 923551e0fc279dfe87c5304cf91077da4c6a2fdf Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 12 Oct 2023 19:38:55 +0500 Subject: [PATCH 29/32] fix(tasks): rework offsets ensurance --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 37 +-- .../KafkaConsumerOffsetsEnsurer.cs | 222 +++++++----------- src/Sitko.Core.Kafka/KafkaModule.cs | 20 +- .../ApplicationExtensions.cs | 1 + .../KafkaTasksModule.cs | 64 ++--- .../Execution/BaseTaskExecutor.cs | 40 ++-- 6 files changed, 188 insertions(+), 196 deletions(-) diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index e6c0465c3..e71bd48ce 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -5,19 +5,24 @@ namespace Sitko.Core.Kafka; public class KafkaConfigurator { - private readonly string name; private readonly string[] brokers; + private readonly List> consumerActions = new(); + private readonly HashSet consumers = new(); + private readonly string name; + private readonly Dictionary> producerActions = new(); + private readonly Dictionary topics = new(); + private bool ensureOffsets; + internal KafkaConfigurator(string name, string[] brokers) { this.name = name; this.brokers = brokers; } - private readonly List> consumerActions = new(); - private readonly Dictionary> producerActions = new(); - private readonly Dictionary topics = new(); - private bool ensureOffsets; + internal string[] Brokers => brokers; + internal HashSet Consumers => consumers; + internal bool NeedToEnsureOffsets => ensureOffsets; public KafkaConfigurator AddProducer(string producerName, Action configure) { @@ -25,8 +30,10 @@ public KafkaConfigurator AddProducer(string producerName, Action configure) + public KafkaConfigurator AddConsumer(string consumerName, string groupId, TopicInfo[] topics, + Action configure) { + consumers.Add(new ConsumerRegistration(consumerName, groupId, topics)); consumerActions.Add(configure); return this; } @@ -51,10 +58,14 @@ public void Build(IKafkaConfigurationBuilder builder) => clusterBuilder .WithName(name) .WithBrokers(brokers); - foreach (var (topic, config) in topics) + if (!ensureOffsets) { - clusterBuilder.CreateTopicIfNotExists(topic, config.Partitions, config.ReplicationFactor); + foreach (var (topic, config) in topics) + { + clusterBuilder.CreateTopicIfNotExists(topic, config.Partitions, config.ReplicationFactor); + } } + foreach (var (producerName, configure) in producerActions) { clusterBuilder.AddProducer(producerName, configurationBuilder => @@ -68,13 +79,11 @@ public void Build(IKafkaConfigurationBuilder builder) => clusterBuilder.AddConsumer(consumerBuilder => { consumerAction(consumerBuilder); - if (ensureOffsets) - { - consumerBuilder.WithPartitionsAssignedHandler((resolver, list) => - resolver.Resolve() - .EnsureOffsets(brokers, name, list)); - } }); } }); } + +internal record ConsumerRegistration(string Name, string GroupId, TopicInfo[] Topics); + +public record TopicInfo(string Name, int PartitionsCount, short ReplicationFactor); diff --git a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs index 030d6754a..2f4f85045 100644 --- a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs +++ b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs @@ -1,167 +1,127 @@ -using System.Collections.Concurrent; -using System.Reflection; -using Confluent.Kafka; -using KafkaFlow.Consumers; +using Confluent.Kafka; +using Confluent.Kafka.Admin; using Microsoft.Extensions.Logging; namespace Sitko.Core.Kafka; internal class KafkaConsumerOffsetsEnsurer { - private static FieldInfo? consumerManagerField; - private static FieldInfo? directConsumerField; - private static PropertyInfo? consumerProperty; - private static readonly HashSet ProcessedPartitions = new(); + private readonly ILogger logger; - private static readonly - ConcurrentDictionary confluentConsumer - )> Consumers = new(); + public KafkaConsumerOffsetsEnsurer(ILogger logger) => this.logger = logger; - private readonly IConsumerAccessor consumerAccessor; - private readonly ConcurrentDictionary tasks = new(); - private IAdminClient? adminClient; - private ILogger logger; - - public KafkaConsumerOffsetsEnsurer(IConsumerAccessor consumerAccessor, ILogger logger) - { - this.consumerAccessor = consumerAccessor; - this.logger = logger; - } - - private IAdminClient GetAdminClient(string[] brokers) - { - if (adminClient is null) - { - var adminClientConfig = new AdminClientConfig - { - BootstrapServers = string.Join(",", brokers), ClientId = "AdminClient" - }; - adminClient = new AdminClientBuilder(adminClientConfig) - .SetLogHandler((_, m) => logger.LogInformation("{Message}", m.Message)) - .SetErrorHandler((_, error) => logger.LogError("Kafka Consumer Error: {Error}", error)) - .Build(); - } - - return adminClient; - } - - public void EnsureOffsets( - string[] brokers, - string name, - List list - ) + public async Task EnsureOffsetsAsync(KafkaConfigurator configurator) { - foreach (var partition in list) + var adminClient = GetAdminClient(configurator.Brokers); + foreach (var consumer in configurator.Consumers) { - var key = $"{name}/{partition.Partition.Value}"; - if (ProcessedPartitions.Contains(key)) + foreach (var topic in consumer.Topics) { - continue; + await EnsureTopicOffsetsAsync(consumer, adminClient, topic, configurator.Brokers); } - - tasks.GetOrAdd( - key, _ => { return Task.Run(async () => await ProcessPartition(brokers, name, partition)); } - ); - ProcessedPartitions.Add(key); } } - private async Task ProcessPartition(string[] brokers, string name, TopicPartition partition) + private async Task EnsureTopicOffsetsAsync(ConsumerRegistration consumer, IAdminClient adminClient, TopicInfo topic, + string[] brokers) { - var messageConsumer = consumerAccessor.GetConsumer(name); - messageConsumer.Pause(new[] { partition }); + logger.LogDebug("Try to create topic {Topic}", topic); try { - var (kafkaFlowConsumer, confluentConsumer) = GetConsumers(messageConsumer); - - var commited = await GetAdminClient(brokers).ListConsumerGroupOffsetsAsync(new[] + await adminClient.CreateTopicsAsync(new[] { - new ConsumerGroupTopicPartitions(messageConsumer.GroupId, new List { partition }) + new TopicSpecification + { + Name = topic.Name, + NumPartitions = topic.PartitionsCount, + ReplicationFactor = topic.ReplicationFactor + } }); - if (!commited.Any()) + logger.LogInformation("Topic {Topic} created", topic); + } + catch (Exception ex) + { + if (ex is CreateTopicsException createTopicsException && + createTopicsException.Results.First().Error.Reason.Contains("already exists")) { - logger.LogWarning( - "Не получилось найти оффсеты для назначенных партиций консьюмера {Consumer}", - messageConsumer.ConsumerName); + logger.LogDebug("Topic {Topic} already exists", topic); + } + else + { + logger.LogError(ex, "Error creating topic {Topic}: {ErrorText}", topic.Name, ex.Message); return; } + } - var currentOffset = commited.First().Partitions.FirstOrDefault( - partitionOffset => - partitionOffset.TopicPartition == partition - ); + var topicInfo = adminClient.GetMetadata(topic.Name, TimeSpan.FromSeconds(30)).Topics.First(); + if (topicInfo is null || !topicInfo.Partitions.Any()) + { + logger.LogError("Still no metadata for topic {Topic}", topic.Name); + return; + } - if (currentOffset is null || currentOffset.Offset == Offset.Unset) + var partitions = topicInfo.Partitions.Select(metadata => + new TopicPartition(topic.Name, new Partition(metadata.PartitionId))).ToList(); + var commited = await adminClient.ListConsumerGroupOffsetsAsync(new[] + { + new ConsumerGroupTopicPartitions(consumer.GroupId, partitions) + }); + if (!commited.Any()) + { + logger.LogWarning( + "Can't find offsets for group {ConsumerGroup} and topic {Topic}", + consumer.GroupId, topic); + return; + } + + var badPartitions = commited.First().Partitions.Where( + partitionOffset => + partitionOffset.Offset == Offset.Unset + ).Select(error => error.Partition).ToList(); + if (badPartitions.Any()) + { + var consumerConfig = new ConsumerConfig { - var partitionOffset = confluentConsumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(30)); - var newOffset = new TopicPartitionOffset(partition, partitionOffset.High); + BootstrapServers = string.Join(",", brokers), GroupId = consumer.GroupId + }; + var confluentConsumer = new ConsumerBuilder(consumerConfig).Build(); + var toCommit = new List(); + foreach (var partition in badPartitions) + { + var topicPartition = new TopicPartition(topic.Name, partition); + var partitionOffset = + confluentConsumer.QueryWatermarkOffsets(topicPartition, TimeSpan.FromSeconds(30)); + var newOffset = new TopicPartitionOffset(topicPartition, partitionOffset.High); logger.LogWarning( - "Сохраняем отсутствующий оффсет для партиции {Partition} консьюмера {Consumer}: {Offset}", - partition, name, newOffset.Offset); - kafkaFlowConsumer.Commit(new[] { newOffset }); + "Ensure {Partition} offset for consumer {ConsumerGroup}: {Offset}", + partition, consumer.GroupId, newOffset.Offset); + toCommit.Add(newOffset); } + + confluentConsumer.Commit(toCommit); + logger.LogInformation("Offsets for consumer group {ConsumerGroupName} in topic {Topic} ensured", + consumer.GroupId, topic); } - catch (Exception ex) - { - logger.LogError(ex, "Error process partition {Partition}: {Error}", partition, ex); - throw; - } - finally + else { - messageConsumer.Resume(new[] { partition }); + logger.LogDebug( + "No partitions without stored offsets in topic {Topic} for consumer group {ConsumerGroupName}", topic, + consumer.GroupId); } } - private static (IConsumer kafkaFlowConsumer, IConsumer confluentConsumer) GetConsumers( - IMessageConsumer consumer) => - Consumers.GetOrAdd( - consumer, messageConsumer => - { - consumerManagerField ??= messageConsumer.GetType().GetField( - "consumerManager", - BindingFlags.Instance | - BindingFlags.NonPublic - ) ?? - throw new InvalidOperationException( - "Can't find field consumerManager" - ); - var consumerManager = - consumerManagerField.GetValue(messageConsumer) ?? - throw new InvalidOperationException( - "Can't get consumerManager" - ); - consumerProperty ??= consumerManager.GetType() - .GetProperty( - "Consumer", - BindingFlags.Instance | - BindingFlags.Public - ) ?? - throw new InvalidOperationException( - "Can't find field consumer" - ); - var flowConsumer = - consumerProperty.GetValue(consumerManager) as IConsumer ?? - throw new InvalidOperationException( - "Can't get flowConsumer" - ); + private IAdminClient GetAdminClient(string[] brokers) + { + var adminClientConfig = new AdminClientConfig + { + BootstrapServers = string.Join(",", brokers), ClientId = "AdminClient" + }; + var adminClient = new AdminClientBuilder(adminClientConfig) + .SetLogHandler((_, m) => logger.LogInformation("{Message}", m.Message)) + .SetErrorHandler((_, error) => logger.LogError("Kafka Consumer Error: {Error}", error)) + .Build(); - directConsumerField ??= flowConsumer.GetType() - .GetField( - "consumer", - BindingFlags.Instance | - BindingFlags.NonPublic - ) ?? - throw new InvalidOperationException( - "Can't find field directConsumer" - ); - var confluentConsumer = - directConsumerField.GetValue(flowConsumer) as - IConsumer ?? - throw new InvalidOperationException( - "Can't getdirectConsumer" - ); - return (flowConsumer, confluentConsumer); - } - ); + return adminClient; + } } diff --git a/src/Sitko.Core.Kafka/KafkaModule.cs b/src/Sitko.Core.Kafka/KafkaModule.cs index bf1b01f48..cd2ed384d 100644 --- a/src/Sitko.Core.Kafka/KafkaModule.cs +++ b/src/Sitko.Core.Kafka/KafkaModule.cs @@ -1,6 +1,5 @@ using KafkaFlow; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Sitko.Core.App; namespace Sitko.Core.Kafka; @@ -10,16 +9,16 @@ public class KafkaModule : BaseApplicationModule private static readonly Dictionary Configurators = new(); public override bool AllowMultiple => false; + public override string OptionsKey => "Kafka"; + public static KafkaConfigurator CreateConfigurator(string name, string[] brokers) => Configurators.SafeGetOrAdd(name, _ => new KafkaConfigurator(name, brokers)); - public override string OptionsKey => "Kafka"; - public override void PostConfigureServices(IApplicationContext applicationContext, IServiceCollection services, BaseApplicationModuleOptions startupOptions) { base.ConfigureServices(applicationContext, services, startupOptions); - services.TryAddSingleton(); + services.AddSingleton(); services.AddKafkaFlowHostedService(builder => { foreach (var (_, configurator) in Configurators) @@ -28,4 +27,17 @@ public override void PostConfigureServices(IApplicationContext applicationContex } }); } + + public override async Task InitAsync(IApplicationContext applicationContext, IServiceProvider serviceProvider) + { + await base.InitAsync(applicationContext, serviceProvider); + var offsetsEnsurer = serviceProvider.GetRequiredService(); + foreach (var (_, configurator) in Configurators) + { + if (configurator.NeedToEnsureOffsets) + { + await offsetsEnsurer.EnsureOffsetsAsync(configurator); + } + } + } } diff --git a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs index d678e2de0..fbc2c26c2 100644 --- a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs +++ b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs @@ -20,6 +20,7 @@ public static Application AddKafkaTasks(this Application { application.AddModule(); } + return application; } } diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 5de4f84a0..659b83073 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -45,7 +45,7 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio EventsRegistry.Register(executor.EventType, kafkaTopic, producerName); } - var kafkaConfigurator = KafkaModule.CreateConfigurator($"Kafka_Tasks_Cluster", startupOptions.Brokers); + var kafkaConfigurator = KafkaModule.CreateConfigurator("Kafka_Tasks_Cluster", startupOptions.Brokers); kafkaConfigurator .AutoCreateTopic(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) .EnsureOffsets() @@ -63,37 +63,41 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio $"{applicationContext.Name}/{applicationContext.Id}/{typeof(TBaseTask).Name}/{commonRegistration.GroupId}"; var parallelThreadCount = groupConsumers.Max(r => r.ParallelThreadCount); var bufferSize = groupConsumers.Max(r => r.BufferSize); - kafkaConfigurator.AddConsumer(consumerBuilder => - { - var groupId = $"{kafkaGroupPrefix}_{commonRegistration.GroupId}".Replace(".", "_"); - consumerBuilder.Topic(kafkaTopic); - consumerBuilder.WithName(name); - consumerBuilder.WithGroupId(groupId); - consumerBuilder.WithWorkersCount(parallelThreadCount); - consumerBuilder.WithBufferSize(bufferSize); - // для гарантии порядка событий - consumerBuilder - .WithWorkDistributionStrategy(); - var consumerConfig = new ConsumerConfig + var groupId = $"{kafkaGroupPrefix}_{commonRegistration.GroupId}".Replace(".", "_"); + kafkaConfigurator.AddConsumer(name, groupId, + new[] + { + new TopicInfo(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) + }, consumerBuilder => { - AutoOffsetReset = AutoOffsetReset.Latest, - ClientId = name, - GroupInstanceId = name, - PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky - }; - consumerBuilder.WithConsumerConfig(consumerConfig); - consumerBuilder.AddMiddlewares( - middlewares => + consumerBuilder.Topic(kafkaTopic); + consumerBuilder.WithName(name); + consumerBuilder.WithGroupId(groupId); + consumerBuilder.WithWorkersCount(parallelThreadCount); + consumerBuilder.WithBufferSize(bufferSize); + // для гарантии порядка событий + consumerBuilder + .WithWorkDistributionStrategy(); + var consumerConfig = new ConsumerConfig { - middlewares - .AddSerializer(); - middlewares.AddTypedHandlers(handlers => - handlers.AddHandlers(groupConsumers.Select(r => - executorType.MakeGenericType(r.EventType, r.ExecutorType))) - .WithHandlerLifetime(InstanceLifetime.Scoped)); - } - ); - }); + AutoOffsetReset = AutoOffsetReset.Latest, + ClientId = name, + GroupInstanceId = name, + PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky + }; + consumerBuilder.WithConsumerConfig(consumerConfig); + consumerBuilder.AddMiddlewares( + middlewares => + { + middlewares + .AddSerializer(); + middlewares.AddTypedHandlers(handlers => + handlers.AddHandlers(groupConsumers.Select(r => + executorType.MakeGenericType(r.EventType, r.ExecutorType))) + .WithHandlerLifetime(InstanceLifetime.Scoped)); + } + ); + }); } } } diff --git a/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs index 8613d6f05..681856119 100644 --- a/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs +++ b/src/Sitko.Core.Tasks/Execution/BaseTaskExecutor.cs @@ -13,21 +13,22 @@ public abstract class BaseTaskExecutor : ITaskExecutor< where TConfig : BaseTaskConfig, new() where TResult : BaseTaskResult, new() { - private readonly ITracer? tracer; - private readonly ILogger> logger; private readonly CancellationTokenSource activityTaskCts = new(); private readonly IRepository repository; private readonly IServiceScopeFactory serviceScopeFactory; + private readonly ITracer? tracer; protected BaseTaskExecutor(ILogger> logger, IServiceScopeFactory serviceScopeFactory, IRepository repository, ITracer? tracer = null) { - this.logger = logger; + Logger = logger; this.tracer = tracer; this.serviceScopeFactory = serviceScopeFactory; this.repository = repository; } + protected ILogger> Logger { get; } + public async Task ExecuteAsync(Guid id, CancellationToken cancellationToken) { using (LogContext.PushProperty("TaskId", id.ToString())) @@ -55,11 +56,11 @@ await tracer.CaptureTransaction($"Tasks/{typeof(TTask)}", "Task", async transact } catch (JobFailedException) { - logger.LogDebug("Mark transaction as failed"); + Logger.LogDebug("Mark transaction as failed"); } catch (Exception ex) { - logger.LogError(ex, "Task {JobId} ( {JobType} ) failed: {ErrorText}", id, typeof(TTask).Name, + Logger.LogError(ex, "Task {JobId} ( {JobType} ) failed: {ErrorText}", id, typeof(TTask).Name, ex.ToString()); } } @@ -67,7 +68,7 @@ await tracer.CaptureTransaction($"Tasks/{typeof(TTask)}", "Task", async transact private async Task ExecuteTaskAsync(Guid id, CancellationToken cancellationToken) { - logger.LogInformation("Start job {JobId}", id); + Logger.LogInformation("Start job {JobId}", id); var task = await repository.GetByIdAsync(id, cancellationToken); if (task is null) { @@ -79,13 +80,14 @@ private async Task ExecuteTaskAsync(Guid id, CancellationToken cancellati var groupInfo = TaskExecutorHelper.GetGroupInfo(GetType()); if (groupInfo is not { allowRetry: true }) { - logger.LogInformation("Skip retry task {JobId}", id); + Logger.LogInformation("Skip retry task {JobId}", id); return task; } - logger.LogInformation("Retry task {JobId}", id); + + Logger.LogInformation("Retry task {JobId}", id); } - logger.LogInformation("Set job {JobId} status to in progress", id); + Logger.LogInformation("Set job {JobId} status to in progress", id); task.ExecuteDateStart = DateTimeOffset.UtcNow; task.TaskStatus = TaskStatus.InProgress; task.LastActivityDate = DateTimeOffset.UtcNow; @@ -111,24 +113,24 @@ private async Task ExecuteTaskAsync(Guid id, CancellationToken cancellati }, activityTaskCts.Token); try { - logger.LogInformation("Try to execute job {JobId}", id); + Logger.LogInformation("Try to execute job {JobId}", id); result = await ExecuteAsync(task, cancellationToken); if (result.IsSuccess) { - logger.LogInformation("Job {JobId} executed successfully", id); + Logger.LogInformation("Job {JobId} executed successfully", id); status = result.HasWarnings ? TaskStatus.SuccessWithWarnings : TaskStatus.Success; } else { - logger.LogInformation("Job {JobId} execution failed", id); + Logger.LogInformation("Job {JobId} execution failed", id); status = TaskStatus.Fails; } } catch (Exception ex) { - logger.LogInformation(ex, "Job {JobId} execution failed with error: {ErrorText}", id, ex.ToString()); + Logger.LogInformation(ex, "Job {JobId} execution failed with error: {ErrorText}", id, ex.ToString()); status = TaskStatus.Fails; result = new TResult { IsSuccess = false, ErrorMessage = ex.Message }; } @@ -144,22 +146,26 @@ private async Task ExecuteTaskAsync(Guid id, CancellationToken cancellati } catch (Exception ex) { - logger.LogInformation(ex, "Activity task error: {ErrorText}", ex.ToString()); + Logger.LogInformation(ex, "Activity task error: {ErrorText}", ex.ToString()); } - logger.LogInformation("Set job {JobId} result and save", id); + Logger.LogInformation("Set job {JobId} result and save", id); task = await repository.RefreshAsync(task, cancellationToken); task.Result = result; task.TaskStatus = status; task.ExecuteDateEnd = DateTimeOffset.UtcNow; await repository.UpdateAsync(task, cancellationToken); - logger.LogInformation("Job {JobId} finished", id); + Logger.LogInformation("Job {JobId} finished", id); return task; } protected abstract Task ExecuteAsync(TTask task, CancellationToken cancellationToken); } -public record ExecutorRegistration(Type ExecutorType, Type EventType, string GroupId, int ParallelThreadCount, +public record ExecutorRegistration( + Type ExecutorType, + Type EventType, + string GroupId, + int ParallelThreadCount, int BufferSize); From 782c2cd3911ad951813fdfd4874d14e671a23561 Mon Sep 17 00:00:00 2001 From: nadia Date: Tue, 17 Oct 2023 09:26:24 +0500 Subject: [PATCH 30/32] feat(tasks): configure timeouts for kafka --- src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs | 11 ++++++++++- src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 659b83073..7b21e3bf7 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -70,6 +70,11 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio new TopicInfo(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) }, consumerBuilder => { + if (startupOptions.MaxPollIntervalMs > 0) + { + consumerBuilder.WithMaxPollIntervalMs(startupOptions.MaxPollIntervalMs); + } + consumerBuilder.Topic(kafkaTopic); consumerBuilder.WithName(name); consumerBuilder.WithGroupId(groupId); @@ -83,8 +88,12 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio AutoOffsetReset = AutoOffsetReset.Latest, ClientId = name, GroupInstanceId = name, - PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky + PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky, }; + if (startupOptions.SessionTimeoutMs > 0) + { + consumerConfig.SessionTimeoutMs = startupOptions.SessionTimeoutMs; + } consumerBuilder.WithConsumerConfig(consumerConfig); consumerBuilder.AddMiddlewares( middlewares => diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs index 1eb278e02..ef7e728c0 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs @@ -16,4 +16,8 @@ public class KafkaTasksModuleOptions : TasksModuleOptions public short TopicReplicationFactor { get; set; } = 1; public bool AddConsumerGroupPrefix { get; set; } = true; public string ConsumerGroupPrefix { get; set; } = ""; + + public int SessionTimeoutMs { get; set; } + + public int MaxPollIntervalMs { get; set; } } From eacff8b6dc798ae2c6583b9bdc9b9fa4cfe81ac0 Mon Sep 17 00:00:00 2001 From: George Drak Date: Wed, 18 Oct 2023 11:54:33 +0500 Subject: [PATCH 31/32] feat(kafka): extend and unify options --- src/Sitko.Core.Kafka/KafkaConfigurator.cs | 73 +++++++++++++------ .../KafkaConsumerOffsetsEnsurer.cs | 6 +- src/Sitko.Core.Kafka/KafkaModule.cs | 43 +++++++++-- .../ApplicationExtensions.cs | 8 +- .../KafkaTasksModule.cs | 34 +-------- .../KafkaTasksModuleOptions.cs | 5 -- 6 files changed, 98 insertions(+), 71 deletions(-) diff --git a/src/Sitko.Core.Kafka/KafkaConfigurator.cs b/src/Sitko.Core.Kafka/KafkaConfigurator.cs index e71bd48ce..ba08f524e 100644 --- a/src/Sitko.Core.Kafka/KafkaConfigurator.cs +++ b/src/Sitko.Core.Kafka/KafkaConfigurator.cs @@ -1,40 +1,39 @@ -using KafkaFlow; +using Confluent.Kafka; +using KafkaFlow; using KafkaFlow.Configuration; +using KafkaFlow.Consumers.DistributionStrategies; namespace Sitko.Core.Kafka; public class KafkaConfigurator { - private readonly string[] brokers; + private readonly Dictionary> + consumerActions = new(); - private readonly List> consumerActions = new(); private readonly HashSet consumers = new(); - private readonly string name; - private readonly Dictionary> producerActions = new(); + private readonly string clusterName; + private readonly Dictionary> producerActions = new(); private readonly Dictionary topics = new(); private bool ensureOffsets; - internal KafkaConfigurator(string name, string[] brokers) - { - this.name = name; - this.brokers = brokers; - } + internal KafkaConfigurator(string clusterName) => this.clusterName = clusterName; - internal string[] Brokers => brokers; internal HashSet Consumers => consumers; internal bool NeedToEnsureOffsets => ensureOffsets; - public KafkaConfigurator AddProducer(string producerName, Action configure) + public KafkaConfigurator AddProducer(string producerName, + Action configure) { producerActions[producerName] = configure; return this; } - public KafkaConfigurator AddConsumer(string consumerName, string groupId, TopicInfo[] topics, - Action configure) + public KafkaConfigurator AddConsumer(string consumerName, string groupId, TopicInfo[] consumerTopics, + Action configure) { - consumers.Add(new ConsumerRegistration(consumerName, groupId, topics)); - consumerActions.Add(configure); + var registration = new ConsumerRegistration(consumerName, groupId, consumerTopics); + consumers.Add(registration); + consumerActions[registration] = configure; return this; } @@ -50,14 +49,14 @@ public KafkaConfigurator EnsureOffsets(bool enable = true) return this; } - public void Build(IKafkaConfigurationBuilder builder) => + public void Build(IKafkaConfigurationBuilder builder, KafkaModuleOptions options) => builder .UseMicrosoftLog() .AddCluster(clusterBuilder => { clusterBuilder - .WithName(name) - .WithBrokers(brokers); + .WithName(clusterName) + .WithBrokers(options.Brokers); if (!ensureOffsets) { foreach (var (topic, config) in topics) @@ -68,17 +67,45 @@ public void Build(IKafkaConfigurationBuilder builder) => foreach (var (producerName, configure) in producerActions) { - clusterBuilder.AddProducer(producerName, configurationBuilder => + clusterBuilder.AddProducer(producerName, producerBuilder => { - configure(configurationBuilder); + var producerConfig = new ProducerConfig + { + ClientId = producerName, + MessageTimeoutMs = (int)options.MessageTimeout.TotalMilliseconds, + MessageMaxBytes = options.MessageMaxBytes, + EnableIdempotence = options.EnableIdempotence, + SocketNagleDisable = options.SocketNagleDisable, + Acks = options.Acks + }; + producerBuilder.WithProducerConfig(producerConfig); + producerBuilder.WithLingerMs(options.MaxProducingTimeout.TotalMilliseconds); + configure(producerBuilder, producerConfig); }); } - foreach (var consumerAction in consumerActions) + foreach (var (registration, configureAction) in consumerActions) { clusterBuilder.AddConsumer(consumerBuilder => { - consumerAction(consumerBuilder); + consumerBuilder.WithName(registration.Name); + consumerBuilder.Topics(registration.Topics.Select(info => info.Name)); + consumerBuilder.WithGroupId(registration.GroupId); + consumerBuilder + .WithWorkDistributionStrategy(); // guarantee events order + consumerBuilder.WithMaxPollIntervalMs((int)options.MaxPollInterval.TotalMilliseconds); + var consumerConfig = new ConsumerConfig + { + MaxPartitionFetchBytes = options.MaxPartitionFetchBytes, + AutoOffsetReset = options.AutoOffsetReset, + ClientId = registration.Name, + GroupInstanceId = registration.Name, + BootstrapServers = string.Join(",", options.Brokers), + SessionTimeoutMs = (int)options.SessionTimeout.TotalMilliseconds, + PartitionAssignmentStrategy = options.PartitionAssignmentStrategy + }; + consumerBuilder.WithConsumerConfig(consumerConfig); + configureAction(consumerBuilder, consumerConfig); }); } }); diff --git a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs index 2f4f85045..418e425a5 100644 --- a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs +++ b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs @@ -10,14 +10,14 @@ internal class KafkaConsumerOffsetsEnsurer public KafkaConsumerOffsetsEnsurer(ILogger logger) => this.logger = logger; - public async Task EnsureOffsetsAsync(KafkaConfigurator configurator) + public async Task EnsureOffsetsAsync(KafkaConfigurator configurator, KafkaModuleOptions options) { - var adminClient = GetAdminClient(configurator.Brokers); + var adminClient = GetAdminClient(options.Brokers); foreach (var consumer in configurator.Consumers) { foreach (var topic in consumer.Topics) { - await EnsureTopicOffsetsAsync(consumer, adminClient, topic, configurator.Brokers); + await EnsureTopicOffsetsAsync(consumer, adminClient, topic, options.Brokers); } } } diff --git a/src/Sitko.Core.Kafka/KafkaModule.cs b/src/Sitko.Core.Kafka/KafkaModule.cs index cd2ed384d..e8bf0816a 100644 --- a/src/Sitko.Core.Kafka/KafkaModule.cs +++ b/src/Sitko.Core.Kafka/KafkaModule.cs @@ -1,21 +1,24 @@ -using KafkaFlow; +using Confluent.Kafka; +using FluentValidation; +using KafkaFlow; using Microsoft.Extensions.DependencyInjection; using Sitko.Core.App; +using Acks = Confluent.Kafka.Acks; namespace Sitko.Core.Kafka; -public class KafkaModule : BaseApplicationModule +public class KafkaModule : BaseApplicationModule { private static readonly Dictionary Configurators = new(); public override bool AllowMultiple => false; public override string OptionsKey => "Kafka"; - public static KafkaConfigurator CreateConfigurator(string name, string[] brokers) => - Configurators.SafeGetOrAdd(name, _ => new KafkaConfigurator(name, brokers)); + public static KafkaConfigurator CreateConfigurator(string name) => + Configurators.SafeGetOrAdd(name, _ => new KafkaConfigurator(name)); public override void PostConfigureServices(IApplicationContext applicationContext, IServiceCollection services, - BaseApplicationModuleOptions startupOptions) + KafkaModuleOptions startupOptions) { base.ConfigureServices(applicationContext, services, startupOptions); services.AddSingleton(); @@ -23,7 +26,7 @@ public override void PostConfigureServices(IApplicationContext applicationContex { foreach (var (_, configurator) in Configurators) { - configurator.Build(builder); + configurator.Build(builder, startupOptions); } }); } @@ -32,12 +35,38 @@ public override async Task InitAsync(IApplicationContext applicationContext, ISe { await base.InitAsync(applicationContext, serviceProvider); var offsetsEnsurer = serviceProvider.GetRequiredService(); + var options = GetOptions(serviceProvider); foreach (var (_, configurator) in Configurators) { if (configurator.NeedToEnsureOffsets) { - await offsetsEnsurer.EnsureOffsetsAsync(configurator); + await offsetsEnsurer.EnsureOffsetsAsync(configurator, options); } } } } + +public class KafkaModuleOptions : BaseModuleOptions +{ + public string[] Brokers { get; set; } = Array.Empty(); + public TimeSpan SessionTimeout { get; set; } = TimeSpan.FromSeconds(15); + public TimeSpan MaxPollInterval { get; set; } = TimeSpan.FromMinutes(5); + public int MaxPartitionFetchBytes { get; set; } = 5 * 1024 * 1024; + public Confluent.Kafka.AutoOffsetReset AutoOffsetReset { get; set; } = Confluent.Kafka.AutoOffsetReset.Latest; + + public PartitionAssignmentStrategy PartitionAssignmentStrategy { get; set; } = + PartitionAssignmentStrategy.CooperativeSticky; + + public TimeSpan MessageTimeout { get; set; } = TimeSpan.FromSeconds(12); + public int MessageMaxBytes { get; set; } = 5 * 1024 * 1024; + public TimeSpan MaxProducingTimeout { get; set; } = TimeSpan.FromMilliseconds(100); + public bool EnableIdempotence { get; set; } = true; + public bool SocketNagleDisable { get; set; } = true; + public Acks Acks { get; set; } = Acks.All; +} + +public class KafkaModuleOptionsValidator : AbstractValidator +{ + public KafkaModuleOptionsValidator() => + RuleFor(options => options.Brokers).NotEmpty().WithMessage("Specify Kafka brokers"); +} diff --git a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs index fbc2c26c2..0f8add91d 100644 --- a/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs +++ b/src/Sitko.Core.Tasks.Kafka/ApplicationExtensions.cs @@ -10,15 +10,17 @@ public static class ApplicationExtensions { public static Application AddKafkaTasks(this Application application, Action> configure, bool configurePostgres = false, - Action>? configurePostgresAction = null) where TBaseTask : BaseTask + Action>? configurePostgresAction = null, + bool configureKafka = true, + Action? configureKafkaAction = null) where TBaseTask : BaseTask where TDbContext : TasksDbContext { application.AddTasks(configurePostgres, configurePostgresAction); application.AddModule, KafkaTasksModuleOptions>( configure); - if (!application.HasModule()) + if (configureKafka && !application.HasModule()) { - application.AddModule(); + application.AddModule(configureKafkaAction); } return application; diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 7b21e3bf7..3bcefe78c 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -45,11 +45,11 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio EventsRegistry.Register(executor.EventType, kafkaTopic, producerName); } - var kafkaConfigurator = KafkaModule.CreateConfigurator("Kafka_Tasks_Cluster", startupOptions.Brokers); + var kafkaConfigurator = KafkaModule.CreateConfigurator("Kafka_Tasks_Cluster"); kafkaConfigurator .AutoCreateTopic(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) .EnsureOffsets() - .AddProducer(producerName, builder => + .AddProducer(producerName, (builder, _) => { builder.DefaultTopic(kafkaTopic); builder.AddMiddlewares(middlewareBuilder => @@ -68,33 +68,10 @@ protected override void ConfigureServicesInternal(IApplicationContext applicatio new[] { new TopicInfo(kafkaTopic, startupOptions.TopicPartitions, startupOptions.TopicReplicationFactor) - }, consumerBuilder => + }, (consumerBuilder, _) => { - if (startupOptions.MaxPollIntervalMs > 0) - { - consumerBuilder.WithMaxPollIntervalMs(startupOptions.MaxPollIntervalMs); - } - - consumerBuilder.Topic(kafkaTopic); - consumerBuilder.WithName(name); - consumerBuilder.WithGroupId(groupId); consumerBuilder.WithWorkersCount(parallelThreadCount); consumerBuilder.WithBufferSize(bufferSize); - // для гарантии порядка событий - consumerBuilder - .WithWorkDistributionStrategy(); - var consumerConfig = new ConsumerConfig - { - AutoOffsetReset = AutoOffsetReset.Latest, - ClientId = name, - GroupInstanceId = name, - PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky, - }; - if (startupOptions.SessionTimeoutMs > 0) - { - consumerConfig.SessionTimeoutMs = startupOptions.SessionTimeoutMs; - } - consumerBuilder.WithConsumerConfig(consumerConfig); consumerBuilder.AddMiddlewares( middlewares => { @@ -116,9 +93,6 @@ public class KafkaTasksModuleOptions> where TBaseTask : BaseTask where TDbContext : TasksDbContext { - public KafkaModuleOptionsValidator() - { - RuleFor(options => options.Brokers).NotEmpty().WithMessage("Specify Kafka brokers"); + public KafkaModuleOptionsValidator() => RuleFor(options => options.TasksTopic).NotEmpty().WithMessage("Specify Kafka topic"); - } } diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs index ef7e728c0..23e250a9a 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModuleOptions.cs @@ -8,7 +8,6 @@ public class KafkaTasksModuleOptions : TasksModuleOptions where TDbContext : TasksDbContext { public override Type GetValidatorType() => typeof(KafkaModuleOptionsValidator); - public string[] Brokers { get; set; } = Array.Empty(); public string TasksTopic { get; set; } = ""; public bool AddTopicPrefix { get; set; } = true; public string TopicPrefix { get; set; } = ""; @@ -16,8 +15,4 @@ public class KafkaTasksModuleOptions : TasksModuleOptions public short TopicReplicationFactor { get; set; } = 1; public bool AddConsumerGroupPrefix { get; set; } = true; public string ConsumerGroupPrefix { get; set; } = ""; - - public int SessionTimeoutMs { get; set; } - - public int MaxPollIntervalMs { get; set; } } From 8353b3723ebe560f10a0d5e41e5a415043722786 Mon Sep 17 00:00:00 2001 From: nadia Date: Wed, 18 Oct 2023 16:54:48 +0500 Subject: [PATCH 32/32] feat(tasks): optimize commit offsets ensurer --- .../KafkaConsumerOffsetsEnsurer.cs | 55 ++++++++++++++----- .../KafkaTasksModule.cs | 5 +- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs index 418e425a5..da7709029 100644 --- a/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs +++ b/src/Sitko.Core.Kafka/KafkaConsumerOffsetsEnsurer.cs @@ -7,6 +7,7 @@ namespace Sitko.Core.Kafka; internal class KafkaConsumerOffsetsEnsurer { private readonly ILogger logger; + private readonly TimeSpan lockTimeout = TimeSpan.FromSeconds(30); public KafkaConsumerOffsetsEnsurer(ILogger logger) => this.logger = logger; @@ -62,11 +63,11 @@ await adminClient.CreateTopicsAsync(new[] var partitions = topicInfo.Partitions.Select(metadata => new TopicPartition(topic.Name, new Partition(metadata.PartitionId))).ToList(); - var commited = await adminClient.ListConsumerGroupOffsetsAsync(new[] + var committed = await adminClient.ListConsumerGroupOffsetsAsync(new[] { new ConsumerGroupTopicPartitions(consumer.GroupId, partitions) }); - if (!commited.Any()) + if (!committed.Any()) { logger.LogWarning( "Can't find offsets for group {ConsumerGroup} and topic {Topic}", @@ -74,31 +75,55 @@ await adminClient.CreateTopicsAsync(new[] return; } - var badPartitions = commited.First().Partitions.Where( + var badPartitions = committed.First().Partitions.Where( partitionOffset => partitionOffset.Offset == Offset.Unset ).Select(error => error.Partition).ToList(); + if (badPartitions.Any()) { var consumerConfig = new ConsumerConfig { - BootstrapServers = string.Join(",", brokers), GroupId = consumer.GroupId + BootstrapServers = string.Join(",", brokers), GroupId = consumer.GroupId, EnableAutoCommit = false }; - var confluentConsumer = new ConsumerBuilder(consumerConfig).Build(); - var toCommit = new List(); - foreach (var partition in badPartitions) + var cts = new CancellationTokenSource(); + using var confluentConsumer = new ConsumerBuilder(consumerConfig) + .SetPartitionsAssignedHandler((_, _) => { cts.Cancel(); }) + .Build(); + var watermarkOffsetsTasks = badPartitions.Select( + partition => Task.Run( + () => + { + var topicPartition = new TopicPartition(topic.Name, partition); + var partitionOffset = + // ReSharper disable once AccessToDisposedClosure + confluentConsumer.QueryWatermarkOffsets(topicPartition, lockTimeout); + var newOffset = new TopicPartitionOffset(topicPartition, partitionOffset.High); + logger.LogWarning("Ensure {Partition} offset for consumer {GroupId}: {Offset}", partition, consumer.GroupId, newOffset.Offset); + return newOffset; + }, CancellationToken.None + ) + ).ToList(); + await Task.WhenAll(watermarkOffsetsTasks); + var toCommit = watermarkOffsetsTasks.Select(task => task.Result); + + // Add consumer to group + confluentConsumer.Subscribe(topic.Name); + // Wait for rebalance + try { - var topicPartition = new TopicPartition(topic.Name, partition); - var partitionOffset = - confluentConsumer.QueryWatermarkOffsets(topicPartition, TimeSpan.FromSeconds(30)); - var newOffset = new TopicPartitionOffset(topicPartition, partitionOffset.High); - logger.LogWarning( - "Ensure {Partition} offset for consumer {ConsumerGroup}: {Offset}", - partition, consumer.GroupId, newOffset.Offset); - toCommit.Add(newOffset); + confluentConsumer.Consume(cts.Token); + } + catch (OperationCanceledException) + { + // Rebalance complete } + // Commit offsets confluentConsumer.Commit(toCommit); + // Remove consumer from group and trigger rebalance + confluentConsumer.Close(); + logger.LogInformation("Offsets for consumer group {ConsumerGroupName} in topic {Topic} ensured", consumer.GroupId, topic); } diff --git a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs index 3bcefe78c..e8c19eeae 100644 --- a/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs +++ b/src/Sitko.Core.Tasks.Kafka/KafkaTasksModule.cs @@ -1,7 +1,5 @@ -using Confluent.Kafka; -using FluentValidation; +using FluentValidation; using KafkaFlow; -using KafkaFlow.Consumers.DistributionStrategies; using KafkaFlow.Serializer; using KafkaFlow.TypedHandler; using Microsoft.Extensions.DependencyInjection; @@ -12,7 +10,6 @@ using Sitko.Core.Tasks.Execution; using Sitko.Core.Tasks.Kafka.Execution; using Sitko.Core.Tasks.Kafka.Scheduling; -using AutoOffsetReset = Confluent.Kafka.AutoOffsetReset; namespace Sitko.Core.Tasks.Kafka;