From 7a96d052acc62a79d4b7353914c22aab470cda68 Mon Sep 17 00:00:00 2001 From: Jesper Justesen <1972142+FirestarJes@users.noreply.github.com> Date: Sun, 8 Sep 2024 16:13:08 +0200 Subject: [PATCH] Implemented some job types (status and id) updated repository to save jobId --- .../DatabaseMigration.csproj | 1 + ...09081607_Add_JobId_To_SettlementReport.sql | 3 ++ .../IRequestSettlemenReportJobHandler.cs | 4 +- .../RequestSettlementReportHandler.cs | 10 ++--- .../GetSettlementReportsHandler.cs | 5 ++- .../ISettlementReportFileRepository.cs | 2 + .../SettlementReports_v2/SettlementReport.cs | 25 +++++++++++- .../SettlementReportInitializeHandler.cs | 2 +- .../Helpers/DatabricksJobsHelper.cs | 38 ++++++++++++++++--- .../RemoveExpiredSettlementReports.cs | 15 ++++++-- .../SettlementReportFileBlobStorage.cs | 6 +++ ...ettlementReportsHandlerIntegrationTests.cs | 4 +- .../SettlementReportRepositoryTests.cs | 2 +- .../Helpers/IDatabricksJobsHelper.cs | 4 +- .../ISettlementReportInitializeHandler.cs | 2 +- .../SettlementReports_v2/Models/JobRunId.cs | 17 +++++++++ .../Models/JobRunStatus.cs | 24 ++++++++++++ .../Models/RequestedSettlementReportDto.cs | 5 ++- .../SettlementReportsController.cs | 5 ++- 19 files changed, 145 insertions(+), 29 deletions(-) create mode 100644 source/settlement-report/DatabaseMigration/Scripts/202409081607_Add_JobId_To_SettlementReport.sql create mode 100644 source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunId.cs create mode 100644 source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunStatus.cs diff --git a/source/settlement-report/DatabaseMigration/DatabaseMigration.csproj b/source/settlement-report/DatabaseMigration/DatabaseMigration.csproj index 7956a7e..a29a3d4 100644 --- a/source/settlement-report/DatabaseMigration/DatabaseMigration.csproj +++ b/source/settlement-report/DatabaseMigration/DatabaseMigration.csproj @@ -26,6 +26,7 @@ limitations under the License. + diff --git a/source/settlement-report/DatabaseMigration/Scripts/202409081607_Add_JobId_To_SettlementReport.sql b/source/settlement-report/DatabaseMigration/Scripts/202409081607_Add_JobId_To_SettlementReport.sql new file mode 100644 index 0000000..5e2d896 --- /dev/null +++ b/source/settlement-report/DatabaseMigration/Scripts/202409081607_Add_JobId_To_SettlementReport.sql @@ -0,0 +1,3 @@ +ALTER TABLE [settlementreports].[SettlementReport] +ADD [JobId] [BigInt] NULL; +GO \ No newline at end of file diff --git a/source/settlement-report/SettlementReports.Application/Handlers/IRequestSettlemenReportJobHandler.cs b/source/settlement-report/SettlementReports.Application/Handlers/IRequestSettlemenReportJobHandler.cs index 7a0eb67..d43ca2d 100644 --- a/source/settlement-report/SettlementReports.Application/Handlers/IRequestSettlemenReportJobHandler.cs +++ b/source/settlement-report/SettlementReports.Application/Handlers/IRequestSettlemenReportJobHandler.cs @@ -25,6 +25,6 @@ public interface IRequestSettlemenReportJobHandler /// /// /// - /// A long value representing the job id of the requested settlement report. - Task HandleAsync(SettlementReportRequestDto command, Guid userId, Guid actorId, bool isFas); + /// A JobRunId value representing the run id of the requested settlement report. + Task HandleAsync(SettlementReportRequestDto command, Guid userId, Guid actorId, bool isFas); } diff --git a/source/settlement-report/SettlementReports.Application/Handlers/RequestSettlementReportHandler.cs b/source/settlement-report/SettlementReports.Application/Handlers/RequestSettlementReportHandler.cs index 3557356..1e81759 100644 --- a/source/settlement-report/SettlementReports.Application/Handlers/RequestSettlementReportHandler.cs +++ b/source/settlement-report/SettlementReports.Application/Handlers/RequestSettlementReportHandler.cs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Energinet.DataHub.SettlementReport.Application.Helpers; +using Energinet.DataHub.SettlementReport.Interfaces.Helpers; using Energinet.DataHub.SettlementReport.Interfaces.SettlementReports_v2; using Energinet.DataHub.SettlementReport.Interfaces.SettlementReports_v2.Models; @@ -31,18 +31,18 @@ public RequestSettlementReportHandler( _settlementReportInitializeHandler = settlementReportInitializeHandler; } - public async Task HandleAsync(SettlementReportRequestDto request, Guid userId, Guid actorId, bool isFas) + public async Task HandleAsync(SettlementReportRequestDto request, Guid userId, Guid actorId, bool isFas) { - var jobId = await _jobHelper.RunSettlementReportsJobAsync(request).ConfigureAwait(false); + var runId = await _jobHelper.RunSettlementReportsJobAsync(request).ConfigureAwait(false); await _settlementReportInitializeHandler .InitializeFromJobAsync( userId, actorId, isFas, - jobId, + runId, request) .ConfigureAwait(false); - return jobId; + return runId; } } diff --git a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/GetSettlementReportsHandler.cs b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/GetSettlementReportsHandler.cs index 32abdf0..327e3b8 100644 --- a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/GetSettlementReportsHandler.cs +++ b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/GetSettlementReportsHandler.cs @@ -55,7 +55,7 @@ public async Task> GetAsync(Guid actor private static RequestedSettlementReportDto Map(SettlementReport report) { return new RequestedSettlementReportDto( - new SettlementReportRequestId(report.RequestId), + report.RequestId is not null ? new SettlementReportRequestId(report.RequestId) : null, report.CalculationType, report.PeriodStart.ToDateTimeOffset(), report.PeriodEnd.ToDateTimeOffset(), @@ -63,6 +63,7 @@ private static RequestedSettlementReportDto Map(SettlementReport report) report.GridAreaCount, 0, report.ActorId, - report.ContainsBasisData); + report.ContainsBasisData, + report.JobId is not null ? new JobRunId(report.JobId.Value) : null); } } diff --git a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/ISettlementReportFileRepository.cs b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/ISettlementReportFileRepository.cs index 4c09fa5..0b2ac50 100644 --- a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/ISettlementReportFileRepository.cs +++ b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/ISettlementReportFileRepository.cs @@ -24,5 +24,7 @@ public interface ISettlementReportFileRepository Task DeleteAsync(SettlementReportRequestId reportRequestId, string fileName); + Task DeleteAsync(JobRunId reportRequestId, string fileName); + Task DownloadAsync(SettlementReportRequestId reportRequestId, string fileName, Stream downloadStream); } diff --git a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReport.cs b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReport.cs index 7718fc6..fbd81c8 100644 --- a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReport.cs +++ b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReport.cs @@ -23,7 +23,7 @@ public sealed class SettlementReport { public int Id { get; init; } - public string RequestId { get; init; } = null!; + public string? RequestId { get; init; } public Guid UserId { get; init; } @@ -47,6 +47,8 @@ public sealed class SettlementReport public string? BlobFileName { get; private set; } + public long? JobId { get; init; } + public SettlementReport( IClock clock, Guid userId, @@ -68,6 +70,27 @@ public SettlementReport( GridAreaCount = request.Filter.GridAreas.Count; } + public SettlementReport( + IClock clock, + Guid userId, + Guid actorId, + bool hideReport, + JobRunId jobRunId, + SettlementReportRequestDto request) + { + JobId = jobRunId.Id; + UserId = userId; + ActorId = actorId; + IsHiddenFromActor = hideReport; + CreatedDateTime = clock.GetCurrentInstant(); + Status = SettlementReportStatus.InProgress; + CalculationType = request.Filter.CalculationType; + ContainsBasisData = request.IncludeBasisData; + PeriodStart = request.Filter.PeriodStart.ToInstant(); + PeriodEnd = request.Filter.PeriodEnd.ToInstant(); + GridAreaCount = request.Filter.GridAreas.Count; + } + // EF Core Constructor. // ReSharper disable once UnusedMember.Local private SettlementReport() diff --git a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReportInitializeHandler.cs b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReportInitializeHandler.cs index b423b0a..4b3d044 100644 --- a/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReportInitializeHandler.cs +++ b/source/settlement-report/SettlementReports.Application/SettlementReports_v2/SettlementReportInitializeHandler.cs @@ -38,7 +38,7 @@ public Task InitializeAsync( return _repository.AddOrUpdateAsync(settlementReport); } - public Task InitializeFromJobAsync(Guid userId, Guid actorId, bool hideReport, long jobId, SettlementReportRequestDto request) + public Task InitializeFromJobAsync(Guid userId, Guid actorId, bool hideReport, JobRunId jobId, SettlementReportRequestDto request) { var settlementReport = new SettlementReport(SystemClock.Instance, userId, actorId, hideReport, new SettlementReportRequestId(jobId.ToString()), request); return _repository.AddOrUpdateAsync(settlementReport); diff --git a/source/settlement-report/SettlementReports.Infrastructure/Helpers/DatabricksJobsHelper.cs b/source/settlement-report/SettlementReports.Infrastructure/Helpers/DatabricksJobsHelper.cs index d49264c..d95e0fc 100644 --- a/source/settlement-report/SettlementReports.Infrastructure/Helpers/DatabricksJobsHelper.cs +++ b/source/settlement-report/SettlementReports.Infrastructure/Helpers/DatabricksJobsHelper.cs @@ -14,8 +14,8 @@ using Energinet.DataHub.Core.Databricks.Jobs.Abstractions; using Energinet.DataHub.SettlementReport.Application.Handlers; -using Energinet.DataHub.SettlementReport.Application.Helpers; using Energinet.DataHub.SettlementReport.Infrastructure.SqlStatements.Mappers; +using Energinet.DataHub.SettlementReport.Interfaces.Helpers; using Energinet.DataHub.SettlementReport.Interfaces.Models; using Energinet.DataHub.SettlementReport.Interfaces.SettlementReports_v2.Models; using Microsoft.Azure.Databricks.Client.Models; @@ -31,10 +31,16 @@ public DatabricksJobsHelper(IJobsApiClient jobsApiClient) _jobsApiClient = jobsApiClient; } - public async Task RunSettlementReportsJobAsync(SettlementReportRequestDto request) + public async Task RunSettlementReportsJobAsync(SettlementReportRequestDto request) { var job = await GetSettlementReportsJobAsync(GetJobName(request.Filter.CalculationType)).ConfigureAwait(false); - return await _jobsApiClient.Jobs.RunNow(job.JobId, CreateParameters(request)).ConfigureAwait(false); + return new JobRunId(await _jobsApiClient.Jobs.RunNow(job.JobId, CreateParameters(request)).ConfigureAwait(false)); + } + + public async Task GetSettlementReportsJobStatusAsync(long runId) + { + var jobRun = await _jobsApiClient.Jobs.RunsGet(runId, false).ConfigureAwait(false); + return ConvertJobStatus(jobRun.Item1); } private string GetJobName(CalculationType calculationType) @@ -53,12 +59,12 @@ private string GetJobName(CalculationType calculationType) private async Task GetSettlementReportsJobAsync(string jobName) { - var calculatorJob = await _jobsApiClient.Jobs + var settlementJob = await _jobsApiClient.Jobs .ListPageable(name: jobName) .SingleAsync() .ConfigureAwait(false); - return await _jobsApiClient.Jobs.Get(calculatorJob.JobId).ConfigureAwait(false); + return await _jobsApiClient.Jobs.Get(settlementJob.JobId).ConfigureAwait(false); } private RunParameters CreateParameters(SettlementReportRequestDto request) @@ -75,4 +81,26 @@ private RunParameters CreateParameters(SettlementReportRequestDto request) return RunParameters.CreatePythonParams(jobParameters); } + + private static JobRunStatus ConvertJobStatus(Run jobRun) + { + if (jobRun.State == null) + { + return JobRunStatus.Queued; + } + + if (jobRun.State.ResultState == RunResultState.SUCCESS || jobRun.IsCompleted) + { + return JobRunStatus.Completed; + } + + return jobRun.State.LifeCycleState switch + { + RunLifeCycleState.RUNNING => JobRunStatus.Running, + RunLifeCycleState.QUEUED or RunLifeCycleState.PENDING => JobRunStatus.Queued, + RunLifeCycleState.TERMINATED => JobRunStatus.Canceled, + RunLifeCycleState.INTERNAL_ERROR => JobRunStatus.Failed, + _ => JobRunStatus.Queued, + }; + } } diff --git a/source/settlement-report/SettlementReports.Infrastructure/SettlementReports_v2/RemoveExpiredSettlementReports.cs b/source/settlement-report/SettlementReports.Infrastructure/SettlementReports_v2/RemoveExpiredSettlementReports.cs index a56ccee..21a61aa 100644 --- a/source/settlement-report/SettlementReports.Infrastructure/SettlementReports_v2/RemoveExpiredSettlementReports.cs +++ b/source/settlement-report/SettlementReports.Infrastructure/SettlementReports_v2/RemoveExpiredSettlementReports.cs @@ -46,9 +46,18 @@ public async Task RemoveExpiredAsync(IList OpenForReadingAsync(SettlementReportRequestId reportRequestI return blobClient.OpenReadAsync(); } + // TODO: Impelement this method when we know the blob storage for job reports + public Task DeleteAsync(JobRunId reportRequestId, string fileName) + { + throw new NotImplementedException(); + } + public async Task DownloadAsync(SettlementReportRequestId reportRequestId, string fileName, Stream downloadStream) { var blobName = GetBlobName(reportRequestId, fileName); diff --git a/source/settlement-report/SettlementReports.IntegrationTests/Application/SettlementReports/GetSettlementReportsHandlerIntegrationTests.cs b/source/settlement-report/SettlementReports.IntegrationTests/Application/SettlementReports/GetSettlementReportsHandlerIntegrationTests.cs index 844b5fd..cd73651 100644 --- a/source/settlement-report/SettlementReports.IntegrationTests/Application/SettlementReports/GetSettlementReportsHandlerIntegrationTests.cs +++ b/source/settlement-report/SettlementReports.IntegrationTests/Application/SettlementReports/GetSettlementReportsHandlerIntegrationTests.cs @@ -127,12 +127,12 @@ public async Task GetAsync_SingleUser_ReturnsOwnRows() item => { Assert.Equal(targetActorId, item.RequestedByActorId); - Assert.Equal(requestId2.ToString(), item.RequestId.Id); + Assert.Equal(requestId2.ToString(), item.RequestId!.Id); }, item => { Assert.Equal(targetActorId, item.RequestedByActorId); - Assert.Equal(requestId3.ToString(), item.RequestId.Id); + Assert.Equal(requestId3.ToString(), item.RequestId!.Id); }); } diff --git a/source/settlement-report/SettlementReports.IntegrationTests/Infrastructure/SettlementReports_v2/Persistence/SettlementReportRepositoryTests.cs b/source/settlement-report/SettlementReports.IntegrationTests/Infrastructure/SettlementReports_v2/Persistence/SettlementReportRepositoryTests.cs index 6c9aedc..92f9fc4 100644 --- a/source/settlement-report/SettlementReports.IntegrationTests/Infrastructure/SettlementReports_v2/Persistence/SettlementReportRepositoryTests.cs +++ b/source/settlement-report/SettlementReports.IntegrationTests/Infrastructure/SettlementReports_v2/Persistence/SettlementReportRepositoryTests.cs @@ -135,7 +135,7 @@ public async Task GetAsync_RequestExistsWithSuppliedRequestId_ReturnsRequest() var repository = new SettlementReportRepository(context); // act - var actual = await repository.GetAsync(expectedRequest.RequestId); + var actual = await repository.GetAsync(expectedRequest.RequestId!); // assert Assert.NotNull(actual); diff --git a/source/settlement-report/SettlementReports.Interfaces/Helpers/IDatabricksJobsHelper.cs b/source/settlement-report/SettlementReports.Interfaces/Helpers/IDatabricksJobsHelper.cs index 08339f1..0213079 100644 --- a/source/settlement-report/SettlementReports.Interfaces/Helpers/IDatabricksJobsHelper.cs +++ b/source/settlement-report/SettlementReports.Interfaces/Helpers/IDatabricksJobsHelper.cs @@ -14,9 +14,9 @@ using Energinet.DataHub.SettlementReport.Interfaces.SettlementReports_v2.Models; -namespace Energinet.DataHub.SettlementReport.Application.Helpers; +namespace Energinet.DataHub.SettlementReport.Interfaces.Helpers; public interface IDatabricksJobsHelper { - Task RunSettlementReportsJobAsync(SettlementReportRequestDto request); + Task RunSettlementReportsJobAsync(SettlementReportRequestDto request); } diff --git a/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/ISettlementReportInitializeHandler.cs b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/ISettlementReportInitializeHandler.cs index a0a9832..12be19e 100644 --- a/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/ISettlementReportInitializeHandler.cs +++ b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/ISettlementReportInitializeHandler.cs @@ -29,6 +29,6 @@ Task InitializeFromJobAsync( Guid userId, Guid actorId, bool hideReport, - long jobId, + JobRunId jobId, SettlementReportRequestDto request); } diff --git a/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunId.cs b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunId.cs new file mode 100644 index 0000000..c3da02d --- /dev/null +++ b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunId.cs @@ -0,0 +1,17 @@ +// Copyright 2020 Energinet DataHub A/S +// +// Licensed under the Apache License, Version 2.0 (the "License2"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Energinet.DataHub.SettlementReport.Interfaces.SettlementReports_v2.Models; + +public sealed record JobRunId(long Id); diff --git a/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunStatus.cs b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunStatus.cs new file mode 100644 index 0000000..8bc21a4 --- /dev/null +++ b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/JobRunStatus.cs @@ -0,0 +1,24 @@ +// Copyright 2020 Energinet DataHub A/S +// +// Licensed under the Apache License, Version 2.0 (the "License2"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Energinet.DataHub.SettlementReport.Interfaces.SettlementReports_v2.Models; + +public enum JobRunStatus +{ + Queued, + Running, + Completed, + Failed, + Canceled, +} diff --git a/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/RequestedSettlementReportDto.cs b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/RequestedSettlementReportDto.cs index 4545cbb..e9b7a35 100644 --- a/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/RequestedSettlementReportDto.cs +++ b/source/settlement-report/SettlementReports.Interfaces/SettlementReports_v2/Models/RequestedSettlementReportDto.cs @@ -18,7 +18,7 @@ namespace Energinet.DataHub.SettlementReport.Interfaces.SettlementReports_v2.Models; public sealed record RequestedSettlementReportDto( - SettlementReportRequestId RequestId, + SettlementReportRequestId? RequestId, CalculationType CalculationType, DateTimeOffset PeriodStart, DateTimeOffset PeriodEnd, @@ -26,4 +26,5 @@ public sealed record RequestedSettlementReportDto( int GridAreaCount, double Progress, Guid RequestedByActorId, - bool ContainsBasisData); + bool ContainsBasisData, + JobRunId? JobId); diff --git a/source/settlement-report/SettlementReports.WebAPI/Controllers/SettlementReportsController.cs b/source/settlement-report/SettlementReports.WebAPI/Controllers/SettlementReportsController.cs index cd33550..7005a33 100644 --- a/source/settlement-report/SettlementReports.WebAPI/Controllers/SettlementReportsController.cs +++ b/source/settlement-report/SettlementReports.WebAPI/Controllers/SettlementReportsController.cs @@ -91,9 +91,10 @@ public async Task RequestSettlementReport([FromBody] SettlementRep [HttpGet] [Route("status/{jobId:long}")] [Authorize] - public async Task RequestSettlementReport(long jobId) + public async Task RequestSettlementReport(long jobId) { - return await Task.FromResult("running").ConfigureAwait(false); + var jobRunId = new JobRunId(jobId); + return await Task.FromResult(JobRunStatus.Running).ConfigureAwait(false); } private bool IsValid(SettlementReportRequestDto req)