Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

$Import cleanup (stage 2) #3270

Merged
merged 41 commits into from
May 5, 2023
Merged

$Import cleanup (stage 2) #3270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
d2a7243
Removed MaxRunningProcessingJobCount
SergeyGaluzo Apr 24, 2023
eedeaa4
Removed checkpoints
SergeyGaluzo Apr 25, 2023
16ff9ea
Removed sequence id
SergeyGaluzo Apr 25, 2023
bb2491c
Removed create date from coord definition
SergeyGaluzo Apr 25, 2023
6154933
Removed transaction date from coord result
SergeyGaluzo Apr 25, 2023
ce0ad5a
SqlImporter
SergeyGaluzo Apr 25, 2023
3c8a7f8
Removed cleanup
SergeyGaluzo Apr 25, 2023
520d073
Coord result cleanup
SergeyGaluzo Apr 26, 2023
7862ed6
index rebuild tests
SergeyGaluzo Apr 26, 2023
51cbda0
size = 0
SergeyGaluzo Apr 26, 2023
0af7585
Remove id sequence generator
SergeyGaluzo Apr 26, 2023
fae498d
Removed output channel from .Import method and remaned to IImporter
SergeyGaluzo Apr 29, 2023
298aa1a
Increased merge timeout
SergeyGaluzo Apr 29, 2023
459df66
retries on timeouts
SergeyGaluzo Apr 29, 2023
13a4ead
try log event
SergeyGaluzo Apr 30, 2023
3734eee
Removed data generators
SergeyGaluzo May 1, 2023
099fc0e
Allow parallel imports
SergeyGaluzo May 1, 2023
1f2678f
removed not valid test
SergeyGaluzo May 1, 2023
2b0c9ac
Removed "bulk" classes
SergeyGaluzo May 1, 2023
e828d8e
Correct names in results
SergeyGaluzo May 1, 2023
b5088b1
Cleanup ImportResource class
SergeyGaluzo May 1, 2023
4d012a4
merge
SergeyGaluzo May 1, 2023
58128fd
fixed merge
SergeyGaluzo May 1, 2023
389f868
Renaming ImportResource.Resource ip recource wrapper
SergeyGaluzo May 2, 2023
2882056
Removed context accessor from ImportProcessingJob class
SergeyGaluzo May 2, 2023
99f5913
context back
SergeyGaluzo May 2, 2023
1dbaf30
Cleanup import configuration
SergeyGaluzo May 2, 2023
c3a8bd7
making retries optional
SergeyGaluzo May 2, 2023
502f041
merge after revert
SergeyGaluzo May 2, 2023
0cee87c
fixing merge
SergeyGaluzo May 2, 2023
5c03203
frequency in seconds
SergeyGaluzo May 3, 2023
ca53d50
frequency in proc
SergeyGaluzo May 3, 2023
8329c25
Removed invalid test
SergeyGaluzo May 3, 2023
19e4b84
10 -> 30
SergeyGaluzo May 3, 2023
8f6672c
fixed typo
SergeyGaluzo May 3, 2023
e33cdaa
Added now to last updated to make string replacements to work
SergeyGaluzo May 4, 2023
faa6249
back to task
SergeyGaluzo May 4, 2023
9e9791a
Cleaning processing job definition
SergeyGaluzo May 4, 2023
b9eb097
Corrected test based on idempoence rules.
SergeyGaluzo May 4, 2023
0d0ff9a
Adding group id to processing job definition
SergeyGaluzo May 4, 2023
493a09d
GroupId = 1
SergeyGaluzo May 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 31 additions & 28 deletions ...oft.Health.Fhir.Core.UnitTests/Features/Operations/Import/GetImportRequestHandlerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
// -------------------------------------------------------------------------------------------------

using System;
using System.Collections.Generic;
using System.Net;
using System.Threading;
Expand Down Expand Up @@ -46,31 +45,34 @@ public GetImportRequestHandlerTests()
[Fact]
public async Task GivenAFhirMediator_WhenGettingAnExistingBulkImportJobWithCompletedStatus_ThenHttpResponseCodeShouldBeOk()
{
ImportOrchestratorJobResult orchestratorJobResult = new ImportOrchestratorJobResult()
var coordResult = new ImportOrchestratorJobResult()
{
TransactionTime = DateTime.Now,
Request = "Request",
};

JobInfo orchestratorJob = new JobInfo()
var orchestratorJob = new JobInfo()
{
Id = 0,
GroupId = 0,
Status = JobStatus.Completed,
Result = JsonConvert.SerializeObject(orchestratorJobResult),
Result = JsonConvert.SerializeObject(coordResult),
Definition = JsonConvert.SerializeObject(new ImportOrchestratorJobDefinition()),
};

ImportProcessingJobResult processingJobResult = new ImportProcessingJobResult()
var processingJobResult = new ImportProcessingJobResult()
{
ResourceLocation = "http://ResourceLocation",
ResourceType = "Patient",
SucceedCount = 1,
FailedCount = 1,
ErrorLogLocation = "http://ResourceLocation",
SucceededResources = 1,
FailedResources = 1,
ErrorLogLocation = "http://ResourceErrorLogLocation",
};

JobInfo processingJob = new JobInfo()
var processingJob = new JobInfo()
{
Id = 1,
GroupId = 0,
Status = JobStatus.Completed,
Result = JsonConvert.SerializeObject(processingJobResult),
Definition = JsonConvert.SerializeObject(new ImportProcessingJobDefinition() { ResourceLocation = "http://ResourceLocation" }),
};

GetImportResponse result = await SetupAndExecuteGetBulkImportJobByIdAsync(orchestratorJob, new List<JobInfo>() { processingJob });
Expand All @@ -83,13 +85,13 @@ public async Task GivenAFhirMediator_WhenGettingAnExistingBulkImportJobWithCompl
[Fact]
public async Task GivenAFhirMediator_WhenGettingAnCompletedImportJobWithFailure_ThenHttpResponseCodeShouldBeExpected()
{
ImportOrchestratorJobErrorResult orchestratorJobResult = new ImportOrchestratorJobErrorResult()
var orchestratorJobResult = new ImportOrchestratorJobErrorResult()
{
HttpStatusCode = HttpStatusCode.BadRequest,
ErrorMessage = "error",
};

JobInfo orchestratorJob = new JobInfo()
var orchestratorJob = new JobInfo()
{
Status = JobStatus.Failed,
Result = JsonConvert.SerializeObject(orchestratorJobResult),
Expand All @@ -104,7 +106,7 @@ public async Task GivenAFhirMediator_WhenGettingAnCompletedImportJobWithFailure_
[Fact]
public async Task GivenAFhirMediator_WhenGettingAnExistingBulkImportJobThatWasCanceled_ThenOperationFailedExceptionIsThrownWithBadRequestHttpResponseCode()
{
JobInfo orchestratorJob = new JobInfo()
var orchestratorJob = new JobInfo()
{
Status = JobStatus.Cancelled,
};
Expand All @@ -116,51 +118,52 @@ public async Task GivenAFhirMediator_WhenGettingAnExistingBulkImportJobThatWasCa
[Fact]
public async Task GivenAFhirMediator_WhenGettingAnExistingBulkImportJobWithNotCompletedStatus_ThenHttpResponseCodeShouldBeAccepted()
{
ImportOrchestratorJobResult orchestratorJobResult = new ImportOrchestratorJobResult()
var orchestratorJobResult = new ImportOrchestratorJobResult()
{
TransactionTime = DateTime.Now,
Request = "Request",
};

JobInfo orchestratorJob = new JobInfo()
var orchestratorJob = new JobInfo()
{
Id = 1,
GroupId = 1,
Status = JobStatus.Running,
Result = JsonConvert.SerializeObject(orchestratorJobResult),
Definition = JsonConvert.SerializeObject(new ImportOrchestratorJobDefinition()),
};

ImportProcessingJobResult processingJobResult = new ImportProcessingJobResult()
var processingJobResult = new ImportProcessingJobResult()
{
ResourceLocation = "http://ResourceLocation",
ResourceType = "Patient",
SucceedCount = 1,
FailedCount = 1,
ErrorLogLocation = "http://ResourceLocation",
SucceededResources = 1,
FailedResources = 1,
ErrorLogLocation = "http://ResourceErrorLogLocation",
};

JobInfo processingJob1 = new JobInfo()
var processingJob1 = new JobInfo()
{
Id = 2,
GroupId = 1,
Status = JobStatus.Completed,
Result = JsonConvert.SerializeObject(processingJobResult),
Definition = JsonConvert.SerializeObject(new ImportProcessingJobDefinition() { ResourceLocation = "http://ResourceLocation" }),
};

JobInfo processingJob2 = new JobInfo()
var processingJob2 = new JobInfo()
{
Id = 3,
GroupId = 1,
Status = JobStatus.Completed,
Result = JsonConvert.SerializeObject(processingJobResult),
Definition = JsonConvert.SerializeObject(new ImportProcessingJobDefinition() { ResourceLocation = "http://ResourceLocation" }),
};

JobInfo processingJob3 = new JobInfo()
var processingJob3 = new JobInfo()
{
Id = 4,
GroupId = 1,
Status = JobStatus.Running,
Result = JsonConvert.SerializeObject(processingJobResult),
Definition = JsonConvert.SerializeObject(new ImportProcessingJobDefinition() { ResourceLocation = "http://ResourceLocation" }),
};

GetImportResponse result = await SetupAndExecuteGetBulkImportJobByIdAsync(orchestratorJob, new List<JobInfo>() { processingJob1, processingJob2, processingJob3 });
Expand All @@ -180,7 +183,7 @@ private async Task<GetImportResponse> SetupAndExecuteGetBulkImportJobByIdAsync(J
{
_queueClient.GetJobByIdAsync(Arg.Any<byte>(), Arg.Any<long>(), Arg.Any<bool>(), Arg.Any<CancellationToken>()).Returns(orchestratorJobInfo);

List<JobInfo> allJobs = new List<JobInfo>(processingJobInfos);
var allJobs = new List<JobInfo>(processingJobInfos);
allJobs.Add(orchestratorJobInfo);
_queueClient.GetJobByGroupIdAsync(Arg.Any<byte>(), Arg.Any<long>(), Arg.Any<bool>(), Arg.Any<CancellationToken>()).Returns(allJobs);

Expand Down
Loading