From c209b587b6a47d84e8cfaeb61dc1d58d6db52ae9 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:18:02 +0000 Subject: [PATCH 01/19] Update target java version to java 11 --- java/pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 6afd0253af..b3e36aca8e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -51,7 +51,7 @@ UTF-8 - 1.8 + 11 2.12 2.12.17 3.3.0 @@ -94,7 +94,7 @@ ${surefire.plugin.version} 3.3.0 3.2.0 - 3.8.1 + 3.10.1 3.0.0 dir 4.7.2.0 @@ -399,8 +399,7 @@ maven-compiler-plugin ${compiler.plugin.version} - ${java.version} - ${java.version} + ${java.version} From 29973df117f57e1b9eea4681ab18bde89015e7a9 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:18:27 +0000 Subject: [PATCH 02/19] Fix spotbugs issues --- .../src/main/java/sleeper/cdk/stack/DashboardStack.java | 2 ++ .../query/model/output/WebSocketResultsOutput.java | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java index bee948b837..2f528461b1 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java @@ -15,6 +15,7 @@ */ package sleeper.cdk.stack; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import sleeper.cdk.Utils; import sleeper.configuration.properties.InstanceProperties; import sleeper.configuration.properties.UserDefinedInstanceProperty; @@ -46,6 +47,7 @@ import static sleeper.configuration.properties.UserDefinedInstanceProperty.DASHBOARD_TIME_WINDOW_MINUTES; +@SuppressFBWarnings("MC_OVERRIDABLE_METHOD_CALL_IN_CONSTRUCTOR") public class DashboardStack extends NestedStack { private final IngestStack ingestStack; private final CompactionStack compactionStack; diff --git a/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java b/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java index 5ef503f74a..0181e06b6b 100644 --- a/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java +++ b/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java @@ -44,13 +44,8 @@ public WebSocketResultsOutput(Map config) { String maxBatchSize = config.get(MAX_BATCH_SIZE); this.maxBatchSize = maxBatchSize != null && !maxBatchSize.isEmpty() ? Long.parseLong(maxBatchSize) : null; - - addOutputLocations(); - } - - private void addOutputLocations() { - this.outputLocations.add(new ResultsOutputLocation("websocket-endpoint", this.getEndpoint())); - this.outputLocations.add(new ResultsOutputLocation("websocket-connection-id", this.getConnectionId())); + this.outputLocations.add(new ResultsOutputLocation("websocket-endpoint", config.get(ENDPOINT))); + this.outputLocations.add(new ResultsOutputLocation("websocket-connection-id", config.get(CONNECTION_ID))); } @Override From 1a24ba75df8bb4ef63ca9005c60b29c83c5a1163 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:28:38 +0000 Subject: [PATCH 03/19] Update GitHub actions to use java 17 --- .github/workflows/check-status.yaml | 4 ++-- .github/workflows/chunk.yaml | 4 ++-- .github/workflows/maven-full.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-status.yaml b/.github/workflows/check-status.yaml index 66f3e90999..d8025b1c83 100644 --- a/.github/workflows/check-status.yaml +++ b/.github/workflows/check-status.yaml @@ -9,8 +9,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'corretto' - name: Cache dependencies uses: actions/cache@v3 with: diff --git a/.github/workflows/chunk.yaml b/.github/workflows/chunk.yaml index 5059e5d171..2c1f90002d 100644 --- a/.github/workflows/chunk.yaml +++ b/.github/workflows/chunk.yaml @@ -17,8 +17,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'corretto' - name: Cache dependencies uses: actions/cache@v3 with: diff --git a/.github/workflows/maven-full.yaml b/.github/workflows/maven-full.yaml index c4da5d8fec..33d75ab763 100644 --- a/.github/workflows/maven-full.yaml +++ b/.github/workflows/maven-full.yaml @@ -13,8 +13,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'corretto' - name: Cache dependencies uses: actions/cache@v3 with: From 15393e87667c5bd85e48b9987757b083f131f660 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 13:09:16 +0000 Subject: [PATCH 04/19] Make java packages accessible to existing tests --- java/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/java/pom.xml b/java/pom.xml index b3e36aca8e..1cedd7bc9b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -322,6 +322,14 @@ org.apache.maven.plugins maven-surefire-plugin ${surefire.plugin.version} + + + --add-opens java.base/java.lang.invoke=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + + org.apache.maven.plugins From b2c8adebacfdf0046525f252d63d245304b705f1 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 14:21:41 +0000 Subject: [PATCH 05/19] Avoid overwriting argLine in maven surefire plugin --- java/pom.xml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 1cedd7bc9b..785d0db2d7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -99,6 +99,12 @@ dir 4.7.2.0 1.2.8 + + --add-opens java.base/java.lang.invoke=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + @@ -322,14 +328,6 @@ org.apache.maven.plugins maven-surefire-plugin ${surefire.plugin.version} - - - --add-opens java.base/java.lang.invoke=ALL-UNNAMED - --add-opens java.base/java.nio=ALL-UNNAMED - --add-opens java.base/java.util=ALL-UNNAMED - --add-opens java.base/sun.nio.ch=ALL-UNNAMED - - org.apache.maven.plugins From 6898b08f7150e9f19a65c13d697f42f124c67eec Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:21:21 +0000 Subject: [PATCH 06/19] Use hardcoded Instants in ingest job tests --- .../job/QueryIngestJobStatusByPeriodIT.java | 26 +++++++++---------- .../job/QueryIngestJobStatusByTaskIdIT.java | 4 +-- .../store/job/StoreIngestJobStartedIT.java | 4 +-- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java index 0009f9f7c5..6e6450beb6 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java @@ -33,8 +33,8 @@ public void shouldReturnIngestJobsInPeriod() { // Given IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithFiles("file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -54,14 +54,14 @@ public void shouldReturnIngestJobsInPeriod() { public void shouldExcludeIngestJobOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); - Instant startedTime = Instant.now(); + Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); // Then - Instant periodStart = Instant.now().plus(Period.ofDays(1)); - Instant periodEnd = periodStart.plus(Period.ofDays(1)); + Instant periodStart = Instant.parse("2023-01-01T14:00:00.001Z"); + Instant periodEnd = Instant.parse("2023-01-02T14:00:00.001Z"); assertThat(store.getJobsInTimePeriod(tableName, periodStart, periodEnd)).isEmpty(); } @@ -70,8 +70,8 @@ public void shouldExcludeIngestJobInOtherTable() { // Given IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithTableAndFiles("other-table", "file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -86,18 +86,16 @@ public void shouldExcludeIngestJobInOtherTable() { } @Test - public void shouldIncludeFinishedStatusUpdateOutsidePeriod() throws Exception { + public void shouldIncludeFinishedStatusUpdateOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); - Instant periodStart = Instant.now().minus(Period.ofDays(1)); - Instant startedTime = Instant.now(); + Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant periodStart = startedTime.minus(Period.ofDays(1)); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); - Thread.sleep(1); - Instant periodEnd = Instant.now(); - Thread.sleep(1); - Instant finishedTime = Instant.now(); + Instant finishedTime = Instant.parse("2023-01-03T14:56:00.001Z"); + Instant periodEnd = finishedTime.plus(Period.ofDays(1)); store.jobFinished(DEFAULT_TASK_ID, job, defaultSummary(startedTime, finishedTime)); // Then diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java index f7dd7539d0..3ec8c25668 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java @@ -34,8 +34,8 @@ public void shouldReturnIngestJobsByTaskId() { String searchingTaskId = "test-task"; IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithFiles("file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2022-12-14T13:51:12.001Z"); + Instant startedTime2 = Instant.parse("2022-12-14T13:52:12.001Z"); // When store.jobStarted(searchingTaskId, job1, startedTime1); diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java index 209b3ddc07..4f28effb61 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java @@ -61,8 +61,8 @@ public void shouldReportSeveralIngestJobsStarted() { // Given IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithFiles("file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2022-12-14T13:51:12.001Z"); + Instant startedTime2 = Instant.parse("2022-12-14T13:56:12.001Z"); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); From add2374b2c08b5805777dc3ee487713bda36375c Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:31:14 +0000 Subject: [PATCH 07/19] Add error flag to mvn test step --- .github/workflows/chunk.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/chunk.yaml b/.github/workflows/chunk.yaml index 2c1f90002d..f56771db53 100644 --- a/.github/workflows/chunk.yaml +++ b/.github/workflows/chunk.yaml @@ -41,7 +41,7 @@ jobs: run: mvn --batch-mode clean install -am -pl ${{ steps.config.outputs.moduleList }} -Pquick,skipShade -Dmaven.repo.local=.m2/repository working-directory: ./java - name: Test - run: mvn --batch-mode --fail-at-end verify -pl ${{ steps.config.outputs.moduleList }} -Dmaven.repo.local=.m2/repository + run: mvn --batch-mode --fail-at-end verify -pl ${{ steps.config.outputs.moduleList }} -Dmaven.repo.local=.m2/repository -e working-directory: ./java - name: Check rate limits id: rate-limit From 971c5857e0569cb798b7986c7124bb0edb04678c Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:45:53 +0000 Subject: [PATCH 08/19] Use late replacement in surefire argLine --- java/pom.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 785d0db2d7..6bd81b5173 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -99,12 +99,12 @@ dir 4.7.2.0 1.2.8 - + --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED - + @@ -328,6 +328,9 @@ org.apache.maven.plugins maven-surefire-plugin ${surefire.plugin.version} + + @{argLine} ${surefire.args} + org.apache.maven.plugins From c01b67dd75fc4eeaaf2574f09719173c1b197cbd Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:49:00 +0000 Subject: [PATCH 09/19] Add missing definition of argLine in pom --- java/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/pom.xml b/java/pom.xml index 6bd81b5173..230a2fb9b7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -105,6 +105,8 @@ --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED + + From 7e110bcb609c2fcc82ba3055f573b6e2d7548ab2 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:54:46 +0000 Subject: [PATCH 10/19] Add missing configuration for failsafe-plugin --- java/pom.xml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 230a2fb9b7..d12b12713f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -99,12 +99,12 @@ dir 4.7.2.0 1.2.8 - + --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED - + @@ -331,13 +331,16 @@ maven-surefire-plugin ${surefire.plugin.version} - @{argLine} ${surefire.args} + @{argLine} ${module.access.args} org.apache.maven.plugins maven-failsafe-plugin ${failsafe.plugin.version} + + @{argLine} ${module.access.args} + integration-test From 9334d6738014db6f06b35173b4aa9ce0e9a69638 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Wed, 4 Jan 2023 10:56:01 +0000 Subject: [PATCH 11/19] Update surefire and failsafe plugins to 3.0.0-M6 --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index d12b12713f..866f4a5c07 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -90,7 +90,7 @@ 2.17 3.0.0-M4 - 3.0.0-M5 + 3.0.0-M6 ${surefire.plugin.version} 3.3.0 3.2.0 From 741de1007aa3b54458aa25438a646e0599c96e57 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:14:10 +0000 Subject: [PATCH 12/19] Update lambda java runtimes to use java 11 --- java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java | 2 +- .../src/main/java/sleeper/cdk/stack/CompactionStack.java | 6 +++--- .../main/java/sleeper/cdk/stack/GarbageCollectorStack.java | 2 +- java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java | 2 +- .../java/sleeper/cdk/stack/PartitionSplittingStack.java | 4 ++-- .../src/main/java/sleeper/cdk/stack/PropertiesStack.java | 2 +- java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java | 4 ++-- java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java | 4 ++-- java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java | 2 +- .../cdk/stack/bulkimport/AbstractEmrBulkImportStack.java | 2 +- .../sleeper/cdk/stack/bulkimport/EksBulkImportStack.java | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java index 8a96166386..658d53a85f 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java @@ -173,7 +173,7 @@ private Function createConnector(String className, String instanceId, int logRet .memorySize(memory) .timeout(Duration.seconds(timeout)) .code(s3Code) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .logRetention(Utils.getRetentionDays(logRetentionDays)) .handler(className) .environment(env) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java index 34e6182bd9..67f7af9d90 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java @@ -302,7 +302,7 @@ private void lambdaToFindCompactionJobsThatShouldBeCreated(IBucket configBucket, .create(this, "JobCreationLambda") .functionName(functionName) .description("Scan DynamoDB looking for files that need merging and create appropriate job specs in DynamoDB") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(COMPACTION_JOB_CREATION_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(COMPACTION_JOB_CREATION_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) @@ -481,7 +481,7 @@ private void lambdaToCreateCompactionTasks(IBucket configBucket, .create(this, "CompactionTasksCreator") .functionName(functionName) .description("If there are compaction jobs on queue create tasks to run them") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(TASK_RUNNER_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) @@ -539,7 +539,7 @@ private void lambdaToCreateSplittingCompactionTasks(IBucket configBucket, .create(this, "SplittingCompactionTasksCreator") .functionName(functionName) .description("If there are splitting compaction jobs on queue create tasks to run them") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(TASK_RUNNER_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java index 306b1b6bcc..bbf7e8eb87 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java @@ -73,7 +73,7 @@ public GarbageCollectorStack( .create(this, "GarbageCollectorLambda") .functionName(functionName) .description("Scan DynamoDB looking for files that need deleting and delete them") - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .memorySize(instanceProperties.getInt(GARBAGE_COLLECTOR_LAMBDA_MEMORY_IN_MB)) // Timeout is set to 90% of the period with which this runs to avoid 2 running simultaneously, // with a maximum of 900 seconds (15 minutes) which is the maximum execution time diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java index b4462a6bcf..79aa245a88 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java @@ -274,7 +274,7 @@ private void lambdaToCreateIngestTasks(IBucket configBucket, Queue ingestJobQueu .create(this, "IngestTasksCreator") .functionName(functionName) .description("If there are ingest jobs on queue create tasks to run them") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(TASK_RUNNER_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java index 114c639aee..6a4fb9cb10 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java @@ -139,7 +139,7 @@ public PartitionSplittingStack(Construct scope, .create(this, "FindPartitionsToSplitLambda") .functionName(functionName) .description("Scan DynamoDB looking for partitions that need splitting") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(FIND_PARTITIONS_TO_SPLIT_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(FIND_PARTITIONS_TO_SPLIT_TIMEOUT_IN_SECONDS))) .code(code) @@ -175,7 +175,7 @@ public PartitionSplittingStack(Construct scope, .create(this, "SplitPartitionLambda") .functionName(functionName) .description("Triggered by an SQS event that contains a partition to split") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(SPLIT_PARTITIONS_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(SPLIT_PARTITIONS_TIMEOUT_IN_SECONDS))) .code(code) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java index f9cd7d478b..0d092de36b 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java @@ -72,7 +72,7 @@ public PropertiesStack(Construct scope, .environment(Utils.createDefaultEnvironment(instanceProperties)) .description("Lambda for writing instance properties to S3 upon initialisation and teardown") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .build()); configBucket.grantWrite(propertiesWriterLambda); diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java index feef6e8a29..f071836f1c 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java @@ -216,7 +216,7 @@ public QueryStack(Construct scope, .create(this, "QueryExecutorLambda") .functionName(functionName) .description("When a query arrives on the query SQS queue, this lambda is invoked to perform the query") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(QUERY_PROCESSOR_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(QUERY_PROCESSOR_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) @@ -282,7 +282,7 @@ protected void setupWebSocketApi(Code queryCode, InstanceProperties instanceProp .memorySize(256) .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) .timeout(Duration.seconds(29)) - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .build(); queriesQueue.grantSendMessages(handler); configBucket.grantRead(handler); diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java index c42d2fc54d..cbcc71bdf4 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java @@ -96,7 +96,7 @@ public TableStack( .environment(Utils.createDefaultEnvironment(instanceProperties)) .description("Lambda for handling initialisation and teardown of Sleeper Tables") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .build()); configBucket.grantReadWrite(sleeperTableLambda); @@ -207,7 +207,7 @@ private void createTable(InstanceProperties instanceProperties, Function tableMetricsPublisher = Function.Builder.create(this, tableName + "MetricsPublisher") .description("Generates metrics for a Sleeper table based on info in its state store, and publishes them to CloudWatch") .code(Code.fromBucket(jarsBucket, "metrics-" + instanceProperties.get(VERSION) + ".jar")) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .handler("sleeper.metrics.TableMetricsLambda::handleRequest") .memorySize(256) .timeout(Duration.seconds(60)) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java index 74ba2da3f4..d1a9ff3af7 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java @@ -71,7 +71,7 @@ public VpcStack(Construct scope, String id, InstanceProperties instancePropertie .memorySize(2048) .description("Lambda for checking the VPC has an associated S3 endpoint") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .build()); vpcCheckLambda.addToRolePolicy(new PolicyStatement(new PolicyStatementProps.Builder() diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java index e3dedf9a08..6db43ce0c4 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java @@ -336,7 +336,7 @@ protected void createBulkImportJobStarterFunction() { .memorySize(1024) .timeout(Duration.seconds(20)) .environment(env) - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .handler("sleeper.bulkimport.starter.BulkImportStarter") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) .events(Lists.newArrayList(new SqsEventSource(bulkImportJobQueue))) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java index f81604f586..482cb9df80 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java @@ -149,7 +149,7 @@ public EksBulkImportStack( .memorySize(1024) .timeout(Duration.seconds(10)) .environment(env) - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .handler("sleeper.bulkimport.starter.BulkImportStarter") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(UserDefinedInstanceProperty.LOG_RETENTION_IN_DAYS))) .events(Lists.newArrayList(new SqsEventSource(bulkImportJobQueue))) From fadbbd23376bb5615c969f2c5ec3b08c1d538d66 Mon Sep 17 00:00:00 2001 From: kr565370 <112558283+kr565370@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:33:35 +0000 Subject: [PATCH 13/19] Update dockerfiles to use java 11 corretto images --- java/bulk-import/bulk-import-runner/docker/Dockerfile | 4 ++-- java/compaction/compaction-job-execution/docker/Dockerfile | 2 +- java/ingest/ingest-runner/docker/Dockerfile | 2 +- java/system-test/docker/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/java/bulk-import/bulk-import-runner/docker/Dockerfile b/java/bulk-import/bulk-import-runner/docker/Dockerfile index 141ee39c19..2f93513779 100644 --- a/java/bulk-import/bulk-import-runner/docker/Dockerfile +++ b/java/bulk-import/bulk-import-runner/docker/Dockerfile @@ -14,8 +14,8 @@ ARG BUILDER_IMAGE_NAME=maven ARG BUILDER_IMAGE_TAG=3.8-openjdk-8-slim -ARG BASE_IMAGE_NAME=openjdk -ARG BASE_IMAGE_TAG=8-jre-slim +ARG BASE_IMAGE_NAME=amazoncorretto +ARG BASE_IMAGE_TAG=11 ARG SPARK_VERSION=3.1.2 ARG HADOOP_VERSION=3.2.1 diff --git a/java/compaction/compaction-job-execution/docker/Dockerfile b/java/compaction/compaction-job-execution/docker/Dockerfile index 1eee521aca..c3b5c62da1 100644 --- a/java/compaction/compaction-job-execution/docker/Dockerfile +++ b/java/compaction/compaction-job-execution/docker/Dockerfile @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM openjdk:8-jre +FROM amazoncorretto:11 RUN apt-get update diff --git a/java/ingest/ingest-runner/docker/Dockerfile b/java/ingest/ingest-runner/docker/Dockerfile index c861e1ba1c..4836b21e49 100644 --- a/java/ingest/ingest-runner/docker/Dockerfile +++ b/java/ingest/ingest-runner/docker/Dockerfile @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM openjdk:8-jre +FROM amazoncorretto:11 RUN apt-get update diff --git a/java/system-test/docker/Dockerfile b/java/system-test/docker/Dockerfile index 21e1078aa3..2a87987bc7 100644 --- a/java/system-test/docker/Dockerfile +++ b/java/system-test/docker/Dockerfile @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM openjdk:8-jre +FROM amazoncorretto:11 RUN apt-get update From de2d7b07f3d98144851581dc6877106201ca2edd Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Wed, 4 Jan 2023 16:22:41 +0000 Subject: [PATCH 14/19] Remove uses of apt-get in Dockerfiles (still need to sort bulk import) --- java/compaction/compaction-job-execution/docker/Dockerfile | 2 -- java/ingest/ingest-runner/docker/Dockerfile | 2 -- java/system-test/docker/Dockerfile | 2 -- 3 files changed, 6 deletions(-) diff --git a/java/compaction/compaction-job-execution/docker/Dockerfile b/java/compaction/compaction-job-execution/docker/Dockerfile index c3b5c62da1..6c55b56f7b 100644 --- a/java/compaction/compaction-job-execution/docker/Dockerfile +++ b/java/compaction/compaction-job-execution/docker/Dockerfile @@ -13,8 +13,6 @@ # limitations under the License. FROM amazoncorretto:11 -RUN apt-get update - COPY compaction-job-execution.jar /compaction-job-execution.jar COPY run.sh /run.sh RUN ["chmod", "+x", "/run.sh"] diff --git a/java/ingest/ingest-runner/docker/Dockerfile b/java/ingest/ingest-runner/docker/Dockerfile index 4836b21e49..8fca50bff7 100644 --- a/java/ingest/ingest-runner/docker/Dockerfile +++ b/java/ingest/ingest-runner/docker/Dockerfile @@ -13,8 +13,6 @@ # limitations under the License. FROM amazoncorretto:11 -RUN apt-get update - COPY ingest.jar /ingest.jar COPY run.sh /run.sh RUN ["chmod", "+x", "/run.sh"] diff --git a/java/system-test/docker/Dockerfile b/java/system-test/docker/Dockerfile index 2a87987bc7..f860ad962c 100644 --- a/java/system-test/docker/Dockerfile +++ b/java/system-test/docker/Dockerfile @@ -13,8 +13,6 @@ # limitations under the License. FROM amazoncorretto:11 -RUN apt-get update - COPY system-test.jar /system-test.jar COPY run.sh /run.sh RUN ["chmod", "+x", "/run.sh"] From 334f22de0895a3ab2f011ec684b84e4184b775d0 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Thu, 5 Jan 2023 10:36:46 +0000 Subject: [PATCH 15/19] Fix update times in QueryIngestJobStatusByPeriodIT --- .../store/job/QueryIngestJobStatusByPeriodIT.java | 15 ++++++++++++--- .../DynamoDBIngestJobStatusStoreTestBase.java | 6 +++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java index 6e6450beb6..db89437587 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Crown Copyright + * Copyright 2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import org.junit.Test; import sleeper.ingest.job.IngestJob; +import sleeper.ingest.job.status.IngestJobStatusStore; import sleeper.ingest.status.store.testutils.DynamoDBIngestJobStatusStoreTestBase; import java.time.Instant; @@ -35,6 +36,8 @@ public void shouldReturnIngestJobsInPeriod() { IngestJob job2 = jobWithFiles("file2"); Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); + // Fix update times to be same as happened times + IngestJobStatusStore store = storeWithUpdateTimes(startedTime1, startedTime2); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -55,6 +58,8 @@ public void shouldExcludeIngestJobOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); + // Fix update time to be same as happened time + IngestJobStatusStore store = storeWithUpdateTimes(startedTime); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); @@ -72,6 +77,8 @@ public void shouldExcludeIngestJobInOtherTable() { IngestJob job2 = jobWithTableAndFiles("other-table", "file2"); Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); + // Fix update times to be same as happened times + IngestJobStatusStore store = storeWithUpdateTimes(startedTime1, startedTime2); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -90,12 +97,14 @@ public void shouldIncludeFinishedStatusUpdateOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant finishedTime = Instant.parse("2023-01-03T14:56:00.001Z"); Instant periodStart = startedTime.minus(Period.ofDays(1)); + Instant periodEnd = finishedTime.plus(Period.ofDays(1)); + // Fix update times to be same as happened times + IngestJobStatusStore store = storeWithUpdateTimes(startedTime, finishedTime); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); - Instant finishedTime = Instant.parse("2023-01-03T14:56:00.001Z"); - Instant periodEnd = finishedTime.plus(Period.ofDays(1)); store.jobFinished(DEFAULT_TASK_ID, job, defaultSummary(startedTime, finishedTime)); // Then diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java index ace1829f23..cc2e0d12e6 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Crown Copyright + * Copyright 2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,6 +71,10 @@ public void tearDown() { protected IngestJobStatusStore storeWithTimeToLiveAndUpdateTimes(Duration timeToLive, Instant... updateTimes) { instanceProperties.set(INGEST_JOB_STATUS_TTL_IN_SECONDS, "" + timeToLive.getSeconds()); + return storeWithUpdateTimes(updateTimes); + } + + protected IngestJobStatusStore storeWithUpdateTimes(Instant... updateTimes) { return new DynamoDBIngestJobStatusStore(dynamoDBClient, instanceProperties, Arrays.stream(updateTimes).iterator()::next); } From 418899b239d6a6aa5d6fe7df455c397c5d99f9c0 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Thu, 5 Jan 2023 10:42:39 +0000 Subject: [PATCH 16/19] Tidy dates in shouldExcludeIngestJobOutsidePeriod --- .../status/store/job/QueryIngestJobStatusByPeriodIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java index db89437587..72e7aa01df 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java @@ -57,6 +57,8 @@ public void shouldReturnIngestJobsInPeriod() { public void shouldExcludeIngestJobOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); + Instant periodStart = Instant.parse("2023-01-01T14:00:00.001Z"); + Instant periodEnd = Instant.parse("2023-01-02T14:00:00.001Z"); Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); // Fix update time to be same as happened time IngestJobStatusStore store = storeWithUpdateTimes(startedTime); @@ -65,8 +67,6 @@ public void shouldExcludeIngestJobOutsidePeriod() { store.jobStarted(DEFAULT_TASK_ID, job, startedTime); // Then - Instant periodStart = Instant.parse("2023-01-01T14:00:00.001Z"); - Instant periodEnd = Instant.parse("2023-01-02T14:00:00.001Z"); assertThat(store.getJobsInTimePeriod(tableName, periodStart, periodEnd)).isEmpty(); } From 7cc7b342a4a4bcd55a8d8ad8b88a7231b74f8c1e Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Thu, 5 Jan 2023 10:43:40 +0000 Subject: [PATCH 17/19] Make shouldIncludeFinishedStatusUpdateOutsidePeriod valid --- .../status/store/job/QueryIngestJobStatusByPeriodIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java index 72e7aa01df..451a4a0b53 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java @@ -96,10 +96,10 @@ public void shouldExcludeIngestJobInOtherTable() { public void shouldIncludeFinishedStatusUpdateOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); + Instant periodStart = Instant.parse("2023-01-02T14:52:00.001Z"); Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant periodEnd = Instant.parse("2023-01-03T14:52:00.001Z"); Instant finishedTime = Instant.parse("2023-01-03T14:56:00.001Z"); - Instant periodStart = startedTime.minus(Period.ofDays(1)); - Instant periodEnd = finishedTime.plus(Period.ofDays(1)); // Fix update times to be same as happened times IngestJobStatusStore store = storeWithUpdateTimes(startedTime, finishedTime); From 21edc925bd55c025b978a54e33ff23db38d6b100 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Thu, 5 Jan 2023 10:55:59 +0000 Subject: [PATCH 18/19] State update times explicitly in QueryIngestJobStatusByPeriodIT --- .../job/QueryIngestJobStatusByPeriodIT.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java index 451a4a0b53..867f3061a2 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java @@ -35,9 +35,10 @@ public void shouldReturnIngestJobsInPeriod() { IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithFiles("file2"); Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedUpdateTime1 = Instant.parse("2023-01-03T14:50:00.123Z"); Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); - // Fix update times to be same as happened times - IngestJobStatusStore store = storeWithUpdateTimes(startedTime1, startedTime2); + Instant startedUpdateTime2 = Instant.parse("2023-01-03T14:55:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime1, startedUpdateTime2); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -60,8 +61,8 @@ public void shouldExcludeIngestJobOutsidePeriod() { Instant periodStart = Instant.parse("2023-01-01T14:00:00.001Z"); Instant periodEnd = Instant.parse("2023-01-02T14:00:00.001Z"); Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); - // Fix update time to be same as happened time - IngestJobStatusStore store = storeWithUpdateTimes(startedTime); + Instant startedUpdateTime = Instant.parse("2023-01-03T14:50:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); @@ -76,9 +77,10 @@ public void shouldExcludeIngestJobInOtherTable() { IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithTableAndFiles("other-table", "file2"); Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedUpdateTime1 = Instant.parse("2023-01-03T14:50:00.123Z"); Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); - // Fix update times to be same as happened times - IngestJobStatusStore store = storeWithUpdateTimes(startedTime1, startedTime2); + Instant startedUpdateTime2 = Instant.parse("2023-01-03T14:55:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime1, startedUpdateTime2); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -98,10 +100,11 @@ public void shouldIncludeFinishedStatusUpdateOutsidePeriod() { IngestJob job = jobWithFiles("file"); Instant periodStart = Instant.parse("2023-01-02T14:52:00.001Z"); Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedUpdateTime = Instant.parse("2023-01-03T14:50:00.123Z"); Instant periodEnd = Instant.parse("2023-01-03T14:52:00.001Z"); Instant finishedTime = Instant.parse("2023-01-03T14:56:00.001Z"); - // Fix update times to be same as happened times - IngestJobStatusStore store = storeWithUpdateTimes(startedTime, finishedTime); + Instant finishedUpdateTime = Instant.parse("2023-01-03T14:56:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime, finishedUpdateTime); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); From 4eb3652c779a5af8f38dc9267a177d6f36f70e82 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 10 Jan 2023 10:03:17 +0000 Subject: [PATCH 19/19] Update incorrect copyright notices --- .../src/main/java/sleeper/status/report/StatusReport.java | 2 +- .../sleeper/status/report/job/StandardProcessRunReporter.java | 2 +- .../report/partitions/PartitionsStatusReportArguments.java | 2 +- .../sleeper/status/report/job/FormatDurationStringTest.java | 2 +- .../ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java | 2 +- .../store/testutils/DynamoDBIngestJobStatusStoreTestBase.java | 2 +- .../src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/clients/src/main/java/sleeper/status/report/StatusReport.java b/java/clients/src/main/java/sleeper/status/report/StatusReport.java index 0479f5f747..0948601c08 100644 --- a/java/clients/src/main/java/sleeper/status/report/StatusReport.java +++ b/java/clients/src/main/java/sleeper/status/report/StatusReport.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java b/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java index 4593067b73..49962fda3c 100644 --- a/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java +++ b/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java b/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java index fb73f03f1a..3d0dd56b0a 100644 --- a/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java +++ b/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java b/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java index acc386591c..883cab59ea 100644 --- a/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java +++ b/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java index 867f3061a2..b7bd5b6c51 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java index cc2e0d12e6..e860af5be9 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java b/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java index 9510113ee5..003937ffa7 100644 --- a/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java +++ b/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.