diff --git a/.github/workflows/check-status.yaml b/.github/workflows/check-status.yaml index 66f3e90999..d8025b1c83 100644 --- a/.github/workflows/check-status.yaml +++ b/.github/workflows/check-status.yaml @@ -9,8 +9,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'corretto' - name: Cache dependencies uses: actions/cache@v3 with: diff --git a/.github/workflows/chunk.yaml b/.github/workflows/chunk.yaml index 5059e5d171..f56771db53 100644 --- a/.github/workflows/chunk.yaml +++ b/.github/workflows/chunk.yaml @@ -17,8 +17,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'corretto' - name: Cache dependencies uses: actions/cache@v3 with: @@ -41,7 +41,7 @@ jobs: run: mvn --batch-mode clean install -am -pl ${{ steps.config.outputs.moduleList }} -Pquick,skipShade -Dmaven.repo.local=.m2/repository working-directory: ./java - name: Test - run: mvn --batch-mode --fail-at-end verify -pl ${{ steps.config.outputs.moduleList }} -Dmaven.repo.local=.m2/repository + run: mvn --batch-mode --fail-at-end verify -pl ${{ steps.config.outputs.moduleList }} -Dmaven.repo.local=.m2/repository -e working-directory: ./java - name: Check rate limits id: rate-limit diff --git a/.github/workflows/maven-full.yaml b/.github/workflows/maven-full.yaml index c4da5d8fec..33d75ab763 100644 --- a/.github/workflows/maven-full.yaml +++ b/.github/workflows/maven-full.yaml @@ -13,8 +13,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'corretto' - name: Cache dependencies uses: actions/cache@v3 with: diff --git a/java/bulk-import/bulk-import-runner/docker/Dockerfile b/java/bulk-import/bulk-import-runner/docker/Dockerfile index 141ee39c19..2f93513779 100644 --- a/java/bulk-import/bulk-import-runner/docker/Dockerfile +++ b/java/bulk-import/bulk-import-runner/docker/Dockerfile @@ -14,8 +14,8 @@ ARG BUILDER_IMAGE_NAME=maven ARG BUILDER_IMAGE_TAG=3.8-openjdk-8-slim -ARG BASE_IMAGE_NAME=openjdk -ARG BASE_IMAGE_TAG=8-jre-slim +ARG BASE_IMAGE_NAME=amazoncorretto +ARG BASE_IMAGE_TAG=11 ARG SPARK_VERSION=3.1.2 ARG HADOOP_VERSION=3.2.1 diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java index 8a96166386..658d53a85f 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java @@ -173,7 +173,7 @@ private Function createConnector(String className, String instanceId, int logRet .memorySize(memory) .timeout(Duration.seconds(timeout)) .code(s3Code) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .logRetention(Utils.getRetentionDays(logRetentionDays)) .handler(className) .environment(env) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java index 34e6182bd9..67f7af9d90 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/CompactionStack.java @@ -302,7 +302,7 @@ private void lambdaToFindCompactionJobsThatShouldBeCreated(IBucket configBucket, .create(this, "JobCreationLambda") .functionName(functionName) .description("Scan DynamoDB looking for files that need merging and create appropriate job specs in DynamoDB") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(COMPACTION_JOB_CREATION_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(COMPACTION_JOB_CREATION_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) @@ -481,7 +481,7 @@ private void lambdaToCreateCompactionTasks(IBucket configBucket, .create(this, "CompactionTasksCreator") .functionName(functionName) .description("If there are compaction jobs on queue create tasks to run them") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(TASK_RUNNER_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) @@ -539,7 +539,7 @@ private void lambdaToCreateSplittingCompactionTasks(IBucket configBucket, .create(this, "SplittingCompactionTasksCreator") .functionName(functionName) .description("If there are splitting compaction jobs on queue create tasks to run them") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(TASK_RUNNER_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java index bee948b837..2f528461b1 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java @@ -15,6 +15,7 @@ */ package sleeper.cdk.stack; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import sleeper.cdk.Utils; import sleeper.configuration.properties.InstanceProperties; import sleeper.configuration.properties.UserDefinedInstanceProperty; @@ -46,6 +47,7 @@ import static sleeper.configuration.properties.UserDefinedInstanceProperty.DASHBOARD_TIME_WINDOW_MINUTES; +@SuppressFBWarnings("MC_OVERRIDABLE_METHOD_CALL_IN_CONSTRUCTOR") public class DashboardStack extends NestedStack { private final IngestStack ingestStack; private final CompactionStack compactionStack; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java index 306b1b6bcc..bbf7e8eb87 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/GarbageCollectorStack.java @@ -73,7 +73,7 @@ public GarbageCollectorStack( .create(this, "GarbageCollectorLambda") .functionName(functionName) .description("Scan DynamoDB looking for files that need deleting and delete them") - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .memorySize(instanceProperties.getInt(GARBAGE_COLLECTOR_LAMBDA_MEMORY_IN_MB)) // Timeout is set to 90% of the period with which this runs to avoid 2 running simultaneously, // with a maximum of 900 seconds (15 minutes) which is the maximum execution time diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java index 48a3e20601..8972ab1275 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/IngestStack.java @@ -275,7 +275,7 @@ private void lambdaToCreateIngestTasks(IBucket configBucket, Queue ingestJobQueu .create(this, "IngestTasksCreator") .functionName(functionName) .description("If there are ingest jobs on queue create tasks to run them") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(TASK_RUNNER_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java index 114c639aee..6a4fb9cb10 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/PartitionSplittingStack.java @@ -139,7 +139,7 @@ public PartitionSplittingStack(Construct scope, .create(this, "FindPartitionsToSplitLambda") .functionName(functionName) .description("Scan DynamoDB looking for partitions that need splitting") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(FIND_PARTITIONS_TO_SPLIT_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(FIND_PARTITIONS_TO_SPLIT_TIMEOUT_IN_SECONDS))) .code(code) @@ -175,7 +175,7 @@ public PartitionSplittingStack(Construct scope, .create(this, "SplitPartitionLambda") .functionName(functionName) .description("Triggered by an SQS event that contains a partition to split") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(SPLIT_PARTITIONS_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(SPLIT_PARTITIONS_TIMEOUT_IN_SECONDS))) .code(code) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java index f9cd7d478b..0d092de36b 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/PropertiesStack.java @@ -72,7 +72,7 @@ public PropertiesStack(Construct scope, .environment(Utils.createDefaultEnvironment(instanceProperties)) .description("Lambda for writing instance properties to S3 upon initialisation and teardown") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .build()); configBucket.grantWrite(propertiesWriterLambda); diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java index feef6e8a29..f071836f1c 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/QueryStack.java @@ -216,7 +216,7 @@ public QueryStack(Construct scope, .create(this, "QueryExecutorLambda") .functionName(functionName) .description("When a query arrives on the query SQS queue, this lambda is invoked to perform the query") - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .memorySize(instanceProperties.getInt(QUERY_PROCESSOR_LAMBDA_MEMORY_IN_MB)) .timeout(Duration.seconds(instanceProperties.getInt(QUERY_PROCESSOR_LAMBDA_TIMEOUT_IN_SECONDS))) .code(code) @@ -282,7 +282,7 @@ protected void setupWebSocketApi(Code queryCode, InstanceProperties instanceProp .memorySize(256) .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) .timeout(Duration.seconds(29)) - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .build(); queriesQueue.grantSendMessages(handler); configBucket.grantRead(handler); diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java index c42d2fc54d..cbcc71bdf4 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/TableStack.java @@ -96,7 +96,7 @@ public TableStack( .environment(Utils.createDefaultEnvironment(instanceProperties)) .description("Lambda for handling initialisation and teardown of Sleeper Tables") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .build()); configBucket.grantReadWrite(sleeperTableLambda); @@ -207,7 +207,7 @@ private void createTable(InstanceProperties instanceProperties, Function tableMetricsPublisher = Function.Builder.create(this, tableName + "MetricsPublisher") .description("Generates metrics for a Sleeper table based on info in its state store, and publishes them to CloudWatch") .code(Code.fromBucket(jarsBucket, "metrics-" + instanceProperties.get(VERSION) + ".jar")) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .handler("sleeper.metrics.TableMetricsLambda::handleRequest") .memorySize(256) .timeout(Duration.seconds(60)) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java index 74ba2da3f4..d1a9ff3af7 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/VpcStack.java @@ -71,7 +71,7 @@ public VpcStack(Construct scope, String id, InstanceProperties instancePropertie .memorySize(2048) .description("Lambda for checking the VPC has an associated S3 endpoint") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) - .runtime(Runtime.JAVA_8) + .runtime(Runtime.JAVA_11) .build()); vpcCheckLambda.addToRolePolicy(new PolicyStatement(new PolicyStatementProps.Builder() diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java index e3dedf9a08..6db43ce0c4 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/AbstractEmrBulkImportStack.java @@ -336,7 +336,7 @@ protected void createBulkImportJobStarterFunction() { .memorySize(1024) .timeout(Duration.seconds(20)) .environment(env) - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .handler("sleeper.bulkimport.starter.BulkImportStarter") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(LOG_RETENTION_IN_DAYS))) .events(Lists.newArrayList(new SqsEventSource(bulkImportJobQueue))) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java index f81604f586..482cb9df80 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java @@ -149,7 +149,7 @@ public EksBulkImportStack( .memorySize(1024) .timeout(Duration.seconds(10)) .environment(env) - .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_8) + .runtime(software.amazon.awscdk.services.lambda.Runtime.JAVA_11) .handler("sleeper.bulkimport.starter.BulkImportStarter") .logRetention(Utils.getRetentionDays(instanceProperties.getInt(UserDefinedInstanceProperty.LOG_RETENTION_IN_DAYS))) .events(Lists.newArrayList(new SqsEventSource(bulkImportJobQueue))) diff --git a/java/clients/src/main/java/sleeper/status/report/StatusReport.java b/java/clients/src/main/java/sleeper/status/report/StatusReport.java index 0479f5f747..0948601c08 100644 --- a/java/clients/src/main/java/sleeper/status/report/StatusReport.java +++ b/java/clients/src/main/java/sleeper/status/report/StatusReport.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java b/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java index 4593067b73..49962fda3c 100644 --- a/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java +++ b/java/clients/src/main/java/sleeper/status/report/job/StandardProcessRunReporter.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java b/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java index fb73f03f1a..3d0dd56b0a 100644 --- a/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java +++ b/java/clients/src/main/java/sleeper/status/report/partitions/PartitionsStatusReportArguments.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java b/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java index acc386591c..883cab59ea 100644 --- a/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java +++ b/java/clients/src/test/java/sleeper/status/report/job/FormatDurationStringTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/compaction/compaction-job-execution/docker/Dockerfile b/java/compaction/compaction-job-execution/docker/Dockerfile index 1eee521aca..6c55b56f7b 100644 --- a/java/compaction/compaction-job-execution/docker/Dockerfile +++ b/java/compaction/compaction-job-execution/docker/Dockerfile @@ -11,9 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM openjdk:8-jre - -RUN apt-get update +FROM amazoncorretto:11 COPY compaction-job-execution.jar /compaction-job-execution.jar COPY run.sh /run.sh diff --git a/java/ingest/ingest-runner/docker/Dockerfile b/java/ingest/ingest-runner/docker/Dockerfile index c861e1ba1c..8fca50bff7 100644 --- a/java/ingest/ingest-runner/docker/Dockerfile +++ b/java/ingest/ingest-runner/docker/Dockerfile @@ -11,9 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM openjdk:8-jre - -RUN apt-get update +FROM amazoncorretto:11 COPY ingest.jar /ingest.jar COPY run.sh /run.sh diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java index 0009f9f7c5..b7bd5b6c51 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByPeriodIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import org.junit.Test; import sleeper.ingest.job.IngestJob; +import sleeper.ingest.job.status.IngestJobStatusStore; import sleeper.ingest.status.store.testutils.DynamoDBIngestJobStatusStoreTestBase; import java.time.Instant; @@ -33,8 +34,11 @@ public void shouldReturnIngestJobsInPeriod() { // Given IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithFiles("file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedUpdateTime1 = Instant.parse("2023-01-03T14:50:00.123Z"); + Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); + Instant startedUpdateTime2 = Instant.parse("2023-01-03T14:55:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime1, startedUpdateTime2); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -54,14 +58,16 @@ public void shouldReturnIngestJobsInPeriod() { public void shouldExcludeIngestJobOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); - Instant startedTime = Instant.now(); + Instant periodStart = Instant.parse("2023-01-01T14:00:00.001Z"); + Instant periodEnd = Instant.parse("2023-01-02T14:00:00.001Z"); + Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedUpdateTime = Instant.parse("2023-01-03T14:50:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); // Then - Instant periodStart = Instant.now().plus(Period.ofDays(1)); - Instant periodEnd = periodStart.plus(Period.ofDays(1)); assertThat(store.getJobsInTimePeriod(tableName, periodStart, periodEnd)).isEmpty(); } @@ -70,8 +76,11 @@ public void shouldExcludeIngestJobInOtherTable() { // Given IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithTableAndFiles("other-table", "file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedUpdateTime1 = Instant.parse("2023-01-03T14:50:00.123Z"); + Instant startedTime2 = Instant.parse("2023-01-03T14:55:00.001Z"); + Instant startedUpdateTime2 = Instant.parse("2023-01-03T14:55:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime1, startedUpdateTime2); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); @@ -86,18 +95,19 @@ public void shouldExcludeIngestJobInOtherTable() { } @Test - public void shouldIncludeFinishedStatusUpdateOutsidePeriod() throws Exception { + public void shouldIncludeFinishedStatusUpdateOutsidePeriod() { // Given IngestJob job = jobWithFiles("file"); - Instant periodStart = Instant.now().minus(Period.ofDays(1)); - Instant startedTime = Instant.now(); + Instant periodStart = Instant.parse("2023-01-02T14:52:00.001Z"); + Instant startedTime = Instant.parse("2023-01-03T14:50:00.001Z"); + Instant startedUpdateTime = Instant.parse("2023-01-03T14:50:00.123Z"); + Instant periodEnd = Instant.parse("2023-01-03T14:52:00.001Z"); + Instant finishedTime = Instant.parse("2023-01-03T14:56:00.001Z"); + Instant finishedUpdateTime = Instant.parse("2023-01-03T14:56:00.123Z"); + IngestJobStatusStore store = storeWithUpdateTimes(startedUpdateTime, finishedUpdateTime); // When store.jobStarted(DEFAULT_TASK_ID, job, startedTime); - Thread.sleep(1); - Instant periodEnd = Instant.now(); - Thread.sleep(1); - Instant finishedTime = Instant.now(); store.jobFinished(DEFAULT_TASK_ID, job, defaultSummary(startedTime, finishedTime)); // Then diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java index f7dd7539d0..3ec8c25668 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/QueryIngestJobStatusByTaskIdIT.java @@ -34,8 +34,8 @@ public void shouldReturnIngestJobsByTaskId() { String searchingTaskId = "test-task"; IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithFiles("file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2022-12-14T13:51:12.001Z"); + Instant startedTime2 = Instant.parse("2022-12-14T13:52:12.001Z"); // When store.jobStarted(searchingTaskId, job1, startedTime1); diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java index 209b3ddc07..4f28effb61 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/job/StoreIngestJobStartedIT.java @@ -61,8 +61,8 @@ public void shouldReportSeveralIngestJobsStarted() { // Given IngestJob job1 = jobWithFiles("file1"); IngestJob job2 = jobWithFiles("file2"); - Instant startedTime1 = Instant.now(); - Instant startedTime2 = Instant.now(); + Instant startedTime1 = Instant.parse("2022-12-14T13:51:12.001Z"); + Instant startedTime2 = Instant.parse("2022-12-14T13:56:12.001Z"); // When store.jobStarted(DEFAULT_TASK_ID, job1, startedTime1); diff --git a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java index ace1829f23..e860af5be9 100644 --- a/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java +++ b/java/ingest/ingest-status-store/src/test/java/sleeper/ingest/status/store/testutils/DynamoDBIngestJobStatusStoreTestBase.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,6 +71,10 @@ public void tearDown() { protected IngestJobStatusStore storeWithTimeToLiveAndUpdateTimes(Duration timeToLive, Instant... updateTimes) { instanceProperties.set(INGEST_JOB_STATUS_TTL_IN_SECONDS, "" + timeToLive.getSeconds()); + return storeWithUpdateTimes(updateTimes); + } + + protected IngestJobStatusStore storeWithUpdateTimes(Instant... updateTimes) { return new DynamoDBIngestJobStatusStore(dynamoDBClient, instanceProperties, Arrays.stream(updateTimes).iterator()::next); } diff --git a/java/pom.xml b/java/pom.xml index d397684312..509628745a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -51,7 +51,7 @@ UTF-8 - 1.8 + 11 2.12 2.12.17 3.3.0 @@ -93,15 +93,23 @@ 2.17 3.0.0-M4 - 3.0.0-M5 + 3.0.0-M6 ${surefire.plugin.version} 3.3.0 3.2.0 - 3.8.1 + 3.10.1 3.0.0 dir 4.7.2.0 1.2.8 + + --add-opens java.base/java.lang.invoke=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + + + @@ -346,11 +354,17 @@ org.apache.maven.plugins maven-surefire-plugin ${surefire.plugin.version} + + @{argLine} ${module.access.args} + org.apache.maven.plugins maven-failsafe-plugin ${failsafe.plugin.version} + + @{argLine} ${module.access.args} + integration-test @@ -423,8 +437,7 @@ maven-compiler-plugin ${compiler.plugin.version} - ${java.version} - ${java.version} + ${java.version} diff --git a/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java b/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java index 5ef503f74a..0181e06b6b 100644 --- a/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java +++ b/java/query/src/main/java/sleeper/query/model/output/WebSocketResultsOutput.java @@ -44,13 +44,8 @@ public WebSocketResultsOutput(Map config) { String maxBatchSize = config.get(MAX_BATCH_SIZE); this.maxBatchSize = maxBatchSize != null && !maxBatchSize.isEmpty() ? Long.parseLong(maxBatchSize) : null; - - addOutputLocations(); - } - - private void addOutputLocations() { - this.outputLocations.add(new ResultsOutputLocation("websocket-endpoint", this.getEndpoint())); - this.outputLocations.add(new ResultsOutputLocation("websocket-connection-id", this.getConnectionId())); + this.outputLocations.add(new ResultsOutputLocation("websocket-endpoint", config.get(ENDPOINT))); + this.outputLocations.add(new ResultsOutputLocation("websocket-connection-id", config.get(CONNECTION_ID))); } @Override diff --git a/java/system-test/docker/Dockerfile b/java/system-test/docker/Dockerfile index 21e1078aa3..f860ad962c 100644 --- a/java/system-test/docker/Dockerfile +++ b/java/system-test/docker/Dockerfile @@ -11,9 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM openjdk:8-jre - -RUN apt-get update +FROM amazoncorretto:11 COPY system-test.jar /system-test.jar COPY run.sh /run.sh diff --git a/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java b/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java index 9510113ee5..003937ffa7 100644 --- a/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java +++ b/java/system-test/src/main/java/sleeper/systemtest/ingest/TaskStatusJson.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Crown Copyright + * Copyright 2022-2023 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.