From da561809187956ad8d735a41f57408cb5088e4ef Mon Sep 17 00:00:00 2001 From: David Venable Date: Thu, 19 May 2022 16:53:07 -0500 Subject: [PATCH] Build on Java 11 with Java 8 as the compilation toolchain (#1406) Updated the project for building on Java 11 using Java 8 as the toolchain for compilation. Updated GitHub Actions to build using Java 11 and for end-to-end tests, run Data Prepper against multiple versions of Java. Signed-off-by: David Venable --- ...per-log-analytics-basic-grok-e2e-tests.yml | 8 +++---- ...prepper-performance-test-compile-check.yml | 2 +- ...ytics-raw-span-compatibility-e2e-tests.yml | 24 +++++++++---------- ...per-trace-analytics-raw-span-e2e-tests.yml | 10 ++++---- ...-trace-analytics-service-map-e2e-tests.yml | 10 ++++---- .github/workflows/gradle.yml | 2 +- ...arch-sink-opendistro-integration-tests.yml | 2 +- .../opensearch-sink-os-integration-tests.yml | 2 +- .github/workflows/release.yml | 2 +- build.gradle | 21 +++++++++++++++- .../mapdb-benchmarks/build.gradle | 2 -- .../build.gradle | 2 -- .../mapdb-prepper-state/build.gradle | 2 -- data-prepper-plugins/opensearch/build.gradle | 9 +++++++ .../service-map-stateful/build.gradle | 5 ---- docs/developer_guide.md | 20 +++++++++------- e2e-test/build.gradle | 2 ++ e2e-test/log/build.gradle | 2 +- e2e-test/trace/build.gradle | 2 +- .../zipkin-opensearch-to-otel/build.gradle | 2 -- 20 files changed, 75 insertions(+), 56 deletions(-) diff --git a/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml b/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml index 64c535a4e5..301447d17e 100644 --- a/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml +++ b/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml @@ -13,16 +13,16 @@ jobs: build: strategy: matrix: - java: [14] + java: [8, 11, 17] runs-on: ubuntu-latest steps: - - name: Set up JDK ${{ matrix.java }} + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: ${{ matrix.java }} + java-version: 11 - name: Checkout Data-Prepper uses: actions/checkout@v2 - name: Run basic grok end-to-end tests with Gradle - run: ./gradlew :e2e-test:log:basicLogEndToEndTest \ No newline at end of file + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:basicLogEndToEndTest \ No newline at end of file diff --git a/.github/workflows/data-prepper-performance-test-compile-check.yml b/.github/workflows/data-prepper-performance-test-compile-check.yml index 2373749b9d..968f867dde 100644 --- a/.github/workflows/data-prepper-performance-test-compile-check.yml +++ b/.github/workflows/data-prepper-performance-test-compile-check.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [14] + java: [11] runs-on: ubuntu-latest diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml index c76dbddbc6..62998d52d7 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml @@ -13,19 +13,19 @@ jobs: build: strategy: matrix: - java: [14] + java: [11, 17] runs-on: ubuntu-latest steps: - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - name: Checkout Data-Prepper - uses: actions/checkout@v2 - # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) - - name: Run raw-span OTLP record type latest release compatibility end-to-end tests with Gradle - run: ./gradlew :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest - - name: Run raw-span Event record type latest release compatibility end-to-end tests with Gradle - run: ./gradlew :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Checkout Data-Prepper + uses: actions/checkout@v2 + # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) + - name: Run raw-span OTLP record type latest release compatibility end-to-end tests with Gradle + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest + - name: Run raw-span Event record type latest release compatibility end-to-end tests with Gradle + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml index 110bbb8e15..fb3bc1b716 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml @@ -13,19 +13,19 @@ jobs: build: strategy: matrix: - java: [14] + java: [11, 17] runs-on: ubuntu-latest steps: - - name: Set up JDK ${{ matrix.java }} + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: ${{ matrix.java }} + java-version: 11 - name: Checkout Data-Prepper uses: actions/checkout@v2 # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) - name: Run raw-span OTLP record type end-to-end tests with Gradle - run: ./gradlew :e2e-test:trace:rawSpanOTLPEndToEndTest + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPEndToEndTest - name: Run raw-span OTLP and Event end-to-end tests with Gradle - run: ./gradlew :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest \ No newline at end of file + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest \ No newline at end of file diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml index 9b28662f0a..3e558f0eaf 100644 --- a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml @@ -13,19 +13,19 @@ jobs: build: strategy: matrix: - java: [14] + java: [11, 17] runs-on: ubuntu-latest steps: - - name: Set up JDK ${{ matrix.java }} + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: ${{ matrix.java }} + java-version: 11 - name: Checkout Data-Prepper uses: actions/checkout@v2 # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) - name: Run service-map OTLP record type end-to-end tests with Gradle - run: ./gradlew :e2e-test:trace:serviceMapOTLPEndToEndTest + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:serviceMapOTLPEndToEndTest - name: Run service-map OTLP and Event end-to-end tests with Gradle - run: ./gradlew :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest \ No newline at end of file + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest \ No newline at end of file diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index b936b1f0ee..29463044b5 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -12,7 +12,7 @@ jobs: build: strategy: matrix: - java: [14] + java: [11] runs-on: ubuntu-latest diff --git a/.github/workflows/opensearch-sink-opendistro-integration-tests.yml b/.github/workflows/opensearch-sink-opendistro-integration-tests.yml index 92e38a5035..c9adb0f4e5 100644 --- a/.github/workflows/opensearch-sink-opendistro-integration-tests.yml +++ b/.github/workflows/opensearch-sink-opendistro-integration-tests.yml @@ -13,7 +13,7 @@ jobs: integration-tests: strategy: matrix: - java: [14] + java: [11] opendistro: [1.6.0, 1.8.0, 1.9.0, 1.11.0, 1.12.0, 1.13.3] runs-on: ubuntu-latest diff --git a/.github/workflows/opensearch-sink-os-integration-tests.yml b/.github/workflows/opensearch-sink-os-integration-tests.yml index 3189e00c1e..b25489e293 100644 --- a/.github/workflows/opensearch-sink-os-integration-tests.yml +++ b/.github/workflows/opensearch-sink-os-integration-tests.yml @@ -9,7 +9,7 @@ jobs: integration-tests: strategy: matrix: - java: [14] + java: [11] opensearch: [1.0.1, 1.1.0, 1.2.4, 1.3.0] runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f62e6e01bd..6009d58883 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v1 with: - java-version: 14 + java-version: 11 - name: Checkout Data-Prepper uses: actions/checkout@v2 - name: Get Version diff --git a/build.gradle b/build.gradle index d7cfefc4fc..ba65ef344f 100644 --- a/build.gradle +++ b/build.gradle @@ -66,7 +66,26 @@ subprojects { apply plugin: 'java' apply plugin: 'maven-publish' apply plugin: 'jacoco' - sourceCompatibility = '1.8' + + java { + toolchain { + languageVersion = JavaLanguageVersion.of(8) + } + } + tasks.withType(JavaCompile).findAll { task -> task.name.containsIgnoreCase('test') }.forEach { task -> + task.configure { + javaCompiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(11) + } + } + } + tasks.withType(Test).configureEach() { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(11) + } + } + + spotless { java { targetExclude 'build/generated-src/antlr/**' diff --git a/data-prepper-benchmarks/mapdb-benchmarks/build.gradle b/data-prepper-benchmarks/mapdb-benchmarks/build.gradle index c59adf4db4..c1e32589ff 100644 --- a/data-prepper-benchmarks/mapdb-benchmarks/build.gradle +++ b/data-prepper-benchmarks/mapdb-benchmarks/build.gradle @@ -11,8 +11,6 @@ plugins { group 'com.amazon' version '0.1-beta' -sourceCompatibility = 1.8 - repositories { mavenCentral() } diff --git a/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle b/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle index 01e64a6f0c..e4b6dfcebb 100644 --- a/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle +++ b/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle @@ -11,8 +11,6 @@ plugins { group 'com.amazon' version '0.1-beta' -sourceCompatibility = 1.8 - repositories { mavenCentral() } diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index f6f40e9d92..c5d44b33a3 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -7,8 +7,6 @@ plugins { id 'java' } -sourceCompatibility = 1.8 - repositories { mavenCentral() } diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 95f5d14043..b8587c1f9d 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -76,11 +76,20 @@ configurations { integrationTestRuntime.extendsFrom testRuntime } +tasks.withType(JavaCompile).getByName('compileIntegrationTestJava').configure { + javaCompiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(11) + } +} + task integrationTest(type: Test) { group = 'verification' testClassesDirs = sourceSets.integrationTest.output.classesDirs useJUnitPlatform() + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(11) + } classpath = sourceSets.integrationTest.runtimeClasspath systemProperty 'tests.opensearch.host', System.getProperty('tests.opensearch.host') diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index 3772e0769e..e99424dbfb 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -7,11 +7,6 @@ plugins { id 'java' } -group 'com.amazon' -version '0.1-beta' - -sourceCompatibility = 1.8 - repositories { mavenCentral() } diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 3e7f2d8296..91688225c9 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -8,27 +8,30 @@ First, please read our [contribution guide](../CONTRIBUTING.md) for more informa ## Installation Prerequisites -### JDK Versions +### Java Versions -Running Data Prepper requires JDK 8 and above. +Building Data Prepper requires JDK 11. The Data Prepper Gradle build runs in a Java 11 JVM, but uses +[Gradle toolchains](https://docs.gradle.org/current/userguide/toolchains.html) to compile the Java +code using Java 8. If you have a JDK 8 installed locally, Gradle will use your installed JDK 8. If you +do not, Gradle will install JDK 8. -Running the integration tests requires JDK 14 or 15. +All main source code builds on JDK 8, so it must be compatible with Java 8. The test code +(unit and integration tests) runs using JDK 11 and may use Java 11 features. ## Building from source The assemble task will build the Jar files without running the integration tests. You can use these jar files for running DataPrepper. If you are just -looking to use DataPrepper and modify it, this build -is faster than running the integration test suite and only requires JDK 8+. +looking to build Data Prepper from source, this build +is faster than running the integration test suite. -To build the project from source, run +To build the project from source, run the following command from the project root: ``` ./gradlew assemble ``` -from the project root. ### Full Project Build @@ -36,13 +39,12 @@ Running the build command will assemble the Jar files needed for running DataPrepper. It will also run the integration test suite. -To build, run +To build, run the following command from the project root: ``` ./gradlew build ``` -from the project root. ## Running the project diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle index ec082f53b1..2233900c4a 100644 --- a/e2e-test/build.gradle +++ b/e2e-test/build.gradle @@ -44,5 +44,7 @@ subprojects { ext { dataPrepperJarFilepath = "${project.buildDir.name}/bin/${DATA_PREPPER_CORE_JAR.archiveName}" + targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11' + dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre-alpine" } } \ No newline at end of file diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index 412ea1b35a..5041577d38 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -38,7 +38,7 @@ def BASIC_GROK_PIPELINE_YAML = "basic-grok-e2e-pipeline.yml" task createDataPrepperDockerFile(type: Dockerfile) { dependsOn copyDataPrepperJar destFile = project.file('build/docker/Dockerfile') - from("adoptopenjdk/openjdk14:jre-14.0.1_7-alpine") + from(dataPrepperBaseImage) workingDir("/app") copyFile("${dataPrepperJarFilepath}", "/app/data-prepper.jar") copyFile("src/integrationTest/resources/${BASIC_GROK_PIPELINE_YAML}", "/app/${BASIC_GROK_PIPELINE_YAML}") diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle index 680009c2e8..1438a50cd1 100644 --- a/e2e-test/trace/build.gradle +++ b/e2e-test/trace/build.gradle @@ -42,7 +42,7 @@ def SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML = "service-map-e2e-pipeline-event-type. task createDataPrepperDockerFile(type: Dockerfile) { dependsOn copyDataPrepperJar destFile = project.file('build/docker/Dockerfile') - from("adoptopenjdk/openjdk14:jre-14.0.1_7-alpine") + from(dataPrepperBaseImage) exposePort(21890) exposePort(4900) workingDir("/app") diff --git a/research/zipkin-opensearch-to-otel/build.gradle b/research/zipkin-opensearch-to-otel/build.gradle index 5ef42fccee..9fd348d601 100644 --- a/research/zipkin-opensearch-to-otel/build.gradle +++ b/research/zipkin-opensearch-to-otel/build.gradle @@ -11,8 +11,6 @@ plugins { group 'com.amazon' version '0.1-beta' -sourceCompatibility = 1.8 - repositories { mavenCentral() }