Build on Java 11 with Java 8 as the compilation toolchain (opensearch…

…-project#1406) Updated the project for building on Java 11 using Java 8 as the toolchain for compilation. Updated GitHub Actions to build using Java 11 and for end-to-end tests, run Data Prepper against multiple versions of Java. Signed-off-by: David Venable <[email protected]>
asifsmohammed · May 19, 2022 · da56180 · da56180
1 parent 1f95d08
commit da56180
Show file tree

Hide file tree

Showing 20 changed files with 75 additions and 56 deletions.
diff --git a/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml b/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml
@@ -13,16 +13,16 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [14]
+        java: [8, 11, 17]
 
     runs-on: ubuntu-latest
 
     steps:
-      - name: Set up JDK ${{ matrix.java }}
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: ${{ matrix.java }}
+          java-version: 11
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Run basic grok end-to-end tests with Gradle
-        run: ./gradlew :e2e-test:log:basicLogEndToEndTest
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:basicLogEndToEndTest
diff --git a/.github/workflows/data-prepper-performance-test-compile-check.yml b/.github/workflows/data-prepper-performance-test-compile-check.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [14]
+        java: [11]
 
     runs-on: ubuntu-latest
 

diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml
@@ -13,19 +13,19 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [14]
+        java: [11, 17]
 
     runs-on: ubuntu-latest
 
     steps:
-    - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v1
-      with:
-        java-version: ${{ matrix.java }}
-    - name: Checkout Data-Prepper
-      uses: actions/checkout@v2
-      # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
-    - name: Run raw-span OTLP record type latest release compatibility end-to-end tests with Gradle
-      run: ./gradlew :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest
-    - name: Run raw-span Event record type latest release compatibility end-to-end tests with Gradle
-      run: ./gradlew :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Checkout Data-Prepper
+        uses: actions/checkout@v2
+        # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+      - name: Run raw-span OTLP record type latest release compatibility end-to-end tests with Gradle
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest
+      - name: Run raw-span Event record type latest release compatibility end-to-end tests with Gradle
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml
@@ -13,19 +13,19 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [14]
+        java: [11, 17]
 
     runs-on: ubuntu-latest
 
     steps:
-      - name: Set up JDK ${{ matrix.java }}
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: ${{ matrix.java }}
+          java-version: 11
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
         # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
       - name: Run raw-span OTLP record type end-to-end tests with Gradle
-        run: ./gradlew :e2e-test:trace:rawSpanOTLPEndToEndTest
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPEndToEndTest
       - name: Run raw-span OTLP and Event end-to-end tests with Gradle
-        run: ./gradlew :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml
@@ -13,19 +13,19 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [14]
+        java: [11, 17]
 
     runs-on: ubuntu-latest
 
     steps:
-      - name: Set up JDK ${{ matrix.java }}
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: ${{ matrix.java }}
+          java-version: 11
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
         # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
       - name: Run service-map OTLP record type end-to-end tests with Gradle
-        run: ./gradlew :e2e-test:trace:serviceMapOTLPEndToEndTest
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:serviceMapOTLPEndToEndTest
       - name: Run service-map OTLP and Event end-to-end tests with Gradle
-        run: ./gradlew :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest
+        run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [14]
+        java: [11]
 
     runs-on: ubuntu-latest
 

diff --git a/.github/workflows/opensearch-sink-opendistro-integration-tests.yml b/.github/workflows/opensearch-sink-opendistro-integration-tests.yml
@@ -13,7 +13,7 @@ jobs:
   integration-tests:
     strategy:
       matrix:
-        java: [14]
+        java: [11]
         opendistro: [1.6.0, 1.8.0, 1.9.0, 1.11.0, 1.12.0, 1.13.3]
 
     runs-on: ubuntu-latest

diff --git a/.github/workflows/opensearch-sink-os-integration-tests.yml b/.github/workflows/opensearch-sink-os-integration-tests.yml
@@ -9,7 +9,7 @@ jobs:
   integration-tests:
     strategy:
       matrix:
-        java: [14]
+        java: [11]
         opensearch: [1.0.1, 1.1.0, 1.2.4, 1.3.0]
 
     runs-on: ubuntu-latest

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -15,7 +15,7 @@ jobs:
     - name: Set up JDK
       uses: actions/setup-java@v1
       with:
-        java-version: 14
+        java-version: 11
     - name: Checkout Data-Prepper
       uses: actions/checkout@v2
     - name: Get Version

diff --git a/build.gradle b/build.gradle
@@ -66,7 +66,26 @@ subprojects {
     apply plugin: 'java'
     apply plugin: 'maven-publish'
     apply plugin: 'jacoco'
-    sourceCompatibility = '1.8'
+
+    java {
+        toolchain {
+            languageVersion = JavaLanguageVersion.of(8)
+        }
+    }
+    tasks.withType(JavaCompile).findAll { task -> task.name.containsIgnoreCase('test') }.forEach { task ->
+        task.configure {
+            javaCompiler = javaToolchains.compilerFor {
+                languageVersion = JavaLanguageVersion.of(11)
+            }
+        }
+    }
+    tasks.withType(Test).configureEach() {
+        javaLauncher = javaToolchains.launcherFor {
+            languageVersion = JavaLanguageVersion.of(11)
+        }
+    }
+
+
     spotless {
         java {
             targetExclude 'build/generated-src/antlr/**'

diff --git a/data-prepper-benchmarks/mapdb-benchmarks/build.gradle b/data-prepper-benchmarks/mapdb-benchmarks/build.gradle
@@ -11,8 +11,6 @@ plugins {
 group 'com.amazon'
 version '0.1-beta'
 
-sourceCompatibility = 1.8
-
 repositories {
     mavenCentral()
 }

diff --git a/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle b/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle
@@ -11,8 +11,6 @@ plugins {
 group 'com.amazon'
 version '0.1-beta'
 
-sourceCompatibility = 1.8
-
 repositories {
     mavenCentral()
 }

diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle
@@ -7,8 +7,6 @@ plugins {
     id 'java'
 }
 
-sourceCompatibility = 1.8
-
 repositories {
     mavenCentral()
 }

diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle
@@ -76,11 +76,20 @@ configurations {
     integrationTestRuntime.extendsFrom testRuntime
 }
 
+tasks.withType(JavaCompile).getByName('compileIntegrationTestJava').configure {
+    javaCompiler = javaToolchains.compilerFor {
+        languageVersion = JavaLanguageVersion.of(11)
+    }
+}
+
 task integrationTest(type: Test) {
     group = 'verification'
     testClassesDirs = sourceSets.integrationTest.output.classesDirs
 
     useJUnitPlatform()
+    javaLauncher = javaToolchains.launcherFor {
+        languageVersion = JavaLanguageVersion.of(11)
+    }
 
     classpath = sourceSets.integrationTest.runtimeClasspath
     systemProperty 'tests.opensearch.host', System.getProperty('tests.opensearch.host')

diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle
@@ -7,11 +7,6 @@ plugins {
     id 'java'
 }
 
-group 'com.amazon'
-version '0.1-beta'
-
-sourceCompatibility = 1.8
-
 repositories {
     mavenCentral()
 }

diff --git a/docs/developer_guide.md b/docs/developer_guide.md
@@ -8,41 +8,43 @@ First, please read our [contribution guide](../CONTRIBUTING.md) for more informa
 
 ## Installation Prerequisites
 
-### JDK Versions
+### Java Versions
 
-Running Data Prepper requires JDK 8 and above.
+Building Data Prepper requires JDK 11. The Data Prepper Gradle build runs in a Java 11 JVM, but uses
+[Gradle toolchains](https://docs.gradle.org/current/userguide/toolchains.html) to compile the Java
+code using Java 8. If you have a JDK 8 installed locally, Gradle will use your installed JDK 8. If you
+do not, Gradle will install JDK 8.
 
-Running the integration tests requires JDK 14 or 15.
+All main source code builds on JDK 8, so it must be compatible with Java 8. The test code
+(unit and integration tests) runs using JDK 11 and may use Java 11 features.
 
 
 ## Building from source
 
 The assemble task will build the Jar files without running the integration
 tests. You can use these jar files for running DataPrepper. If you are just
-looking to use DataPrepper and modify it, this build
-is faster than running the integration test suite and only requires JDK 8+.
+looking to build Data Prepper from source, this build
+is faster than running the integration test suite.
 
-To build the project from source, run
+To build the project from source, run the following command from the project root:
 
 ```
 ./gradlew assemble
 ```
 
-from the project root. 
 
 ### Full Project Build
 
 Running the build command will assemble the Jar files needed
 for running DataPrepper. It will also run the integration test
 suite.
 
-To build, run
+To build, run the following command from the project root:
 
 ```
 ./gradlew build
 ```
 
-from the project root.
 
 ## Running the project
 

diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle
@@ -44,5 +44,7 @@ subprojects {
 
     ext {
         dataPrepperJarFilepath = "${project.buildDir.name}/bin/${DATA_PREPPER_CORE_JAR.archiveName}"
+        targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11'
+        dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre-alpine"
     }
 }
diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle
@@ -38,7 +38,7 @@ def BASIC_GROK_PIPELINE_YAML = "basic-grok-e2e-pipeline.yml"
 task createDataPrepperDockerFile(type: Dockerfile) {
     dependsOn copyDataPrepperJar
     destFile = project.file('build/docker/Dockerfile')
-    from("adoptopenjdk/openjdk14:jre-14.0.1_7-alpine")
+    from(dataPrepperBaseImage)
     workingDir("/app")
     copyFile("${dataPrepperJarFilepath}", "/app/data-prepper.jar")
     copyFile("src/integrationTest/resources/${BASIC_GROK_PIPELINE_YAML}", "/app/${BASIC_GROK_PIPELINE_YAML}")

diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle
@@ -42,7 +42,7 @@ def SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML = "service-map-e2e-pipeline-event-type.
 task createDataPrepperDockerFile(type: Dockerfile) {
     dependsOn copyDataPrepperJar
     destFile = project.file('build/docker/Dockerfile')
-    from("adoptopenjdk/openjdk14:jre-14.0.1_7-alpine")
+    from(dataPrepperBaseImage)
     exposePort(21890)
     exposePort(4900)
     workingDir("/app")

diff --git a/research/zipkin-opensearch-to-otel/build.gradle b/research/zipkin-opensearch-to-otel/build.gradle
@@ -11,8 +11,6 @@ plugins {
 group 'com.amazon'
 version '0.1-beta'
 
-sourceCompatibility = 1.8
-
 repositories {
     mavenCentral()
 }