Update end-to-end tests to use the release Docker image or a custom i…

…mage with a specific Java version. Resolves opensearch-project#3566 Signed-off-by: David Venable <[email protected]>
dlvenable · Nov 1, 2023 · 32aa89a · 32aa89a
1 parent ca3d6ac
commit 32aa89a
Show file tree

Hide file tree

Showing 15 changed files with 442 additions and 347 deletions.
diff --git a/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml b/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [11, 17, 21, docker]
         test: ['basicLogEndToEndTest', 'parallelGrokStringSubstituteTest']
       fail-fast: false
 

diff --git a/.github/workflows/data-prepper-peer-forwarder-local-node-e2e-tests.yml b/.github/workflows/data-prepper-peer-forwarder-local-node-e2e-tests.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [11, 17, 21, docker]
       fail-fast: false
 
     runs-on: ubuntu-latest

diff --git a/.github/workflows/data-prepper-peer-forwarder-static-e2e-tests.yml b/.github/workflows/data-prepper-peer-forwarder-static-e2e-tests.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [11, 17, 21, docker]
         test: ['staticAggregateEndToEndTest', 'staticLogMetricsEndToEndTest']
       fail-fast: false
 

diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [11, 17, 21, docker]
       fail-fast: false
 
     runs-on: ubuntu-latest

diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [11, 17, 21, docker]
         otelVersion: ['0.9.0-alpha', '0.16.0-alpha']
       fail-fast: false
 

diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-peer-forwarder-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-peer-forwarder-e2e-tests.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [11, 17, 21, docker]
       fail-fast: false
 
     runs-on: ubuntu-latest

diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [11, 17, 21, docker]
       fail-fast: false
 
     runs-on: ubuntu-latest

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -0,0 +1,9 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+plugins {
+    id 'java-gradle-plugin'
+    id 'java'
+}
diff --git a/buildSrc/src/main/java/org/opensearch/dataprepper/gradle/end_to_end/DockerProviderTask.java b/buildSrc/src/main/java/org/opensearch/dataprepper/gradle/end_to_end/DockerProviderTask.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.gradle.end_to_end;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.Input;
+
+/**
+ * A task which can provide a Docker image to use for an end-to-end test.
+ */
+public abstract class DockerProviderTask extends DefaultTask {
+    /**
+     * The Docker image with both the name and tag in the standard string
+     * format - <i>my-image:mytag</i>
+     *
+     * @return The Docker image
+     */
+    @Input
+    abstract Property<String> getImageId();
+}
diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle
@@ -7,21 +7,24 @@
 import com.bmuschko.gradle.docker.tasks.container.DockerCreateContainer
 import com.bmuschko.gradle.docker.tasks.container.DockerStartContainer
 import com.bmuschko.gradle.docker.tasks.container.DockerStopContainer
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 import com.bmuschko.gradle.docker.tasks.image.Dockerfile
 import com.bmuschko.gradle.docker.tasks.image.DockerPullImage
 import com.bmuschko.gradle.docker.tasks.network.DockerCreateNetwork
 import com.bmuschko.gradle.docker.tasks.network.DockerRemoveNetwork
+import org.opensearch.dataprepper.gradle.end_to_end.DockerProviderTask
 
 plugins {
     id 'com.bmuschko.docker-remote-api' version '9.3.2'
 }
 
+
 subprojects {
     apply plugin: 'com.bmuschko.docker-remote-api'
 
     ext {
         dataPrepperJarImageFilepath = 'bin/data-prepper/'
-        targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11'
+        targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : 'docker'
         targetOpenTelemetryVersion = project.hasProperty('openTelemetryVersion') ? project.getProperty('openTelemetryVersion') : "${libs.versions.opentelemetry.get()}"
         dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre"
     }
@@ -46,65 +49,85 @@ subprojects {
         integrationTestRuntime.extendsFrom testRuntime
     }
 
-    task copyDataPrepperJar(type: Copy) {
-        dependsOn project(':data-prepper-main').jar
-        dependsOn project(':data-prepper-plugins').jar
+    tasks.register('copyDataPrepperArchive', Copy) {
+        dependsOn ':release:archives:linux:linuxx64DistTar'
         duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-        from project(':data-prepper-main').jar.archivePath
-        from project(':data-prepper-main').configurations.runtimeClasspath
-        into("${project.buildDir}/docker/${dataPrepperJarImageFilepath}")
+        from project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archivePath
+        into("${project.buildDir}/docker/")
     }
 
-    task createDataPrepperDockerFile(type: Dockerfile) {
-        dependsOn copyDataPrepperJar
+    tasks.register('createDataPrepperDockerFile', Dockerfile) {
+        dependsOn copyDataPrepperArchive
+        dependsOn ':release:archives:linux:linuxx64DistTar'
         destFile = project.file('build/docker/Dockerfile')
+
         from(dataPrepperBaseImage)
-        workingDir('/app/data-prepper')
-        copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib')
-        defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute')
+        runCommand('mkdir -p /var/log/data-prepper')
+        addFile(project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get(), '/usr/share')
+        runCommand("mv /usr/share/${project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get().replace('.tar.gz', '')} /usr/share/data-prepper")
+        workingDir('/usr/share/data-prepper')
+        defaultCommand('bin/data-prepper')
+    }
+
+    tasks.register('buildDataPrepperDockerImage', DockerBuildImage) {
+        dependsOn createDataPrepperDockerFile
+        dockerFile = file('build/docker/Dockerfile')
+        images.add('e2e-test-data-prepper')
+    }
+
+
+    tasks.register('dataPrepperDockerImage', DockerProviderTask) {
+        if(targetJavaVersion == 'docker') {
+            dependsOn ':release:docker:docker'
+            imageId = "${project.rootProject.name}:${project.version}"
+        }
+        else {
+            dependsOn 'createDataPrepperDockerFile'
+            imageId = buildDataPrepperDockerImage.getImageId()
+        }
     }
 
-    task createDataPrepperNetwork(type: DockerCreateNetwork) {
+    tasks.register('createDataPrepperNetwork', DockerCreateNetwork) {
         networkName = 'data_prepper_network'
     }
 
-    task removeDataPrepperNetwork(type: DockerRemoveNetwork) {
+    tasks.register('removeDataPrepperNetwork', DockerRemoveNetwork) {
         dependsOn createDataPrepperNetwork
         networkId = createDataPrepperNetwork.getNetworkId()
     }
 
     /**
      * OpenSearch Docker tasks
      */
-    task pullOpenSearchDockerImage(type: DockerPullImage) {
+    tasks.register('pullOpenSearchDockerImage', DockerPullImage) {
         image = "opensearchproject/opensearch:${libs.versions.opensearch.get()}"
     }
 
-    task createOpenSearchDockerContainer(type: DockerCreateContainer) {
+    tasks.register('createOpenSearchDockerContainer', DockerCreateContainer) {
         dependsOn createDataPrepperNetwork
         dependsOn pullOpenSearchDockerImage
         targetImageId pullOpenSearchDockerImage.image
         containerName = 'node-0.example.com'
         hostConfig.portBindings = ['9200:9200', '9600:9600']
         hostConfig.autoRemove = true
         hostConfig.network = createDataPrepperNetwork.getNetworkName()
-        envVars = ['discovery.type':'single-node']
+        envVars = ['discovery.type': 'single-node']
     }
 
-    task startOpenSearchDockerContainer(type: DockerStartContainer) {
+    tasks.register('startOpenSearchDockerContainer', DockerStartContainer) {
         dependsOn createOpenSearchDockerContainer
         targetContainerId createOpenSearchDockerContainer.getContainerId()
 
         doLast {
-            sleep(90*1000)
+            sleep(90 * 1000)
         }
     }
 
-    task stopOpenSearchDockerContainer(type: DockerStopContainer) {
+    tasks.register('stopOpenSearchDockerContainer', DockerStopContainer) {
         targetContainerId createOpenSearchDockerContainer.getContainerId()
 
         doLast {
-            sleep(5*1000)
+            sleep(5 * 1000)
         }
     }