Run trace end-to-end tests using the release Docker image. opensearch…

…-project#3566 This is not working for rawSpanLatestReleaseCompatibilityEndToEndTest, at least not locally. Maybe an ARM issue?

Signed-off-by: David Venable <[email protected]>

e2e-test/log/build.gradle

@@ -92,7 +92,7 @@ logTestConfigurations.each { testConfiguration ->
         dependsOn startOpenSearchDockerContainer
         dependsOn "start${testConfiguration.testName}"
 
-        // wait for Data Prepper image to be ready
+        // Wait for Data Prepper image to be ready
         doFirst {
             sleep(15 * 1000)
         }

e2e-test/trace/build.gradle

@@ -11,14 +11,202 @@ import com.bmuschko.gradle.docker.tasks.container.DockerStopContainer
 import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 import com.bmuschko.gradle.docker.tasks.image.DockerPullImage
 
+class TraceTestContainerConfiguration {
+    TraceTestContainerConfiguration(
+            String image,
+            String sourceTask,
+            String pipelineConfiguration,
+            String dataPrepperConfiguration) {
+        this.image = image
+        this.sourceTask = sourceTask
+        this.pipelineConfiguration = pipelineConfiguration
+        this.dataPrepperConfiguration = dataPrepperConfiguration
+    }
+    String image
+    String sourceTask
+    String pipelineConfiguration
+    String dataPrepperConfiguration
+}
+
+/**
+ * Represents the configurations needed for any end-to-end log test.
+ */
+class TraceTestConfiguration {
+    TraceTestConfiguration(
+            String testName,
+            String description,
+            String testFilters,
+            List<TraceTestContainerConfiguration> containerConfigurations,
+            String containerNamePrefix) {
+        this.testName = testName
+        this.description = description
+        this.testFilters = testFilters
+        this.containerConfigurations = containerConfigurations
+        this.containerNamePrefix = containerNamePrefix
+    }
+    String testName
+    String description
+    String testFilters
+    List<TraceTestContainerConfiguration> containerConfigurations
+    String containerNamePrefix
+}
+
+def RAW_SPAN_PIPELINE_YAML = 'raw-span-e2e-pipeline.yml'
+def DATA_PREPPER_CONFIG_YAML = 'data_prepper.yml'
+def RAW_SPAN_TESTS_PATTERN = 'org.opensearch.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*'
+def DATA_PREPPER_CONFIG_STATIC_YAML = 'data_prepper_static.yml'
+def RELEASED_DATA_PREPPER_DOCKER_IMAGE = 'opensearchproject/data-prepper:latest'
+
+List<TraceTestConfiguration> traceTestConfigurations = [
+        new TraceTestConfiguration(
+                'rawSpanEndToEndTest',
+                'Runs the raw span integration tests.',
+                RAW_SPAN_TESTS_PATTERN,
+                [new TraceTestContainerConfiguration(
+                        "${project.rootProject.name}:${project.version}",
+                        ':release:docker:docker',
+                        RAW_SPAN_PIPELINE_YAML,
+                        DATA_PREPPER_CONFIG_YAML),
+                 new TraceTestContainerConfiguration(
+                         "${project.rootProject.name}:${project.version}",
+                         ':release:docker:docker',
+                         RAW_SPAN_PIPELINE_YAML,
+                         DATA_PREPPER_CONFIG_YAML)],
+                'data-prepper-raw',
+        ),
+        new TraceTestConfiguration(
+                'rawSpanPeerForwarderEndToEndTest',
+                'Runs the raw span with peer-forwarder integration tests.',
+                RAW_SPAN_TESTS_PATTERN,
+                [new TraceTestContainerConfiguration(
+                        "${project.rootProject.name}:${project.version}",
+                        ':release:docker:docker',
+                        RAW_SPAN_PIPELINE_YAML,
+                        DATA_PREPPER_CONFIG_STATIC_YAML),
+                 new TraceTestContainerConfiguration(
+                         "${project.rootProject.name}:${project.version}",
+                         ':release:docker:docker',
+                         RAW_SPAN_PIPELINE_YAML,
+                         DATA_PREPPER_CONFIG_STATIC_YAML)],
+                'data-prepper'
+        ),
+        new TraceTestConfiguration(
+                'rawSpanLatestReleaseCompatibilityEndToEndTest',
+                'Runs the raw span integration tests with the latest released Data Prepper as a peer.',
+                RAW_SPAN_TESTS_PATTERN,
+                [new TraceTestContainerConfiguration(
+                        "${project.rootProject.name}:${project.version}",
+                        ':release:docker:docker',
+                        'raw-span-e2e-pipeline-from-build.yml',
+                        DATA_PREPPER_CONFIG_STATIC_YAML),
+                 new TraceTestContainerConfiguration(
+                         RELEASED_DATA_PREPPER_DOCKER_IMAGE,
+                         'pullDataPrepperDockerImage',
+                         'raw-span-e2e-pipeline-latest-release.yml',
+                         DATA_PREPPER_CONFIG_STATIC_YAML)
+                ],
+                'data-prepper'
+        ),
+        new TraceTestConfiguration(
+                'serviceMapPeerForwarderEndToEndTest',
+                'Runs the service map with peer-forwarder integration tests.',
+                'org.opensearch.dataprepper.integration.trace.EndToEndServiceMapTest.testPipelineEndToEnd*',
+                [new TraceTestContainerConfiguration(
+                        "${project.rootProject.name}:${project.version}",
+                        ':release:docker:docker',
+                        'service-map-e2e-pipeline.yml',
+                        DATA_PREPPER_CONFIG_STATIC_YAML),
+                 new TraceTestContainerConfiguration(
+                         "${project.rootProject.name}:${project.version}",
+                         ':release:docker:docker',
+                         'service-map-e2e-pipeline.yml',
+                         DATA_PREPPER_CONFIG_STATIC_YAML)],
+                'data-prepper'
+        )
+]
+
+tasks.register('pullDataPrepperDockerImage', DockerPullImage) {
+    image = RELEASED_DATA_PREPPER_DOCKER_IMAGE
+}
+
+
+traceTestConfigurations.each { testConfiguration ->
+    tasks.register("start${testConfiguration.testName}All")
+    tasks.register("remove${testConfiguration.testName}All")
+
+    (0..<testConfiguration.containerConfigurations.size()).each { containerIndex ->
+        tasks.register("create${testConfiguration.testName}${containerIndex}", DockerCreateContainer) {
+            dependsOn testConfiguration.containerConfigurations.get(containerIndex).sourceTask
+            dependsOn createDataPrepperNetwork
+            containerName = testConfiguration.containerNamePrefix
+            containerName = "${testConfiguration.containerNamePrefix}-${containerIndex}"
+            exposePorts('tcp', [21890])
+            hostConfig.portBindings = ["${21890+containerIndex}:21890"]
+            hostConfig.binds = [
+                    (project.file("src/integrationTest/resources/${testConfiguration.containerConfigurations.get(containerIndex).pipelineConfiguration}").toString())   : '/usr/share/data-prepper/pipelines/trace-pipeline.yaml',
+                    (project.file("src/integrationTest/resources/${testConfiguration.containerConfigurations.get(containerIndex).dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml'
+            ]
+            hostConfig.network = createDataPrepperNetwork.getNetworkName()
+            targetImageId testConfiguration.containerConfigurations.get(containerIndex).image
+        }
+
+        tasks.register("start${testConfiguration.testName}${containerIndex}", DockerStartContainer) {
+            dependsOn "create${testConfiguration.testName}${containerIndex}"
+            dependsOn 'startOpenSearchDockerContainer'
+            mustRunAfter 'startOpenSearchDockerContainer'
+            targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId()
+        }
+
+        tasks.named("start${testConfiguration.testName}All").configure {
+            dependsOn "start${testConfiguration.testName}${containerIndex}"
+        }
+
+        tasks.register("stop${testConfiguration.testName}${containerIndex}", DockerStopContainer) {
+            dependsOn "${testConfiguration.testName}"
+            targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId()
+        }
+
+        tasks.register("remove${testConfiguration.testName}${containerIndex}", DockerRemoveContainer) {
+            dependsOn "stop${testConfiguration.testName}${containerIndex}"
+            targetContainerId tasks.getByName("stop${testConfiguration.testName}${containerIndex}").getContainerId()
+        }
+
+        tasks.named("remove${testConfiguration.testName}All").configure {
+            dependsOn "remove${testConfiguration.testName}${containerIndex}"
+        }
+    }
+
+    tasks.register(testConfiguration.testName, Test) {
+        dependsOn build
+        dependsOn startOpenSearchDockerContainer
+        dependsOn "start${testConfiguration.testName}All"
+
+        // Wait for Data Prepper image to be ready
+        doFirst {
+            sleep(15 * 1000)
+        }
+
+        description = testConfiguration.description
+        group = 'verification'
+        testClassesDirs = sourceSets.integrationTest.output.classesDirs
+        classpath = sourceSets.integrationTest.runtimeClasspath
+
+        filter {
+            includeTestsMatching testConfiguration.testFilters
+        }
+
+        finalizedBy stopOpenSearchDockerContainer
+        finalizedBy "remove${testConfiguration.testName}All"
+        finalizedBy removeDataPrepperNetwork
+    }
+}
+
+
 /**
  * End-to-end test docker network
  */
 
-def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml"
 def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml"
-def DATA_PREPPER_CONFIG_YAML = "data_prepper.yml"
-def DATA_PREPPER_CONFIG_STATIC_YAML = "data_prepper_static.yml"
 def RAW_SPAN_PIPELINE_FROM_BUILD_YAML = "raw-span-e2e-pipeline-from-build.yml"
 def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml"
 
@@ -47,29 +235,6 @@ def createDataPrepperDockerContainer(final String taskBaseName, final String dat
     }
 }
 
-def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) {
-    return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) {
-        dependsOn createDataPrepperDockerContainerTask
-        targetContainerId createDataPrepperDockerContainerTask.getContainerId()
-    }
-}
-
-def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) {
-    return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) {
-        targetContainerId startDataPrepperDockerContainerTask.getContainerId()
-    }
-}
-
-def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) {
-    return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) {
-        targetContainerId stopDataPrepperDockerContainerTask.getContainerId()
-    }
-}
-
-task pullDataPrepperDockerImage(type: DockerPullImage) {
-    image = 'opensearchproject/data-prepper:latest'
-}
-
 def createDataPrepperDockerContainerFromPullImage(final String taskBaseName, final String dataPrepperName, final int grpcPort,
                                                   final String pipelineConfigYAML, final String dataPrepperConfigYAML) {
     return tasks.create("create${taskBaseName}", DockerCreateContainer) {
@@ -84,87 +249,21 @@ def createDataPrepperDockerContainerFromPullImage(final String taskBaseName, fin
         targetImageId pullDataPrepperDockerImage.image
     }
 }
-/**
- * End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test
- * Stops the docker containers when finished
- */
-def createEndToEndTest(final String testName, final String includeTestsMatchPattern,
-                       final DockerCreateContainer createDataPrepper1Task, final DockerCreateContainer createDataPrepper2Task) {
-    return tasks.create(testName, Test) {
-        dependsOn build
-        dependsOn startOpenSearchDockerContainer
-        def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
-        def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
-        dependsOn startDataPrepper1Task
-        dependsOn startDataPrepper2Task
-        startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer'
-        startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer'
-        // wait for data-preppers to be ready
-        doFirst {
-            sleep(15*1000)
-        }
-
-        description = 'Runs the raw span integration tests.'
-        group = 'verification'
-        testClassesDirs = sourceSets.integrationTest.output.classesDirs
-        classpath = sourceSets.integrationTest.runtimeClasspath
-
-        filter {
-            includeTestsMatching includeTestsMatchPattern
-        }
-
-        def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
-        def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
-        finalizedBy stopDataPrepper1Task
-        finalizedBy stopDataPrepper2Task
-        finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
-        finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
-        finalizedBy stopOpenSearchDockerContainer
-        finalizedBy removeDataPrepperNetwork
-    }
-}
 
 // raw span e2e test
 def includeRawSpanTestsMatchPattern = "org.opensearch.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*"
 
-def createRawSpanDataPrepper1Task = createDataPrepperDockerContainer(
-        "rawSpanDataPrepper1", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}")
-def createRawSpanDataPrepper2Task = createDataPrepperDockerContainer(
-        "rawSpanDataPrepper2", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}")
-
-def rawSpanEndToEndTest = createEndToEndTest("rawSpanEndToEndTest", includeRawSpanTestsMatchPattern,
-        createRawSpanDataPrepper1Task, createRawSpanDataPrepper2Task)
-
-// raw span with peer forwarding e2e test
-def createRawSpanPeerForwarderDataPrepper1Task = createDataPrepperDockerContainer(
-        "rawSpanPeerForwarderDataPrepper1", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}")
-def createRawSpanPeerForwarderDataPrepper2Task = createDataPrepperDockerContainer(
-        "rawSpanPeerForwarderDataPrepper2", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}")
-
-def rawSpanPeerForwarderEndToEndTest = createEndToEndTest("rawSpanPeerForwarderEndToEndTest", includeRawSpanTestsMatchPattern,
-        createRawSpanPeerForwarderDataPrepper1Task, createRawSpanPeerForwarderDataPrepper2Task)
 
 // raw span compatibility e2e test
 def rawSpanDataPrepperEventFromBuild = createDataPrepperDockerContainer(
         "rawSpanDataPrepperEventFromBuild", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_FROM_BUILD_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}")
 def rawSpanDataPrepperLatestFromPull = createDataPrepperDockerContainerFromPullImage(
         "rawSpanDataPrepperLatestFromPull", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}")
 
-def rawSpanLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanLatestReleaseCompatibilityEndToEndTest",
-        includeRawSpanTestsMatchPattern,
-        rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull)
-
-// service map e2e
-def includeServiceMapTestsMatchPattern = "org.opensearch.dataprepper.integration.trace.EndToEndServiceMapTest.testPipelineEndToEnd*"
-
-// service map with peer forwarding e2e test
-def createServiceMapPeerForwarderDataPrepper1Task = createDataPrepperDockerContainer(
-        "serviceMapPeerForwarderDataPrepper1", "dataprepper1", 21890, "${SERVICE_MAP_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}")
-def createServiceMapPeerForwarderDataPrepper2Task = createDataPrepperDockerContainer(
-        "serviceMapPeerForwarderDataPrepper2", "dataprepper2", 21891, "${SERVICE_MAP_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}")
+//def rawSpanLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanLatestReleaseCompatibilityEndToEndTest",
+//        includeRawSpanTestsMatchPattern,
+//        rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull)
 
-def serviceMapPeerForwarderEndToEndTest = createEndToEndTest("serviceMapPeerForwarderEndToEndTest", includeServiceMapTestsMatchPattern,
-        createServiceMapPeerForwarderDataPrepper1Task, createServiceMapPeerForwarderDataPrepper2Task)
 
 dependencies {
     integrationTestImplementation project(':data-prepper-api')

e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml

@@ -3,4 +3,4 @@ peer_forwarder:
   port: 4994
   ssl: false
   discovery_mode: static
-  static_endpoints: ["dataprepper1", "dataprepper2"]
+  static_endpoints: ["data-prepper-0", "data-prepper-1"]