diff --git a/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml b/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml index 5449ed7f6c..0ac943a231 100644 --- a/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml +++ b/.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21, docker] test: ['basicLogEndToEndTest', 'parallelGrokStringSubstituteTest'] fail-fast: false diff --git a/.github/workflows/data-prepper-peer-forwarder-local-node-e2e-tests.yml b/.github/workflows/data-prepper-peer-forwarder-local-node-e2e-tests.yml index de36390665..36f4aea9a6 100644 --- a/.github/workflows/data-prepper-peer-forwarder-local-node-e2e-tests.yml +++ b/.github/workflows/data-prepper-peer-forwarder-local-node-e2e-tests.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21, docker] fail-fast: false runs-on: ubuntu-latest diff --git a/.github/workflows/data-prepper-peer-forwarder-static-e2e-tests.yml b/.github/workflows/data-prepper-peer-forwarder-static-e2e-tests.yml index 10efb39f01..0d81b0a615 100644 --- a/.github/workflows/data-prepper-peer-forwarder-static-e2e-tests.yml +++ b/.github/workflows/data-prepper-peer-forwarder-static-e2e-tests.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21, docker] test: ['staticAggregateEndToEndTest', 'staticLogMetricsEndToEndTest'] fail-fast: false diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml index e85d4b17ef..0dc90a9917 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21, docker] fail-fast: false runs-on: ubuntu-latest diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml index d7d705c93e..9538f3eb09 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21, docker] otelVersion: ['0.9.0-alpha', '0.16.0-alpha'] fail-fast: false diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-peer-forwarder-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-peer-forwarder-e2e-tests.yml index 44291cdbf2..3cbe0d53a7 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-peer-forwarder-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-peer-forwarder-e2e-tests.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21, docker] fail-fast: false runs-on: ubuntu-latest diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml index 0c5aa7eb16..6d6f7c1fc3 100644 --- a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - java: [11, 17, 21] + java: [11, 17, 21, docker] fail-fast: false runs-on: ubuntu-latest diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 0000000000..9d29afb944 --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java-gradle-plugin' + id 'java' +} diff --git a/buildSrc/src/main/java/org/opensearch/dataprepper/gradle/end_to_end/DockerProviderTask.java b/buildSrc/src/main/java/org/opensearch/dataprepper/gradle/end_to_end/DockerProviderTask.java new file mode 100644 index 0000000000..eb264c1fd7 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/dataprepper/gradle/end_to_end/DockerProviderTask.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.gradle.end_to_end; + +import org.gradle.api.DefaultTask; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; + +/** + * A task which can provide a Docker image to use for an end-to-end test. + */ +public abstract class DockerProviderTask extends DefaultTask { + /** + * The Docker image with both the name and tag in the standard string + * format - my-image:mytag + * + * @return The Docker image + */ + @Input + abstract Property getImageId(); +} diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle index bc4331a09d..23578f3656 100644 --- a/e2e-test/build.gradle +++ b/e2e-test/build.gradle @@ -7,21 +7,24 @@ import com.bmuschko.gradle.docker.tasks.container.DockerCreateContainer import com.bmuschko.gradle.docker.tasks.container.DockerStartContainer import com.bmuschko.gradle.docker.tasks.container.DockerStopContainer +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.bmuschko.gradle.docker.tasks.image.Dockerfile import com.bmuschko.gradle.docker.tasks.image.DockerPullImage import com.bmuschko.gradle.docker.tasks.network.DockerCreateNetwork import com.bmuschko.gradle.docker.tasks.network.DockerRemoveNetwork +import org.opensearch.dataprepper.gradle.end_to_end.DockerProviderTask plugins { id 'com.bmuschko.docker-remote-api' version '9.3.2' } + subprojects { apply plugin: 'com.bmuschko.docker-remote-api' ext { dataPrepperJarImageFilepath = 'bin/data-prepper/' - targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11' + targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : 'docker' targetOpenTelemetryVersion = project.hasProperty('openTelemetryVersion') ? project.getProperty('openTelemetryVersion') : "${libs.versions.opentelemetry.get()}" dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre" } @@ -46,29 +49,49 @@ subprojects { integrationTestRuntime.extendsFrom testRuntime } - task copyDataPrepperJar(type: Copy) { - dependsOn project(':data-prepper-main').jar - dependsOn project(':data-prepper-plugins').jar + tasks.register('copyDataPrepperArchive', Copy) { + dependsOn ':release:archives:linux:linuxx64DistTar' duplicatesStrategy = DuplicatesStrategy.EXCLUDE - from project(':data-prepper-main').jar.archivePath - from project(':data-prepper-main').configurations.runtimeClasspath - into("${project.buildDir}/docker/${dataPrepperJarImageFilepath}") + from project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archivePath + into("${project.buildDir}/docker/") } - task createDataPrepperDockerFile(type: Dockerfile) { - dependsOn copyDataPrepperJar + tasks.register('createDataPrepperDockerFile', Dockerfile) { + dependsOn copyDataPrepperArchive + dependsOn ':release:archives:linux:linuxx64DistTar' destFile = project.file('build/docker/Dockerfile') + from(dataPrepperBaseImage) - workingDir('/app/data-prepper') - copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib') - defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute') + runCommand('mkdir -p /var/log/data-prepper') + addFile(project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get(), '/usr/share') + runCommand("mv /usr/share/${project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get().replace('.tar.gz', '')} /usr/share/data-prepper") + workingDir('/usr/share/data-prepper') + defaultCommand('bin/data-prepper') + } + + tasks.register('buildDataPrepperDockerImage', DockerBuildImage) { + dependsOn createDataPrepperDockerFile + dockerFile = file('build/docker/Dockerfile') + images.add('e2e-test-data-prepper') + } + + + tasks.register('dataPrepperDockerImage', DockerProviderTask) { + if(targetJavaVersion == 'docker') { + dependsOn ':release:docker:docker' + imageId = "${project.rootProject.name}:${project.version}" + } + else { + dependsOn 'createDataPrepperDockerFile' + imageId = buildDataPrepperDockerImage.getImageId() + } } - task createDataPrepperNetwork(type: DockerCreateNetwork) { + tasks.register('createDataPrepperNetwork', DockerCreateNetwork) { networkName = 'data_prepper_network' } - task removeDataPrepperNetwork(type: DockerRemoveNetwork) { + tasks.register('removeDataPrepperNetwork', DockerRemoveNetwork) { dependsOn createDataPrepperNetwork networkId = createDataPrepperNetwork.getNetworkId() } @@ -76,11 +99,11 @@ subprojects { /** * OpenSearch Docker tasks */ - task pullOpenSearchDockerImage(type: DockerPullImage) { + tasks.register('pullOpenSearchDockerImage', DockerPullImage) { image = "opensearchproject/opensearch:${libs.versions.opensearch.get()}" } - task createOpenSearchDockerContainer(type: DockerCreateContainer) { + tasks.register('createOpenSearchDockerContainer', DockerCreateContainer) { dependsOn createDataPrepperNetwork dependsOn pullOpenSearchDockerImage targetImageId pullOpenSearchDockerImage.image @@ -88,23 +111,23 @@ subprojects { hostConfig.portBindings = ['9200:9200', '9600:9600'] hostConfig.autoRemove = true hostConfig.network = createDataPrepperNetwork.getNetworkName() - envVars = ['discovery.type':'single-node'] + envVars = ['discovery.type': 'single-node'] } - task startOpenSearchDockerContainer(type: DockerStartContainer) { + tasks.register('startOpenSearchDockerContainer', DockerStartContainer) { dependsOn createOpenSearchDockerContainer targetContainerId createOpenSearchDockerContainer.getContainerId() doLast { - sleep(90*1000) + sleep(90 * 1000) } } - task stopOpenSearchDockerContainer(type: DockerStopContainer) { + tasks.register('stopOpenSearchDockerContainer', DockerStopContainer) { targetContainerId createOpenSearchDockerContainer.getContainerId() doLast { - sleep(5*1000) + sleep(5 * 1000) } } diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index 45624939fe..8198e41bee 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -8,120 +8,109 @@ import com.bmuschko.gradle.docker.tasks.container.DockerCreateContainer import com.bmuschko.gradle.docker.tasks.container.DockerRemoveContainer import com.bmuschko.gradle.docker.tasks.container.DockerStartContainer import com.bmuschko.gradle.docker.tasks.container.DockerStopContainer -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage /** - * End-to-end test docker network + * Represents the configurations needed for any end-to-end log test. */ - -def BASIC_GROK_PIPELINE_YAML = "basic-grok-e2e-pipeline.yml" -def PARALLEL_GROK_SUBSTITUTE_PIPELINE_YAML = "parallel-grok-substitute-e2e-pipeline.yml" -def DATA_PREPPER_CONFIG_YAML = "data_prepper.yml" - -/** - * DataPrepper Docker tasks - */ -task buildDataPrepperDockerImage(type: DockerBuildImage) { - dependsOn createDataPrepperDockerFile - dockerFile = file('build/docker/Dockerfile') - images.add('e2e-test-log-pipeline-image') +class LogTestConfiguration { + LogTestConfiguration( + String testName, + String description, + String testFilters, + String containerName, + String pipelineConfiguration, + String dataPrepperConfiguration) { + this.testName = testName + this.description = description + this.testFilters = testFilters + this.containerName = containerName + this.pipelineConfiguration = pipelineConfiguration + this.dataPrepperConfiguration = dataPrepperConfiguration + } + String testName + String description + String testFilters + String containerName + String pipelineConfiguration + String dataPrepperConfiguration } -def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int sourcePort, - final int serverPort, final String pipelineConfigYAML, final String dataPrepperConfigYAML) { - return tasks.create("create${taskBaseName}", DockerCreateContainer) { - dependsOn buildDataPrepperDockerImage +List logTestConfigurations = [ + new LogTestConfiguration( + 'basicLogEndToEndTest', + 'Runs the basic grok end-to-end test.', + 'org.opensearch.dataprepper.integration.log.EndToEndBasicLogTest.testPipelineEndToEnd*', + 'data-prepper-basic-log', + 'basic-grok-e2e-pipeline.yml', + 'data_prepper.yml' + ), + new LogTestConfiguration( + 'parallelGrokStringSubstituteTest', + 'Runs the parallel grok and string substitute end-to-end test.', + 'org.opensearch.dataprepper.integration.log.ParallelGrokStringSubstituteLogTest.testPipelineEndToEnd*', + 'data-prepper-parallel-log', + 'parallel-grok-substitute-e2e-pipeline.yml', + 'data_prepper.yml' + ) +] + + +logTestConfigurations.each { testConfiguration -> + tasks.register("create${testConfiguration.testName}", DockerCreateContainer) { + dependsOn dataPrepperDockerImage dependsOn createDataPrepperNetwork - containerName = dataPrepperName - exposePorts("tcp", [2021, 4900]) - hostConfig.portBindings = [String.format('%d:2021', sourcePort), String.format('%d:4900', serverPort)] - hostConfig.binds = [(project.file("src/integrationTest/resources/${pipelineConfigYAML}").toString()):"/app/data-prepper/pipelines/pipelines.yaml", - (project.file("src/integrationTest/resources/${dataPrepperConfigYAML}").toString()):"/app/data-prepper/config/data-prepper-config.yaml"] + containerName = testConfiguration.containerName + exposePorts('tcp', [2021, 4900]) + hostConfig.portBindings = ['2021:2021', '4900:4900'] + hostConfig.binds = [ + (project.file("src/integrationTest/resources/${testConfiguration.pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/log-pipeline.yaml', + (project.file("src/integrationTest/resources/${testConfiguration.dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml' + ] hostConfig.network = createDataPrepperNetwork.getNetworkName() - cmd = ['java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute'] - targetImageId buildDataPrepperDockerImage.getImageId() + targetImageId dataPrepperDockerImage.imageId } -} -def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) { - return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) { - dependsOn createDataPrepperDockerContainerTask - targetContainerId createDataPrepperDockerContainerTask.getContainerId() + tasks.register("start${testConfiguration.testName}", DockerStartContainer) { + dependsOn "create${testConfiguration.testName}" + dependsOn 'startOpenSearchDockerContainer' + mustRunAfter 'startOpenSearchDockerContainer' + targetContainerId tasks.getByName("create${testConfiguration.testName}").getContainerId() } -} -def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) { - return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) { - targetContainerId startDataPrepperDockerContainerTask.getContainerId() + tasks.register("stop${testConfiguration.testName}", DockerStopContainer) { + dependsOn "${testConfiguration.testName}" + targetContainerId tasks.getByName("create${testConfiguration.testName}").getContainerId() } -} -def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) { - return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) { - targetContainerId stopDataPrepperDockerContainerTask.getContainerId() + tasks.register("remove${testConfiguration.testName}", DockerRemoveContainer) { + dependsOn "stop${testConfiguration.testName}" + targetContainerId tasks.getByName("stop${testConfiguration.testName}").getContainerId() } -} -/** - * End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test - * Stops the docker containers when finished - */ -task basicLogEndToEndTest(type: Test) { - dependsOn build - dependsOn startOpenSearchDockerContainer - def createDataPrepperTask = createDataPrepperDockerContainer( - "basicLogDataPrepper", "dataprepper", 2021, 4900, "${BASIC_GROK_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}") - def startDataPrepperTask = startDataPrepperDockerContainer(createDataPrepperTask as DockerCreateContainer) - dependsOn startDataPrepperTask - startDataPrepperTask.mustRunAfter 'startOpenSearchDockerContainer' - // wait for data-preppers to be ready - doFirst { - sleep(15*1000) - } - - description = 'Runs the basic grok end-to-end test.' - group = 'verification' - testClassesDirs = sourceSets.integrationTest.output.classesDirs - classpath = sourceSets.integrationTest.runtimeClasspath + tasks.register(testConfiguration.testName, Test) { + dependsOn build + dependsOn startOpenSearchDockerContainer + dependsOn "start${testConfiguration.testName}" - filter { - includeTestsMatching "org.opensearch.dataprepper.integration.log.EndToEndBasicLogTest.testPipelineEndToEnd*" - } - - finalizedBy stopOpenSearchDockerContainer - def stopDataPrepperTask = stopDataPrepperDockerContainer(startDataPrepperTask as DockerStartContainer) - finalizedBy stopDataPrepperTask - finalizedBy removeDataPrepperDockerContainer(stopDataPrepperTask as DockerStopContainer) - finalizedBy removeDataPrepperNetwork -} + // Wait for Data Prepper image to be ready + doFirst { + sleep(15 * 1000) + } -task parallelGrokStringSubstituteTest(type: Test) { - dependsOn build - dependsOn startOpenSearchDockerContainer - def createDataPrepperTask = createDataPrepperDockerContainer( - "ParallelGrokSubstLogDataPrepper", "dataprepper-pgsts-test", 2021, 4900, "${PARALLEL_GROK_SUBSTITUTE_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}") - def startDataPrepperTask = startDataPrepperDockerContainer(createDataPrepperTask as DockerCreateContainer) - dependsOn startDataPrepperTask - startDataPrepperTask.mustRunAfter 'startOpenSearchDockerContainer' - // wait for data-preppers to be ready - doFirst { - sleep(15*1000) - } + description = testConfiguration.description + group = 'verification' + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath - description = 'Runs the parallel grok and string substitute end-to-end test.' - group = 'verification' - testClassesDirs = sourceSets.integrationTest.output.classesDirs - classpath = sourceSets.integrationTest.runtimeClasspath + filter { + includeTestsMatching testConfiguration.testFilters + } - filter { - includeTestsMatching "org.opensearch.dataprepper.integration.log.ParallelGrokStringSubstituteLogTest.testPipelineEndToEnd*" + finalizedBy stopOpenSearchDockerContainer + finalizedBy "remove${testConfiguration.testName}" + finalizedBy removeDataPrepperNetwork } - finalizedBy stopOpenSearchDockerContainer - def stopDataPrepperTask = stopDataPrepperDockerContainer(startDataPrepperTask as DockerStartContainer) - finalizedBy stopDataPrepperTask - finalizedBy removeDataPrepperDockerContainer(stopDataPrepperTask as DockerStopContainer) - finalizedBy removeDataPrepperNetwork } dependencies { diff --git a/e2e-test/peerforwarder/build.gradle b/e2e-test/peerforwarder/build.gradle index fd017b03fc..2754f4ab34 100644 --- a/e2e-test/peerforwarder/build.gradle +++ b/e2e-test/peerforwarder/build.gradle @@ -8,139 +8,155 @@ import com.bmuschko.gradle.docker.tasks.container.DockerCreateContainer import com.bmuschko.gradle.docker.tasks.container.DockerRemoveContainer import com.bmuschko.gradle.docker.tasks.container.DockerStartContainer import com.bmuschko.gradle.docker.tasks.container.DockerStopContainer -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage /** * End-to-end test docker network */ -def AGGREGATE_PIPELINE_YAML = "aggregate-e2e-pipeline.yml" -def LOG_METRICS_PIPELINE_YAML = "log-metrics-pipeline.yml" -def DATA_PREPPER_CONFIG_LOCAL_NODE = "data_prepper_local_node.yml" -def DATA_PREPPER_CONFIG_STATIC = "data_prepper_static.yml" +def AGGREGATE_PIPELINE_YAML = 'aggregate-e2e-pipeline.yml' +def DATA_PREPPER_CONFIG_LOCAL_NODE = 'data_prepper_local_node.yml' +def DATA_PREPPER_CONFIG_STATIC = 'data_prepper_static.yml' + /** - * DataPrepper Docker tasks + * Represents the configurations needed for peer-forwarder end-to-end tests. */ - -task buildDataPrepperDockerImage(type: DockerBuildImage) { - dependsOn createDataPrepperDockerFile - dockerFile = file('build/docker/Dockerfile') - images.add('integ-test-pipeline-image') -} - -def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int sourcePort, - final String pipelineConfigYAML, final String dataPrepperConfigYAML) { - return tasks.create("create${taskBaseName}", DockerCreateContainer) { - dependsOn buildDataPrepperDockerImage - dependsOn createDataPrepperNetwork - containerName = dataPrepperName - exposePorts('tcp', [2021]) - hostConfig.portBindings = [String.format('%d:2021', sourcePort)] - hostConfig.network = createDataPrepperNetwork.getNetworkName() - hostConfig.binds = [ - (project.file("src/integrationTest/resources/${pipelineConfigYAML}").toString()) : "/app/data-prepper/pipelines/pipelines.yaml", - "/tmp" : "/tmp", - (project.file("src/integrationTest/resources/${dataPrepperConfigYAML}").toString()): "/app/data-prepper/config/data-prepper-config.yaml", - (project.file('src/integrationTest/resources/default_certificate.pem').toString()) : '/app/data-prepper/config/default_certificate.pem', - (project.file('src/integrationTest/resources/default_private_key.pem').toString()): '/app/data-prepper/config/default_private_key.pem' - ] - cmd = ['java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute'] - targetImageId buildDataPrepperDockerImage.getImageId() +class PeerForwarderTestConfiguration { + PeerForwarderTestConfiguration( + String testName, + String description, + String testFilters, + String containerName, + String pipelineConfiguration, + String dataPrepperConfiguration) { + this.testName = testName + this.description = description + this.testFilters = testFilters + this.containerName = containerName + this.pipelineConfiguration = pipelineConfiguration + this.dataPrepperConfiguration = dataPrepperConfiguration } + String testName + String description + String testFilters + String containerName + String pipelineConfiguration + String dataPrepperConfiguration } -def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) { - return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) { - dependsOn createDataPrepperDockerContainerTask - targetContainerId createDataPrepperDockerContainerTask.getContainerId() - } -} +List peerForwarderTestConfigurations = [ + new PeerForwarderTestConfiguration( + 'localAggregateEndToEndTest', + 'Runs the local log aggregation end-to-end test.', + 'org.opensearch.dataprepper.integration.peerforwarder.EndToEndPeerForwarderTest.testAggregatePipelineWithSingleNodeEndToEnd*', + 'data-prepper', + AGGREGATE_PIPELINE_YAML, + DATA_PREPPER_CONFIG_LOCAL_NODE + ), + new PeerForwarderTestConfiguration( + 'staticAggregateEndToEndTest', + 'Runs the local log aggregation end-to-end test.', + 'org.opensearch.dataprepper.integration.peerforwarder.EndToEndPeerForwarderTest.testAggregatePipelineWithMultipleNodesEndToEnd*', + 'node.data-prepper.example.com', + AGGREGATE_PIPELINE_YAML, + DATA_PREPPER_CONFIG_STATIC + ), + new PeerForwarderTestConfiguration( + 'staticLogMetricsEndToEndTest', + 'Runs the local log aggregation end-to-end test.', + 'org.opensearch.dataprepper.integration.peerforwarder.EndToEndLogMetricsTest.testLogMetricsPipelineWithMultipleNodesEndToEnd*', + 'node.data-prepper.example.com', + 'log-metrics-pipeline.yml', + DATA_PREPPER_CONFIG_STATIC + ) +] -def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) { - return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) { - targetContainerId startDataPrepperDockerContainerTask.getContainerId() - } +/** + * Creates a container name from a base name. If the name looks like a DNS entry + * it adds the index after the first part. Otherwise, it appends the index to the end. + * + * @param name The base container name + * @param index The index of the container to create + * @return The container name to use. + */ +static def createContainerName(String name, int index) { + def nameParts = name.split('\\.', 2) + + return "${nameParts[0]}${index}${nameParts.length==2 ? '.' + nameParts[1] : ''}" } -def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) { - return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) { - targetContainerId stopDataPrepperDockerContainerTask.getContainerId() +peerForwarderTestConfigurations.each { testConfiguration -> + tasks.register("start${testConfiguration.testName}All") + tasks.register("remove${testConfiguration.testName}All") + + (0..1).each { containerIndex -> + tasks.register("create${testConfiguration.testName}${containerIndex}", DockerCreateContainer) { + dependsOn dataPrepperDockerImage + dependsOn createDataPrepperNetwork + containerName = createContainerName(testConfiguration.containerName, containerIndex) + exposePorts('tcp', [2021]) + hostConfig.portBindings = ["${2021+containerIndex}:2021"] + hostConfig.binds = [ + (project.file("src/integrationTest/resources/${testConfiguration.pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/test-pipeline.yaml', + (project.file("src/integrationTest/resources/${testConfiguration.dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml', + (project.file('src/integrationTest/resources/default_certificate.pem').toString()) : '/usr/share/data-prepper/config/default_certificate.pem', + (project.file('src/integrationTest/resources/default_private_key.pem').toString()): '/usr/share/data-prepper/config/default_private_key.pem' + ] + hostConfig.network = createDataPrepperNetwork.getNetworkName() + targetImageId dataPrepperDockerImage.imageId + } + + tasks.register("start${testConfiguration.testName}${containerIndex}", DockerStartContainer) { + dependsOn "create${testConfiguration.testName}${containerIndex}" + dependsOn 'startOpenSearchDockerContainer' + mustRunAfter 'startOpenSearchDockerContainer' + targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.named("start${testConfiguration.testName}All").configure { + dependsOn "start${testConfiguration.testName}${containerIndex}" + } + + tasks.register("stop${testConfiguration.testName}${containerIndex}", DockerStopContainer) { + dependsOn "${testConfiguration.testName}" + targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.register("remove${testConfiguration.testName}${containerIndex}", DockerRemoveContainer) { + dependsOn "stop${testConfiguration.testName}${containerIndex}" + targetContainerId tasks.getByName("stop${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.named("remove${testConfiguration.testName}All").configure { + dependsOn "remove${testConfiguration.testName}${containerIndex}" + } } -} -/** - * End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test - * Stops the docker containers when finished - */ -def createEndToEndTest(final String testName, final String includeTestsMatchPattern, - final DockerCreateContainer createDataPrepper1Task, final DockerCreateContainer createDataPrepper2Task) { - return tasks.create(testName, Test) { + tasks.register(testConfiguration.testName, Test) { dependsOn build dependsOn startOpenSearchDockerContainer - def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) - def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) - dependsOn startDataPrepper1Task - dependsOn startDataPrepper2Task - startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer' - startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer' - // wait for data-preppers to be ready + dependsOn "start${testConfiguration.testName}All" + + // Wait for Data Prepper image to be ready doFirst { sleep(15 * 1000) } - description = 'Runs the raw span integration tests.' + description = testConfiguration.description group = 'verification' testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath filter { - includeTestsMatching includeTestsMatchPattern + includeTestsMatching testConfiguration.testFilters } finalizedBy stopOpenSearchDockerContainer - def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) - def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) - finalizedBy stopDataPrepper1Task - finalizedBy stopDataPrepper2Task - finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) - finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) + finalizedBy "remove${testConfiguration.testName}All" finalizedBy removeDataPrepperNetwork } } -// Discovery mode: LOCAL_NODE -def includeLocalAggregateTestsMatchPattern = "org.opensearch.dataprepper.integration.peerforwarder.EndToEndPeerForwarderTest.testAggregatePipelineWithSingleNodeEndToEnd*" - -def createLocalAggregateDataPrepper1Task = createDataPrepperDockerContainer( - "localAggregateDataPrepper1", "dataprepper1", 2021, "${AGGREGATE_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_LOCAL_NODE}") -def createLocalAggregateDataPrepper2Task = createDataPrepperDockerContainer( - "localAggregateDataPrepper2", "dataprepper2", 2022, "${AGGREGATE_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_LOCAL_NODE}") - -def localAggregateEndToEndTest = createEndToEndTest("localAggregateEndToEndTest", includeLocalAggregateTestsMatchPattern, - createLocalAggregateDataPrepper1Task, createLocalAggregateDataPrepper2Task) - -// Discovery mode: STATIC with SSL & mTLS -def includeStaticAggregateTestsMatchPattern = "org.opensearch.dataprepper.integration.peerforwarder.EndToEndPeerForwarderTest.testAggregatePipelineWithMultipleNodesEndToEnd*" - -def createAggregateDataPrepper1Task = createDataPrepperDockerContainer( - "staticAggregateDataPrepper1", "node1.data-prepper.example.com", 2021, "${AGGREGATE_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC}") -def createAggregateDataPrepper2Task = createDataPrepperDockerContainer( - "staticAggregateDataPrepper2", "node2.data-prepper.example.com", 2022, "${AGGREGATE_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC}") - -def staticAggregateEndToEndTest = createEndToEndTest("staticAggregateEndToEndTest", includeStaticAggregateTestsMatchPattern, - createAggregateDataPrepper1Task, createAggregateDataPrepper2Task) - -// Discovery mode: STATIC - Log pipeline metrics e2e test -def includeLocalLogMetricsTestsMatchPattern = "org.opensearch.dataprepper.integration.peerforwarder.EndToEndLogMetricsTest.testLogMetricsPipelineWithMultipleNodesEndToEnd*" - -def createStaticLogMetricsDataPrepper1Task = createDataPrepperDockerContainer( - "staticLogMetricsDataPrepper1", "node1.data-prepper.example.com", 2021, "${LOG_METRICS_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC}") -def createStaticLogMetricsDataPrepper2Task = createDataPrepperDockerContainer( - "staticLogMetricsDataPrepper2", "node2.data-prepper.example.com", 2022, "${LOG_METRICS_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC}") - -def staticLogMetricsEndToEndTest = createEndToEndTest("staticLogMetricsEndToEndTest", includeLocalLogMetricsTestsMatchPattern, - createStaticLogMetricsDataPrepper1Task, createStaticLogMetricsDataPrepper2Task) dependencies { integrationTestImplementation project(':data-prepper-api') diff --git a/e2e-test/peerforwarder/src/integrationTest/resources/data_prepper_static.yml b/e2e-test/peerforwarder/src/integrationTest/resources/data_prepper_static.yml index 910ef43191..e16b43b45c 100644 --- a/e2e-test/peerforwarder/src/integrationTest/resources/data_prepper_static.yml +++ b/e2e-test/peerforwarder/src/integrationTest/resources/data_prepper_static.yml @@ -2,9 +2,9 @@ ssl: false peer_forwarder: port: 4994 ssl: true - ssl_certificate_file: "/app/data-prepper/config/default_certificate.pem" - ssl_key_file: "/app/data-prepper/config/default_private_key.pem" + ssl_certificate_file: "/usr/share/data-prepper/config/default_certificate.pem" + ssl_key_file: "/usr/share/data-prepper/config/default_private_key.pem" discovery_mode: static - static_endpoints: ["node1.data-prepper.example.com", "node2.data-prepper.example.com"] + static_endpoints: ["node0.data-prepper.example.com", "node1.data-prepper.example.com"] authentication: mutual_tls: diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle index c0ff41e62f..b83d977973 100644 --- a/e2e-test/trace/build.gradle +++ b/e2e-test/trace/build.gradle @@ -8,163 +8,197 @@ import com.bmuschko.gradle.docker.tasks.container.DockerCreateContainer import com.bmuschko.gradle.docker.tasks.container.DockerRemoveContainer import com.bmuschko.gradle.docker.tasks.container.DockerStartContainer import com.bmuschko.gradle.docker.tasks.container.DockerStopContainer -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.bmuschko.gradle.docker.tasks.image.DockerPullImage +import org.opensearch.dataprepper.gradle.end_to_end.DockerProviderTask /** - * End-to-end test docker network + * Represents a configuration for a trace test container. */ - -def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml" -def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" -def DATA_PREPPER_CONFIG_YAML = "data_prepper.yml" -def DATA_PREPPER_CONFIG_STATIC_YAML = "data_prepper_static.yml" -def RAW_SPAN_PIPELINE_FROM_BUILD_YAML = "raw-span-e2e-pipeline-from-build.yml" -def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml" +class TraceTestContainerConfiguration { + TraceTestContainerConfiguration( + TaskProvider imageTask, + String pipelineConfiguration, + String dataPrepperConfiguration) { + this.imageTask = imageTask + this.pipelineConfiguration = pipelineConfiguration + this.dataPrepperConfiguration = dataPrepperConfiguration + } + TaskProvider imageTask + String pipelineConfiguration + String dataPrepperConfiguration +} /** - * DataPrepper Docker tasks + * Represents the configurations needed for any end-to-end trace test. */ -task buildDataPrepperDockerImage(type: DockerBuildImage) { - dependsOn createDataPrepperDockerFile - dockerFile = file('build/docker/Dockerfile') - images.add('integ-test-pipeline-image') -} - -def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int grpcPort, - final String pipelineConfigYAML, final String dataPrepperConfigYAML) { - return tasks.create("create${taskBaseName}", DockerCreateContainer) { - dependsOn buildDataPrepperDockerImage - dependsOn createDataPrepperNetwork - containerName = dataPrepperName - exposePorts("tcp", [21890]) - hostConfig.portBindings = [String.format('%d:21890', grpcPort)] - hostConfig.network = createDataPrepperNetwork.getNetworkName() - hostConfig.binds = [(project.file("src/integrationTest/resources/${pipelineConfigYAML}").toString()):"/app/data-prepper/pipelines/pipelines.yaml", - (project.file("src/integrationTest/resources/${dataPrepperConfigYAML}").toString()):"/app/data-prepper/config/data-prepper-config.yaml"] - cmd = ['java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute'] - targetImageId buildDataPrepperDockerImage.getImageId() +class TraceTestConfiguration { + TraceTestConfiguration( + String testName, + String description, + String testFilters, + List containerConfigurations, + String containerNamePrefix) { + this.testName = testName + this.description = description + this.testFilters = testFilters + this.containerConfigurations = containerConfigurations + this.containerNamePrefix = containerNamePrefix } + String testName + String description + String testFilters + List containerConfigurations + String containerNamePrefix } -def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) { - return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) { - dependsOn createDataPrepperDockerContainerTask - targetContainerId createDataPrepperDockerContainerTask.getContainerId() - } -} +def RAW_SPAN_PIPELINE_YAML = 'raw-span-e2e-pipeline.yml' +def DATA_PREPPER_CONFIG_YAML = 'data_prepper.yml' +def RAW_SPAN_TESTS_PATTERN = 'org.opensearch.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*' +def DATA_PREPPER_CONFIG_STATIC_YAML = 'data_prepper_static.yml' +def RELEASED_DATA_PREPPER_DOCKER_IMAGE = 'opensearchproject/data-prepper:latest' -def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) { - return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) { - targetContainerId startDataPrepperDockerContainerTask.getContainerId() - } -} -def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) { - return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) { - targetContainerId stopDataPrepperDockerContainerTask.getContainerId() - } +tasks.register('pullDataPrepperDockerImage', DockerPullImage) { + image = RELEASED_DATA_PREPPER_DOCKER_IMAGE } -task pullDataPrepperDockerImage(type: DockerPullImage) { - image = 'opensearchproject/data-prepper:latest' +tasks.register('latestDataPrepperDockerImage', DockerProviderTask) { + dependsOn 'pullDataPrepperDockerImage' + imageId = RELEASED_DATA_PREPPER_DOCKER_IMAGE } -def createDataPrepperDockerContainerFromPullImage(final String taskBaseName, final String dataPrepperName, final int grpcPort, - final String pipelineConfigYAML, final String dataPrepperConfigYAML) { - return tasks.create("create${taskBaseName}", DockerCreateContainer) { - dependsOn createDataPrepperNetwork - dependsOn pullDataPrepperDockerImage - containerName = dataPrepperName - hostConfig.portBindings = [String.format('%d:21890', grpcPort)] - exposePorts('tcp', [21890]) - hostConfig.network = createDataPrepperNetwork.getNetworkName() - hostConfig.binds = [(project.file("src/integrationTest/resources/${pipelineConfigYAML}").toString()):"/usr/share/data-prepper/pipelines.yaml", - (project.file("src/integrationTest/resources/${dataPrepperConfigYAML}").toString()):"/usr/share/data-prepper/data-prepper-config.yaml"] - targetImageId pullDataPrepperDockerImage.image + +List traceTestConfigurations = [ + new TraceTestConfiguration( + 'rawSpanEndToEndTest', + 'Runs the raw span integration tests.', + RAW_SPAN_TESTS_PATTERN, + [new TraceTestContainerConfiguration( + tasks.named('dataPrepperDockerImage'), + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_YAML), + new TraceTestContainerConfiguration( + tasks.named('dataPrepperDockerImage'), + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_YAML)], + 'data-prepper-raw', + ), + new TraceTestConfiguration( + 'rawSpanPeerForwarderEndToEndTest', + 'Runs the raw span with peer-forwarder integration tests.', + RAW_SPAN_TESTS_PATTERN, + [new TraceTestContainerConfiguration( + tasks.named('dataPrepperDockerImage'), + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_STATIC_YAML), + new TraceTestContainerConfiguration( + tasks.named('dataPrepperDockerImage'), + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_STATIC_YAML)], + 'data-prepper' + ), + new TraceTestConfiguration( + 'rawSpanLatestReleaseCompatibilityEndToEndTest', + 'Runs the raw span integration tests with the latest released Data Prepper as a peer.', + RAW_SPAN_TESTS_PATTERN, + [new TraceTestContainerConfiguration( + tasks.named('dataPrepperDockerImage'), + 'raw-span-e2e-pipeline-from-build.yml', + DATA_PREPPER_CONFIG_STATIC_YAML), + new TraceTestContainerConfiguration( + tasks.named('latestDataPrepperDockerImage'), + 'raw-span-e2e-pipeline-latest-release.yml', + DATA_PREPPER_CONFIG_STATIC_YAML) + ], + 'data-prepper' + ), + new TraceTestConfiguration( + 'serviceMapPeerForwarderEndToEndTest', + 'Runs the service map with peer-forwarder integration tests.', + 'org.opensearch.dataprepper.integration.trace.EndToEndServiceMapTest.testPipelineEndToEnd*', + [new TraceTestContainerConfiguration( + tasks.named('dataPrepperDockerImage'), + 'service-map-e2e-pipeline.yml', + DATA_PREPPER_CONFIG_STATIC_YAML), + new TraceTestContainerConfiguration( + tasks.named('dataPrepperDockerImage'), + 'service-map-e2e-pipeline.yml', + DATA_PREPPER_CONFIG_STATIC_YAML)], + 'data-prepper' + ) +] + + +traceTestConfigurations.each { testConfiguration -> + tasks.register("start${testConfiguration.testName}All") + tasks.register("remove${testConfiguration.testName}All") + + (0.. + tasks.register("create${testConfiguration.testName}${containerIndex}", DockerCreateContainer) { + dependsOn testConfiguration.containerConfigurations.get(containerIndex).imageTask + dependsOn createDataPrepperNetwork + containerName = "${testConfiguration.containerNamePrefix}-${containerIndex}" + exposePorts('tcp', [21890]) + hostConfig.portBindings = ["${21890+containerIndex}:21890"] + hostConfig.binds = [ + (project.file("src/integrationTest/resources/${testConfiguration.containerConfigurations.get(containerIndex).pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/trace-pipeline.yaml', + (project.file("src/integrationTest/resources/${testConfiguration.containerConfigurations.get(containerIndex).dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml' + ] + hostConfig.network = createDataPrepperNetwork.getNetworkName() + targetImageId testConfiguration.containerConfigurations.get(containerIndex).imageTask.get().imageId + } + + tasks.register("start${testConfiguration.testName}${containerIndex}", DockerStartContainer) { + dependsOn "create${testConfiguration.testName}${containerIndex}" + dependsOn 'startOpenSearchDockerContainer' + mustRunAfter 'startOpenSearchDockerContainer' + targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.named("start${testConfiguration.testName}All").configure { + dependsOn "start${testConfiguration.testName}${containerIndex}" + } + + tasks.register("stop${testConfiguration.testName}${containerIndex}", DockerStopContainer) { + dependsOn "${testConfiguration.testName}" + targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.register("remove${testConfiguration.testName}${containerIndex}", DockerRemoveContainer) { + dependsOn "stop${testConfiguration.testName}${containerIndex}" + targetContainerId tasks.getByName("stop${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.named("remove${testConfiguration.testName}All").configure { + dependsOn "remove${testConfiguration.testName}${containerIndex}" + } } -} -/** - * End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test - * Stops the docker containers when finished - */ -def createEndToEndTest(final String testName, final String includeTestsMatchPattern, - final DockerCreateContainer createDataPrepper1Task, final DockerCreateContainer createDataPrepper2Task) { - return tasks.create(testName, Test) { + + tasks.register(testConfiguration.testName, Test) { dependsOn build dependsOn startOpenSearchDockerContainer - def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) - def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) - dependsOn startDataPrepper1Task - dependsOn startDataPrepper2Task - startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer' - startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer' - // wait for data-preppers to be ready + dependsOn "start${testConfiguration.testName}All" + + // Wait for Data Prepper image to be ready doFirst { - sleep(15*1000) + sleep(15 * 1000) } - description = 'Runs the raw span integration tests.' + description = testConfiguration.description group = 'verification' testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath filter { - includeTestsMatching includeTestsMatchPattern + includeTestsMatching testConfiguration.testFilters } - def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) - def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) - finalizedBy stopDataPrepper1Task - finalizedBy stopDataPrepper2Task - finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) - finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) finalizedBy stopOpenSearchDockerContainer + finalizedBy "remove${testConfiguration.testName}All" finalizedBy removeDataPrepperNetwork } } -// raw span e2e test -def includeRawSpanTestsMatchPattern = "org.opensearch.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" - -def createRawSpanDataPrepper1Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper1", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}") -def createRawSpanDataPrepper2Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper2", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}") - -def rawSpanEndToEndTest = createEndToEndTest("rawSpanEndToEndTest", includeRawSpanTestsMatchPattern, - createRawSpanDataPrepper1Task, createRawSpanDataPrepper2Task) - -// raw span with peer forwarding e2e test -def createRawSpanPeerForwarderDataPrepper1Task = createDataPrepperDockerContainer( - "rawSpanPeerForwarderDataPrepper1", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") -def createRawSpanPeerForwarderDataPrepper2Task = createDataPrepperDockerContainer( - "rawSpanPeerForwarderDataPrepper2", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") - -def rawSpanPeerForwarderEndToEndTest = createEndToEndTest("rawSpanPeerForwarderEndToEndTest", includeRawSpanTestsMatchPattern, - createRawSpanPeerForwarderDataPrepper1Task, createRawSpanPeerForwarderDataPrepper2Task) - -// raw span compatibility e2e test -def rawSpanDataPrepperEventFromBuild = createDataPrepperDockerContainer( - "rawSpanDataPrepperEventFromBuild", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_FROM_BUILD_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") -def rawSpanDataPrepperLatestFromPull = createDataPrepperDockerContainerFromPullImage( - "rawSpanDataPrepperLatestFromPull", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") - -def rawSpanLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanLatestReleaseCompatibilityEndToEndTest", - includeRawSpanTestsMatchPattern, - rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull) - -// service map e2e -def includeServiceMapTestsMatchPattern = "org.opensearch.dataprepper.integration.trace.EndToEndServiceMapTest.testPipelineEndToEnd*" - -// service map with peer forwarding e2e test -def createServiceMapPeerForwarderDataPrepper1Task = createDataPrepperDockerContainer( - "serviceMapPeerForwarderDataPrepper1", "dataprepper1", 21890, "${SERVICE_MAP_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") -def createServiceMapPeerForwarderDataPrepper2Task = createDataPrepperDockerContainer( - "serviceMapPeerForwarderDataPrepper2", "dataprepper2", 21891, "${SERVICE_MAP_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") - -def serviceMapPeerForwarderEndToEndTest = createEndToEndTest("serviceMapPeerForwarderEndToEndTest", includeServiceMapTestsMatchPattern, - createServiceMapPeerForwarderDataPrepper1Task, createServiceMapPeerForwarderDataPrepper2Task) dependencies { integrationTestImplementation project(':data-prepper-api') diff --git a/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml b/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml index 9add9cf0b2..b40680acc1 100644 --- a/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml +++ b/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml @@ -3,4 +3,4 @@ peer_forwarder: port: 4994 ssl: false discovery_mode: static - static_endpoints: ["dataprepper1", "dataprepper2"] + static_endpoints: ["data-prepper-0", "data-prepper-1"]