diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index a0158819e3..f576301e85 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -92,7 +92,7 @@ logTestConfigurations.each { testConfiguration -> dependsOn startOpenSearchDockerContainer dependsOn "start${testConfiguration.testName}" - // wait for Data Prepper image to be ready + // Wait for Data Prepper image to be ready doFirst { sleep(15 * 1000) } diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle index c0ff41e62f..5e8b7194fb 100644 --- a/e2e-test/trace/build.gradle +++ b/e2e-test/trace/build.gradle @@ -11,14 +11,202 @@ import com.bmuschko.gradle.docker.tasks.container.DockerStopContainer import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.bmuschko.gradle.docker.tasks.image.DockerPullImage +class TraceTestContainerConfiguration { + TraceTestContainerConfiguration( + String image, + String sourceTask, + String pipelineConfiguration, + String dataPrepperConfiguration) { + this.image = image + this.sourceTask = sourceTask + this.pipelineConfiguration = pipelineConfiguration + this.dataPrepperConfiguration = dataPrepperConfiguration + } + String image + String sourceTask + String pipelineConfiguration + String dataPrepperConfiguration +} + +/** + * Represents the configurations needed for any end-to-end log test. + */ +class TraceTestConfiguration { + TraceTestConfiguration( + String testName, + String description, + String testFilters, + List containerConfigurations, + String containerNamePrefix) { + this.testName = testName + this.description = description + this.testFilters = testFilters + this.containerConfigurations = containerConfigurations + this.containerNamePrefix = containerNamePrefix + } + String testName + String description + String testFilters + List containerConfigurations + String containerNamePrefix +} + +def RAW_SPAN_PIPELINE_YAML = 'raw-span-e2e-pipeline.yml' +def DATA_PREPPER_CONFIG_YAML = 'data_prepper.yml' +def RAW_SPAN_TESTS_PATTERN = 'org.opensearch.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*' +def DATA_PREPPER_CONFIG_STATIC_YAML = 'data_prepper_static.yml' +def RELEASED_DATA_PREPPER_DOCKER_IMAGE = 'opensearchproject/data-prepper:latest' + +List traceTestConfigurations = [ + new TraceTestConfiguration( + 'rawSpanEndToEndTest', + 'Runs the raw span integration tests.', + RAW_SPAN_TESTS_PATTERN, + [new TraceTestContainerConfiguration( + "${project.rootProject.name}:${project.version}", + ':release:docker:docker', + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_YAML), + new TraceTestContainerConfiguration( + "${project.rootProject.name}:${project.version}", + ':release:docker:docker', + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_YAML)], + 'data-prepper-raw', + ), + new TraceTestConfiguration( + 'rawSpanPeerForwarderEndToEndTest', + 'Runs the raw span with peer-forwarder integration tests.', + RAW_SPAN_TESTS_PATTERN, + [new TraceTestContainerConfiguration( + "${project.rootProject.name}:${project.version}", + ':release:docker:docker', + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_STATIC_YAML), + new TraceTestContainerConfiguration( + "${project.rootProject.name}:${project.version}", + ':release:docker:docker', + RAW_SPAN_PIPELINE_YAML, + DATA_PREPPER_CONFIG_STATIC_YAML)], + 'data-prepper' + ), + new TraceTestConfiguration( + 'rawSpanLatestReleaseCompatibilityEndToEndTest', + 'Runs the raw span integration tests with the latest released Data Prepper as a peer.', + RAW_SPAN_TESTS_PATTERN, + [new TraceTestContainerConfiguration( + "${project.rootProject.name}:${project.version}", + ':release:docker:docker', + 'raw-span-e2e-pipeline-from-build.yml', + DATA_PREPPER_CONFIG_STATIC_YAML), + new TraceTestContainerConfiguration( + RELEASED_DATA_PREPPER_DOCKER_IMAGE, + 'pullDataPrepperDockerImage', + 'raw-span-e2e-pipeline-latest-release.yml', + DATA_PREPPER_CONFIG_STATIC_YAML) + ], + 'data-prepper' + ), + new TraceTestConfiguration( + 'serviceMapPeerForwarderEndToEndTest', + 'Runs the service map with peer-forwarder integration tests.', + 'org.opensearch.dataprepper.integration.trace.EndToEndServiceMapTest.testPipelineEndToEnd*', + [new TraceTestContainerConfiguration( + "${project.rootProject.name}:${project.version}", + ':release:docker:docker', + 'service-map-e2e-pipeline.yml', + DATA_PREPPER_CONFIG_STATIC_YAML), + new TraceTestContainerConfiguration( + "${project.rootProject.name}:${project.version}", + ':release:docker:docker', + 'service-map-e2e-pipeline.yml', + DATA_PREPPER_CONFIG_STATIC_YAML)], + 'data-prepper' + ) +] + +tasks.register('pullDataPrepperDockerImage', DockerPullImage) { + image = RELEASED_DATA_PREPPER_DOCKER_IMAGE +} + + +traceTestConfigurations.each { testConfiguration -> + tasks.register("start${testConfiguration.testName}All") + tasks.register("remove${testConfiguration.testName}All") + + (0.. + tasks.register("create${testConfiguration.testName}${containerIndex}", DockerCreateContainer) { + dependsOn testConfiguration.containerConfigurations.get(containerIndex).sourceTask + dependsOn createDataPrepperNetwork + containerName = testConfiguration.containerNamePrefix + containerName = "${testConfiguration.containerNamePrefix}-${containerIndex}" + exposePorts('tcp', [21890]) + hostConfig.portBindings = ["${21890+containerIndex}:21890"] + hostConfig.binds = [ + (project.file("src/integrationTest/resources/${testConfiguration.containerConfigurations.get(containerIndex).pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/trace-pipeline.yaml', + (project.file("src/integrationTest/resources/${testConfiguration.containerConfigurations.get(containerIndex).dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml' + ] + hostConfig.network = createDataPrepperNetwork.getNetworkName() + targetImageId testConfiguration.containerConfigurations.get(containerIndex).image + } + + tasks.register("start${testConfiguration.testName}${containerIndex}", DockerStartContainer) { + dependsOn "create${testConfiguration.testName}${containerIndex}" + dependsOn 'startOpenSearchDockerContainer' + mustRunAfter 'startOpenSearchDockerContainer' + targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.named("start${testConfiguration.testName}All").configure { + dependsOn "start${testConfiguration.testName}${containerIndex}" + } + + tasks.register("stop${testConfiguration.testName}${containerIndex}", DockerStopContainer) { + dependsOn "${testConfiguration.testName}" + targetContainerId tasks.getByName("create${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.register("remove${testConfiguration.testName}${containerIndex}", DockerRemoveContainer) { + dependsOn "stop${testConfiguration.testName}${containerIndex}" + targetContainerId tasks.getByName("stop${testConfiguration.testName}${containerIndex}").getContainerId() + } + + tasks.named("remove${testConfiguration.testName}All").configure { + dependsOn "remove${testConfiguration.testName}${containerIndex}" + } + } + + tasks.register(testConfiguration.testName, Test) { + dependsOn build + dependsOn startOpenSearchDockerContainer + dependsOn "start${testConfiguration.testName}All" + + // Wait for Data Prepper image to be ready + doFirst { + sleep(15 * 1000) + } + + description = testConfiguration.description + group = 'verification' + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath + + filter { + includeTestsMatching testConfiguration.testFilters + } + + finalizedBy stopOpenSearchDockerContainer + finalizedBy "remove${testConfiguration.testName}All" + finalizedBy removeDataPrepperNetwork + } +} + + /** * End-to-end test docker network */ -def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml" def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" -def DATA_PREPPER_CONFIG_YAML = "data_prepper.yml" -def DATA_PREPPER_CONFIG_STATIC_YAML = "data_prepper_static.yml" def RAW_SPAN_PIPELINE_FROM_BUILD_YAML = "raw-span-e2e-pipeline-from-build.yml" def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml" @@ -47,29 +235,6 @@ def createDataPrepperDockerContainer(final String taskBaseName, final String dat } } -def startDataPrepperDockerContainer(final DockerCreateContainer createDataPrepperDockerContainerTask) { - return tasks.create("start${createDataPrepperDockerContainerTask.getName()}", DockerStartContainer) { - dependsOn createDataPrepperDockerContainerTask - targetContainerId createDataPrepperDockerContainerTask.getContainerId() - } -} - -def stopDataPrepperDockerContainer(final DockerStartContainer startDataPrepperDockerContainerTask) { - return tasks.create("stop${startDataPrepperDockerContainerTask.getName()}", DockerStopContainer) { - targetContainerId startDataPrepperDockerContainerTask.getContainerId() - } -} - -def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDockerContainerTask) { - return tasks.create("remove${stopDataPrepperDockerContainerTask.getName()}", DockerRemoveContainer) { - targetContainerId stopDataPrepperDockerContainerTask.getContainerId() - } -} - -task pullDataPrepperDockerImage(type: DockerPullImage) { - image = 'opensearchproject/data-prepper:latest' -} - def createDataPrepperDockerContainerFromPullImage(final String taskBaseName, final String dataPrepperName, final int grpcPort, final String pipelineConfigYAML, final String dataPrepperConfigYAML) { return tasks.create("create${taskBaseName}", DockerCreateContainer) { @@ -84,65 +249,10 @@ def createDataPrepperDockerContainerFromPullImage(final String taskBaseName, fin targetImageId pullDataPrepperDockerImage.image } } -/** - * End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test - * Stops the docker containers when finished - */ -def createEndToEndTest(final String testName, final String includeTestsMatchPattern, - final DockerCreateContainer createDataPrepper1Task, final DockerCreateContainer createDataPrepper2Task) { - return tasks.create(testName, Test) { - dependsOn build - dependsOn startOpenSearchDockerContainer - def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) - def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) - dependsOn startDataPrepper1Task - dependsOn startDataPrepper2Task - startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer' - startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer' - // wait for data-preppers to be ready - doFirst { - sleep(15*1000) - } - - description = 'Runs the raw span integration tests.' - group = 'verification' - testClassesDirs = sourceSets.integrationTest.output.classesDirs - classpath = sourceSets.integrationTest.runtimeClasspath - - filter { - includeTestsMatching includeTestsMatchPattern - } - - def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) - def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) - finalizedBy stopDataPrepper1Task - finalizedBy stopDataPrepper2Task - finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) - finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) - finalizedBy stopOpenSearchDockerContainer - finalizedBy removeDataPrepperNetwork - } -} // raw span e2e test def includeRawSpanTestsMatchPattern = "org.opensearch.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" -def createRawSpanDataPrepper1Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper1", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}") -def createRawSpanDataPrepper2Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper2", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_YAML}") - -def rawSpanEndToEndTest = createEndToEndTest("rawSpanEndToEndTest", includeRawSpanTestsMatchPattern, - createRawSpanDataPrepper1Task, createRawSpanDataPrepper2Task) - -// raw span with peer forwarding e2e test -def createRawSpanPeerForwarderDataPrepper1Task = createDataPrepperDockerContainer( - "rawSpanPeerForwarderDataPrepper1", "dataprepper1", 21890, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") -def createRawSpanPeerForwarderDataPrepper2Task = createDataPrepperDockerContainer( - "rawSpanPeerForwarderDataPrepper2", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") - -def rawSpanPeerForwarderEndToEndTest = createEndToEndTest("rawSpanPeerForwarderEndToEndTest", includeRawSpanTestsMatchPattern, - createRawSpanPeerForwarderDataPrepper1Task, createRawSpanPeerForwarderDataPrepper2Task) // raw span compatibility e2e test def rawSpanDataPrepperEventFromBuild = createDataPrepperDockerContainer( @@ -150,21 +260,10 @@ def rawSpanDataPrepperEventFromBuild = createDataPrepperDockerContainer( def rawSpanDataPrepperLatestFromPull = createDataPrepperDockerContainerFromPullImage( "rawSpanDataPrepperLatestFromPull", "dataprepper2", 21891, "${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") -def rawSpanLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanLatestReleaseCompatibilityEndToEndTest", - includeRawSpanTestsMatchPattern, - rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull) - -// service map e2e -def includeServiceMapTestsMatchPattern = "org.opensearch.dataprepper.integration.trace.EndToEndServiceMapTest.testPipelineEndToEnd*" - -// service map with peer forwarding e2e test -def createServiceMapPeerForwarderDataPrepper1Task = createDataPrepperDockerContainer( - "serviceMapPeerForwarderDataPrepper1", "dataprepper1", 21890, "${SERVICE_MAP_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") -def createServiceMapPeerForwarderDataPrepper2Task = createDataPrepperDockerContainer( - "serviceMapPeerForwarderDataPrepper2", "dataprepper2", 21891, "${SERVICE_MAP_PIPELINE_YAML}", "${DATA_PREPPER_CONFIG_STATIC_YAML}") +//def rawSpanLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanLatestReleaseCompatibilityEndToEndTest", +// includeRawSpanTestsMatchPattern, +// rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull) -def serviceMapPeerForwarderEndToEndTest = createEndToEndTest("serviceMapPeerForwarderEndToEndTest", includeServiceMapTestsMatchPattern, - createServiceMapPeerForwarderDataPrepper1Task, createServiceMapPeerForwarderDataPrepper2Task) dependencies { integrationTestImplementation project(':data-prepper-api') diff --git a/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml b/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml index 9add9cf0b2..b40680acc1 100644 --- a/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml +++ b/e2e-test/trace/src/integrationTest/resources/data_prepper_static.yml @@ -3,4 +3,4 @@ peer_forwarder: port: 4994 ssl: false discovery_mode: static - static_endpoints: ["dataprepper1", "dataprepper2"] + static_endpoints: ["data-prepper-0", "data-prepper-1"]