-
Notifications
You must be signed in to change notification settings - Fork 210
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
TST: trace event migration backward compatibility e2e tests #1264
Changes from 5 commits
9084dad
46869a7
e355d2a
d10649d
8220b61
4564080
456fdeb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,8 +30,11 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) { | |
} | ||
|
||
def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml" | ||
def RAW_SPAN_PIPELINE_EVENT_TYPE_YAML = "raw-span-e2e-pipeline-event-type.yml" | ||
def RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML = "raw-span-e2e-pipeline-peer-forwarder-event-type.yml" | ||
def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml" | ||
def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" | ||
def SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML = "service-map-e2e-pipeline-event-type.yml" | ||
|
||
/** | ||
* DataPrepper Docker tasks | ||
|
@@ -45,8 +48,11 @@ task createDataPrepperDockerFile(type: Dockerfile) { | |
workingDir("/app") | ||
copyFile("${dataPrepperJarFilepath}", "/app/data-prepper.jar") | ||
copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}") | ||
copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}", "/app/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}") | ||
copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}", "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}") | ||
copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") | ||
copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}") | ||
copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}", "/app/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}") | ||
copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml") | ||
defaultCommand("java", "-jar", "data-prepper.jar", "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml") | ||
} | ||
|
@@ -148,120 +154,104 @@ task stopOpenSearchDockerContainer(type: DockerStopContainer) { | |
* End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test | ||
* Stops the docker containers when finished | ||
*/ | ||
task rawSpanEndToEndTest(type: Test) { | ||
dependsOn build | ||
dependsOn startOpenSearchDockerContainer | ||
def createDataPrepper1Task = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") | ||
def createDataPrepper2Task = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") | ||
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) | ||
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) | ||
dependsOn startDataPrepper1Task | ||
dependsOn startDataPrepper2Task | ||
startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
// wait for data-preppers to be ready | ||
doFirst { | ||
sleep(10*1000) | ||
def createEndToEndTest(final String testName, final String includeTestsMatchPattern, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nice work removing the repetitive code blocks. |
||
final DockerCreateContainer createDataPrepper1Task, final DockerCreateContainer createDataPrepper2Task) { | ||
return tasks.create(testName, Test) { | ||
dependsOn build | ||
dependsOn startOpenSearchDockerContainer | ||
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) | ||
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) | ||
dependsOn startDataPrepper1Task | ||
dependsOn startDataPrepper2Task | ||
startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
// wait for data-preppers to be ready | ||
doFirst { | ||
sleep(10*1000) | ||
} | ||
|
||
description = 'Runs the raw span integration tests.' | ||
group = 'verification' | ||
testClassesDirs = sourceSets.integrationTest.output.classesDirs | ||
classpath = sourceSets.integrationTest.runtimeClasspath | ||
|
||
filter { | ||
includeTestsMatching includeTestsMatchPattern | ||
} | ||
|
||
finalizedBy stopOpenSearchDockerContainer | ||
def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) | ||
def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) | ||
finalizedBy stopDataPrepper1Task | ||
finalizedBy stopDataPrepper2Task | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperNetwork | ||
} | ||
} | ||
|
||
description = 'Runs the raw span integration tests.' | ||
group = 'verification' | ||
testClassesDirs = sourceSets.integrationTest.output.classesDirs | ||
classpath = sourceSets.integrationTest.runtimeClasspath | ||
def includeRawSpanTestsMatchPattern = "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" | ||
|
||
filter { | ||
includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" | ||
} | ||
def createRawSpanDataPrepperOTLP1Task = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") | ||
def createRawSpanDataPrepperOTLP2Task = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") | ||
|
||
finalizedBy stopOpenSearchDockerContainer | ||
def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) | ||
def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) | ||
finalizedBy stopDataPrepper1Task | ||
finalizedBy stopDataPrepper2Task | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperNetwork | ||
} | ||
// TODO: replace with Event type in 2.0 | ||
def rawSpanOTLPEndToEndTest = createEndToEndTest("rawSpanOTLPEndToEndTest", includeRawSpanTestsMatchPattern, | ||
createRawSpanDataPrepperOTLP1Task, createRawSpanDataPrepperOTLP2Task) | ||
|
||
task rawSpanCompatibilityEndToEndTest(type: Test) { | ||
dependsOn build | ||
dependsOn startOpenSearchDockerContainer | ||
def createDataPrepper1Task = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepperFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") | ||
def createDataPrepper2Task = createDataPrepperDockerContainerFromPullImage( | ||
"rawSpanDataPrepperFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") | ||
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) | ||
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) | ||
dependsOn startDataPrepper1Task | ||
dependsOn startDataPrepper2Task | ||
startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
// wait for data-preppers to be ready | ||
doFirst { | ||
sleep(10*1000) | ||
} | ||
def createRawSpanDataPrepperOTLPTask = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepperOTLP", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") | ||
def createRawSpanDataPrepperEventTask = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepperEvent", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}") | ||
|
||
description = 'Runs the raw span compatibility integration tests.' | ||
group = 'verification' | ||
testClassesDirs = sourceSets.integrationTest.output.classesDirs | ||
classpath = sourceSets.integrationTest.runtimeClasspath | ||
// TODO: remove in 2.0 | ||
def rawSpanOTLPAndEventEndToEndTest = createEndToEndTest("rawSpanOTLPAndEventEndToEndTest", includeRawSpanTestsMatchPattern, | ||
createRawSpanDataPrepperOTLPTask, createRawSpanDataPrepperEventTask) | ||
|
||
filter { | ||
includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" | ||
} | ||
def rawSpanDataPrepperOTLPFromBuild = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepperOTLPFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") | ||
def rawSpanDataPrepperOTLPFromPull = createDataPrepperDockerContainerFromPullImage( | ||
"rawSpanDataPrepperOTLPFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") | ||
|
||
finalizedBy stopOpenSearchDockerContainer | ||
def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) | ||
def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) | ||
finalizedBy stopDataPrepper1Task | ||
finalizedBy stopDataPrepper2Task | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperNetwork | ||
} | ||
// TODO: remove in 2.0 | ||
def rawSpanOTLPLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanOTLPLatestReleaseCompatibilityEndToEndTest", | ||
includeRawSpanTestsMatchPattern, | ||
rawSpanDataPrepperOTLPFromBuild, rawSpanDataPrepperOTLPFromPull) | ||
|
||
task serviceMapEndToEndTest(type: Test) { | ||
dependsOn build | ||
dependsOn startOpenSearchDockerContainer | ||
def createDataPrepper1Task = createDataPrepperDockerContainer( | ||
"serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") | ||
def createDataPrepper2Task = createDataPrepperDockerContainer( | ||
"serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") | ||
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) | ||
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) | ||
dependsOn startDataPrepper1Task | ||
dependsOn startDataPrepper2Task | ||
startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer' | ||
// wait for data-preppers to be ready | ||
doFirst { | ||
sleep(10*1000) | ||
} | ||
def rawSpanDataPrepperEventFromBuild = createDataPrepperDockerContainer( | ||
"rawSpanDataPrepperEventFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}") | ||
def rawSpanDataPrepperLatestFromPull = createDataPrepperDockerContainerFromPullImage( | ||
"rawSpanDataPrepperLatestFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") | ||
|
||
description = 'Runs the service-map integration tests.' | ||
group = 'verification' | ||
testClassesDirs = sourceSets.integrationTest.output.classesDirs | ||
classpath = sourceSets.integrationTest.runtimeClasspath | ||
def rawSpanEventLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanEventLatestReleaseCompatibilityEndToEndTest", | ||
includeRawSpanTestsMatchPattern, | ||
rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull) | ||
|
||
filter { | ||
includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndServiceMapTest*" | ||
} | ||
def includeServiceMapTestsMatchPattern = "com.amazon.dataprepper.integration.trace.EndToEndServiceMapTest*" | ||
|
||
finalizedBy stopOpenSearchDockerContainer | ||
def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) | ||
def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) | ||
finalizedBy stopDataPrepper1Task | ||
finalizedBy stopDataPrepper2Task | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) | ||
finalizedBy removeDataPrepperNetwork | ||
} | ||
def createServiceMapDataPrepperOTLP1Task = createDataPrepperDockerContainer( | ||
"serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") | ||
def createServiceMapDataPrepperOTLP2Task = createDataPrepperDockerContainer( | ||
"serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") | ||
|
||
// TODO: replace with Event type in 2.0 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's provide a link to a Github issue and include a reference to this work in that issue. This way we can easily track the work required when we pick it up. |
||
def serviceMapOTLPEndToEndTest = createEndToEndTest("serviceMapOTLPEndToEndTest", includeServiceMapTestsMatchPattern, | ||
createServiceMapDataPrepperOTLP1Task, createServiceMapDataPrepperOTLP2Task) | ||
|
||
def serviceMapDataPrepperOTLP = createDataPrepperDockerContainer( | ||
"serviceMapDataPrepperOTLP", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") | ||
def serviceMapDataPrepperEvent = createDataPrepperDockerContainer( | ||
"serviceMapDataPrepperEvent", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}") | ||
|
||
// TODO: remove in 2.0 | ||
def serviceMapOTLPAndEventEndToEndTest = createEndToEndTest("serviceMapOTLPAndEventEndToEndTest", includeServiceMapTestsMatchPattern, | ||
serviceMapDataPrepperOTLP, serviceMapDataPrepperEvent) | ||
|
||
dependencies { | ||
integrationTestImplementation project(':data-prepper-plugins:opensearch') | ||
integrationTestImplementation project(':data-prepper-plugins:otel-trace-group-prepper') | ||
integrationTestImplementation project(':data-prepper-plugins:otel-trace-group-processor') | ||
integrationTestImplementation "org.awaitility:awaitility:4.1.1" | ||
integrationTestImplementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}" | ||
integrationTestImplementation 'com.google.protobuf:protobuf-java-util:3.19.4' | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I propose that we keep the
rawSpanCompatibilityEndToEndTest
. It can just be a task which depends on the other two tests.This approach allows for developers to have few test commands to run locally.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So the current rawSpanCompatibilityEndToEndTest is renamed as rawSpanOTLPLatestReleaseCompatibilityEndToEndTest. I only added a similar test with event type named rawSpanEventLatestReleaseCompatibilityEndToEndTest. Can you specify what two other tests this should depend on?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You have renamed
rawSpanCompatibilityEndToEndTest
torawSpanOTLPLatestReleaseCompatibilityEndToEndTest
. So keep that.But, you can also add a task named
rawSpanCompatibilityEndToEndTest
. Just make it depend onrawSpanOTLPLatestReleaseCompatibilityEndToEndTest
andrawSpanEventLatestReleaseCompatibilityEndToEndTest
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Tried the following:
but unfortunately the container conflict between the two tasks remains:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
OK - it appears it isn't quite so easy due to using Docker. Maybe we can improve this later.