Skip to content

Commit

Permalink
Run OSD integ test ci-groups in parallel (#5179)
Browse files Browse the repository at this point in the history 
Signed-off-by: Rishabh Singh <[email protected]>
  • Loading branch information
@rishabh6788
rishabh6788 authored Dec 3, 2024
1 parent 0ad66a0 commit 14f0225
Show file tree
Hide file tree
Showing 8 changed files with 1,348 additions and 453 deletions.
145 changes: 83 additions & 62 deletions jenkins/opensearch-dashboards/integ-test.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* compatible open source license.
*/

lib = library(identifier: 'jenkins@7.3.0', retriever: modernSCM([
lib = library(identifier: 'jenkins@8.0.0', retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))
Expand Down Expand Up @@ -230,73 +230,94 @@ pipeline {
// Must use local variable due to groovy for loop and closure scope
// Or the stage will be fixed to the last item in return when new stages are triggered here
// https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/
def local_component = component.trim()
def local_component_index = componentList.indexOf(local_component)
def wait_seconds = local_component_index * 20

echo "Adding Component: ${local_component}"
componentTests["Run Integtest ${local_component}"] = {
// Using scripted pipelines to trigger dynamic parallel stages
timeout(time: 6, unit: 'HOURS') {
// Changes to run OpenSearch-Dashboards ci-groups in parallel.
// There are total 9 ci-groups
if (component == "OpenSearch-Dashboards") {
for (int i = 1; i <= 9; i++) {
def local_component = 'OpenSearch-Dashboards'
def wait_seconds = i * 20
def ciNum = i.toString()

echo "Adding Component: ${local_component}-ci-group-${ciNum}"

componentTests["Run Integtest ${local_component}-ci-group-${ciNum}"] = {
timeout(time: 3, unit: 'HOURS') {
node(AGENT_LABEL) {
docker.withRegistry('https://public.ecr.aws/') {
docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
try {
stage("${local_component}") {
// Jenkins tend to not clean up workspace at times even though ws clean is called
// Since docker is mounted on the agent node directly so it can communicated with the agent
// This sometimes causes the workspace to retain last run test-results and ends with build failures
// https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99
sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}")
unstash "integtest-opensearch-dashboards-$BUILD_NUMBER"
if (env.platform == 'windows') {
echo "On Windows Platform, unstash repository and download the artifacts"
echo "Downloading from S3: ${artifactPathOpenSearch}"
downloadFromS3(
assumedRoleName: 'opensearch-bundle',
roleAccountNumberCred: 'jenkins-aws-account-public',
downloadPath: "${artifactPathOpenSearch}/",
bucketName: "${ARTIFACT_BUCKET_NAME}",
localPath: "${WORKSPACE}/artifacts",
force: true
)
sh("cp -a $WORKSPACE/artifacts/${artifactPathOpenSearch} $WORKSPACE")
docker.withRegistry('https://public.ecr.aws/') {
docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
try {
stage("${local_component}-ci-group-${ciNum}") {
sh("echo ${local_component}-ci-group-${ciNum} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}")
runIntegTestScriptForOSD(
localComponent: "${local_component}",
switchUserNonRoot: "${switch_user_non_root}",
ciGroup: "${ciNum}",
artifactPathOpenSearch: "${artifactPathOpenSearch}",
artifactPath: "${artifactPath}",
artifactBucketName: "${ARTIFACT_BUCKET_NAME}",
distribution: "${distribution}",
buildManifest: "${BUILD_MANIFEST}",
testManifest: "${TEST_MANIFEST}"
)
}
} catch (e) {
throw new Exception("Error running integtest for component ${local_component}-ci-group-${ciNum}", e)
} finally {
echo "Completed running integtest for component ${local_component}-ci-group-${ciNum}"
uploadTestResults(
buildManifestFileName: BUILD_MANIFEST,
jobName: JOB_NAME
)
postCleanup()
}
}
}
}
}
}
}
} else {
def local_component = component.trim()
def local_component_index = componentList.indexOf(local_component)
def wait_seconds = local_component_index * 20

echo "Downloading from S3: ${artifactPath}"
downloadFromS3(
assumedRoleName: 'opensearch-bundle',
roleAccountNumberCred: 'jenkins-aws-account-public',
downloadPath: "${artifactPath}/",
bucketName: "${ARTIFACT_BUCKET_NAME}",
localPath: "${WORKSPACE}/artifacts",
force: true
)
sh("cp -a $WORKSPACE/artifacts/${artifactPath} $WORKSPACE")
sh("rm -rf $WORKSPACE/artifacts")
echo "Adding Component: ${local_component}"
componentTests["Run Integtest ${local_component}"] = {
// Using scripted pipelines to trigger dynamic parallel stages
timeout(time: 6, unit: 'HOURS') {
node(AGENT_LABEL) {
docker.withRegistry('https://public.ecr.aws/') {
docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
try {
stage("${local_component}") {
// Jenkins tend to not clean up workspace at times even though ws clean is called
// Since docker is mounted on the agent node directly so it can communicated with the agent
// This sometimes causes the workspace to retain last run test-results and ends with build failures
// https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99
sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}")
runIntegTestScriptForOSD(
localComponent: local_component,
switchUserNonRoot: switch_user_non_root,
ciGroup: "",
artifactPathOpenSearch: "${artifactPathOpenSearch}",
artifactPath: "${artifactPath}",
artifactBucketName: "${ARTIFACT_BUCKET_NAME}",
distribution: "${distribution}",
buildManifest: "${BUILD_MANIFEST}",
testManifest: "${TEST_MANIFEST}"
)
}
else {
echo "Not on Windows, unstash repository+artifacts"
}

sh("rm -rf test-results")
runIntegTestScript(
jobName: "$BUILD_JOB_NAME",
componentName: "${local_component}",
buildManifest: "$BUILD_MANIFEST",
testManifest: "manifests/${TEST_MANIFEST}",
localPath: "${WORKSPACE}/${distribution}",
switchUserNonRoot: "${switch_user_non_root}"
} catch (e) {
throw new Exception("Error running integtest for component ${local_component}", e)
} finally {
echo "Completed running integtest for component ${local_component}"
uploadTestResults(
buildManifestFileName: BUILD_MANIFEST,
jobName: JOB_NAME
)
postCleanup()
}
} catch (e) {
throw new Exception("Error running integtest for component ${local_component}", e)
} finally {
echo "Completed running integtest for component ${local_component}"
uploadTestResults(
buildManifestFileName: BUILD_MANIFEST,
jobName: JOB_NAME
)
postCleanup()
}
}
}
Expand Down
19 changes: 6 additions & 13 deletions src/test_workflow/integ_test/integ_test_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,19 +43,12 @@ def run(self) -> TestSuiteResults:
if component.name in self.test_manifest.components:
test_config = self.test_manifest.components[component.name]
if test_config.integ_test:
if 'ci-groups' in test_config.integ_test.keys():
orig_component_name = component.name
for i in range(1, test_config.integ_test['ci-groups'] + 1):
component.name = f"{orig_component_name}-ci-group-{i}"
test_suite = self.__create_test_suite__(component, test_config, work_dir.path)
test_results = test_suite.execute_tests()
[self.test_recorder.test_results_logs.generate_component_yml(result_data) for result_data in test_suite.result_data]
all_results.append(component.name, test_results)
else:
test_suite = self.__create_test_suite__(component, test_config, work_dir.path)
test_results = test_suite.execute_tests()
[self.test_recorder.test_results_logs.generate_component_yml(result_data) for result_data in test_suite.result_data]
all_results.append(component.name, test_results)
if self.args.ci_group:
component.name = f"{component.name}-ci-group-{self.args.ci_group}"
test_suite = self.__create_test_suite__(component, test_config, work_dir.path)
test_results = test_suite.execute_tests()
[self.test_recorder.test_results_logs.generate_component_yml(result_data) for result_data in test_suite.result_data]
all_results.append(component.name, test_results)
else:
logging.info(f"Skipping integ-tests for {component.name}, as it is currently not supported")
else:
Expand Down
3 changes: 3 additions & 0 deletions src/test_workflow/test_args.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class TestArgs:
logging_level: int
test_manifest_path: str
paths: dict
ci_group: str

def __init__(self) -> None:
parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle")
Expand All @@ -36,6 +37,7 @@ def __init__(self) -> None:
parser.add_argument(
"-v", "--verbose", help="Show more verbose output.", action="store_const", default=logging.INFO, const=logging.DEBUG, dest="logging_level"
)
parser.add_argument("--ci-group", type=str, default=None, help="ci group number.")
args = parser.parse_args()
self.test_run_id = args.test_run_id or uuid.uuid4().hex
self.components = args.components
Expand All @@ -44,6 +46,7 @@ def __init__(self) -> None:
self.test_manifest_path = args.test_manifest_path
self.paths = args.paths
self.base_path = args.base_path
self.ci_group = args.ci_group


TestArgs.__test__ = False # type:ignore
Loading

0 comments on commit 14f0225

Please sign in to comment.