Skip to content

Commit

Permalink
Jenkins Job to run OSDInteg in Parallel (#3465)
Browse files Browse the repository at this point in the history 
Signed-off-by: Divya Madala <[email protected]>
  • Loading branch information
@Divyaasm
Divyaasm authored May 4, 2023
1 parent 904abbc commit 936f2dd
Show file tree
Hide file tree
Showing 3 changed files with 586 additions and 70 deletions.
109 changes: 83 additions & 26 deletions jenkins/opensearch-dashboards/integ-test.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,19 @@ pipeline {
ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name')
}
parameters {
string(
name: 'COMPONENT_NAME',
description: 'If this field contains one or more component names (e.g. notificationsDashboards indexManagementDashboards ...) separated by space, will test with "--component ...", else test everything in the TEST_MANIFEST..',
trim: true
)
string(
name: 'TEST_MANIFEST',
description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.',
trim: true
)
string(
name: 'BUILD_MANIFEST_URL',
description: 'The build manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.5.0/5367/linux/x64/tar/builds/opensearch-dashboards/manifest.yml',
description: 'The build manifest URL for OpenSearch Dashboards, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.5.0/5367/linux/x64/tar/builds/opensearch-dashboards/manifest.yml.',
trim: true
)
string(
Expand Down Expand Up @@ -98,29 +103,38 @@ pipeline {
currentBuild.result = 'ABORTED'
error("OSD Version $version does not match OS Version $versionOpenSearch")
}

}
}
}
stage('integ-test') {
agent {
docker {
label AGENT_LABEL
image docker_images["$distribution"]
args docker_args["$distribution"]
registryUrl 'https://public.ecr.aws/'
alwaysPull true
post {
always {
postCleanup()
}
}
}
stage('integ-test') {
// Need to run this directly on agent node here in order to trigger stages with docker container and avoid docker within docker situation
// Can only be run in runner that is at least 50GB per container
agent { label AGENT_LABEL }
steps {
script {
def buildManifestObj = downloadBuildManifest(

downloadBuildManifest(
url: BUILD_MANIFEST_URL,
path: BUILD_MANIFEST
)

def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
def componentDefaultList = buildManifestObj.getNames()
def componentList = COMPONENT_NAME ? COMPONENT_NAME.trim().split(" ") as List : componentDefaultList
String switch_user_non_root = (distribution.equals('rpm') || distribution.equals('deb')) ? 'true' : 'false'
echo "switch_user_non_root: ${switch_user_non_root}"
echo "componentList: ${componentList}"

for (component_check in componentList) {
if (!componentDefaultList.contains(component_check)) {
error("${component_check} is not in build manifest: ${componentDefaultList}, exit 1")
}
}

echo "Downloading from S3: ${artifactPathOpenSearch}"
downloadFromS3(
Expand All @@ -144,24 +158,67 @@ pipeline {
)
sh("cp -a $WORKSPACE/artifacts/${artifactPath} $WORKSPACE")

runIntegTestScript(
jobName: "$BUILD_JOB_NAME",
componentName: 'functionalTestDashboards',
buildManifest: "$BUILD_MANIFEST",
testManifest: "manifests/${TEST_MANIFEST}",
localPath: "${WORKSPACE}/${distribution}",
switchUserNonRoot: "${switch_user_non_root}"
)
// Stash the current working directory files, aka opensearch-build repo
// Unstash later in each triggered stage to run integTest
stash includes: "**", name: "integtest-opensearch-dashboards-$BUILD_NUMBER"

componentTests = [:]

for (component in componentList) {
// Must use local variable due to groovy for loop and closure scope
// Or the stage will be fixed to the last item in return when new stages are triggered here
// https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/
def local_component = component.trim()
def local_component_index = componentList.indexOf(local_component)
def wait_seconds = local_component_index * 10

echo "Adding Component: ${local_component}"
componentTests["Run Integtest ${local_component}"] = {
// Using scripted pipelines to trigger dynamic parallel stages
timeout(time: 2, unit: 'HOURS') {
node(AGENT_LABEL) {
docker.withRegistry('https://public.ecr.aws/') {
docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
try {
stage("${local_component}") {
// Jenkins tend to not clean up workspace at times even though ws clean is called
// Since docker is mounted on the agent node directly so it can communicated with the agent
// This sometimes causes the workspace to retain last run test-results and ends with build failures
// https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99
sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}")
unstash "integtest-opensearch-dashboards-$BUILD_NUMBER"
sh("rm -rf test-results")
runIntegTestScript(
jobName: "$BUILD_JOB_NAME",
componentName: "${local_component}",
buildManifest: "$BUILD_MANIFEST",
testManifest: "manifests/${TEST_MANIFEST}",
localPath: "${WORKSPACE}/${distribution}",
switchUserNonRoot: "${switch_user_non_root}"
)
}
} catch (e) {
echo "Error running integtest for component ${local_component}"
throw e
} finally {
echo "Completed running integtest for component ${local_component}"
uploadTestResults(
buildManifestFileName: BUILD_MANIFEST,
jobName: JOB_NAME
)
postCleanup()
}
}
}
}
}
}
}
parallel componentTests
}
}
post {
always {
script {
uploadTestResults(
buildManifestFileName: BUILD_MANIFEST,
jobName: JOB_NAME
)
}
postCleanup()
}
}
Expand Down
53 changes: 50 additions & 3 deletions tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ import org.junit.Before
import org.junit.Test
import org.yaml.snakeyaml.Yaml
import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
import static org.hamcrest.CoreMatchers.hasItem
import static org.hamcrest.CoreMatchers.hasItems
import static org.hamcrest.MatcherAssert.assertThat
import static org.junit.jupiter.api.Assertions.assertThrows

class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {

Expand All @@ -32,8 +37,9 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
def testManifest = "tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml"
def buildId = 215
def buildManifest = "tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml"
def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/${buildId}/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz"
def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/${buildId}/linux/x64/tar/builds/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz"
def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host"
def bucketName = 'job-s3-bucket-name'

binding.setVariable('env', ['BUILD_NUMBER': '215'])
binding.setVariable('ARTIFACT_BUCKET_NAME', 'DUMMY_BUCKET_NAME')
Expand All @@ -53,7 +59,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
binding.setVariable('BUILD_MANIFEST', buildManifest)
binding.setVariable('BUILD_ID', "${buildId}")
def env = binding.getVariable('env')
env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11']
env['DOCKER_AGENT'] = [image:'opensearchstaging/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2', args:'-e JAVA_HOME=/opt/java/openjdk-11']
binding.getVariable('currentBuild').upstreamBuilds = [[fullProjectName: jobName]]

helper.registerAllowedMethod("s3Download", [Map])
Expand All @@ -73,6 +79,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
closure.delegate = delegate
return helper.callClosure(closure)
})
helper.addFileExistsMock("manifests/${testManifest}", true)
helper.registerAllowedMethod("s3Upload", [Map])
helper.registerAllowedMethod('fileExists', [String.class], { args ->
return true;
Expand All @@ -83,8 +90,48 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
}

@Test
void integTests_runs_consistently() {
void integTests_runs_for_all_components() {
super.testPipeline('jenkins/opensearch-dashboards/integ-test.jenkinsfile',
'tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile')
assertThat(getCommandExecutions('sh', 'test.sh'), hasItems(
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component ganttChartDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component indexManagementDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component anomalyDetectionDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component securityDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component functionalTestDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component OpenSearch-Dashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component alertingDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component queryWorkbenchDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component reportsDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component observabilityDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString()
))
}

@Test
void checkUploadResults() {
runScript('jenkins/opensearch-dashboards/integ-test.jenkinsfile')
assertThat(getCommandExecutions('s3Upload', ''), hasItem('{file=test-results, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/1.2.0/215/linux/x64/tar/test-results}'))
}

@Test
void checkIfRunningInParallel(){
runScript('jenkins/opensearch-dashboards/integ-test.jenkinsfile')
assertThat(getCommandExecutions('parallel', ''), hasItem('{Run Integtest ganttChartDashboards=groovy.lang.Closure, Run Integtest indexManagementDashboards=groovy.lang.Closure, Run Integtest anomalyDetectionDashboards=groovy.lang.Closure, Run Integtest OpenSearch-Dashboards=groovy.lang.Closure, Run Integtest securityDashboards=groovy.lang.Closure, Run Integtest functionalTestDashboards=groovy.lang.Closure, Run Integtest alertingDashboards=groovy.lang.Closure, Run Integtest queryWorkbenchDashboards=groovy.lang.Closure, Run Integtest reportsDashboards=groovy.lang.Closure, Run Integtest observabilityDashboards=groovy.lang.Closure}'))
}

def getCommandExecutions(methodName, command) {
def shCommands = helper.callStack.findAll {
call ->
call.methodName == methodName
}.
collect {
call ->
callArgsToString(call)
}.findAll {
shCommand ->
shCommand.contains(command)
}

return shCommands
}
}
Loading

0 comments on commit 936f2dd

Please sign in to comment.