From a803ec9070af9c8541b893aad4f5e23f76cf71bf Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 2 Feb 2022 14:04:28 -0800 Subject: [PATCH] Add integ tests to OpenSearch distribution build jenkins job (#1517) * Hook up integ tests to distribution-build Signed-off-by: Tyler Ohlsen * yamlfix test manifest file Signed-off-by: Tyler Ohlsen * Remove unnecessary BuildManifest fns Signed-off-by: Tyler Ohlsen * Remove runIntegTests(); refactor buildDockerImage vars as params Signed-off-by: Tyler Ohlsen * Comment out notification block on dashboards dist build job Signed-off-by: Tyler Ohlsen * Test out stabler integ test job Signed-off-by: Tyler Ohlsen * Add test pipeline for pulling nested build info Signed-off-by: Tyler Ohlsen * Remove test pipeline Signed-off-by: Tyler Ohlsen * Add back notifications in OSD dist build Signed-off-by: Tyler Ohlsen * Remove few extra test output lines Signed-off-by: Tyler Ohlsen * Clean up buildDockerImage tests Signed-off-by: Tyler Ohlsen * Update params in check-for-build cron; add icon in notifications Signed-off-by: Tyler Ohlsen * Simplify notification Signed-off-by: Tyler Ohlsen * Change references back to prod jenkins jobs Signed-off-by: Tyler Ohlsen --- jenkins/check-for-build.jenkinsfile | 24 +++-- .../distribution-build.jenkinsfile | 23 ++++- .../opensearch/distribution-build.jenkinsfile | 94 ++++++++++++++++--- jenkins/opensearch/integ-test.jenkinsfile | 2 +- src/jenkins/BuildManifest.groovy | 24 +++++ src/jenkins/InputManifest.groovy | 20 ---- tests/jenkins/data/opensearch-1.3.0-test.yml | 91 ++++++++++++++++++ .../jobs/ArchiveAssembleUpload_Jenkinsfile | 2 +- .../ArchiveAssembleUpload_Jenkinsfile.txt | 20 ++-- tests/jenkins/jobs/AssembleUpload_Jenkinsfile | 2 +- .../jobs/AssembleUpload_Jenkinsfile.txt | 12 +-- tests/jenkins/jobs/BuildArchive_Jenkinsfile | 2 +- .../jenkins/jobs/BuildArchive_Jenkinsfile.txt | 4 +- .../jobs/BuildAssembleUpload_Jenkinsfile | 2 +- .../jobs/BuildAssembleUpload_Jenkinsfile.txt | 18 ++-- .../jenkins/jobs/BuildDockerImage_Jenkinsfile | 4 +- ...ildDockerImage_Jenkinsfile_builds_both.txt | 5 +- ...ildDockerImage_Jenkinsfile_skips_arm64.txt | 5 +- ...uildDockerImage_Jenkinsfile_skips_both.txt | 5 +- ...BuildDockerImage_Jenkinsfile_skips_x64.txt | 5 +- tests/jenkins/jobs/BuildManifest_Jenkinsfile | 3 + .../jobs/BuildManifest_Jenkinsfile.txt | 5 + .../jobs/BuildUploadManifestSHA_Jenkinsfile | 2 +- .../BuildUploadManifestSHA_Jenkinsfile.txt | 6 +- .../Build_OpenSearch_Dashboards_Jenkinsfile | 21 ++++- tests/jenkins/jobs/GetManifestSHA_Jenkinsfile | 2 +- ...ManifestSHA_Jenkinsfile_does_not_exist.txt | 4 +- .../GetManifestSHA_Jenkinsfile_exists.txt | 4 +- tests/jenkins/jobs/InputManifest_Jenkinsfile | 4 - .../jobs/InputManifest_Jenkinsfile.txt | 6 -- vars/README.md | 5 + vars/archiveAssembleUpload.groovy | 13 ++- vars/assembleManifest.groovy | 4 +- vars/assembleUpload.groovy | 9 +- vars/buildArchive.groovy | 2 +- vars/buildAssembleUpload.groovy | 13 ++- vars/buildDockerImage.groovy | 15 +-- vars/buildManifest.groovy | 2 +- vars/getManifestSHA.groovy | 10 +- vars/uploadArtifacts.groovy | 2 +- 40 files changed, 337 insertions(+), 159 deletions(-) create mode 100644 tests/jenkins/data/opensearch-1.3.0-test.yml diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index a194fb7d40..b87a3a612d 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -8,14 +8,14 @@ pipeline { triggers { parameterizedCron ''' H/10 * * * * %INPUT_MANIFEST=1.3.0/opensearch-dashboards-1.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards - H/10 * * * * %INPUT_MANIFEST=1.2.4/opensearch-1.2.4.yml;TARGET_JOB_NAME=distribution-build-opensearch - H/10 * * * * %INPUT_MANIFEST=1.3.0/opensearch-1.3.0.yml;TARGET_JOB_NAME=distribution-build-opensearch - H 1 * * * %INPUT_MANIFEST=1.1.1/opensearch-1.1.1.yml;TARGET_JOB_NAME=distribution-build-opensearch + H/10 * * * * %INPUT_MANIFEST=1.2.4/opensearch-1.2.4.yml;TEST_MANIFEST=1.2.4/opensearch-1.2.4-test.yml;TARGET_JOB_NAME=distribution-build-opensearch + H/10 * * * * %INPUT_MANIFEST=1.3.0/opensearch-1.3.0.yml;TEST_MANIFEST=1.3.0/opensearch-1.3.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch + H 1 * * * %INPUT_MANIFEST=1.1.1/opensearch-1.1.1.yml;TEST_MANIFEST=1.1.1/opensearch-1.1.1-test.yml;TARGET_JOB_NAME=distribution-build-opensearch H 1 * * * %INPUT_MANIFEST=1.1.1/opensearch-dashboards-1.1.1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards - H 1 * * * %INPUT_MANIFEST=1.2.1/opensearch-1.2.1.yml;TARGET_JOB_NAME=distribution-build-opensearch - H 1 * * * %INPUT_MANIFEST=1.2.2/opensearch-1.2.2.yml;TARGET_JOB_NAME=distribution-build-opensearch - H 1 * * * %INPUT_MANIFEST=1.2.3/opensearch-1.2.3.yml;TARGET_JOB_NAME=distribution-build-opensearch - H 1 * * * %INPUT_MANIFEST=2.0.0/opensearch-2.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch + H 1 * * * %INPUT_MANIFEST=1.2.1/opensearch-1.2.1.yml;TEST_MANIFEST=1.2.1/opensearch-1.2.1-test.yml;TARGET_JOB_NAME=distribution-build-opensearch + H 1 * * * %INPUT_MANIFEST=1.2.2/opensearch-1.2.2.yml;TEST_MANIFEST=1.2.2/opensearch-1.2.2-test.yml;TARGET_JOB_NAME=distribution-build-opensearch + H 1 * * * %INPUT_MANIFEST=1.2.3/opensearch-1.2.3.yml;TEST_MANIFEST=1.2.3/opensearch-1.2.3-test.yml;TARGET_JOB_NAME=distribution-build-opensearch + H 1 * * * %INPUT_MANIFEST=2.0.0/opensearch-2.0.0.yml;TEST_MANIFEST=2.0.0/opensearch-2.0.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch H 1 * * * %INPUT_MANIFEST=2.0.0/opensearch-dashboards-2.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards ''' } @@ -58,7 +58,10 @@ pipeline { steps { script { lock(resource: "CheckForBuild-${INPUT_MANIFEST}-${TARGET_JOB_NAME}", skipIfLocked: true) { - def sha = getManifestSHA(jobName: "${TARGET_JOB_NAME}") + def sha = getManifestSHA( + inputManifest: "manifests/${INPUT_MANIFEST}", + jobName: "${TARGET_JOB_NAME}" + ) if (sha.exists) { echo "Skipping, ${sha.path} already exists." } else { @@ -67,7 +70,10 @@ pipeline { string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}") ], wait: true echo "Build succeeded, uploading build SHA for that job" - buildUploadManifestSHA(jobName: "${TARGET_JOB_NAME}") + buildUploadManifestSHA( + inputManifest: "manifests/${INPUT_MANIFEST}", + jobName: "${TARGET_JOB_NAME}" + ) } catch (err) { echo "${TARGET_JOB_NAME} failed" } diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index c976a21506..dff843c6f6 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -42,10 +42,14 @@ pipeline { } steps { script { - buildAssembleUpload( + def buildManifestObj = buildAssembleUpload( + inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64' ) + String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) + env.ARTIFACT_URL_X64 = artifactUrl + echo "artifactUrl (x64): ${artifactUrl}" } } post { @@ -69,6 +73,7 @@ pipeline { steps { script { buildArchive( + inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64' ) @@ -91,10 +96,15 @@ pipeline { } steps { script { - archiveAssembleUpload( + def buildManifestObj = archiveAssembleUpload( + inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64' ) + + String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) + env.ARTIFACT_URL_ARM64 = artifactUrl + echo "artifactUrl (arm64): ${artifactUrl}" } } post { @@ -111,7 +121,14 @@ pipeline { steps { node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { script { - buildDockerImage() + echo "env.ARTIFACT_URL_X64: ${env.ARTIFACT_URL_X64}" + echo "env.ARTIFACT_URL_ARM64: ${env.ARTIFACT_URL_ARM64}" + + buildDockerImage( + inputManifest: "manifests/${INPUT_MANIFEST}", + artifactUrlX64: env.ARTIFACT_URL_X64, + artifactUrlArm64: env.ARTIFACT_URL_ARM64 + ) } } } diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 020811a80f..89008b3327 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -2,12 +2,22 @@ lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm)) pipeline { agent none + environment { + AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + INTEG_TEST_JOB_NAME = 'integ-test' + } parameters { string( name: 'INPUT_MANIFEST', description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.', trim: true ) + string( + name: 'TEST_MANIFEST', + description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', + trim: true + ) } stages { stage('detect docker image + args') { @@ -21,7 +31,7 @@ pipeline { steps { script { dockerAgent = detectDockerAgent() - currentBuild.description = "$INPUT_MANIFEST" + currentBuild.description = INPUT_MANIFEST } } } @@ -43,6 +53,7 @@ pipeline { script { git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' buildManifest( + inputManifest: "manifests/${INPUT_MANIFEST}", snapshot: true ) if (fileExists("$WORKSPACE/builds/opensearch/maven")) { @@ -77,6 +88,7 @@ pipeline { script { git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' buildManifest( + inputManifest: "manifests/${INPUT_MANIFEST}", snapshot: true ) echo("Uploading min snapshots to S3") @@ -91,10 +103,10 @@ pipeline { } } } - stage('build-x64') { + stage('build-and-test-x64') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label AGENT_X64 image dockerAgent.image args dockerAgent.args alwaysPull true @@ -102,7 +114,33 @@ pipeline { } steps { script { - buildAssembleUpload() + def buildManifestObj = buildAssembleUpload( + inputManifest: "manifests/${INPUT_MANIFEST}" + ) + String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) + String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) + env.ARTIFACT_URL_X64 = artifactUrl + + echo "buildManifestUrl (x64): ${buildManifestUrl}" + echo "artifactUrl (x64): ${artifactUrl}" + + def integTestResults = + build job: INTEG_TEST_JOB_NAME, + propagate: false, + wait: true, + parameters: [ + string(name: 'TEST_MANIFEST', value: "manifests/${TEST_MANIFEST}"), + string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), + string(name: 'AGENT_LABEL', value: AGENT_X64) + ] + + String status = integTestResults.getResult() + String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:' + lib.jenkins.Messages.new(this).add( + STAGE_NAME, + lib.jenkins.Messages.new(this).get([STAGE_NAME]) + + "\nInteg Tests (x64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}" + ) } } post { @@ -111,10 +149,10 @@ pipeline { } } } - stage('build-arm64') { + stage('build-and-test-arm64') { agent { docker { - label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + label AGENT_ARM64 image dockerAgent.image args dockerAgent.args alwaysPull true @@ -122,7 +160,33 @@ pipeline { } steps { script { - buildAssembleUpload() + def buildManifestObj = buildAssembleUpload( + inputManifest: "manifests/${INPUT_MANIFEST}" + ) + String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) + String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) + env.ARTIFACT_URL_ARM64 = artifactUrl + + echo "buildManifestUrl (arm64): ${buildManifestUrl}" + echo "artifactUrl (arm64): ${artifactUrl}" + + def integTestResults = + build job: INTEG_TEST_JOB_NAME, + propagate: false, + wait: true, + parameters: [ + string(name: 'TEST_MANIFEST', value: "manifests/${TEST_MANIFEST}"), + string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), + string(name: 'AGENT_LABEL', value: AGENT_ARM64) + ] + + String status = integTestResults.getResult() + String icon = status == 'SUCCESS' ? ':white_check_mark:' : ':warning:' + lib.jenkins.Messages.new(this).add( + STAGE_NAME, + lib.jenkins.Messages.new(this).get([STAGE_NAME]) + + "\nInteg Tests (arm64): ${icon} ${status} ${integTestResults.getAbsoluteUrl()}" + ) } } post { @@ -137,7 +201,15 @@ pipeline { steps { node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { script { - buildDockerImage() + + echo "env.ARTIFACT_URL_X64: ${env.ARTIFACT_URL_X64}" + echo "env.ARTIFACT_URL_ARM64: ${env.ARTIFACT_URL_ARM64}" + + buildDockerImage( + inputManifest: "manifests/${INPUT_MANIFEST}", + artifactUrlX64: env.ARTIFACT_URL_X64, + artifactUrlArm64: env.ARTIFACT_URL_ARM64 + ) } } } @@ -145,9 +217,9 @@ pipeline { } post { success { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node(AGENT_X64) { script { - def stashed = lib.jenkins.Messages.new(this).get(['build-x64', 'build-arm64']) + def stashed = lib.jenkins.Messages.new(this).get(['build-and-test-x64', 'build-and-test-arm64']) publishNotification( icon: ':white_check_mark:', @@ -162,7 +234,7 @@ pipeline { } } failure { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node(AGENT_X64) { script { publishNotification( icon: ':warning:', diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index e4a8c0a059..4d54603b9f 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -5,7 +5,7 @@ pipeline { environment { AWS_ROLE_ARN = "arn:aws:iam::${AWS_ACCOUNT_PUBLIC}:role/opensearch-test" AWS_ROLE_SESSION_NAME = "jenkins-test-session" - BUILD_MANIFEST= "build-manifest.yml" + BUILD_MANIFEST = "build-manifest.yml" } tools { jdk "JDK14" diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy index d73ccbad9b..6a613a944e 100644 --- a/src/jenkins/BuildManifest.groovy +++ b/src/jenkins/BuildManifest.groovy @@ -28,6 +28,12 @@ class BuildManifest implements Serializable { return this.name.toLowerCase().replaceAll(' ', '-') } + String getFilenameWithExtension(String platform = null, String architecture = null) { + String resolvedPlatform = platform ?: this.platform + String resolvedArchitecture = architecture ?: this.architecture + return "${this.getFilename()}-${this.version}-${resolvedPlatform}-${resolvedArchitecture}.${resolvedPlatform == 'windows' ? 'zip' : 'tar.gz'}" + } + String getPackageName() { return [ this.getFilename(), @@ -92,6 +98,24 @@ class BuildManifest implements Serializable { ].join('/') } + public String getUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) { + return [ + this.getArtifactRootUrl(publicArtifactUrl, jobName, buildNumber), + 'builds', + this.build.getFilename(), + 'manifest.yml' + ].join("/") + } + + public String getArtifactUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) { + return [ + this.getArtifactRootUrl(publicArtifactUrl, jobName, buildNumber), + 'dist', + this.build.getFilename(), + this.build.getFilenameWithExtension() + ].join("/") + } + public String getArtifactArchitecture() { return this.build.architecture } diff --git a/src/jenkins/InputManifest.groovy b/src/jenkins/InputManifest.groovy index 28e7a7f501..75fe9e14d7 100644 --- a/src/jenkins/InputManifest.groovy +++ b/src/jenkins/InputManifest.groovy @@ -48,12 +48,6 @@ class InputManifest { String getFilename() { return this.name.toLowerCase().replaceAll(' ', '-') } - - String getFilenameWithExtension(String platform = null, String architecture = null) { - String resolvedPlatform = platform ?: this.platform - String resolvedArchitecture = architecture ?: this.architecture - return "${this.getFilename()}-${this.version}-${resolvedPlatform}-${resolvedArchitecture}.${resolvedPlatform == 'windows' ? 'zip' : 'tar.gz'}" - } } Build build @@ -64,20 +58,6 @@ class InputManifest { this.ci = data.ci ? new InputManifest.Ci(data.ci) : null } - String getPublicDistUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber, String platform = null, String architecture = null) { - return [ - publicArtifactUrl, - jobName, - this.build.version, - buildNumber, - platform ?: this.build.platform, - architecture ?: this.build.architecture, - 'dist', - this.build.getFilename(), - this.build.getFilenameWithExtension(platform, architecture) - ].join("/") - } - String getSHAsRoot(String jobName) { return [ jobName, diff --git a/tests/jenkins/data/opensearch-1.3.0-test.yml b/tests/jenkins/data/opensearch-1.3.0-test.yml new file mode 100644 index 0000000000..14d7235d5b --- /dev/null +++ b/tests/jenkins/data/opensearch-1.3.0-test.yml @@ -0,0 +1,91 @@ +--- +name: OpenSearch +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security + - without-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security + + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + bwc-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + bwc-test: + test-configs: + - with-security + - without-security + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security + + - name: dashboards-reports + working-directory: reports-scheduler + integ-test: + test-configs: + - without-security + bwc-test: + test-configs: + - without-security + + - name: dashboards-notebooks + working-directory: opensearch-notebooks + integ-test: + test-configs: + - without-security + bwc-test: + test-configs: + - without-security +schema-version: '1.0' diff --git a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile index 9dacf1235a..bd3994564c 100644 --- a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile +++ b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile @@ -5,7 +5,7 @@ pipeline { steps { script { archiveAssembleUpload( - manifest: 'tests/jenkins/data/opensearch-1.3.0.yml', + inputManifest: 'tests/jenkins/data/opensearch-1.3.0.yml', platform: 'linux', architecture: 'x64' ) diff --git a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt index 44e6261014..c38e8796a6 100644 --- a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt @@ -3,26 +3,28 @@ ArchiveAssembleUpload_Jenkinsfile.echo(Executing on agent [label:none]) ArchiveAssembleUpload_Jenkinsfile.stage(assemble from archive, groovy.lang.Closure) ArchiveAssembleUpload_Jenkinsfile.script(groovy.lang.Closure) - ArchiveAssembleUpload_Jenkinsfile.archiveAssembleUpload({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + ArchiveAssembleUpload_Jenkinsfile.archiveAssembleUpload({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) archiveAssembleUpload.legacySCM(groovy.lang.Closure) archiveAssembleUpload.library({identifier=jenkins@20211123, retriever=null}) + archiveAssembleUpload.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) + InputManifest.asBoolean() archiveAssembleUpload.echo(Assembling tests/jenkins/data/opensearch-1.3.0.yml) archiveAssembleUpload.specific(33) archiveAssembleUpload.copyArtifacts({filter=*.zip, fingerprintArtifacts=true, projectName=vars-build, selector=null}) archiveAssembleUpload.unzip({zipFile=archived-builds.zip}) - archiveAssembleUpload.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) - InputManifest.asBoolean() - archiveAssembleUpload.assembleUpload({manifest=builds/opensearch/manifest.yml, platform=linux, architecture=x64}) + archiveAssembleUpload.readYaml({file=builds/opensearch/manifest.yml}) + BuildManifest.asBoolean() + archiveAssembleUpload.assembleUpload({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, buildManifest=builds/opensearch/manifest.yml}) assembleUpload.legacySCM(groovy.lang.Closure) assembleUpload.library({identifier=jenkins@20211123, retriever=null}) - assembleUpload.assembleManifest({manifest=builds/opensearch/manifest.yml, platform=linux, architecture=x64}) + assembleUpload.assembleManifest({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, buildManifest=builds/opensearch/manifest.yml}) assembleManifest.legacySCM(groovy.lang.Closure) assembleManifest.library({identifier=jenkins@20211123, retriever=null}) assembleManifest.readYaml({file=builds/opensearch/manifest.yml}) BuildManifest.asBoolean() BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) assembleManifest.sh(./assemble.sh "builds/opensearch/manifest.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64) - assembleUpload.uploadArtifacts({manifest=builds/opensearch/manifest.yml, platform=linux, architecture=x64}) + assembleUpload.uploadArtifacts({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, buildManifest=builds/opensearch/manifest.yml}) uploadArtifacts.legacySCM(groovy.lang.Closure) uploadArtifacts.library({identifier=jenkins@20211123, retriever=null}) uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml}) @@ -47,9 +49,3 @@ https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/mani uploadArtifacts.writeFile({file=messages/stage.msg, text=https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/manifest.yml}) uploadArtifacts.stash({includes=messages/*, name=messages-stage}) - assembleUpload.readYaml({file=builds/opensearch/manifest.yml}) - InputManifest.asBoolean() - assembleUpload.readYaml({file=builds/opensearch/manifest.yml}) - BuildManifest.asBoolean() - InputManifest.getPublicDistUrl(https://ci.opensearch.org/dbc, vars-build, 33, linux, x64) - assembleUpload.echo(Setting env."ARTIFACT_URL_linux_x64"=https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz) diff --git a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile index 7013a15280..36a236eb06 100644 --- a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile +++ b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile @@ -5,7 +5,7 @@ pipeline { steps { script { assembleUpload( - manifest: 'tests/data/opensearch-build-1.1.0.yml', + buildManifest: 'tests/data/opensearch-build-1.1.0.yml', ) } } diff --git a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt index ae57cbd088..10ef941999 100644 --- a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt @@ -3,17 +3,17 @@ AssembleUpload_Jenkinsfile.echo(Executing on agent [label:none]) AssembleUpload_Jenkinsfile.stage(assemble and upload, groovy.lang.Closure) AssembleUpload_Jenkinsfile.script(groovy.lang.Closure) - AssembleUpload_Jenkinsfile.assembleUpload({manifest=tests/data/opensearch-build-1.1.0.yml}) + AssembleUpload_Jenkinsfile.assembleUpload({buildManifest=tests/data/opensearch-build-1.1.0.yml}) assembleUpload.legacySCM(groovy.lang.Closure) assembleUpload.library({identifier=jenkins@20211123, retriever=null}) - assembleUpload.assembleManifest({manifest=tests/data/opensearch-build-1.1.0.yml}) + assembleUpload.assembleManifest({buildManifest=tests/data/opensearch-build-1.1.0.yml}) assembleManifest.legacySCM(groovy.lang.Closure) assembleManifest.library({identifier=jenkins@20211123, retriever=null}) assembleManifest.readYaml({file=tests/data/opensearch-build-1.1.0.yml}) BuildManifest.asBoolean() BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.1.0.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64) - assembleUpload.uploadArtifacts({manifest=tests/data/opensearch-build-1.1.0.yml}) + assembleUpload.uploadArtifacts({buildManifest=tests/data/opensearch-build-1.1.0.yml}) uploadArtifacts.legacySCM(groovy.lang.Closure) uploadArtifacts.library({identifier=jenkins@20211123, retriever=null}) uploadArtifacts.readYaml({file=tests/data/opensearch-build-1.1.0.yml}) @@ -38,9 +38,3 @@ https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/mani uploadArtifacts.writeFile({file=messages/stage.msg, text=https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/manifest.yml}) uploadArtifacts.stash({includes=messages/*, name=messages-stage}) - assembleUpload.readYaml({file=tests/data/opensearch-build-1.1.0.yml}) - InputManifest.asBoolean() - assembleUpload.readYaml({file=tests/data/opensearch-build-1.1.0.yml}) - BuildManifest.asBoolean() - InputManifest.getPublicDistUrl(https://ci.opensearch.org/dbc, vars-build, 33, linux, x64) - assembleUpload.echo(Setting env."ARTIFACT_URL_linux_x64"=https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz) diff --git a/tests/jenkins/jobs/BuildArchive_Jenkinsfile b/tests/jenkins/jobs/BuildArchive_Jenkinsfile index 9b8770d995..05086a8202 100644 --- a/tests/jenkins/jobs/BuildArchive_Jenkinsfile +++ b/tests/jenkins/jobs/BuildArchive_Jenkinsfile @@ -5,7 +5,7 @@ pipeline { steps { script { buildArchive( - manifest: 'tests/jenkins/data/opensearch-1.3.0.yml', + inputManifest: 'tests/jenkins/data/opensearch-1.3.0.yml', platform: 'linux', architecture: 'x64' ) diff --git a/tests/jenkins/jobs/BuildArchive_Jenkinsfile.txt b/tests/jenkins/jobs/BuildArchive_Jenkinsfile.txt index bb06298029..3b6ecfdf64 100644 --- a/tests/jenkins/jobs/BuildArchive_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildArchive_Jenkinsfile.txt @@ -3,10 +3,10 @@ BuildArchive_Jenkinsfile.echo(Executing on agent [label:none]) BuildArchive_Jenkinsfile.stage(build and archive, groovy.lang.Closure) BuildArchive_Jenkinsfile.script(groovy.lang.Closure) - BuildArchive_Jenkinsfile.buildArchive({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + BuildArchive_Jenkinsfile.buildArchive({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) buildArchive.legacySCM(groovy.lang.Closure) buildArchive.library({identifier=jenkins@20211123, retriever=null}) - buildArchive.buildManifest({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + buildArchive.buildManifest({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) buildManifest.sh(./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64) buildArchive.echo(Archiving into zip: builds/**, tests/jenkins/data/opensearch-1.3.0.yml) buildArchive.zip({zipFile=archived-builds.zip, archive=true, glob=builds/**}) diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile index 96f05817e6..d1b899719a 100644 --- a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile +++ b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile @@ -5,7 +5,7 @@ pipeline { steps { script { buildAssembleUpload( - manifest: 'tests/jenkins/data/opensearch-1.3.0.yml', + inputManifest: 'tests/jenkins/data/opensearch-1.3.0.yml', platform: 'linux', architecture: 'x64' ) diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt index c5099d83e6..c9113989df 100644 --- a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt @@ -3,24 +3,26 @@ BuildAssembleUpload_Jenkinsfile.echo(Executing on agent [label:none]) BuildAssembleUpload_Jenkinsfile.stage(build and assemble, groovy.lang.Closure) BuildAssembleUpload_Jenkinsfile.script(groovy.lang.Closure) - BuildAssembleUpload_Jenkinsfile.buildAssembleUpload({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + BuildAssembleUpload_Jenkinsfile.buildAssembleUpload({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) buildAssembleUpload.legacySCM(groovy.lang.Closure) buildAssembleUpload.library({identifier=jenkins@20211123, retriever=null}) buildAssembleUpload.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - buildAssembleUpload.buildManifest({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + buildAssembleUpload.buildManifest({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) buildManifest.sh(./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64) - buildAssembleUpload.assembleUpload({manifest=builds/opensearch/manifest.yml, platform=linux, architecture=x64}) + buildAssembleUpload.readYaml({file=builds/opensearch/manifest.yml}) + BuildManifest.asBoolean() + buildAssembleUpload.assembleUpload({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, buildManifest=builds/opensearch/manifest.yml}) assembleUpload.legacySCM(groovy.lang.Closure) assembleUpload.library({identifier=jenkins@20211123, retriever=null}) - assembleUpload.assembleManifest({manifest=builds/opensearch/manifest.yml, platform=linux, architecture=x64}) + assembleUpload.assembleManifest({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, buildManifest=builds/opensearch/manifest.yml}) assembleManifest.legacySCM(groovy.lang.Closure) assembleManifest.library({identifier=jenkins@20211123, retriever=null}) assembleManifest.readYaml({file=builds/opensearch/manifest.yml}) BuildManifest.asBoolean() BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) assembleManifest.sh(./assemble.sh "builds/opensearch/manifest.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64) - assembleUpload.uploadArtifacts({manifest=builds/opensearch/manifest.yml, platform=linux, architecture=x64}) + assembleUpload.uploadArtifacts({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, buildManifest=builds/opensearch/manifest.yml}) uploadArtifacts.legacySCM(groovy.lang.Closure) uploadArtifacts.library({identifier=jenkins@20211123, retriever=null}) uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml}) @@ -45,9 +47,3 @@ https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/mani uploadArtifacts.writeFile({file=messages/stage.msg, text=https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/manifest.yml}) uploadArtifacts.stash({includes=messages/*, name=messages-stage}) - assembleUpload.readYaml({file=builds/opensearch/manifest.yml}) - InputManifest.asBoolean() - assembleUpload.readYaml({file=builds/opensearch/manifest.yml}) - BuildManifest.asBoolean() - InputManifest.getPublicDistUrl(https://ci.opensearch.org/dbc, vars-build, 33, linux, x64) - assembleUpload.echo(Setting env."ARTIFACT_URL_linux_x64"=https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz) diff --git a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile index 0295ee416a..f4b6e5c4cf 100644 --- a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile +++ b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile @@ -5,7 +5,9 @@ pipeline { steps { script { buildDockerImage( - manifest: 'tests/jenkins/data/opensearch-1.3.0.yml' + inputManifest: 'tests/jenkins/data/opensearch-1.3.0.yml', + artifactUrlX64: env.ARTIFACT_URL_linux_x64, + artifactUrlArm64: env.ARTIFACT_URL_linux_arm64 ) } } diff --git a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt index 9200fb1cc2..9da8786aa5 100644 --- a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt +++ b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_builds_both.txt @@ -3,15 +3,12 @@ BuildDockerImage_Jenkinsfile.echo(Executing on agent [label:none]) BuildDockerImage_Jenkinsfile.stage(build docker image, groovy.lang.Closure) BuildDockerImage_Jenkinsfile.script(groovy.lang.Closure) - BuildDockerImage_Jenkinsfile.buildDockerImage({manifest=tests/jenkins/data/opensearch-1.3.0.yml}) + BuildDockerImage_Jenkinsfile.buildDockerImage({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, artifactUrlX64=opensearch.linux.x64, artifactUrlArm64=opensearch.linux.arm64}) buildDockerImage.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) buildDockerImage.legacySCM(groovy.lang.Closure) buildDockerImage.library({identifier=jenkins@20211123, retriever=null}) buildDockerImage.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - buildDockerImage.echo(filename: opensearch) - buildDockerImage.echo(x64TarGz: opensearch.linux.x64) - buildDockerImage.echo(arm64TarGz: opensearch.linux.arm64) buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build}) buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main}) buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL opensearch.linux.x64 -o opensearch-x64.tgz && curl -sSL opensearch.linux.arm64 -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 1.3.0 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 33}) diff --git a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_arm64.txt b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_arm64.txt index 3097e9963d..ab84ae06b8 100644 --- a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_arm64.txt +++ b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_arm64.txt @@ -3,13 +3,10 @@ BuildDockerImage_Jenkinsfile.echo(Executing on agent [label:none]) BuildDockerImage_Jenkinsfile.stage(build docker image, groovy.lang.Closure) BuildDockerImage_Jenkinsfile.script(groovy.lang.Closure) - BuildDockerImage_Jenkinsfile.buildDockerImage({manifest=tests/jenkins/data/opensearch-1.3.0.yml}) + BuildDockerImage_Jenkinsfile.buildDockerImage({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, artifactUrlX64=opensearch.linux.x64, artifactUrlArm64=null}) buildDockerImage.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) buildDockerImage.legacySCM(groovy.lang.Closure) buildDockerImage.library({identifier=jenkins@20211123, retriever=null}) buildDockerImage.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - buildDockerImage.echo(filename: opensearch) - buildDockerImage.echo(x64TarGz: opensearch.linux.x64) - buildDockerImage.echo(arm64TarGz: null) buildDockerImage.echo(Skipping docker build, one of x64 or arm64 artifacts was not built.) diff --git a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_both.txt b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_both.txt index 306cedcf2d..dce6048d79 100644 --- a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_both.txt +++ b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_both.txt @@ -3,13 +3,10 @@ BuildDockerImage_Jenkinsfile.echo(Executing on agent [label:none]) BuildDockerImage_Jenkinsfile.stage(build docker image, groovy.lang.Closure) BuildDockerImage_Jenkinsfile.script(groovy.lang.Closure) - BuildDockerImage_Jenkinsfile.buildDockerImage({manifest=tests/jenkins/data/opensearch-1.3.0.yml}) + BuildDockerImage_Jenkinsfile.buildDockerImage({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, artifactUrlX64=null, artifactUrlArm64=null}) buildDockerImage.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) buildDockerImage.legacySCM(groovy.lang.Closure) buildDockerImage.library({identifier=jenkins@20211123, retriever=null}) buildDockerImage.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - buildDockerImage.echo(filename: opensearch) - buildDockerImage.echo(x64TarGz: null) - buildDockerImage.echo(arm64TarGz: null) buildDockerImage.echo(Skipping docker build, one of x64 or arm64 artifacts was not built.) diff --git a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_x64.txt b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_x64.txt index e9b5ddc830..f42ba6f0f7 100644 --- a/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_x64.txt +++ b/tests/jenkins/jobs/BuildDockerImage_Jenkinsfile_skips_x64.txt @@ -3,13 +3,10 @@ BuildDockerImage_Jenkinsfile.echo(Executing on agent [label:none]) BuildDockerImage_Jenkinsfile.stage(build docker image, groovy.lang.Closure) BuildDockerImage_Jenkinsfile.script(groovy.lang.Closure) - BuildDockerImage_Jenkinsfile.buildDockerImage({manifest=tests/jenkins/data/opensearch-1.3.0.yml}) + BuildDockerImage_Jenkinsfile.buildDockerImage({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, artifactUrlX64=null, artifactUrlArm64=opensearch.linux.arm64}) buildDockerImage.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) buildDockerImage.legacySCM(groovy.lang.Closure) buildDockerImage.library({identifier=jenkins@20211123, retriever=null}) buildDockerImage.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - buildDockerImage.echo(filename: opensearch) - buildDockerImage.echo(x64TarGz: null) - buildDockerImage.echo(arm64TarGz: opensearch.linux.arm64) buildDockerImage.echo(Skipping docker build, one of x64 or arm64 artifacts was not built.) diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile b/tests/jenkins/jobs/BuildManifest_Jenkinsfile index c83931dddb..4fab50e345 100644 --- a/tests/jenkins/jobs/BuildManifest_Jenkinsfile +++ b/tests/jenkins/jobs/BuildManifest_Jenkinsfile @@ -20,8 +20,11 @@ pipeline { echo buildManifest.build.platform echo buildManifest.build.architecture echo buildManifest.build.getFilename() + echo buildManifest.build.getFilenameWithExtension('windows', 'x86') echo buildManifest.getArtifactRoot('bundle-build', '1') echo buildManifest.getArtifactRootUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1') + echo buildManifest.getUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1') + echo buildManifest.getArtifactUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1') echo buildManifest.build.getPackageName() echo buildManifest.components.getMinArtifact() } diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt index fb9893a0e1..5dcece6e36 100644 --- a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt @@ -11,10 +11,15 @@ BuildManifest_Jenkinsfile.echo(linux) BuildManifest_Jenkinsfile.echo(x64) BuildManifest_Jenkinsfile.echo(opensearch) + BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-windows-x86.zip) BuildManifest.getArtifactRoot(bundle-build, 1) BuildManifest_Jenkinsfile.echo(bundle-build/1.1.0/1/linux/x64) BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1) BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64) + BuildManifest.getUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1) + BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64/builds/opensearch/manifest.yml) + BuildManifest.getArtifactUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1) + BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64/dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz) BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-linux-x64.tar.gz) BuildManifest.getMinArtifact() BuildManifest_Jenkinsfile.echo(dist/opensearch-min-1.1.0-linux-x64.tar.gz) diff --git a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile index 587256d495..21ddbf2acf 100644 --- a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile +++ b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile @@ -5,7 +5,7 @@ pipeline { steps { script { buildUploadManifestSHA( - manifest: 'tests/jenkins/data/opensearch-1.3.0.yml', + inputManifest: 'tests/jenkins/data/opensearch-1.3.0.yml', platform: 'linux', architecture: 'x64' ) diff --git a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt index e30c5b117d..bffbe238c1 100644 --- a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt @@ -3,11 +3,11 @@ BuildUploadManifestSHA_Jenkinsfile.echo(Executing on agent [label:none]) BuildUploadManifestSHA_Jenkinsfile.stage(upload manifest SHA, groovy.lang.Closure) BuildUploadManifestSHA_Jenkinsfile.script(groovy.lang.Closure) - BuildUploadManifestSHA_Jenkinsfile.buildUploadManifestSHA({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + BuildUploadManifestSHA_Jenkinsfile.buildUploadManifestSHA({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) buildUploadManifestSHA.legacySCM(groovy.lang.Closure) buildUploadManifestSHA.library({identifier=jenkins@20211123, retriever=null}) - buildUploadManifestSHA.getManifestSHA({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) - getManifestSHA.buildManifest({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true}) + buildUploadManifestSHA.getManifestSHA({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + getManifestSHA.buildManifest({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true}) buildManifest.sh(./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64 --lock) getManifestSHA.sha1(tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA: sha1) diff --git a/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile b/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile index e543e5094b..e5dfa042d6 100644 --- a/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile +++ b/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile @@ -43,10 +43,15 @@ pipeline { } steps { script { - buildAssembleUpload( + def buildManifestObj = archiveAssembleUpload( + inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64' ) + + String artifactUrl = buildManifestObj.getArtifactUrl('bundle-build', '1') + env.ARTIFACT_URL_X64 = artifactUrl + echo "artifactUrl (x64): ${artifactUrl}" } } post { @@ -70,6 +75,7 @@ pipeline { steps { script { buildArchive( + inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64' ) @@ -92,10 +98,15 @@ pipeline { } steps { script { - archiveAssembleUpload( + def buildManifestObj = archiveAssembleUpload( + inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64' ) + + String artifactUrl = buildManifestObj.getArtifactUrl('bundle-build', '1') + env.ARTIFACT_URL_ARM64 = artifactUrl + echo "artifactUrl (arm64): ${artifactUrl}" } } post { @@ -112,7 +123,11 @@ pipeline { steps { node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { script { - buildDockerImage() + buildDockerImage( + inputManifest: "manifests/${INPUT_MANIFEST}", + artifactUrlX64: env.ARTIFACT_URL_X64, + artifactUrlArm64: env.ARTIFACT_URL_ARM64 + ) } } } diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile index dd2ba6b3fb..3100489237 100644 --- a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile +++ b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile @@ -5,7 +5,7 @@ pipeline { steps { script { def sha = getManifestSHA( - manifest: 'tests/jenkins/data/opensearch-1.3.0.yml', + inputManifest: 'tests/jenkins/data/opensearch-1.3.0.yml', platform: 'linux', architecture: 'x64' ) diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt index 43a585d8d0..ac58fb01af 100644 --- a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt +++ b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt @@ -3,8 +3,8 @@ GetManifestSHA_Jenkinsfile.echo(Executing on agent [label:none]) GetManifestSHA_Jenkinsfile.stage(manifest SHA, groovy.lang.Closure) GetManifestSHA_Jenkinsfile.script(groovy.lang.Closure) - GetManifestSHA_Jenkinsfile.getManifestSHA({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) - getManifestSHA.buildManifest({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true}) + GetManifestSHA_Jenkinsfile.getManifestSHA({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + getManifestSHA.buildManifest({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true}) buildManifest.sh(./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64 --lock) getManifestSHA.sha1(tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA: sha1) diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt index a744adfcb3..81b97f07b5 100644 --- a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt +++ b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt @@ -3,8 +3,8 @@ GetManifestSHA_Jenkinsfile.echo(Executing on agent [label:none]) GetManifestSHA_Jenkinsfile.stage(manifest SHA, groovy.lang.Closure) GetManifestSHA_Jenkinsfile.script(groovy.lang.Closure) - GetManifestSHA_Jenkinsfile.getManifestSHA({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) - getManifestSHA.buildManifest({manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true}) + GetManifestSHA_Jenkinsfile.getManifestSHA({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64}) + getManifestSHA.buildManifest({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true}) buildManifest.sh(./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64 --lock) getManifestSHA.sha1(tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA: sha1) diff --git a/tests/jenkins/jobs/InputManifest_Jenkinsfile b/tests/jenkins/jobs/InputManifest_Jenkinsfile index a0f5d83c69..7b4ecfc423 100644 --- a/tests/jenkins/jobs/InputManifest_Jenkinsfile +++ b/tests/jenkins/jobs/InputManifest_Jenkinsfile @@ -19,8 +19,6 @@ pipeline { echo inputManifest.build.name echo inputManifest.build.version echo inputManifest.build.getFilename() - echo inputManifest.build.getFilenameWithExtension('windows', 'x86') - echo inputManifest.getPublicDistUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1', 'linux', 'x86') echo inputManifest.getSHAsRoot('bundle-build') } } @@ -34,8 +32,6 @@ pipeline { echo inputManifest.build.name echo inputManifest.build.version echo inputManifest.build.getFilename() - echo inputManifest.build.getFilenameWithExtension('windows', 'x86') - echo inputManifest.getPublicDistUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1', 'linux', 'x86') echo inputManifest.getSHAsRoot('bundle-build') } } diff --git a/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt b/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt index f22750800d..2294fd9937 100644 --- a/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt @@ -10,9 +10,6 @@ InputManifest_Jenkinsfile.echo(OpenSearch) InputManifest_Jenkinsfile.echo(1.1.0) InputManifest_Jenkinsfile.echo(opensearch) - InputManifest_Jenkinsfile.echo(opensearch-1.1.0-windows-x86.zip) - InputManifest.getPublicDistUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1, linux, x86) - InputManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x86/dist/opensearch/opensearch-1.1.0-linux-x86.tar.gz) InputManifest.getSHAsRoot(bundle-build) InputManifest_Jenkinsfile.echo(bundle-build/1.1.0/shas) InputManifest_Jenkinsfile.stage(input manifest 1.2.0, groovy.lang.Closure) @@ -24,8 +21,5 @@ InputManifest_Jenkinsfile.echo(OpenSearch) InputManifest_Jenkinsfile.echo(1.2.0) InputManifest_Jenkinsfile.echo(opensearch) - InputManifest_Jenkinsfile.echo(opensearch-1.2.0-windows-x86.zip) - InputManifest.getPublicDistUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1, linux, x86) - InputManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1/linux/x86/dist/opensearch/opensearch-1.2.0-linux-x86.tar.gz) InputManifest.getSHAsRoot(bundle-build) InputManifest_Jenkinsfile.echo(bundle-build/1.2.0/shas) diff --git a/vars/README.md b/vars/README.md index fa7ab47684..b19af3ee77 100644 --- a/vars/README.md +++ b/vars/README.md @@ -6,6 +6,7 @@ - [Avoiding Rebuilding](#avoiding-rebuilding) - [Publishing Notifications](#publishing-notifications) - [Cleanup](#cleanup) + - [Running Integration Tests](#running-integration-tests) - [Uploading Test Results](#uploading-test-results) ## Jenkins Shared Library @@ -43,6 +44,10 @@ See [publishNotification](./publishNotification.groovy). See [postCleanup](./postCleanup.groovy). +### Running Integration Tests + +See [runIntegTests](./runIntegTests.groovy). + ### Uploading Test Results See [uploadTestResults](./uploadTestResults.groovy). diff --git a/vars/archiveAssembleUpload.groovy b/vars/archiveAssembleUpload.groovy index 19ef9785d5..c0f63cc473 100644 --- a/vars/archiveAssembleUpload.groovy +++ b/vars/archiveAssembleUpload.groovy @@ -1,8 +1,8 @@ -void call(Map args = [:]) { +def call(Map args = [:]) { lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) - String manifest = args.manifest ?: "manifests/${INPUT_MANIFEST}" + def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) - echo "Assembling ${manifest}" + echo "Assembling ${args.inputManifest}" copyArtifacts( filter: "*.zip", @@ -13,11 +13,14 @@ void call(Map args = [:]) { unzip(zipFile: "archived-builds.zip") - def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest)) + String buildManifest = "builds/${inputManifestObj.build.getFilename()}/manifest.yml" + def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: buildManifest)) assembleUpload( args + [ - manifest: "builds/${inputManifest.build.getFilename()}/manifest.yml", + buildManifest: buildManifest, ] ) + + return buildManifestObj } diff --git a/vars/assembleManifest.groovy b/vars/assembleManifest.groovy index c404456b71..f301e62eed 100644 --- a/vars/assembleManifest.groovy +++ b/vars/assembleManifest.groovy @@ -1,10 +1,10 @@ void call(Map args = [:]) { lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) - def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.manifest)) + def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.buildManifest)) def baseUrl = buildManifest.getArtifactRootUrl("${PUBLIC_ARTIFACT_URL}", "${JOB_NAME}", "${BUILD_NUMBER}") sh([ './assemble.sh', - "\"${args.manifest}\"", + "\"${args.buildManifest}\"", "--base-url ${baseUrl}" ].join(' ')) } diff --git a/vars/assembleUpload.groovy b/vars/assembleUpload.groovy index bebacdd625..b07278a525 100644 --- a/vars/assembleUpload.groovy +++ b/vars/assembleUpload.groovy @@ -1,13 +1,8 @@ + void call(Map args = [:]) { lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) assembleManifest(args) uploadArtifacts(args) - - def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: args.manifest)) - def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.manifest)) - String artifactUrl = inputManifest.getPublicDistUrl("${PUBLIC_ARTIFACT_URL}", "${JOB_NAME}", "${BUILD_NUMBER}", buildManifest.build.platform, buildManifest.build.architecture) - echo "Setting env.\"ARTIFACT_URL_${buildManifest.build.platform}_${buildManifest.build.architecture}\"=${artifactUrl}" - env."ARTIFACT_URL_${buildManifest.build.platform}_${buildManifest.build.architecture}" = artifactUrl -} +} \ No newline at end of file diff --git a/vars/buildArchive.groovy b/vars/buildArchive.groovy index 0b1e3769fb..aa2bf19756 100644 --- a/vars/buildArchive.groovy +++ b/vars/buildArchive.groovy @@ -2,7 +2,7 @@ void call(Map args = [:]) { def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) buildManifest(args) - echo "Archiving into zip: builds/**, ${args.manifest}" + echo "Archiving into zip: builds/**, ${args.inputManifest}" zip( zipFile: "archived-builds.zip", diff --git a/vars/buildAssembleUpload.groovy b/vars/buildAssembleUpload.groovy index 247bc46f18..fb23415cc3 100644 --- a/vars/buildAssembleUpload.groovy +++ b/vars/buildAssembleUpload.groovy @@ -1,14 +1,17 @@ -void call(Map args = [:]) { +def call(Map args = [:]) { def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) - String manifest = args.manifest ?: "manifests/${INPUT_MANIFEST}" - - def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest)) + def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) buildManifest(args) + String buildManifest = "builds/${inputManifestObj.build.getFilename()}/manifest.yml" + def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: buildManifest)) + assembleUpload( args + [ - manifest: "builds/${inputManifest.build.getFilename()}/manifest.yml", + buildManifest: buildManifest, ] ) + + return buildManifestObj } diff --git a/vars/buildDockerImage.groovy b/vars/buildDockerImage.groovy index 7ad8163230..5f168eb0e5 100644 --- a/vars/buildDockerImage.groovy +++ b/vars/buildDockerImage.groovy @@ -2,18 +2,11 @@ void call(Map args = [:]) { git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) - String manifest = args.manifest ?: "manifests/${INPUT_MANIFEST}" - def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest)) + def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) String filename = inputManifest.build.getFilename() - String x64TarGz = env.ARTIFACT_URL_linux_x64 - String arm64TarGz = env.ARTIFACT_URL_linux_arm64 - echo "filename: ${filename}" - echo "x64TarGz: ${env.ARTIFACT_URL_linux_x64}" - echo "arm64TarGz: ${env.ARTIFACT_URL_linux_arm64}" - - if (x64TarGz == null || arm64TarGz == null) { + if (args.artifactUrlX64 == null || args.artifactUrlArm64 == null) { echo 'Skipping docker build, one of x64 or arm64 artifacts was not built.' } else { dockerBuild: { @@ -25,8 +18,8 @@ void call(Map args = [:]) { 'id', 'pwd', 'cd docker/release', - "curl -sSL ${x64TarGz} -o ${filename}-x64.tgz", - "curl -sSL ${arm64TarGz} -o ${filename}-arm64.tgz", + "curl -sSL ${args.artifactUrlX64} -o ${filename}-x64.tgz", + "curl -sSL ${args.artifactUrlArm64} -o ${filename}-arm64.tgz", [ 'bash', 'build-image-multi-arch.sh', diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index d30c216019..3f79b6a2c6 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -1,7 +1,7 @@ void call(Map args = [:]) { sh(([ './build.sh', - args.manifest ?: "manifests/${INPUT_MANIFEST}", + args.inputManifest ?: "manifests/${INPUT_MANIFEST}", args.platform ? "-p ${args.platform}" : null, args.architecture ? "-a ${args.architecture}" : null, args.snapshot ? '--snapshot' : null, diff --git a/vars/getManifestSHA.groovy b/vars/getManifestSHA.groovy index 33424a9cc9..23a810fde3 100644 --- a/vars/getManifestSHA.groovy +++ b/vars/getManifestSHA.groovy @@ -1,21 +1,21 @@ Map call(Map args = [:]) { - String manifest = args.manifest ?: "manifests/${INPUT_MANIFEST}" + String inputManifest = args.inputManifest ?: "manifests/${INPUT_MANIFEST}" String jobName = args.jobName ?: "${JOB_NAME}" buildManifest( args + [ - manifest: manifest, + inputManifest: inputManifest, lock: true ] ) - String manifestLock = "${manifest}.lock" + String manifestLock = "${inputManifest}.lock" String manifestSHA = sha1(manifestLock) echo "Manifest SHA: ${manifestSHA}" def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) - def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifestLock)) - String shasRoot = inputManifest.getSHAsRoot(jobName) + def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: manifestLock)) + String shasRoot = inputManifestObj.getSHAsRoot(jobName) String manifestSHAPath = "${shasRoot}/${manifestSHA}.yml" echo "Manifest lock: ${manifestLock}" echo "Manifest SHA path: ${manifestSHAPath}" diff --git a/vars/uploadArtifacts.groovy b/vars/uploadArtifacts.groovy index b2870c129a..0bf300969a 100644 --- a/vars/uploadArtifacts.groovy +++ b/vars/uploadArtifacts.groovy @@ -1,7 +1,7 @@ void call(Map args = [:]) { def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) - def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.manifest)) + def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.buildManifest)) def minArtifactPath = buildManifest.getMinArtifact() def productFilename = buildManifest.build.getFilename() def packageName = buildManifest.build.getPackageName()