From 8b7df1261220a27f90f1ac7042f8c5256540597c Mon Sep 17 00:00:00 2001 From: Zelin Hao Date: Thu, 18 Jan 2024 11:38:50 -0800 Subject: [PATCH] Add a new groovy library to download the previous build artifacts (#362) Signed-off-by: Zelin Hao --- .../opensearch-dashboards-input-2.12.0.yml | 18 ++ tests/data/opensearch-input-2.12.0.yml | 205 ++++++++++++++++++ .../jenkins/TestRetrievePreviousBuild.groovy | 64 ++++++ .../jobs/RetrievePreviousBuild_Jenkinsfile | 35 +++ .../RetrievePreviousBuild_Jenkinsfile.txt | 33 +++ .../RetrievePreviousBuildLibTester.groovy | 54 +++++ vars/buildManifest.groovy | 9 + vars/retrievePreviousBuild.groovy | 60 +++++ 8 files changed, 478 insertions(+) create mode 100644 tests/data/opensearch-dashboards-input-2.12.0.yml create mode 100644 tests/data/opensearch-input-2.12.0.yml create mode 100644 tests/jenkins/TestRetrievePreviousBuild.groovy create mode 100644 tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile create mode 100644 tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt create mode 100644 tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy create mode 100644 vars/retrievePreviousBuild.groovy diff --git a/tests/data/opensearch-dashboards-input-2.12.0.yml b/tests/data/opensearch-dashboards-input-2.12.0.yml new file mode 100644 index 000000000..e2084c046 --- /dev/null +++ b/tests/data/opensearch-dashboards-input-2.12.0.yml @@ -0,0 +1,18 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch Dashboards + version: 2.12.0 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-build-v1 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: 361df7741b699e67c03f68a8550a5b8edcb1c838 + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: a0be908272a67dec0d39211b432c9d415eee780c + - name: observabilityDashboards + repository: https://github.com/opensearch-project/dashboards-observability.git + ref: 42a0be2aedc4f28e0e37cde6936783767881d5e7 diff --git a/tests/data/opensearch-input-2.12.0.yml b/tests/data/opensearch-input-2.12.0.yml new file mode 100644 index 000000000..955b85ea5 --- /dev/null +++ b/tests/data/opensearch-input-2.12.0.yml @@ -0,0 +1,205 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 2.12.0 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 05c2befd7d01fab4aef4f0d3d6722d2da240b2c6 + checks: + - gradle:publish + - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: ace0d5a6bf4b489bbeaccad65f62f6ccf20df491 + checks: + - gradle:publish + - gradle:properties:version + platforms: + - linux + - windows + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: aaf09b0211df15dd74ff2756f2590c360b03486b + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: 7ccb10cf5339701217c4019932bd26bf304a93aa + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + - name: geospatial + repository: https://github.com/opensearch-project/geospatial.git + ref: f48c9dabcd4d955e5d88b0670d519cd7d341581c + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - job-scheduler + - name: security + repository: https://github.com/opensearch-project/security.git + ref: ea0d4ff4775ea3d8793458393ccadbe0ec8c0ef0 + platforms: + - linux + - windows + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: ee577a1729dec1b0bb05ddc3b767a54e6e905401 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: 2ee80cde78c7099d5132af7308933724b32c1f0d + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-ml-plugin + platforms: + - linux + - windows + depends_on: + - common-utils + - name: neural-search + repository: https://github.com/opensearch-project/neural-search.git + ref: 5442f84398f8c65378ba71dc009d75550810fce5 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - ml-commons + - k-NN + - name: notifications-core + repository: https://github.com/opensearch-project/notifications.git + ref: cf64c411ce4f85ce67734feb8704467b2232d171 + working_directory: notifications + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-notifications-core + platforms: + - linux + - windows + depends_on: + - common-utils + - name: notifications + repository: https://github.com/opensearch-project/notifications.git + ref: cf64c411ce4f85ce67734feb8704467b2232d171 + working_directory: notifications + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: notifications + platforms: + - linux + - windows + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability.git + ref: 40b0260573d5ba2cd8aa371febddd5816bff4967 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: opensearch-reports + repository: https://github.com/opensearch-project/reporting.git + ref: e8e5025b9577038c6c160b6ea5a1610312b66180 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: e090584440cdf4832b4225241622570fc5a8c80d + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-sql-plugin + platforms: + - linux + - windows + depends_on: + - ml-commons + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: 3fd781eaee620895d10067baf9a368d970257455 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: 72673698e062b061e8c6734c4dd52d43208ffa3d + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: 944abf29a42ce71d8e5d62c61768f826d34b2a45 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: alerting + platforms: + - linux + - windows + depends_on: + - common-utils + - name: security-analytics + repository: https://github.com/opensearch-project/security-analytics.git + ref: 22fa49d9d02734e561be55993ff336eb713318b1 + checks: + - gradle:properties:version + platforms: + - linux + - windows + depends_on: + - common-utils + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: 11aa4469ae2b9fb3be714ae24ef52d9bfb127510 + checks: + - gradle:properties:version + platforms: + - linux + - windows + depends_on: + - common-utils + - job-scheduler diff --git a/tests/jenkins/TestRetrievePreviousBuild.groovy b/tests/jenkins/TestRetrievePreviousBuild.groovy new file mode 100644 index 000000000..4daeec7e2 --- /dev/null +++ b/tests/jenkins/TestRetrievePreviousBuild.groovy @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package jenkins.tests + +import org.junit.* +import jenkins.tests.BuildPipelineTest +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat +import static org.hamcrest.core.IsNot.not + +class TestRetrievePreviousBuild extends BuildPipelineTest { + @Before + void setUp() { + super.setUp() + + binding.setVariable('JOB_NAME', 'dummy_job') + this.registerLibTester(new RetrievePreviousBuildLibTester('tests/data/opensearch-input-2.12.0.yml', 'linux', 'x64', 'tar', '123')) + helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod("s3Download", [Map]) + } + + @Test + void testRetrievePreviousBuild() { + super.testPipeline('tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile') + def shCommands = getCommands('sh', 'mkdir') + assertThat(shCommands, hasItems('mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace')) + assertThat(shCommands, hasItems('mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')) + + assertThat(shCommands, hasItems('mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace')) + assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/'))) + + def s3DownloadCommands = getCommands('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands, hasItems( + "{file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/123/linux/x64/tar/, force=true}".toString())) + + assertThat(s3DownloadCommands, hasItems( + "{file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true}".toString())) + } + + def getCommands(method, text) { + def shCommands = helper.callStack.findAll { call -> + call.methodName == method + }.collect { call -> + callArgsToString(call) + }.findAll { command -> + command.contains(text) + } + return shCommands + } +} + diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile new file mode 100644 index 000000000..a2d334119 --- /dev/null +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('Test build manifest jenkins var') { + steps { + script { + retrievePreviousBuild( + inputManifest: "tests/data/opensearch-input-2.12.0.yml", + distribution: "tar", + architecture: "x64", + platform: "linux", + previousBuildId: "123" + ) + retrievePreviousBuild( + inputManifest: "tests/data/opensearch-dashboards-input-2.12.0.yml", + distribution: "zip", + architecture: "x64", + platform: "windows", + previousBuildId: "1234" + ) + } + } + } + } +} + diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt new file mode 100644 index 000000000..4cd1d6e23 --- /dev/null +++ b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt @@ -0,0 +1,33 @@ + RetrievePreviousBuild_Jenkinsfile.run() + RetrievePreviousBuild_Jenkinsfile.pipeline(groovy.lang.Closure) + RetrievePreviousBuild_Jenkinsfile.echo(Executing on agent [label:none]) + RetrievePreviousBuild_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure) + RetrievePreviousBuild_Jenkinsfile.script(groovy.lang.Closure) + RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, previousBuildId=123}) + retrievePreviousBuild.legacySCM(groovy.lang.Closure) + retrievePreviousBuild.library({identifier=jenkins@main, retriever=null}) + retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-2.12.0.yml}) + InputManifest.asBoolean() + retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/2.12.0/123/linux/x64/tar/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER}) + downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/123/linux/x64/tar/, force=true}) + retrievePreviousBuild.sh(mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace) + retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.) + retrievePreviousBuild.sh(mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/) + RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, previousBuildId=1234}) + retrievePreviousBuild.legacySCM(groovy.lang.Closure) + retrievePreviousBuild.library({identifier=jenkins@main, retriever=null}) + retrievePreviousBuild.readYaml({file=tests/data/opensearch-dashboards-input-2.12.0.yml}) + InputManifest.asBoolean() + retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/2.12.0/1234/windows/x64/zip/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER}) + downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true}) + retrievePreviousBuild.sh(mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace) diff --git a/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy b/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy new file mode 100644 index 000000000..ac3303e2e --- /dev/null +++ b/tests/jenkins/lib-testers/RetrievePreviousBuildLibTester.groovy @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.CoreMatchers.nullValue +import static org.hamcrest.MatcherAssert.assertThat + +class RetrievePreviousBuildLibTester extends LibFunctionTester { + private String inputManifestPath + private String distribution = 'tar' + private String platform = 'linux' + private String architecture = 'tar' + private String previousBuildId = 'latest' + + public RetrievePreviousBuildLibTester(String inputManifestPath, String platform, String architecture, String distribution, String previousBuildId){ + this.inputManifestPath = inputManifestPath + this.platform = platform + this.architecture = architecture + this.distribution = distribution + this.previousBuildId = previousBuildId + } + + @Override + void configure(helper, binding) { + } + + @Override + void parameterInvariantsAssertions(call) { + assertThat(call.args.inputManifest.first(), notNullValue()) + assertThat(call.args.platform.first(), notNullValue()) + assertThat(call.args.architecture.first(), notNullValue()) + assertThat(call.args.distribution.first(), notNullValue()) + assertThat(call.args.previousBuildId.first(), notNullValue()) + } + + @Override + boolean expectedParametersMatcher(call) { + return call.args.inputManifest.first().equals(this.inputManifestPath) + && call.args.platform.first().equals(this.platform) + && call.args.architecture.first().equals(this.architecture) + && call.args.distribution.first().equals(this.distribution) + && call.args.previousBuildId.first().equals(this.previousBuildId) + } + + @Override + String libFunctionName() { + return 'retrievePreviousBuild' + } +} diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy index 5c13372d4..110e24391 100644 --- a/vars/buildManifest.groovy +++ b/vars/buildManifest.groovy @@ -16,8 +16,16 @@ @param args.snapshot - Boolean value. Defaults to null. @param args.lock - Generate a stable reference manifest. Defaults to null. @param args.continueOnError - Do not fail the distribution build on any plugin component failure. Defaults to null + @param args.incremental - Boolean value to enable incremental build. */ void call(Map args = [:]) { + boolean incremental_enabled = args.incremental != null && args.incremental + + if (incremental_enabled) { + echo("Incremental build enabled! Retrieving previous build library.") + retrievePreviousBuild(args) + } + sh(([ './build.sh', args.inputManifest ?: "manifests/${INPUT_MANIFEST}", @@ -28,5 +36,6 @@ void call(Map args = [:]) { args.snapshot ? '--snapshot' : null, args.lock ? '--lock' : null, args.continueOnError ? '--continue-on-error' : null, + incremental_enabled ? '--incremental' : null ] - null).join(' ')) } diff --git a/vars/retrievePreviousBuild.groovy b/vars/retrievePreviousBuild.groovy new file mode 100644 index 000000000..29572d8fd --- /dev/null +++ b/vars/retrievePreviousBuild.groovy @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/** Library to download previous build artifacts into the workspace. + @param Map args = [:] args A map of the following parameters + @param args.inputManifest - Relative path to input manifest containing all the components to build. + @param args.jobName - Relative path to input manifest containing all the components to build. + @param args.platform - Platform of previous build to retrieve. + @param args.architecture - Architecture of previous build to retrieve. + @param args.distribution - Distribution of previous build to retrieve. + @param args.previousBuildId - Build id of previous build for incremental build. Defaults to latest. + */ +void call(Map args = [:]) { + def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) + def inputManifestObj = lib.jenkins.InputManifest.new(readYaml(file: args.inputManifest)) + + def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}" + def version = inputManifestObj.build.version + + def DISTRIBUTION_PLATFORM = args.platform + def DISTRIBUTION_ARCHITECTURE = args.architecture + def distribution = args.distribution + def prefixPath = "${WORKSPACE}/download" + def previousBuildId = args.previousBuildId ?: "latest" + def DISTRIBUTION_BUILD_NUMBER + + if (previousBuildId.equalsIgnoreCase("latest")) { + DISTRIBUTION_BUILD_NUMBER = sh( + script: "curl -sL https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\"", + returnStdout: true + ).trim() + //Once we have new index.json, URL will be changed to: https://ci.opensearch.org/ci/dbc/${DISTRIBUTION_JOB_NAME}/${version}/index/${platform}/${architecture}/${distribution}/index.json + } else { + DISTRIBUTION_BUILD_NUMBER = previousBuildId + } + + def artifactPath = "${DISTRIBUTION_JOB_NAME}/${version}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}" + + withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + downloadFromS3( + assumedRoleName: "opensearch-bundle", + roleAccountNumberCred: "jenkins-aws-account-public", + downloadPath: "${artifactPath}/", + bucketName: "${ARTIFACT_BUCKET_NAME}", + localPath: "${prefixPath}", + force: true, + ) + } + sh("mkdir -p ${distribution} && mv -v ${prefixPath}/${artifactPath} ${WORKSPACE}") + if (inputManifestObj.build.getFilename().equals("opensearch")) { + echo("Setting up Maven Local for OpenSearch build.") + sh("mkdir -p ~/.m2/repository/org/ && cp -r ${distribution}/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/") + } + +}