Add a new groovy library to download the previous build artifacts (op…

…ensearch-project#362) Signed-off-by: Zelin Hao <[email protected]>
peterzhuamazon · Jan 18, 2024 · 8b7df12 · 8b7df12
1 parent b195cb2
commit 8b7df12
Show file tree

Hide file tree

Showing 8 changed files with 478 additions and 0 deletions.
diff --git a/tests/data/opensearch-dashboards-input-2.12.0.yml b/tests/data/opensearch-dashboards-input-2.12.0.yml
@@ -0,0 +1,18 @@
+---
+schema-version: '1.1'
+build:
+  name: OpenSearch Dashboards
+  version: 2.12.0
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-build-v1
+components:
+  - name: OpenSearch-Dashboards
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    ref: 361df7741b699e67c03f68a8550a5b8edcb1c838
+  - name: functionalTestDashboards
+    repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
+    ref: a0be908272a67dec0d39211b432c9d415eee780c
+  - name: observabilityDashboards
+    repository: https://github.com/opensearch-project/dashboards-observability.git
+    ref: 42a0be2aedc4f28e0e37cde6936783767881d5e7
diff --git a/tests/data/opensearch-input-2.12.0.yml b/tests/data/opensearch-input-2.12.0.yml
@@ -0,0 +1,205 @@
+---
+schema-version: '1.1'
+build:
+  name: OpenSearch
+  version: 2.12.0
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: 05c2befd7d01fab4aef4f0d3d6722d2da240b2c6
+    checks:
+      - gradle:publish
+      - gradle:properties:version
+  - name: common-utils
+    repository: https://github.com/opensearch-project/common-utils.git
+    ref: ace0d5a6bf4b489bbeaccad65f62f6ccf20df491
+    checks:
+      - gradle:publish
+      - gradle:properties:version
+    platforms:
+      - linux
+      - windows
+  - name: job-scheduler
+    repository: https://github.com/opensearch-project/job-scheduler.git
+    ref: aaf09b0211df15dd74ff2756f2590c360b03486b
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+  - name: k-NN
+    repository: https://github.com/opensearch-project/k-NN.git
+    ref: 7ccb10cf5339701217c4019932bd26bf304a93aa
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+  - name: geospatial
+    repository: https://github.com/opensearch-project/geospatial.git
+    ref: f48c9dabcd4d955e5d88b0670d519cd7d341581c
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - job-scheduler
+  - name: security
+    repository: https://github.com/opensearch-project/security.git
+    ref: ea0d4ff4775ea3d8793458393ccadbe0ec8c0ef0
+    platforms:
+      - linux
+      - windows
+  - name: cross-cluster-replication
+    repository: https://github.com/opensearch-project/cross-cluster-replication.git
+    ref: ee577a1729dec1b0bb05ddc3b767a54e6e905401
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: ml-commons
+    repository: https://github.com/opensearch-project/ml-commons.git
+    ref: 2ee80cde78c7099d5132af7308933724b32c1f0d
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version: opensearch-ml-plugin
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: neural-search
+    repository: https://github.com/opensearch-project/neural-search.git
+    ref: 5442f84398f8c65378ba71dc009d75550810fce5
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - ml-commons
+      - k-NN
+  - name: notifications-core
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: cf64c411ce4f85ce67734feb8704467b2232d171
+    working_directory: notifications
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version: opensearch-notifications-core
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: notifications
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: cf64c411ce4f85ce67734feb8704467b2232d171
+    working_directory: notifications
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version: notifications
+    platforms:
+      - linux
+      - windows
+  - name: opensearch-observability
+    repository: https://github.com/opensearch-project/observability.git
+    ref: 40b0260573d5ba2cd8aa371febddd5816bff4967
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: opensearch-reports
+    repository: https://github.com/opensearch-project/reporting.git
+    ref: e8e5025b9577038c6c160b6ea5a1610312b66180
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - job-scheduler
+  - name: sql
+    repository: https://github.com/opensearch-project/sql.git
+    ref: e090584440cdf4832b4225241622570fc5a8c80d
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version: opensearch-sql-plugin
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - ml-commons
+  - name: asynchronous-search
+    repository: https://github.com/opensearch-project/asynchronous-search.git
+    ref: 3fd781eaee620895d10067baf9a368d970257455
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: anomaly-detection
+    repository: https://github.com/opensearch-project/anomaly-detection.git
+    ref: 72673698e062b061e8c6734c4dd52d43208ffa3d
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - job-scheduler
+  - name: alerting
+    repository: https://github.com/opensearch-project/alerting.git
+    ref: 944abf29a42ce71d8e5d62c61768f826d34b2a45
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version: alerting
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: security-analytics
+    repository: https://github.com/opensearch-project/security-analytics.git
+    ref: 22fa49d9d02734e561be55993ff336eb713318b1
+    checks:
+      - gradle:properties:version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+  - name: index-management
+    repository: https://github.com/opensearch-project/index-management.git
+    ref: 11aa4469ae2b9fb3be714ae24ef52d9bfb127510
+    checks:
+      - gradle:properties:version
+    platforms:
+      - linux
+      - windows
+    depends_on:
+      - common-utils
+      - job-scheduler
diff --git a/tests/jenkins/TestRetrievePreviousBuild.groovy b/tests/jenkins/TestRetrievePreviousBuild.groovy
@@ -0,0 +1,64 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package jenkins.tests
+
+import org.junit.*
+import jenkins.tests.BuildPipelineTest
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.hasItems
+import static org.hamcrest.MatcherAssert.assertThat
+import static org.hamcrest.core.IsNot.not
+
+class TestRetrievePreviousBuild extends BuildPipelineTest {
+    @Before
+    void setUp() {
+        super.setUp()
+
+        binding.setVariable('JOB_NAME', 'dummy_job')
+        this.registerLibTester(new RetrievePreviousBuildLibTester('tests/data/opensearch-input-2.12.0.yml', 'linux', 'x64', 'tar', '123'))
+        helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
+            closure.delegate = delegate
+            return helper.callClosure(closure)
+        })
+        helper.registerAllowedMethod("s3Download", [Map])
+    }
+
+    @Test
+    void testRetrievePreviousBuild() {
+        super.testPipeline('tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile')
+        def shCommands = getCommands('sh', 'mkdir')
+        assertThat(shCommands, hasItems('mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace'))
+        assertThat(shCommands, hasItems('mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/'))
+
+        assertThat(shCommands, hasItems('mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace'))
+        assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')))
+
+        def s3DownloadCommands = getCommands('s3Download', 'bucket').findAll {
+            shCommand -> shCommand.contains('bucket')
+        }
+
+        assertThat(s3DownloadCommands, hasItems(
+                "{file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/123/linux/x64/tar/, force=true}".toString()))
+
+        assertThat(s3DownloadCommands, hasItems(
+                "{file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true}".toString()))
+    }
+
+    def getCommands(method, text) {
+        def shCommands = helper.callStack.findAll { call ->
+            call.methodName == method
+        }.collect { call ->
+            callArgsToString(call)
+        }.findAll { command ->
+            command.contains(text)
+        }
+        return shCommands
+    }
+}
+
diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile
@@ -0,0 +1,35 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('Test build manifest jenkins var') {
+            steps {
+                script {
+                    retrievePreviousBuild(
+                        inputManifest: "tests/data/opensearch-input-2.12.0.yml",
+                        distribution: "tar",
+                        architecture: "x64",
+                        platform: "linux",
+                        previousBuildId: "123"
+                    )
+                    retrievePreviousBuild(
+                        inputManifest: "tests/data/opensearch-dashboards-input-2.12.0.yml",
+                        distribution: "zip",
+                        architecture: "x64",
+                        platform: "windows",
+                        previousBuildId: "1234"
+                    )
+                }
+            }
+        }
+    }
+}
+
diff --git a/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt b/tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt
@@ -0,0 +1,33 @@
+   RetrievePreviousBuild_Jenkinsfile.run()
+      RetrievePreviousBuild_Jenkinsfile.pipeline(groovy.lang.Closure)
+         RetrievePreviousBuild_Jenkinsfile.echo(Executing on agent [label:none])
+         RetrievePreviousBuild_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure)
+            RetrievePreviousBuild_Jenkinsfile.script(groovy.lang.Closure)
+               RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-2.12.0.yml, distribution=tar, architecture=x64, platform=linux, previousBuildId=123})
+                  retrievePreviousBuild.legacySCM(groovy.lang.Closure)
+                  retrievePreviousBuild.library({identifier=jenkins@main, retriever=null})
+                  retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-2.12.0.yml})
+                  InputManifest.asBoolean()
+                  retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/2.12.0/123/linux/x64/tar/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true})
+                        downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER})
+                        downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure)
+                           downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
+                              downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/123/linux/x64/tar/, force=true})
+                  retrievePreviousBuild.sh(mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/2.12.0/123/linux/x64/tar /tmp/workspace)
+                  retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.)
+                  retrievePreviousBuild.sh(mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/)
+               RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-dashboards-input-2.12.0.yml, distribution=zip, architecture=x64, platform=windows, previousBuildId=1234})
+                  retrievePreviousBuild.legacySCM(groovy.lang.Closure)
+                  retrievePreviousBuild.library({identifier=jenkins@main, retriever=null})
+                  retrievePreviousBuild.readYaml({file=tests/data/opensearch-dashboards-input-2.12.0.yml})
+                  InputManifest.asBoolean()
+                  retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/2.12.0/1234/windows/x64/zip/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true})
+                        downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER})
+                        downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure)
+                           downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
+                              downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true})
+                  retrievePreviousBuild.sh(mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip /tmp/workspace)