Promote min and bundle artifacts to release-candidate after every bun…

…dle build (#1378) * Promote to release candidate after every bundle build Signed-off-by: Yilin Zhang <[email protected]> Signed-off-by: Sayali Gaikawad <[email protected]> Co-authored-by: Sayali Gaikawad <[email protected]>
opensearch-project · Jan 12, 2022 · d0ac211 · d0ac211
1 parent 4debe78
commit d0ac211
Show file tree

Hide file tree

Showing 11 changed files with 95 additions and 14 deletions.
diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -19,7 +19,7 @@ pipeline {
                 }
             }
             steps {
-                script { 
+                script {
                     dockerAgent = detectDockerAgent()
                     currentBuild.description = "$INPUT_MANIFEST"
                 }

diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy
@@ -25,30 +25,72 @@ class BuildManifest implements Serializable {
         String getFilename() {
             return this.name.toLowerCase().replaceAll(' ', '-')
         }
+
+        String getPackageName() {
+            return [
+                    this.getFilename(),
+                    this.version,
+                    this.platform,
+                    this.architecture,
+            ].join('-') + '.tar.gz'
+        }
+    }
+
+    class Components extends HashMap<String, Component> {
+
+        Components(ArrayList data) {
+            data.each { item ->
+                Component component = new Component(item)
+                this[component.name] = component
+            }
+        }
+    }
+
+    class Component implements Serializable {
+        String name
+        String version
+        String ref
+        String commit_id
+        String repository
+        Map<String, ArrayList> artifacts
+
+        Component(Map data) {
+            this.name = data.name
+            this.version = data.version
+            this.ref = data.ref
+            this.commit_id = data.commit_id
+            this.repository = data.repository
+            this.artifacts = new HashMap<>(data.artifacts)
+        }
+
     }
 
     Build build
+    Components components
 
     BuildManifest(Map data) {
         this.build = new BuildManifest.Build(data.build)
+        this.components = new BuildManifest.Components(data.components)
     }
 
     public String getArtifactRoot(String jobName, String buildNumber) {
         return [
-            jobName,
-            this.build.version,
-            buildNumber,
-            this.build.platform,
-            this.build.architecture
+                jobName,
+                this.build.version,
+                buildNumber,
+                this.build.platform,
+                this.build.architecture
         ].join("/")
     }
 
     public String getArtifactRootUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) {
         return [
-            publicArtifactUrl,
-            this.getArtifactRoot(jobName, buildNumber)
+                publicArtifactUrl,
+                this.getArtifactRoot(jobName, buildNumber)
         ].join('/')
     }
-}
-
 
+    public String getMinArtifact() {
+        components.get(build.name.replace(' ','-'))?.artifacts?.get("dist")?.first()
+    }
+}
diff --git a/tests/jenkins/TestArchiveAssembleUpload.groovy b/tests/jenkins/TestArchiveAssembleUpload.groovy
@@ -24,6 +24,9 @@ class TestArchiveAssembleUpload extends BuildPipelineTest {
         binding.setVariable('STAGE_NAME', 'stage')
         binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
         binding.setVariable('BUILD_NUMBER', '33')
+        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
+        binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
+        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 
         helper.registerAllowedMethod("s3Upload", [Map])

diff --git a/tests/jenkins/TestAssembleUpload.groovy b/tests/jenkins/TestAssembleUpload.groovy
@@ -22,6 +22,9 @@ class TestAssembleUpload extends BuildPipelineTest {
         binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')
         binding.setVariable('STAGE_NAME', 'stage')
         binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
+        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
+        binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
+        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
         binding.setVariable('BUILD_NUMBER', '33')
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 

diff --git a/tests/jenkins/TestBuildAssembleUpload.groovy b/tests/jenkins/TestBuildAssembleUpload.groovy
@@ -24,6 +24,9 @@ class TestBuildAssembleUpload extends BuildPipelineTest {
         binding.setVariable('ARTIFACT_BUCKET_NAME', 'artifact-bucket')
         binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')
         binding.setVariable('STAGE_NAME', 'stage')
+        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
+        binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
+        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 
         helper.registerAllowedMethod("withCredentials", [List, Closure], { list, closure ->

diff --git a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt
@@ -27,6 +27,7 @@
                         uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})
                         uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml})
                         BuildManifest.asBoolean()
+                        BuildManifest.getMinArtifact()
                         BuildManifest.getArtifactRoot(vars-build, 33)
                         uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64)
                         uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds})
@@ -35,6 +36,10 @@
                         uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                            uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                               uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                        uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
+                        uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                           uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
+                           uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
                         BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
                         Messages.asBoolean()
                         Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

diff --git a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt
@@ -18,6 +18,7 @@
                      uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})
                      uploadArtifacts.readYaml({file=tests/data/opensearch-build-1.1.0.yml})
                      BuildManifest.asBoolean()
+                     BuildManifest.getMinArtifact()
                      BuildManifest.getArtifactRoot(vars-build, 33)
                      uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64)
                      uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds})
@@ -26,6 +27,10 @@
                      uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                         uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                            uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                     uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
+                     uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
+                        uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
                      BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
                      Messages.asBoolean()
                      Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt
@@ -25,6 +25,7 @@
                         uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})
                         uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml})
                         BuildManifest.asBoolean()
+                        BuildManifest.getMinArtifact()
                         BuildManifest.getArtifactRoot(vars-build, 33)
                         uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64)
                         uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds})
@@ -33,6 +34,10 @@
                         uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                            uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                               uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                        uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
+                        uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                           uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
+                           uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
                         BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
                         Messages.asBoolean()
                         Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile b/tests/jenkins/jobs/BuildManifest_Jenkinsfile
@@ -5,7 +5,7 @@
  * this file be licensed under the Apache-2.0 license or a
  * compatible open source license.
  */
- 
+
 def lib = library("jenkins")
 
 pipeline {
@@ -22,8 +22,10 @@ pipeline {
                     echo buildManifest.build.getFilename()
                     echo buildManifest.getArtifactRoot('bundle-build', '1')
                     echo buildManifest.getArtifactRootUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
+                    echo buildManifest.build.getPackageName()
+                    echo buildManifest.components.getMinArtifact()
                 }
             }
         }
     }
-}
+}
diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt
@@ -15,3 +15,6 @@
                BuildManifest_Jenkinsfile.echo(bundle-build/1.1.0/1/linux/x64)
                BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1)
                BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64)
+               BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-linux-x64.tar.gz)
+               BuildManifest.getMinArtifact()
+               BuildManifest_Jenkinsfile.echo(dist/opensearch-min-1.1.0-linux-x64.tar.gz)
diff --git a/vars/uploadArtifacts.groovy b/vars/uploadArtifacts.groovy
@@ -2,6 +2,9 @@ void call(Map args = [:]) {
     def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
 
     def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.manifest))
+    def minArtifactPath = buildManifest.getMinArtifact()
+    def productFilename = buildManifest.build.getFilename()
+    def packageName = buildManifest.build.getPackageName()
 
     def artifactPath = buildManifest.getArtifactRoot("${JOB_NAME}", "${BUILD_NUMBER}")
     echo "Uploading to s3://${ARTIFACT_BUCKET_NAME}/${artifactPath}"
@@ -18,10 +21,17 @@ void call(Map args = [:]) {
             path: "${artifactPath}/dist"
     )
 
+    echo "Uploading to s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${artifactPath}"
+
+    withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
+        s3Upload(file: "builds/${productFilename}/${minArtifactPath}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/core/${productFilename}/${buildManifest.build.version}/")
+        s3Upload(file: "dist/${productFilename}/${packageName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/bundle/${productFilename}/${buildManifest.build.version}/")
+    }
+
     def baseUrl = buildManifest.getArtifactRootUrl("${PUBLIC_ARTIFACT_URL}", "${JOB_NAME}", "${BUILD_NUMBER}")
     lib.jenkins.Messages.new(this).add("${STAGE_NAME}", [
-            "${baseUrl}/builds/${buildManifest.build.getFilename()}/manifest.yml",
-            "${baseUrl}/dist/${buildManifest.build.getFilename()}/manifest.yml"
+            "${baseUrl}/builds/${productFilename}/manifest.yml",
+            "${baseUrl}/dist/${productFilename}/manifest.yml"
         ].join('\n')
     )
 }