Promote min and bundle artifacts to release-candidate after every bundle build

jenkins/opensearch/distribution-build.jenkinsfile

@@ -19,7 +19,7 @@ pipeline {
                 }
             }
             steps {
-                script { 
+                script {
                     dockerAgent = detectDockerAgent()
                     currentBuild.description = "$INPUT_MANIFEST"
                 }

src/jenkins/BuildManifest.groovy

@@ -25,30 +25,76 @@ class BuildManifest implements Serializable {
         String getFilename() {
             return this.name.toLowerCase().replaceAll(' ', '-')
         }
+
+        String getPackageName() {
+            return [
+                    this.getFilename(),
+                    this.version,
+                    this.platform,
+                    this.architecture,
+            ].join('-') + '.tar.gz'
+        }
+    }
+
+    class Components extends HashMap<String, Component> {
+
+        Components(ArrayList data) {
+            data.each { item ->
+                Component component = new Component(item)
+                this[component.name] = component
+            }
+        }
+
+        public String getMinArtifact(){
+            try {
+                this.get("OpenSearch").artifacts.get("dist").first()
+            } catch (Exception ex){
+                echo("Min artifact cannot be found", ex.getMessage())
+            }
+        }
+    }
+
+    class Component implements Serializable {
+        String name
+        String version
+        String ref
+        String commit_id
+        String repository
+        Map<String, ArrayList> artifacts
+
+        Component(Map data) {
+            this.name = data.name
+            this.version = data.version
+            this.ref = data.ref
+            this.commit_id = data.commit_id
+            this.repository = data.repository
+            this.artifacts = new HashMap<>(data.artifacts)
+        }
+
     }
 
     Build build
+    Components components
 
     BuildManifest(Map data) {
         this.build = new BuildManifest.Build(data.build)
+        this.components = new BuildManifest.Components(data.components)
     }
 
     public String getArtifactRoot(String jobName, String buildNumber) {
         return [
-            jobName,
-            this.build.version,
-            buildNumber,
-            this.build.platform,
-            this.build.architecture
+                jobName,
+                this.build.version,
+                buildNumber,
+                this.build.platform,
+                this.build.architecture
         ].join("/")
     }
 
     public String getArtifactRootUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) {
         return [
-            publicArtifactUrl,
-            this.getArtifactRoot(jobName, buildNumber)
+                publicArtifactUrl,
+                this.getArtifactRoot(jobName, buildNumber)
         ].join('/')
     }
 }
-
-

tests/jenkins/TestArchiveAssembleUpload.groovy

@@ -24,6 +24,9 @@ class TestArchiveAssembleUpload extends BuildPipelineTest {
         binding.setVariable('STAGE_NAME', 'stage')
         binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
         binding.setVariable('BUILD_NUMBER', '33')
+        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
+        binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
+        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 
         helper.registerAllowedMethod("s3Upload", [Map])

tests/jenkins/TestAssembleUpload.groovy

@@ -22,6 +22,9 @@ class TestAssembleUpload extends BuildPipelineTest {
         binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')
         binding.setVariable('STAGE_NAME', 'stage')
         binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
+        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
+        binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
+        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
         binding.setVariable('BUILD_NUMBER', '33')
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 

tests/jenkins/TestBuildAssembleUpload.groovy

@@ -24,6 +24,9 @@ class TestBuildAssembleUpload extends BuildPipelineTest {
         binding.setVariable('ARTIFACT_BUCKET_NAME', 'artifact-bucket')
         binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')
         binding.setVariable('STAGE_NAME', 'stage')
+        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
+        binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
+        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
 
         helper.registerAllowedMethod("withCredentials", [List, Closure], { list, closure ->

tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt

@@ -35,6 +35,10 @@
                         uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                            uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                               uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                        uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
+                        uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                           uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
+                           uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
                         BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
                         Messages.asBoolean()
                         Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt

@@ -26,6 +26,10 @@
                      uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                         uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                            uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                     uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
+                     uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
+                        uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
                      BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
                      Messages.asBoolean()
                      Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt

@@ -33,6 +33,10 @@
                         uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
                            uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                               uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
+                        uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
+                        uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                           uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
+                           uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
                         BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
                         Messages.asBoolean()
                         Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

tests/jenkins/jobs/BuildManifest_Jenkinsfile

@@ -5,7 +5,7 @@
  * this file be licensed under the Apache-2.0 license or a
  * compatible open source license.
  */
- 
+
 def lib = library("jenkins")
 
 pipeline {
@@ -22,8 +22,10 @@ pipeline {
                     echo buildManifest.build.getFilename()
                     echo buildManifest.getArtifactRoot('bundle-build', '1')
                     echo buildManifest.getArtifactRootUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
+                    echo buildManifest.build.getPackageName()
+                    echo buildManifest.components.getMinArtifact()
                 }
             }
         }
     }
-}
+}

tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt

@@ -15,3 +15,5 @@
                BuildManifest_Jenkinsfile.echo(bundle-build/1.1.0/1/linux/x64)
                BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1)
                BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64)
+               BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-linux-x64.tar.gz)
+               BuildManifest_Jenkinsfile.echo(dist/opensearch-min-1.1.0-linux-x64.tar.gz)

vars/uploadArtifacts.groovy

@@ -2,6 +2,9 @@ void call(Map args = [:]) {
     def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
 
     def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.manifest))
+    def minArtifactPath = buildManifest.components.getMinArtifact()
+    def productFilename = buildManifest.build.getFilename()
+    def packageName = buildManifest.build.getPackageName()
 
     def artifactPath = buildManifest.getArtifactRoot("${JOB_NAME}", "${BUILD_NUMBER}")
     echo "Uploading to s3://${ARTIFACT_BUCKET_NAME}/${artifactPath}"
@@ -18,10 +21,17 @@ void call(Map args = [:]) {
             path: "${artifactPath}/dist"
     )
 
+    echo "Uploading to s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${artifactPath}"
+
+    withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
+        s3Upload(file: "builds/${productFilename}/${minArtifactPath}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/core/${productFilename}/${buildManifest.build.version}/")
+        s3Upload(file: "dist/${productFilename}/${packageName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/bundle/${productFilename}/${buildManifest.build.version}/")
+    }
+
     def baseUrl = buildManifest.getArtifactRootUrl("${PUBLIC_ARTIFACT_URL}", "${JOB_NAME}", "${BUILD_NUMBER}")
     lib.jenkins.Messages.new(this).add("${STAGE_NAME}", [
-            "${baseUrl}/builds/${buildManifest.build.getFilename()}/manifest.yml",
-            "${baseUrl}/dist/${buildManifest.build.getFilename()}/manifest.yml"
+            "${baseUrl}/builds/${productFilename}/manifest.yml",
+            "${baseUrl}/dist/${productFilename}/manifest.yml"
         ].join('\n')
     )
 }