Fix location in bundle manifest and update artifacts upload logic (op…

…ensearch-project#1944) * remove distribution from getArtifactRoot Signed-off-by: Tianle Huang <[email protected]> * use new API Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]> * fix field Signed-off-by: Tianle Huang <[email protected]> * fix UTs Signed-off-by: Tianle Huang <[email protected]> * add more Signed-off-by: Tianle Huang <[email protected]> * simplfy Signed-off-by: Tianle Huang <[email protected]> * add test set up Signed-off-by: Tianle Huang <[email protected]> * fix test Signed-off-by: Tianle Huang <[email protected]> * simplify Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]> * fix ut Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]>
kavilla · Apr 8, 2022 · 69ad812 · 69ad812
1 parent ead0194
commit 69ad812
Show file tree

Hide file tree

Showing 20 changed files with 153 additions and 128 deletions.
diff --git a/jenkins/promotion/tar-promotion.jenkinsfile b/jenkins/promotion/tar-promotion.jenkinsfile
@@ -45,7 +45,7 @@ pipeline {
             }
             steps {
                 script {
-                    promoteArtifacts(
+                    promoteTarArtifacts(
                         fileActions: [createSha512Checksums(), createSignatureFiles()]
                     )
                 }

diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy
@@ -93,13 +93,19 @@ class BuildManifest implements Serializable {
         this.components = new BuildManifest.Components(data.components)
     }
 
-    public String getArtifactRoot(String jobName, String buildNumber) {
+    public String getArtifactRootWithoutDistribution(String jobName, String buildNumber) {
         return [
                 jobName,
                 this.build.version,
                 buildNumber,
                 this.build.platform,
-                this.build.architecture,
+                this.build.architecture
+        ].join("/")
+    }
+
+    public String getArtifactRoot(String jobName, String buildNumber) {
+        return [
+                this.getArtifactRootWithoutDistribution(jobName, buildNumber),
                 this.build.distribution
         ].join("/")
     }
@@ -111,10 +117,17 @@ class BuildManifest implements Serializable {
         ].join("/")
     }
 
-    public String getArtifactRootUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) {
+    public String getArtifactRootUrlWithoutDistribution(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) {
         return [
                 publicArtifactUrl,
-                this.getArtifactRoot(jobName, buildNumber)
+                this.getArtifactRootWithoutDistribution(jobName, buildNumber)
+        ].join('/')
+    }
+
+    public String getArtifactRootUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) {
+        return [
+                this.getArtifactRootUrlWithoutDistribution(publicArtifactUrl, jobName, buildNumber),
+                this.build.distribution
         ].join('/')
     }
 

diff --git a/tests/jenkins/TestPromoteArtifacts.groovy b/tests/jenkins/TestPromoteArtifacts.groovy
@@ -55,32 +55,32 @@ class TestPromoteArtifacts extends BuildPipelineTest {
         helper.registerAllowedMethod('findFiles', [Map], { args ->
             return [{}]
         })
-        helper.addFileExistsMock('workspace/artifacts/vars-build/1.3.0/33/linux/x64/builds/opensearch/core-plugins', true)
+        helper.addFileExistsMock('workspace/artifacts/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins', true)
 
-        helper.addShMock('find workspace/artifacts/vars-build/1.3.0/33/linux/x64/builds/opensearch/core-plugins -type f') { script ->
+        helper.addShMock('find workspace/artifacts/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins -type f') { script ->
             return [stdout: "tar_dummy_artifact_1.0.0.tar.gz zip_dummy_artifact_1.1.0.zip dummy_artifact_1.1.0.dummy", exitValue: 0]
         }
         helper.addShMock('sha512sum tar_dummy_artifact_1.0.0.tar.gz') { script ->
-            return [stdout: "shaHashDummy_tar_dummy_artifact_1.0.0.tar.gz  workspace/artifacts/vars-build/1.3.0/33/linux/x64/builds/opensearch/core-plugins/tar_dummy_artifact_1.0.0.tar.gz", exitValue: 0]
+            return [stdout: "shaHashDummy_tar_dummy_artifact_1.0.0.tar.gz  workspace/artifacts/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/tar_dummy_artifact_1.0.0.tar.gz", exitValue: 0]
         }
         helper.addShMock('sha512sum zip_dummy_artifact_1.1.0.zip') { script ->
-            return [stdout: "shaHashDummy_zip_dummy_artifact_1.1.0.zip  workspace/artifacts/vars-build/1.3.0/33/linux/x64/builds/opensearch/core-plugins/zip_dummy_artifact_1.1.0.zip", exitValue: 0]
+            return [stdout: "shaHashDummy_zip_dummy_artifact_1.1.0.zip  workspace/artifacts/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/zip_dummy_artifact_1.1.0.zip", exitValue: 0]
         }
-        helper.addShMock('basename workspace/artifacts/vars-build/1.3.0/33/linux/x64/builds/opensearch/core-plugins/tar_dummy_artifact_1.0.0.tar.gz') { script ->
+        helper.addShMock('basename workspace/artifacts/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/tar_dummy_artifact_1.0.0.tar.gz') { script ->
             return [stdout: "tar_dummy_artifact_1.0.0.tar.gz", exitValue: 0]
         }
-        helper.addShMock('basename workspace/artifacts/vars-build/1.3.0/33/linux/x64/builds/opensearch/core-plugins/zip_dummy_artifact_1.1.0.zip') { script ->
+        helper.addShMock('basename workspace/artifacts/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/zip_dummy_artifact_1.1.0.zip') { script ->
             return [stdout: "zip_dummy_artifact_1.1.0.zip", exitValue: 0]
         }
 
         targetOpenSearch = copy(
             "tests/data/opensearch-build-1.1.0.yml", 
-            "artifacts/vars-build/1.3.0/33/linux/x64/builds/opensearch/manifest.yml"
+            "artifacts/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/manifest.yml"
         );
 
         targetOpenSearchDashboards = copy(
             "tests/data/opensearch-dashboards-build-1.2.0.yml", 
-            "artifacts/vars-build/1.2.0/33/linux/x64/builds/opensearch-dashboards/manifest.yml"
+            "artifacts/vars-build/1.2.0/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml"
         );
     }
 

diff --git a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt
@@ -22,8 +22,8 @@
                         assembleManifest.library({identifier=jenkins@20211123, retriever=null})
                         assembleManifest.readYaml({file=tar/builds/opensearch/manifest.yml})
                         BuildManifest.asBoolean()
-                        BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
-                        assembleManifest.sh(./assemble.sh "tar/builds/opensearch/manifest.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/x64/tar)
+                        BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 33)
+                        assembleManifest.sh(./assemble.sh "tar/builds/opensearch/manifest.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/x64)
                      assembleUpload.uploadArtifacts({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, distribution=tar, buildManifest=tar/builds/opensearch/manifest.yml})
                         uploadArtifacts.legacySCM(groovy.lang.Closure)
                         uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})

diff --git a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt
@@ -11,8 +11,8 @@
                      assembleManifest.library({identifier=jenkins@20211123, retriever=null})
                      assembleManifest.readYaml({file=tests/data/opensearch-build-1.1.0.yml})
                      BuildManifest.asBoolean()
-                     BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
-                     assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.1.0.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/tar)
+                     BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 33)
+                     assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.1.0.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64)
                   assembleUpload.uploadArtifacts({buildManifest=tests/data/opensearch-build-1.1.0.yml})
                      uploadArtifacts.legacySCM(groovy.lang.Closure)
                      uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})

diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt
@@ -20,8 +20,8 @@
                         assembleManifest.library({identifier=jenkins@20211123, retriever=null})
                         assembleManifest.readYaml({file=tar/builds/opensearch/manifest.yml})
                         BuildManifest.asBoolean()
-                        BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
-                        assembleManifest.sh(./assemble.sh "tar/builds/opensearch/manifest.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/x64/tar)
+                        BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 33)
+                        assembleManifest.sh(./assemble.sh "tar/builds/opensearch/manifest.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/x64)
                      assembleUpload.uploadArtifacts({inputManifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, distribution=tar, buildManifest=tar/builds/opensearch/manifest.yml})
                         uploadArtifacts.legacySCM(groovy.lang.Closure)
                         uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})

diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile b/tests/jenkins/jobs/BuildManifest_Jenkinsfile
@@ -22,6 +22,8 @@ pipeline {
                     echo buildManifest.build.getFilename()
                     echo buildManifest.build.getFilenameWithExtension('windows', 'x86')
                     echo buildManifest.getArtifactRoot('bundle-build', '1')
+                    echo buildManifest.getArtifactRootWithoutDistribution('bundle-build', '1')
+                    echo buildManifest.getArtifactRootUrlWithoutDistribution('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
                     echo buildManifest.getArtifactRootUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
                     echo buildManifest.getUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
                     echo buildManifest.getArtifactUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')

diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt
@@ -14,6 +14,10 @@
                BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-windows-x86.tar.gz)
                BuildManifest.getArtifactRoot(bundle-build, 1)
                BuildManifest_Jenkinsfile.echo(bundle-build/1.1.0/1/linux/x64/tar)
+               BuildManifest.getArtifactRootWithoutDistribution(bundle-build, 1)
+               BuildManifest_Jenkinsfile.echo(bundle-build/1.1.0/1/linux/x64)
+               BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/ci/dbc, bundle-build, 1)
+               BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64)
                BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1)
                BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64/tar)
                BuildManifest.getUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1)

diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile_component_no_artifact b/tests/jenkins/jobs/BuildManifest_Jenkinsfile_component_no_artifact
@@ -21,7 +21,9 @@ pipeline {
                     echo buildManifest.build.architecture
                     echo buildManifest.build.getFilename()
                     echo buildManifest.getArtifactRoot('bundle-build', '1')
+                    echo buildManifest.getArtifactRootWithoutDistribution('bundle-build', '1')
                     echo buildManifest.getArtifactRootUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
+                    echo buildManifest.getArtifactRootUrlWithoutDistribution('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
                     echo buildManifest.build.getPackageName()
                     echo buildManifest.getMinArtifact()
                     echo buildManifest.getCommitId("OpenSearch-Dashboards")

diff --git a/tests/jenkins/jobs/BuildManifest_Jenkinsfile_component_no_artifact.txt b/tests/jenkins/jobs/BuildManifest_Jenkinsfile_component_no_artifact.txt
@@ -13,8 +13,12 @@
                BuildManifest_Jenkinsfile_component_no_artifact.echo(opensearch-dashboards)
                BuildManifest.getArtifactRoot(bundle-build, 1)
                BuildManifest_Jenkinsfile_component_no_artifact.echo(bundle-build/1.2.0/1/linux/x64/tar)
+               BuildManifest.getArtifactRootWithoutDistribution(bundle-build, 1)
+               BuildManifest_Jenkinsfile_component_no_artifact.echo(bundle-build/1.2.0/1/linux/x64)
                BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1)
                BuildManifest_Jenkinsfile_component_no_artifact.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1/linux/x64/tar)
+               BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/ci/dbc, bundle-build, 1)
+               BuildManifest_Jenkinsfile_component_no_artifact.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1/linux/x64)
                BuildManifest_Jenkinsfile_component_no_artifact.echo(opensearch-dashboards-1.2.0-linux-x64.tar.gz)
                BuildManifest.getMinArtifact()
                BuildManifest_Jenkinsfile_component_no_artifact.echo(dist/opensearch-dashboards-min-1.2.0-linux-x64.tar.gz)

diff --git a/tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile b/tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile
@@ -4,7 +4,7 @@ pipeline {
         stage('promote') {
             steps {
                 script {
-                    promoteArtifacts(
+                    promoteTarArtifacts(
                         manifest: 'tests/jenkins/data/opensearch-1.3.0.yml'
                     )
                 }