Minor bugfix for signArtifacts.groovy

jenkins/sign-artifacts/Jenkinsfile → jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile

@@ -47,6 +47,15 @@ pipeline {
                             distributionPlatform: DISTRIBUTION_PLATFORM
                     )
 
+                    filenamesForUrls = []
+
+                    println("Note: only supported file types will be signed")
+
+                    for(filename in downloadedFiles){
+                        filenamesForUrls.add(filename)
+                        filenamesForUrls.add(filename + SIGNATURE_TYPE)
+                    }
+
                     finalUploadPath = ([
                             "${JOB_NAME}",
                             "${S3_FILE_UPLOAD_PATH}",
@@ -62,7 +71,10 @@ pipeline {
                             path: finalUploadPath
                     )
 
-                    printUrls(downloadedFiles)
+                    printArtifactDownloadUrlsForStaging(
+                            artifactFileNames: filenamesForUrls,
+                            uploadPath: finalUploadPath
+                    )
 
                 }
             }
@@ -77,19 +89,6 @@ pipeline {
     }
 }
 
-void printUrls(downloadedFiles){
-
-    println("Note: only supported file types will be signed")
-
-    for(filename in downloadedFiles){
-
-        println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" )
-        println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" )
-
-    }
-
-}
-
 List downloadArtifactsFromUrls() {
 
     listOfURls = URLs.split(",")

tests/jenkins/TestPrintArtifactDownloadUrlsForStaging.groovy

@@ -0,0 +1,28 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins.tests
+
+import org.junit.*
+
+
+class TestPrintArtifactDownloadUrlsForStaging extends BuildPipelineTest {
+
+    @Before
+    void setUp() {
+        super.setUp()
+        binding.setVariable('filenamesForUrls', ['dummy_file.tar.gz', 'dummy_file.tar.gz.sig'])
+        binding.setVariable('UPLOAD_PATH', 'dummy/upload/path')
+    }
+
+    @Test
+    void testPrintArtifactDownloadUrlsForStaging() {
+        super.testPipeline("tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile")
+    }
+
+}

tests/jenkins/TestSignArtifacts.groovy

@@ -18,6 +18,28 @@ class TestSignArtifacts extends BuildPipelineTest {
     void setUp() {
         super.setUp()
 
+        binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
+        binding.setVariable('WORKSPACE', 'workspace')
+        binding.setVariable('DISTRIBUTION_PLATFORM', 'linux')
+        binding.setVariable('SIGNATURE_TYPE', '.sig')
+
+        binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role')
+        binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id')
+        binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket')
+        binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket')
+
+        helper.registerAllowedMethod("git", [Map])
+        helper.registerAllowedMethod("withCredentials", [Map])
+
+    }
+
+    @Test
+    void testSignArtifacts() {
+        super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile")
+    }
+
+    @Test
+    void testSignArtifactsJob() {
         binding.setVariable('URLs', 'https://www.dummy.com/dummy_1_artifact.tar.gz,' +
                 ' https://www.dummy.com/dummy_2_artifact.tar.gz')
         binding.setVariable('S3_FILE_UPLOAD_PATH', '/dummy/upload/path/')
@@ -27,26 +49,14 @@ class TestSignArtifacts extends BuildPipelineTest {
         binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'Dummy_Upload_Role')
         binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account')
         binding.setVariable('ARTIFACT_BUCKET_NAME', 'dummy_bucket_name')
-        binding.setVariable('DISTRIBUTION_PLATFORM', 'linux')
-        binding.setVariable('SIGNATURE_TYPE', '.sig')
-        binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
-        binding.setVariable('WORKSPACE', 'workspace')
 
-        binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role')
-        binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id')
-        binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket')
-        binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket')
-
-        helper.registerAllowedMethod("git", [Map])
+        helper.registerAllowedMethod("cleanWs", [Map])
         helper.registerAllowedMethod("s3Upload", [Map])
         helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
             closure.delegate = delegate
             return helper.callClosure(closure)
         })
-    }
 
-    @Test
-    void testSignArtifacts() {
-        super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile")
+        super.testPipeline("jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile", "tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile")
     }
 }

tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile.txt

@@ -0,0 +1,33 @@
+   sign-standalone-artifacts.run()
+      sign-standalone-artifacts.legacySCM(groovy.lang.Closure)
+      sign-standalone-artifacts.library({identifier=jenkins@20211123, retriever=null})
+      sign-standalone-artifacts.pipeline(groovy.lang.Closure)
+         sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
+         sign-standalone-artifacts.stage(sign, groovy.lang.Closure)
+            sign-standalone-artifacts.script(groovy.lang.Closure)
+               sign-standalone-artifacts.sh(mkdir workspace/artifacts)
+               sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz)
+               sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz)
+               sign-standalone-artifacts.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux})
+                  signArtifacts.fileExists(workspace/sign.sh)
+                  signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
+                  signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
+                  signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
+                  signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
+                     signArtifacts.sh(
+            #!/bin/bash
+            set +x
+            export ROLE=dummy_signer_client_role
+            export EXTERNAL_ID=signer_client_external_id
+            export UNSIGNED_BUCKET=signer_client_unsigned_bucket
+            export SIGNED_BUCKET=signer_client_signed_bucket
+
+            workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null
+        )
+               sign-standalone-artifacts.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
+                  uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                     uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
+               sign-standalone-artifacts.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_1_artifact.tar.gz, dummy_1_artifact.tar.gz.sig, dummy_2_artifact.tar.gz, dummy_2_artifact.tar.gz.sig], uploadPath=sign_artifacts_job/dummy/upload/path/20/dist/signed})
+         sign-standalone-artifacts.script(groovy.lang.Closure)
+            sign-standalone-artifacts.postCleanup()
+               postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})

tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile

@@ -0,0 +1,15 @@
+pipeline {
+    agent none
+    stages {
+        stage('print_urls') {
+            steps {
+                script {
+                    printArtifactDownloadUrlsForStaging(
+                            artifactFileNames: filenamesForUrls,
+                            uploadPath: UPLOAD_PATH
+                    )
+                }
+            }
+        }
+    }
+}

tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile.txt

@@ -0,0 +1,6 @@
+   PrintArtifactDownloadUrlsForStaging_Jenkinsfile.run()
+      PrintArtifactDownloadUrlsForStaging_Jenkinsfile.pipeline(groovy.lang.Closure)
+         PrintArtifactDownloadUrlsForStaging_Jenkinsfile.echo(Executing on agent [label:none])
+         PrintArtifactDownloadUrlsForStaging_Jenkinsfile.stage(print_urls, groovy.lang.Closure)
+            PrintArtifactDownloadUrlsForStaging_Jenkinsfile.script(groovy.lang.Closure)
+               PrintArtifactDownloadUrlsForStaging_Jenkinsfile.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_file.tar.gz, dummy_file.tar.gz.sig], uploadPath=dummy/upload/path})

tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt

@@ -28,9 +28,7 @@
                   createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/artifacts/vars-build/1.3.0/33/x64/linux/builds/opensearch/core-plugins})
                      signArtifacts.fileExists(workspace/sign.sh)
                      signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
-                     signArtifacts.fileExists(workspace/opensearch.pgp)
-                     signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
-                     signArtifacts.sh(gpg --import workspace/opensearch.pgp)
+                     signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
                      signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                      signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                         signArtifacts.sh(
@@ -53,9 +51,7 @@
                   createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/workspace/file/found.zip})
                      signArtifacts.fileExists(workspace/sign.sh)
                      signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
-                     signArtifacts.fileExists(workspace/opensearch.pgp)
-                     signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
-                     signArtifacts.sh(gpg --import workspace/opensearch.pgp)
+                     signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
                      signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                      signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                         signArtifacts.sh(

tests/jenkins/jobs/SignArtifacts_Jenkinsfile

@@ -4,70 +4,13 @@ pipeline {
         stage('sign') {
             steps {
                 script {
-                    if (URLs == '' || S3_FILE_UPLOAD_PATH == '') {
-                        currentBuild.result = 'ABORTED'
-                        error('URLs or S3_FILE_UPLOAD_PATH params are not set')
-                    }
-                    S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('/$', "")
-                    S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('^/+', "")
-                    downloadedFiles = downloadArtifactsFromUrls()
                     signArtifacts(
                             artifactPath: "$WORKSPACE/artifacts/",
                             signatureType: SIGNATURE_TYPE,
                             distributionPlatform: DISTRIBUTION_PLATFORM
                     )
-
-                    finalUploadPath = ([
-                            "${JOB_NAME}",
-                            "${S3_FILE_UPLOAD_PATH}",
-                            "${BUILD_NUMBER}",
-                            "dist",
-                            "signed"
-                    ].join('/'))
-
-                    // uploading artifacts with signatures on s3
-                    uploadToS3(
-                            sourcePath: "$WORKSPACE/artifacts",
-                            bucket: "${ARTIFACT_BUCKET_NAME}",
-                            path: finalUploadPath
-                    )
-
-                    printUrls(downloadedFiles)
-
                 }
             }
         }
     }
 }
-
-void printUrls(downloadedFiles){
-
-    println("Note: only supported file types will be signed")
-
-    for(filename in downloadedFiles){
-
-        println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" )
-        println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" )
-
-    }
-
-}
-
-List downloadArtifactsFromUrls() {
-
-    listOfURls = URLs.split(",")
-
-    def downloadedFiles = []
-
-    sh "mkdir ${WORKSPACE}/artifacts"
-
-    for (url in listOfURls) {
-        trimmedUrl = url.trim()
-        filename = trimmedUrl.substring(trimmedUrl.lastIndexOf('/') + 1)
-        downloadedFiles.add(filename)
-        sh "curl -SL ${trimmedUrl} -o ${WORKSPACE}/artifacts/${filename}"
-    }
-
-    return downloadedFiles
-
-}

tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt

@@ -3,15 +3,10 @@
          SignArtifacts_Jenkinsfile.echo(Executing on agent [label:none])
          SignArtifacts_Jenkinsfile.stage(sign, groovy.lang.Closure)
             SignArtifacts_Jenkinsfile.script(groovy.lang.Closure)
-               SignArtifacts_Jenkinsfile.sh(mkdir workspace/artifacts)
-               SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz)
-               SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz)
                SignArtifacts_Jenkinsfile.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux})
                   signArtifacts.fileExists(workspace/sign.sh)
                   signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
-                  signArtifacts.fileExists(workspace/opensearch.pgp)
-                  signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
-                  signArtifacts.sh(gpg --import workspace/opensearch.pgp)
+                  signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
                   signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      signArtifacts.sh(
@@ -24,6 +19,3 @@
 
             workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null
         )
-               SignArtifacts_Jenkinsfile.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
-                  uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
-                     uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})

vars/printArtifactDownloadUrlsForStaging.groovy

@@ -0,0 +1,8 @@
+void call(Map args = [:]){
+
+    for(filename in args.artifactFileNames){
+        url = "https://ci.opensearch.org/ci/dbc/${args.uploadPath}/${filename}"
+        println("File ${filename} can be accessed using the url - ${url}" )
+    }
+
+}

vars/signArtifacts.groovy

@@ -30,8 +30,7 @@ void call(Map args = [:]) {
 }
 
 void importPGPKey(){
-    if( !fileExists("$WORKSPACE/opensearch.pgp")) {
-        sh("curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o $WORKSPACE/opensearch.pgp")
-        sh("gpg --import $WORKSPACE/opensearch.pgp")
-    }
+
+    sh "curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -"
+
 }