diff --git a/jenkins/sign-artifacts/Jenkinsfile b/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile similarity index 85% rename from jenkins/sign-artifacts/Jenkinsfile rename to jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile index 7b08ca4eab..1bb39c70d0 100644 --- a/jenkins/sign-artifacts/Jenkinsfile +++ b/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile @@ -47,6 +47,15 @@ pipeline { distributionPlatform: DISTRIBUTION_PLATFORM ) + filenamesForUrls = [] + + println("Note: only supported file types will be signed") + + for(filename in downloadedFiles){ + filenamesForUrls.add(filename) + filenamesForUrls.add(filename + SIGNATURE_TYPE) + } + finalUploadPath = ([ "${JOB_NAME}", "${S3_FILE_UPLOAD_PATH}", @@ -62,7 +71,10 @@ pipeline { path: finalUploadPath ) - printUrls(downloadedFiles) + printArtifactDownloadUrlsForStaging( + artifactFileNames: filenamesForUrls, + uploadPath: finalUploadPath + ) } } @@ -77,19 +89,6 @@ pipeline { } } -void printUrls(downloadedFiles){ - - println("Note: only supported file types will be signed") - - for(filename in downloadedFiles){ - - println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" ) - println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" ) - - } - -} - List downloadArtifactsFromUrls() { listOfURls = URLs.split(",") diff --git a/tests/jenkins/TestPrintArtifactDownloadUrlsForStaging.groovy b/tests/jenkins/TestPrintArtifactDownloadUrlsForStaging.groovy new file mode 100644 index 0000000000..68cb250813 --- /dev/null +++ b/tests/jenkins/TestPrintArtifactDownloadUrlsForStaging.groovy @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins.tests + +import org.junit.* + + +class TestPrintArtifactDownloadUrlsForStaging extends BuildPipelineTest { + + @Before + void setUp() { + super.setUp() + binding.setVariable('filenamesForUrls', ['dummy_file.tar.gz', 'dummy_file.tar.gz.sig']) + binding.setVariable('UPLOAD_PATH', 'dummy/upload/path') + } + + @Test + void testPrintArtifactDownloadUrlsForStaging() { + super.testPipeline("tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile") + } + +} diff --git a/tests/jenkins/TestSignArtifacts.groovy b/tests/jenkins/TestSignArtifacts.groovy index 52a1e51c11..da6ef1ec8d 100644 --- a/tests/jenkins/TestSignArtifacts.groovy +++ b/tests/jenkins/TestSignArtifacts.groovy @@ -18,6 +18,28 @@ class TestSignArtifacts extends BuildPipelineTest { void setUp() { super.setUp() + binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') + binding.setVariable('WORKSPACE', 'workspace') + binding.setVariable('DISTRIBUTION_PLATFORM', 'linux') + binding.setVariable('SIGNATURE_TYPE', '.sig') + + binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role') + binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id') + binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket') + binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket') + + helper.registerAllowedMethod("git", [Map]) + helper.registerAllowedMethod("withCredentials", [Map]) + + } + + @Test + void testSignArtifacts() { + super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile") + } + + @Test + void testSignArtifactsJob() { binding.setVariable('URLs', 'https://www.dummy.com/dummy_1_artifact.tar.gz,' + ' https://www.dummy.com/dummy_2_artifact.tar.gz') binding.setVariable('S3_FILE_UPLOAD_PATH', '/dummy/upload/path/') @@ -27,26 +49,14 @@ class TestSignArtifacts extends BuildPipelineTest { binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'Dummy_Upload_Role') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') binding.setVariable('ARTIFACT_BUCKET_NAME', 'dummy_bucket_name') - binding.setVariable('DISTRIBUTION_PLATFORM', 'linux') - binding.setVariable('SIGNATURE_TYPE', '.sig') - binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') - binding.setVariable('WORKSPACE', 'workspace') - binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role') - binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id') - binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket') - binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket') - - helper.registerAllowedMethod("git", [Map]) + helper.registerAllowedMethod("cleanWs", [Map]) helper.registerAllowedMethod("s3Upload", [Map]) helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> closure.delegate = delegate return helper.callClosure(closure) }) - } - @Test - void testSignArtifacts() { - super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile") + super.testPipeline("jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile", "tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile") } } diff --git a/tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile.txt new file mode 100644 index 0000000000..68279b9757 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile.txt @@ -0,0 +1,33 @@ + sign-standalone-artifacts.run() + sign-standalone-artifacts.legacySCM(groovy.lang.Closure) + sign-standalone-artifacts.library({identifier=jenkins@20211123, retriever=null}) + sign-standalone-artifacts.pipeline(groovy.lang.Closure) + sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + sign-standalone-artifacts.stage(sign, groovy.lang.Closure) + sign-standalone-artifacts.script(groovy.lang.Closure) + sign-standalone-artifacts.sh(mkdir workspace/artifacts) + sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz) + sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz) + sign-standalone-artifacts.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux}) + signArtifacts.fileExists(workspace/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_signer_client_role + export EXTERNAL_ID=signer_client_external_id + export UNSIGNED_BUCKET=signer_client_unsigned_bucket + export SIGNED_BUCKET=signer_client_signed_bucket + + workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null + ) + sign-standalone-artifacts.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed}) + uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed}) + sign-standalone-artifacts.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_1_artifact.tar.gz, dummy_1_artifact.tar.gz.sig, dummy_2_artifact.tar.gz, dummy_2_artifact.tar.gz.sig], uploadPath=sign_artifacts_job/dummy/upload/path/20/dist/signed}) + sign-standalone-artifacts.script(groovy.lang.Closure) + sign-standalone-artifacts.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile b/tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile new file mode 100644 index 0000000000..c5699c31a3 --- /dev/null +++ b/tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile @@ -0,0 +1,15 @@ +pipeline { + agent none + stages { + stage('print_urls') { + steps { + script { + printArtifactDownloadUrlsForStaging( + artifactFileNames: filenamesForUrls, + uploadPath: UPLOAD_PATH + ) + } + } + } + } +} diff --git a/tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile.txt b/tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile.txt new file mode 100644 index 0000000000..da30e1b952 --- /dev/null +++ b/tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile.txt @@ -0,0 +1,6 @@ + PrintArtifactDownloadUrlsForStaging_Jenkinsfile.run() + PrintArtifactDownloadUrlsForStaging_Jenkinsfile.pipeline(groovy.lang.Closure) + PrintArtifactDownloadUrlsForStaging_Jenkinsfile.echo(Executing on agent [label:none]) + PrintArtifactDownloadUrlsForStaging_Jenkinsfile.stage(print_urls, groovy.lang.Closure) + PrintArtifactDownloadUrlsForStaging_Jenkinsfile.script(groovy.lang.Closure) + PrintArtifactDownloadUrlsForStaging_Jenkinsfile.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_file.tar.gz, dummy_file.tar.gz.sig], uploadPath=dummy/upload/path}) diff --git a/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt index affe2c5331..22e280ccbd 100644 --- a/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt @@ -28,9 +28,7 @@ createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/artifacts/vars-build/1.3.0/33/x64/linux/builds/opensearch/core-plugins}) signArtifacts.fileExists(workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.fileExists(workspace/opensearch.pgp) - signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp) - signArtifacts.sh(gpg --import workspace/opensearch.pgp) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) signArtifacts.sh( @@ -53,9 +51,7 @@ createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/workspace/file/found.zip}) signArtifacts.fileExists(workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.fileExists(workspace/opensearch.pgp) - signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp) - signArtifacts.sh(gpg --import workspace/opensearch.pgp) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) signArtifacts.sh( diff --git a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile index 621d29cbbe..7e815a3737 100644 --- a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile +++ b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile @@ -4,70 +4,13 @@ pipeline { stage('sign') { steps { script { - if (URLs == '' || S3_FILE_UPLOAD_PATH == '') { - currentBuild.result = 'ABORTED' - error('URLs or S3_FILE_UPLOAD_PATH params are not set') - } - S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('/$', "") - S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('^/+', "") - downloadedFiles = downloadArtifactsFromUrls() signArtifacts( artifactPath: "$WORKSPACE/artifacts/", signatureType: SIGNATURE_TYPE, distributionPlatform: DISTRIBUTION_PLATFORM ) - - finalUploadPath = ([ - "${JOB_NAME}", - "${S3_FILE_UPLOAD_PATH}", - "${BUILD_NUMBER}", - "dist", - "signed" - ].join('/')) - - // uploading artifacts with signatures on s3 - uploadToS3( - sourcePath: "$WORKSPACE/artifacts", - bucket: "${ARTIFACT_BUCKET_NAME}", - path: finalUploadPath - ) - - printUrls(downloadedFiles) - } } } } } - -void printUrls(downloadedFiles){ - - println("Note: only supported file types will be signed") - - for(filename in downloadedFiles){ - - println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" ) - println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" ) - - } - -} - -List downloadArtifactsFromUrls() { - - listOfURls = URLs.split(",") - - def downloadedFiles = [] - - sh "mkdir ${WORKSPACE}/artifacts" - - for (url in listOfURls) { - trimmedUrl = url.trim() - filename = trimmedUrl.substring(trimmedUrl.lastIndexOf('/') + 1) - downloadedFiles.add(filename) - sh "curl -SL ${trimmedUrl} -o ${WORKSPACE}/artifacts/${filename}" - } - - return downloadedFiles - -} diff --git a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt index ceadaeb435..6c3a0f78a5 100644 --- a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt +++ b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt @@ -3,15 +3,10 @@ SignArtifacts_Jenkinsfile.echo(Executing on agent [label:none]) SignArtifacts_Jenkinsfile.stage(sign, groovy.lang.Closure) SignArtifacts_Jenkinsfile.script(groovy.lang.Closure) - SignArtifacts_Jenkinsfile.sh(mkdir workspace/artifacts) - SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz) - SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz) SignArtifacts_Jenkinsfile.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux}) signArtifacts.fileExists(workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.fileExists(workspace/opensearch.pgp) - signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp) - signArtifacts.sh(gpg --import workspace/opensearch.pgp) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) signArtifacts.sh( @@ -24,6 +19,3 @@ workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null ) - SignArtifacts_Jenkinsfile.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed}) - uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed}) diff --git a/vars/printArtifactDownloadUrlsForStaging.groovy b/vars/printArtifactDownloadUrlsForStaging.groovy new file mode 100644 index 0000000000..e27e27a0bc --- /dev/null +++ b/vars/printArtifactDownloadUrlsForStaging.groovy @@ -0,0 +1,8 @@ +void call(Map args = [:]){ + + for(filename in args.artifactFileNames){ + url = "https://ci.opensearch.org/ci/dbc/${args.uploadPath}/${filename}" + println("File ${filename} can be accessed using the url - ${url}" ) + } + +} diff --git a/vars/signArtifacts.groovy b/vars/signArtifacts.groovy index 8a1fe77933..8f42d19581 100644 --- a/vars/signArtifacts.groovy +++ b/vars/signArtifacts.groovy @@ -30,8 +30,7 @@ void call(Map args = [:]) { } void importPGPKey(){ - if( !fileExists("$WORKSPACE/opensearch.pgp")) { - sh("curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o $WORKSPACE/opensearch.pgp") - sh("gpg --import $WORKSPACE/opensearch.pgp") - } + + sh "curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -" + }