Skip to content

Commit

Permalink
Minor bugfix for signArtifacts.groovy (#1437)
Browse files Browse the repository at this point in the history
* importing the key outside of if condition

Signed-off-by: Abhinav Gupta <[email protected]>

* added test cases for pgp and sign-artifacts job

Signed-off-by: Abhinav Gupta <[email protected]>

* moved variables for job in the job test case

Signed-off-by: Abhinav Gupta <[email protected]>

* add printurl library

Signed-off-by: Abhinav Gupta <[email protected]>

* added test case for printArtifactUrls

Signed-off-by: Abhinav Gupta <[email protected]>

* single line download and import key

Signed-off-by: Abhinav Gupta <[email protected]>

* remove extra line

Signed-off-by: Abhinav Gupta <[email protected]>

* added regression test file in tests folder

Signed-off-by: Abhinav Gupta <[email protected]>
  • Loading branch information
abhinavGupta16 authored Jan 11, 2022
1 parent 84d8da0 commit 1f2608b
Show file tree
Hide file tree
Showing 11 changed files with 133 additions and 104 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ pipeline {
distributionPlatform: DISTRIBUTION_PLATFORM
)

filenamesForUrls = []

println("Note: only supported file types will be signed")

for(filename in downloadedFiles){
filenamesForUrls.add(filename)
filenamesForUrls.add(filename + SIGNATURE_TYPE)
}

finalUploadPath = ([
"${JOB_NAME}",
"${S3_FILE_UPLOAD_PATH}",
Expand All @@ -62,7 +71,10 @@ pipeline {
path: finalUploadPath
)

printUrls(downloadedFiles)
printArtifactDownloadUrlsForStaging(
artifactFileNames: filenamesForUrls,
uploadPath: finalUploadPath
)

}
}
Expand All @@ -77,19 +89,6 @@ pipeline {
}
}

void printUrls(downloadedFiles){

println("Note: only supported file types will be signed")

for(filename in downloadedFiles){

println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" )
println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" )

}

}

List downloadArtifactsFromUrls() {

listOfURls = URLs.split(",")
Expand Down
28 changes: 28 additions & 0 deletions tests/jenkins/TestPrintArtifactDownloadUrlsForStaging.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package jenkins.tests

import org.junit.*


class TestPrintArtifactDownloadUrlsForStaging extends BuildPipelineTest {

@Before
void setUp() {
super.setUp()
binding.setVariable('filenamesForUrls', ['dummy_file.tar.gz', 'dummy_file.tar.gz.sig'])
binding.setVariable('UPLOAD_PATH', 'dummy/upload/path')
}

@Test
void testPrintArtifactDownloadUrlsForStaging() {
super.testPipeline("tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile")
}

}
38 changes: 24 additions & 14 deletions tests/jenkins/TestSignArtifacts.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,28 @@ class TestSignArtifacts extends BuildPipelineTest {
void setUp() {
super.setUp()

binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
binding.setVariable('WORKSPACE', 'workspace')
binding.setVariable('DISTRIBUTION_PLATFORM', 'linux')
binding.setVariable('SIGNATURE_TYPE', '.sig')

binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role')
binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id')
binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket')
binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket')

helper.registerAllowedMethod("git", [Map])
helper.registerAllowedMethod("withCredentials", [Map])

}

@Test
void testSignArtifacts() {
super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile")
}

@Test
void testSignArtifactsJob() {
binding.setVariable('URLs', 'https://www.dummy.com/dummy_1_artifact.tar.gz,' +
' https://www.dummy.com/dummy_2_artifact.tar.gz')
binding.setVariable('S3_FILE_UPLOAD_PATH', '/dummy/upload/path/')
Expand All @@ -27,26 +49,14 @@ class TestSignArtifacts extends BuildPipelineTest {
binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'Dummy_Upload_Role')
binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account')
binding.setVariable('ARTIFACT_BUCKET_NAME', 'dummy_bucket_name')
binding.setVariable('DISTRIBUTION_PLATFORM', 'linux')
binding.setVariable('SIGNATURE_TYPE', '.sig')
binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
binding.setVariable('WORKSPACE', 'workspace')

binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role')
binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id')
binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket')
binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket')

helper.registerAllowedMethod("git", [Map])
helper.registerAllowedMethod("cleanWs", [Map])
helper.registerAllowedMethod("s3Upload", [Map])
helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
closure.delegate = delegate
return helper.callClosure(closure)
})
}

@Test
void testSignArtifacts() {
super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile")
super.testPipeline("jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile", "tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile")
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
sign-standalone-artifacts.run()
sign-standalone-artifacts.legacySCM(groovy.lang.Closure)
sign-standalone-artifacts.library({identifier=jenkins@20211123, retriever=null})
sign-standalone-artifacts.pipeline(groovy.lang.Closure)
sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
sign-standalone-artifacts.stage(sign, groovy.lang.Closure)
sign-standalone-artifacts.script(groovy.lang.Closure)
sign-standalone-artifacts.sh(mkdir workspace/artifacts)
sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz)
sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz)
sign-standalone-artifacts.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux})
signArtifacts.fileExists(workspace/sign.sh)
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
signArtifacts.sh(
#!/bin/bash
set +x
export ROLE=dummy_signer_client_role
export EXTERNAL_ID=signer_client_external_id
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
export SIGNED_BUCKET=signer_client_signed_bucket

workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null
)
sign-standalone-artifacts.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
sign-standalone-artifacts.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_1_artifact.tar.gz, dummy_1_artifact.tar.gz.sig, dummy_2_artifact.tar.gz, dummy_2_artifact.tar.gz.sig], uploadPath=sign_artifacts_job/dummy/upload/path/20/dist/signed})
sign-standalone-artifacts.script(groovy.lang.Closure)
sign-standalone-artifacts.postCleanup()
postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
15 changes: 15 additions & 0 deletions tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
pipeline {
agent none
stages {
stage('print_urls') {
steps {
script {
printArtifactDownloadUrlsForStaging(
artifactFileNames: filenamesForUrls,
uploadPath: UPLOAD_PATH
)
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.run()
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.pipeline(groovy.lang.Closure)
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.echo(Executing on agent [label:none])
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.stage(print_urls, groovy.lang.Closure)
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.script(groovy.lang.Closure)
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_file.tar.gz, dummy_file.tar.gz.sig], uploadPath=dummy/upload/path})
8 changes: 2 additions & 6 deletions tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@
createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/artifacts/vars-build/1.3.0/33/x64/linux/builds/opensearch/core-plugins})
signArtifacts.fileExists(workspace/sign.sh)
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
signArtifacts.fileExists(workspace/opensearch.pgp)
signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
signArtifacts.sh(gpg --import workspace/opensearch.pgp)
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
signArtifacts.sh(
Expand All @@ -53,9 +51,7 @@
createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/workspace/file/found.zip})
signArtifacts.fileExists(workspace/sign.sh)
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
signArtifacts.fileExists(workspace/opensearch.pgp)
signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
signArtifacts.sh(gpg --import workspace/opensearch.pgp)
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
signArtifacts.sh(
Expand Down
57 changes: 0 additions & 57 deletions tests/jenkins/jobs/SignArtifacts_Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,70 +4,13 @@ pipeline {
stage('sign') {
steps {
script {
if (URLs == '' || S3_FILE_UPLOAD_PATH == '') {
currentBuild.result = 'ABORTED'
error('URLs or S3_FILE_UPLOAD_PATH params are not set')
}
S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('/$', "")
S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('^/+', "")
downloadedFiles = downloadArtifactsFromUrls()
signArtifacts(
artifactPath: "$WORKSPACE/artifacts/",
signatureType: SIGNATURE_TYPE,
distributionPlatform: DISTRIBUTION_PLATFORM
)

finalUploadPath = ([
"${JOB_NAME}",
"${S3_FILE_UPLOAD_PATH}",
"${BUILD_NUMBER}",
"dist",
"signed"
].join('/'))

// uploading artifacts with signatures on s3
uploadToS3(
sourcePath: "$WORKSPACE/artifacts",
bucket: "${ARTIFACT_BUCKET_NAME}",
path: finalUploadPath
)

printUrls(downloadedFiles)

}
}
}
}
}

void printUrls(downloadedFiles){

println("Note: only supported file types will be signed")

for(filename in downloadedFiles){

println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" )
println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" )

}

}

List downloadArtifactsFromUrls() {

listOfURls = URLs.split(",")

def downloadedFiles = []

sh "mkdir ${WORKSPACE}/artifacts"

for (url in listOfURls) {
trimmedUrl = url.trim()
filename = trimmedUrl.substring(trimmedUrl.lastIndexOf('/') + 1)
downloadedFiles.add(filename)
sh "curl -SL ${trimmedUrl} -o ${WORKSPACE}/artifacts/${filename}"
}

return downloadedFiles

}
10 changes: 1 addition & 9 deletions tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,10 @@
SignArtifacts_Jenkinsfile.echo(Executing on agent [label:none])
SignArtifacts_Jenkinsfile.stage(sign, groovy.lang.Closure)
SignArtifacts_Jenkinsfile.script(groovy.lang.Closure)
SignArtifacts_Jenkinsfile.sh(mkdir workspace/artifacts)
SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz)
SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz)
SignArtifacts_Jenkinsfile.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux})
signArtifacts.fileExists(workspace/sign.sh)
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
signArtifacts.fileExists(workspace/opensearch.pgp)
signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
signArtifacts.sh(gpg --import workspace/opensearch.pgp)
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
signArtifacts.sh(
Expand All @@ -24,6 +19,3 @@

workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null
)
SignArtifacts_Jenkinsfile.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
8 changes: 8 additions & 0 deletions vars/printArtifactDownloadUrlsForStaging.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
void call(Map args = [:]){

for(filename in args.artifactFileNames){
url = "https://ci.opensearch.org/ci/dbc/${args.uploadPath}/${filename}"
println("File ${filename} can be accessed using the url - ${url}" )
}

}
7 changes: 3 additions & 4 deletions vars/signArtifacts.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ void call(Map args = [:]) {
}

void importPGPKey(){
if( !fileExists("$WORKSPACE/opensearch.pgp")) {
sh("curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o $WORKSPACE/opensearch.pgp")
sh("gpg --import $WORKSPACE/opensearch.pgp")
}

sh "curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -"

}

0 comments on commit 1f2608b

Please sign in to comment.