From 343d9cb38022f1f0bffec22591d574e03653a52c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Feb 2023 15:08:39 -0500 Subject: [PATCH] Bump version to 2.0.0 and increment version workflow base branch to 2.x (#137) (#145) Update dependency org.jenkins-ci.plugins:junit to v1166.1168.vd6b_8042a_06de (#143) Fix Cve's (#139) Add apt repo and the artifacts promotion setups (#136) --------- Signed-off-by: Peter Zhu Co-authored-by: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> --- .github/workflows/version-increment.yml | 2 +- build.gradle | 10 +- tests/jenkins/TestPromoteRepos.groovy | 62 +++++ .../jobs/AssembleManifest_rpm_Jenkinsfile.txt | 2 +- ...s_Jenkinsfile => PromoteRepos_Jenkinsfile} | 17 +- .../jenkins/jobs/PromoteRepos_Jenkinsfile.txt | 185 +++++++++++++++ .../jobs/SignArtifacts_Jenkinsfile.txt | 2 +- .../PromoteReposLibTester.groovy} | 38 ++-- vars/promoteRepos.groovy | 211 ++++++++++++++++++ vars/signArtifacts.groovy | 2 +- 10 files changed, 507 insertions(+), 24 deletions(-) create mode 100644 tests/jenkins/TestPromoteRepos.groovy rename tests/jenkins/jobs/{PromoteYumRepos_Jenkinsfile => PromoteRepos_Jenkinsfile} (52%) create mode 100644 tests/jenkins/jobs/PromoteRepos_Jenkinsfile.txt rename tests/jenkins/{TestPromoteYumRepos.groovy => lib-testers/PromoteReposLibTester.groovy} (52%) create mode 100644 vars/promoteRepos.groovy diff --git a/.github/workflows/version-increment.yml b/.github/workflows/version-increment.yml index ea83fa626..b6821e991 100644 --- a/.github/workflows/version-increment.yml +++ b/.github/workflows/version-increment.yml @@ -32,7 +32,7 @@ jobs: uses: peter-evans/create-pull-request@v4 with: token: ${{ steps.github_app_token.outputs.token }} - base: '1.x' + base: '2.x' committer: opensearch-ci-bot author: opensearch-ci-bot commit-message: | diff --git a/build.gradle b/build.gradle index 90dfb516a..a860d1000 100644 --- a/build.gradle +++ b/build.gradle @@ -23,7 +23,7 @@ dependencies { testImplementation group: 'junit', name: 'junit', version: '4.13.2' implementation group: 'org.codehaus.groovy', name: 'groovy-all', version: '3.0.14', ext: 'pom' implementation group: 'com.cloudbees', name: 'groovy-cps', version: '1.31' - testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.29' + testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.33' testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.4.1' testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13' } @@ -32,7 +32,7 @@ configurations.all { resolutionStrategy { force group: 'com.google.code.gson', name: 'gson', version: '2.8.9' force group: 'commons-codec', name: 'commons-codec', version: '1.15' - force group: 'com.google.protobuf', name: 'protobuf-java', version: '3.19.3' + force group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.7' } } @@ -66,13 +66,13 @@ sharedLibrary { pluginDependencies { workflowCpsGlobalLibraryPluginVersion = '570.v21311f4951f8' // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/workflow/workflow-cps-global-lib/ // see https://mvnrepository.com/artifact/org.jenkins-ci.plugins/?repo=jenkins-releases for latest - dependency('org.jenkins-ci.plugins.workflow', 'workflow-cps', '2803.v1a_f77ffcc773') + dependency('org.jenkins-ci.plugins.workflow', 'workflow-cps', '3606.v0b_d8b_e512dcf') dependency('org.jenkins-ci.plugins.workflow', 'workflow-multibranch', '2.26.1') dependency('org.jenkins-ci.plugins', 'pipeline-input-step', '456.vd8a_957db_5b_e9') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/pipeline-input-step/ dependency('org.jenkins-ci.plugins', 'script-security', '1229.v4880b_b_e905a_6') dependency('org.jenkins-ci.plugins', 'credentials', '1112.vc87b_7a_3597f6') dependency('org.jenkins-ci.plugins', 'git-client', '3.11.1') - dependency('org.jenkins-ci.plugins', 'junit', '1166.va_436e268e972') + dependency('org.jenkins-ci.plugins', 'junit', '1166.1168.vd6b_8042a_06de') dependency('org.jenkins-ci.plugins', 'mailer', '408.vd726a_1130320') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/mailer/ } } @@ -120,7 +120,7 @@ jacocoTestReport { } } -String version = '1.6.0' +String version = '2.0.0' task updateVersion { doLast { diff --git a/tests/jenkins/TestPromoteRepos.groovy b/tests/jenkins/TestPromoteRepos.groovy new file mode 100644 index 000000000..58a591bf3 --- /dev/null +++ b/tests/jenkins/TestPromoteRepos.groovy @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins.tests + +import org.junit.Before +import org.junit.Test +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat + +class TestPromoteRepos extends BuildPipelineTest { + + @Override + @Before + void setUp() { + this.registerLibTester(new PromoteReposLibTester('opensearch', '123', 'yum')) + this.registerLibTester(new PromoteReposLibTester('opensearch', '123', 'apt')) + super.setUp() + + } + + @Test + public void test() { + super.testPipeline("tests/jenkins/jobs/PromoteRepos_Jenkinsfile") + } + + @Test + void 'yum verification'() { + runScript("tests/jenkins/jobs/PromoteRepos_Jenkinsfile") + assertThat(getShellCommands('sh', 'curl'), hasItems('\n set -e\n set +x\n\n echo \"Pulling 1.3.0 rpm\"\n cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm\n\n ls -l\n ')) + assertThat(getShellCommands('sh', 'aws'), hasItems('aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress')) + assertThat(getShellCommands('signArtifacts', ''), hasItems('{artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.asc, platform=linux}')) + assertThat(getShellCommands('sh', 'repomd.pom.asc'), hasItems('\n set -e\n set +x\n \n cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/\n \n ls -l\n \n mv -v repomd.pom repomd.xml\n mv -v repomd.pom.asc repomd.xml.asc\n \n ls -l\n \n cd -\n ')) + } + + @Test + void 'apt verification'() { + runScript("tests/jenkins/jobs/PromoteRepos_Jenkinsfile") + assertThat(getShellCommands('sh', 'curl'), hasItems('\n set -e\n set +x\n\n echo \"Pulling 1.3.0 deb\"\n cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/deb/dist/opensearch/opensearch-1.3.0-linux-x64.deb\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/deb/dist/opensearch/opensearch-1.3.0-linux-arm64.deb\n\n ls -l\n ')) + assertThat(getShellCommands('sh', 'aws'), hasItems('aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ --no-progress')) + assertThat(getShellCommands('sh', 'aptly'), hasItems('#!/bin/bash\n\n echo \"Start Signing Apt\"\n rm -rf ~/.aptly\n mkdir $ARTIFACT_PATH/base\n find $ARTIFACT_PATH -type f -name \"*.deb\" | xargs -I {} mv -v {} $ARTIFACT_PATH/base\n aptly repo create -distribution=stable -component=main opensearch\n aptly repo add opensearch $ARTIFACT_PATH/base\n aptly repo show -with-packages opensearch\n aptly snapshot create opensearch-1.x from repo opensearch\n aptly publish snapshot -batch=true -passphrase-file=passphrase opensearch-1.x\n echo \"------------------------------------------------------------------------\"\n echo \"Clean up gpg\"\n gpg --batch --yes --delete-secret-keys RPM_SIGNING_KEY_ID\n gpg --batch --yes --delete-keys RPM_SIGNING_KEY_ID\n rm -v passphrase\n echo \"------------------------------------------------------------------------\"\n rm -rf $ARTIFACT_PATH/*\n cp -rvp ~/.aptly/public/* $ARTIFACT_PATH/\n ls $ARTIFACT_PATH\n\n ')) + } + + def getShellCommands(methodName, searchString) { + def shCommands = helper.callStack.findAll { call -> + call.methodName == methodName + }.collect { call -> + callArgsToString(call) + }.findAll { command -> + command.contains(searchString) + } + return shCommands + } +} diff --git a/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt index fc82646e4..f33ca6506 100644 --- a/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt +++ b/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt @@ -18,7 +18,7 @@ signArtifacts.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure) signArtifacts.echo(RPM Add Sign) signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) - signArtifacts.sh( + signArtifacts.sh(#!/bin/bash set -e set +x diff --git a/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile b/tests/jenkins/jobs/PromoteRepos_Jenkinsfile similarity index 52% rename from tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile rename to tests/jenkins/jobs/PromoteRepos_Jenkinsfile index ef68fa57b..76ac15fd7 100644 --- a/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile +++ b/tests/jenkins/jobs/PromoteRepos_Jenkinsfile @@ -10,12 +10,25 @@ pipeline { agent none stages { - stage('promote') { + stage('promote yum repo') { steps { script { - promoteYumRepos( + promoteRepos( + jobName: "opensearch", buildNumber: "123", + distributionRepoType: "yum", + manifest: "tests/data/opensearch-1.3.0.yml" + ) + } + } + } + stage('promote apt repo') { + steps { + script { + promoteRepos( jobName: "opensearch", + buildNumber: "123", + distributionRepoType: "apt", manifest: "tests/data/opensearch-1.3.0.yml" ) } diff --git a/tests/jenkins/jobs/PromoteRepos_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteRepos_Jenkinsfile.txt new file mode 100644 index 000000000..4fdaa2105 --- /dev/null +++ b/tests/jenkins/jobs/PromoteRepos_Jenkinsfile.txt @@ -0,0 +1,185 @@ + PromoteRepos_Jenkinsfile.run() + PromoteRepos_Jenkinsfile.pipeline(groovy.lang.Closure) + PromoteRepos_Jenkinsfile.echo(Executing on agent [label:none]) + PromoteRepos_Jenkinsfile.stage(promote yum repo, groovy.lang.Closure) + PromoteRepos_Jenkinsfile.script(groovy.lang.Closure) + PromoteRepos_Jenkinsfile.promoteRepos({jobName=opensearch, buildNumber=123, distributionRepoType=yum, manifest=tests/data/opensearch-1.3.0.yml}) + promoteRepos.legacySCM(groovy.lang.Closure) + promoteRepos.library({identifier=jenkins@main, retriever=null}) + promoteRepos.readYaml({file=tests/data/opensearch-1.3.0.yml}) + InputManifest.asBoolean() + promoteRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteRepos.println(Pulling Prod yum) + promoteRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress) + promoteRepos.sh( + set -e + set +x + + echo "Pulling 1.3.0 rpm" + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm + + ls -l + ) + promoteRepos.println(Yum Repo Starts) + promoteRepos.sh( + set -e + set +x + + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum + rm -vf repodata/repomd.xml.asc + + echo "Update repo metadata" + createrepo --update . + + # Rename .xml to .pom for signing + # Please do not add .xml to signer filter + # As maven have many .xml and we do not want to sign them + # This is an outlier case for yum repo only + mv -v repodata/repomd.xml repodata/repomd.pom + + echo "Complete metadata update, awaiting signing repomd.xml" + cd - + ) + promoteRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.asc, platform=linux}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(/tmp/workspace/sign.sh) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + + /tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype .asc --platform linux + ) + promoteRepos.sh( + set -e + set +x + + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/ + + ls -l + + mv -v repomd.pom repomd.xml + mv -v repomd.pom.asc repomd.xml.asc + + ls -l + + cd - + ) + promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteRepos.println(Pushing Prod yum) + promoteRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ --no-progress) + PromoteRepos_Jenkinsfile.stage(promote apt repo, groovy.lang.Closure) + PromoteRepos_Jenkinsfile.script(groovy.lang.Closure) + PromoteRepos_Jenkinsfile.promoteRepos({jobName=opensearch, buildNumber=123, distributionRepoType=apt, manifest=tests/data/opensearch-1.3.0.yml}) + promoteRepos.legacySCM(groovy.lang.Closure) + promoteRepos.library({identifier=jenkins@main, retriever=null}) + promoteRepos.readYaml({file=tests/data/opensearch-1.3.0.yml}) + InputManifest.asBoolean() + promoteRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteRepos.println(Pulling Prod apt) + promoteRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ --no-progress) + promoteRepos.sh( + set -e + set +x + + echo "Pulling 1.3.0 deb" + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/deb/dist/opensearch/opensearch-1.3.0-linux-x64.deb + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/deb/dist/opensearch/opensearch-1.3.0-linux-arm64.deb + + ls -l + ) + promoteRepos.println(Apt Repo Starts) + promoteRepos.sh(#!/bin/bash + set -e + set +x + + ARTIFACT_PATH="/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt" + + echo "------------------------------------------------------------------------" + echo "Check Utility Versions" + gpg_version_requirement="2.2.0" + aptly_version_requirement="1.5.0" + + gpg_version_check=`gpg --version | head -n 1 | grep -oE '[0-9.]+'` + gpg_version_check_final=`echo $gpg_version_check $gpg_version_requirement | tr ' ' ' +' | sort -V | head -n 1` + aptly_version_check=`aptly version | head -n 1 | grep -oE '[0-9.]+'` + aptly_version_check_final=`echo $aptly_version_check $aptly_version_requirement | tr ' ' ' +' | sort -V | head -n 1` + + echo -e "gpg_version_requirement gpg_version_check" + echo -e "$gpg_version_requirement $gpg_version_check" + echo -e "aptly_version_requirement aptly_version_check" + echo -e "$aptly_version_requirement $aptly_version_check" + + if [[ $gpg_version_requirement = $gpg_version_check_final ]] && [[ $aptly_version_requirement = $aptly_version_check_final ]]; then + echo "Utility version is equal or greater than set limit, continue." + else + echo "Utility version is lower than set limit, exit 1" + exit 1 + fi + + ) + promoteRepos.string({credentialsId=jenkins-rpm-signing-account-number, variable=RPM_SIGNING_ACCOUNT_NUMBER}) + promoteRepos.string({credentialsId=jenkins-rpm-signing-passphrase-secrets-arn, variable=RPM_SIGNING_PASSPHRASE_SECRETS_ARN}) + promoteRepos.string({credentialsId=jenkins-rpm-signing-secret-key-secrets-arn, variable=RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN}) + promoteRepos.string({credentialsId=jenkins-rpm-signing-key-id, variable=RPM_SIGNING_KEY_ID}) + promoteRepos.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure) + promoteRepos.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) + promoteRepos.sh(#!/bin/bash + + export GPG_TTY=`tty` + + echo "------------------------------------------------------------------------" + echo "Import OpenSearch keys" + aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_PASSPHRASE_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode > passphrase + aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - + + echo "------------------------------------------------------------------------" + ) + promoteRepos.sh(#!/bin/bash + + echo "Start Signing Apt" + rm -rf ~/.aptly + mkdir $ARTIFACT_PATH/base + find $ARTIFACT_PATH -type f -name "*.deb" | xargs -I {} mv -v {} $ARTIFACT_PATH/base + aptly repo create -distribution=stable -component=main opensearch + aptly repo add opensearch $ARTIFACT_PATH/base + aptly repo show -with-packages opensearch + aptly snapshot create opensearch-1.x from repo opensearch + aptly publish snapshot -batch=true -passphrase-file=passphrase opensearch-1.x + echo "------------------------------------------------------------------------" + echo "Clean up gpg" + gpg --batch --yes --delete-secret-keys RPM_SIGNING_KEY_ID + gpg --batch --yes --delete-keys RPM_SIGNING_KEY_ID + rm -v passphrase + echo "------------------------------------------------------------------------" + rm -rf $ARTIFACT_PATH/* + cp -rvp ~/.aptly/public/* $ARTIFACT_PATH/ + ls $ARTIFACT_PATH + + ) + promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteRepos.println(Pushing Prod apt) + promoteRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ --no-progress) diff --git a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt index 13b178810..61694e470 100644 --- a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt +++ b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt @@ -31,7 +31,7 @@ signArtifacts.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure) signArtifacts.echo(RPM Add Sign) signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) - signArtifacts.sh( + signArtifacts.sh(#!/bin/bash set -e set +x diff --git a/tests/jenkins/TestPromoteYumRepos.groovy b/tests/jenkins/lib-testers/PromoteReposLibTester.groovy similarity index 52% rename from tests/jenkins/TestPromoteYumRepos.groovy rename to tests/jenkins/lib-testers/PromoteReposLibTester.groovy index 47222167f..0903c52f0 100644 --- a/tests/jenkins/TestPromoteYumRepos.groovy +++ b/tests/jenkins/lib-testers/PromoteReposLibTester.groovy @@ -6,20 +6,22 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat -package jenkins.tests +class PromoteReposLibTester extends LibFunctionTester { -import org.junit.* -import java.util.* -import java.nio.file.* + private String jobName + private String buildNumber + private String distributionRepoType -class TestPromoteYumRepos extends BuildPipelineTest { - - @Override - @Before - void setUp() { - super.setUp() + public PromoteReposLibTester(jobName, buildNumber, distributionRepoType) { + this.jobName = jobName + this.buildNumber = buildNumber + this.distributionRepoType = distributionRepoType + } + void configure(helper, binding){ binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc') binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') def configs = ["role": "dummy_role", @@ -38,11 +40,21 @@ class TestPromoteYumRepos extends BuildPipelineTest { closure.delegate = delegate return helper.callClosure(closure) }) + } + + void parameterInvariantsAssertions(call){ + assertThat(call.args.jobName.first(), notNullValue()) + assertThat(call.args.buildNumber.first(), notNullValue()) + assertThat(call.args.distributionRepoType.first(), notNullValue()) + } + boolean expectedParametersMatcher(call) { + return call.args.jobName.first().toString().equals(this.jobName) + && call.args.buildNumber.first().toString().equals(this.buildNumber) + && call.args.distributionRepoType.first().toString().equals(this.distributionRepoType) } - @Test - public void testDefault() { - super.testPipeline("tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile") + String libFunctionName() { + return 'promoteRepos' } } diff --git a/vars/promoteRepos.groovy b/vars/promoteRepos.groovy new file mode 100644 index 000000000..51c30ddc3 --- /dev/null +++ b/vars/promoteRepos.groovy @@ -0,0 +1,211 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + + + /** + * Library to sign and promote repositories to artifacts.opensearch.org +@param Map[jobName] - Name of the distribution build workflow on Jenkins that build the artifacts +@param Map[buildNumber] - The build number of the artifacts in above build workflow +@param Map[distributionRepoType] - distribution repository type, e.g. yum / apt +@param Map[manifest] - input manifest of the corresponding OpenSearch/OpenSearch-Dashboards product version (e.g. 2.0.0) + */ + +void call(Map args = [:]) { + def lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) + + String manifest = args.manifest ?: "manifests/${INPUT_MANIFEST}" + def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest)) + + String filename = inputManifest.build.getFilename() + String jobname = args.jobName + + String buildnumber = args.buildNumber ?: 'none' + if (buildnumber == 'none') { + println('User did not enter build number in jenkins parameter, exit 1') + System.exit(1) + } + + def distTypeMap = [ + "yum": "rpm", + "apt": "deb", + ] + String repoType = args.distributionRepoType + String distType = distTypeMap["${repoType}"] + String version = inputManifest.build.version + String majorVersion = version.tokenize('.')[0] + String repoVersion = majorVersion + '.x' + String qualifier = inputManifest.build.qualifier ? '-' + inputManifest.build.qualifier : '' + String revision = version + qualifier + println("Product: ${filename}") + println("Build Number: ${buildnumber}") + println("Input Manifest: ${manifest}") + println("Revision: ${revision}") + println("Major Version: ${majorVersion}") + println("Repo Type: ${repoType}") + println("Dist Type: ${distType}") + println("Repo Version: ${repoVersion}") + + String stagingPkgPathX64 = "${PUBLIC_ARTIFACT_URL}/${jobname}/${revision}/${buildnumber}/linux/x64/${distType}/dist/${filename}/${filename}-${revision}-linux-x64.${distType}" + String stagingPkgPathARM64 = "${PUBLIC_ARTIFACT_URL}/${jobname}/${revision}/${buildnumber}/linux/arm64/${distType}/dist/${filename}/${filename}-${revision}-linux-arm64.${distType}" + + String localPath = "${WORKSPACE}/artifacts" + String RepoProdPath = "releases/bundle/${filename}/${repoVersion}/${repoType}" + String artifactPath = "${localPath}/${RepoProdPath}" + + withCredentials([string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'), + string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), + string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) { + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + println("Pulling Prod ${repoType}") + sh("aws s3 sync s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${RepoProdPath}/ ${artifactPath}/ --no-progress") + } + + sh """ + set -e + set +x + + echo "Pulling ${revision} ${distType}" + cd ${artifactPath} + curl -SLO ${stagingPkgPathX64} + curl -SLO ${stagingPkgPathARM64} + + ls -l + """ + + if (repoType.equals("yum")) { + + println("Yum Repo Starts") + + sh """ + set -e + set +x + + cd ${artifactPath} + rm -vf repodata/repomd.xml.asc + + echo "Update repo metadata" + createrepo --update . + + # Rename .xml to .pom for signing + # Please do not add .xml to signer filter + # As maven have many .xml and we do not want to sign them + # This is an outlier case for yum repo only + mv -v repodata/repomd.xml repodata/repomd.pom + + echo "Complete metadata update, awaiting signing repomd.xml" + cd - + """ + + signArtifacts( + artifactPath: "${artifactPath}/repodata/repomd.pom", + sigtype: '.asc', + platform: 'linux' + ) + + sh """ + set -e + set +x + + cd ${artifactPath}/repodata/ + + ls -l + + mv -v repomd.pom repomd.xml + mv -v repomd.pom.asc repomd.xml.asc + + ls -l + + cd - + """ + } + + if (repoType.equals("apt")) { + + println("Apt Repo Starts") + + sh """#!/bin/bash + set -e + set +x + + ARTIFACT_PATH="${artifactPath}" + + echo "------------------------------------------------------------------------" + echo "Check Utility Versions" + gpg_version_requirement="2.2.0" + aptly_version_requirement="1.5.0" + + gpg_version_check=`gpg --version | head -n 1 | grep -oE '[0-9.]+'` + gpg_version_check_final=`echo \$gpg_version_check \$gpg_version_requirement | tr ' ' '\n' | sort -V | head -n 1` + aptly_version_check=`aptly version | head -n 1 | grep -oE '[0-9.]+'` + aptly_version_check_final=`echo \$aptly_version_check \$aptly_version_requirement | tr ' ' '\n' | sort -V | head -n 1` + + echo -e "gpg_version_requirement gpg_version_check" + echo -e "\$gpg_version_requirement \$gpg_version_check" + echo -e "aptly_version_requirement aptly_version_check" + echo -e "\$aptly_version_requirement \$aptly_version_check" + + if [[ \$gpg_version_requirement = \$gpg_version_check_final ]] && [[ \$aptly_version_requirement = \$aptly_version_check_final ]]; then + echo "Utility version is equal or greater than set limit, continue." + else + echo "Utility version is lower than set limit, exit 1" + exit 1 + fi + + """ + + withCredentials([ + string(credentialsId: 'jenkins-rpm-signing-account-number', variable: 'RPM_SIGNING_ACCOUNT_NUMBER'), + string(credentialsId: 'jenkins-rpm-signing-passphrase-secrets-arn', variable: 'RPM_SIGNING_PASSPHRASE_SECRETS_ARN'), + string(credentialsId: 'jenkins-rpm-signing-secret-key-secrets-arn', variable: 'RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN'), + string(credentialsId: 'jenkins-rpm-signing-key-id', variable: 'RPM_SIGNING_KEY_ID')]) { + withAWS(role: 'jenkins-prod-rpm-signing-assume-role', roleAccount: "${RPM_SIGNING_ACCOUNT_NUMBER}", duration: 900, roleSessionName: 'jenkins-signing-session') { + sh """#!/bin/bash + + export GPG_TTY=`tty` + + echo "------------------------------------------------------------------------" + echo "Import OpenSearch keys" + aws secretsmanager get-secret-value --region us-west-2 --secret-id "${RPM_SIGNING_PASSPHRASE_SECRETS_ARN}" | jq -r .SecretBinary | base64 --decode > passphrase + aws secretsmanager get-secret-value --region us-west-2 --secret-id "${RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN}" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - + + echo "------------------------------------------------------------------------" + """ + } + + sh """#!/bin/bash + + echo "Start Signing Apt" + rm -rf ~/.aptly + mkdir \$ARTIFACT_PATH/base + find \$ARTIFACT_PATH -type f -name "*.deb" | xargs -I {} mv -v {} \$ARTIFACT_PATH/base + aptly repo create -distribution=stable -component=main ${jobname} + aptly repo add ${jobname} \$ARTIFACT_PATH/base + aptly repo show -with-packages ${jobname} + aptly snapshot create ${jobname}-${repoVersion} from repo ${jobname} + aptly publish snapshot -batch=true -passphrase-file=passphrase ${jobname}-${repoVersion} + echo "------------------------------------------------------------------------" + echo "Clean up gpg" + gpg --batch --yes --delete-secret-keys ${RPM_SIGNING_KEY_ID} + gpg --batch --yes --delete-keys ${RPM_SIGNING_KEY_ID} + rm -v passphrase + echo "------------------------------------------------------------------------" + rm -rf \$ARTIFACT_PATH/* + cp -rvp ~/.aptly/public/* \$ARTIFACT_PATH/ + ls \$ARTIFACT_PATH + + """ + } + } + + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + println("Pushing Prod ${repoType}") + sh("aws s3 sync ${artifactPath}/ s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${RepoProdPath}/ --no-progress") + } + } +} diff --git a/vars/signArtifacts.groovy b/vars/signArtifacts.groovy index 14b9e5ea4..ce761b31c 100644 --- a/vars/signArtifacts.groovy +++ b/vars/signArtifacts.groovy @@ -26,7 +26,7 @@ void call(Map args = [:]) { echo 'RPM Add Sign' withAWS(role: 'jenkins-prod-rpm-signing-assume-role', roleAccount: "${RPM_SIGNING_ACCOUNT_NUMBER}", duration: 900, roleSessionName: 'jenkins-signing-session') { - sh """ + sh """#!/bin/bash set -e set +x