Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add apt repo and the artifacts promotion setups #136

Merged
merged 4 commits into from
Feb 22, 2023
Merged

Add apt repo and the artifacts promotion setups #136

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cda8700
Add apt repo and the artifacts promotion setups
peterzhuamazon Feb 8, 2023
2cfaec1
Add Apt Support Test Cases Now
peterzhuamazon Feb 9, 2023
e4ba68a
Update test cases
peterzhuamazon Feb 16, 2023
06ac499
Merge remote-tracking branch 'upstream/main' into apt-repo-automation
peterzhuamazon Feb 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions tests/jenkins/TestPromoteYumRepos.groovy → tests/jenkins/TestPromoteRepos.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import org.junit.*
import java.util.*
import java.nio.file.*

class TestPromoteYumRepos extends BuildPipelineTest {
class TestPromoteRepos extends BuildPipelineTest {

@Override
@Before
Expand Down Expand Up @@ -43,6 +43,6 @@ class TestPromoteYumRepos extends BuildPipelineTest {

@Test
public void testDefault() {
super.testPipeline("tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile")
super.testPipeline("tests/jenkins/jobs/PromoteRepos_Jenkinsfile")
peterzhuamazon marked this conversation as resolved.
Show resolved Hide resolved
}
}
2 changes: 1 addition & 1 deletion tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
signArtifacts.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure)
signArtifacts.echo(RPM Add Sign)
signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure)
signArtifacts.sh(
signArtifacts.sh(#!/bin/bash
set -e
set +x

Expand Down
17 changes: 15 additions & 2 deletions .../jenkins/jobs/PromoteYumRepos_Jenkinsfile → tests/jenkins/jobs/PromoteRepos_Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,23 @@
pipeline {
agent none
stages {
stage('promote') {
stage('promote yum repo') {
steps {
script {
promoteYumRepos(
promoteRepos(
distributionRepoType: "yum",
buildNumber: "123",
jobName: "opensearch",
manifest: "tests/data/opensearch-1.3.0.yml"
)
}
}
}
stage('promote apt repo') {
steps {
script {
promoteRepos(
distributionRepoType: "apt",
buildNumber: "123",
jobName: "opensearch",
manifest: "tests/data/opensearch-1.3.0.yml"
Expand Down
180 changes: 180 additions & 0 deletions tests/jenkins/jobs/PromoteRepos_Jenkinsfile.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
PromoteRepos_Jenkinsfile.run()
PromoteRepos_Jenkinsfile.pipeline(groovy.lang.Closure)
PromoteRepos_Jenkinsfile.echo(Executing on agent [label:none])
PromoteRepos_Jenkinsfile.stage(promote yum repo, groovy.lang.Closure)
PromoteRepos_Jenkinsfile.script(groovy.lang.Closure)
PromoteRepos_Jenkinsfile.promoteRepos({distributionRepoType=yum, buildNumber=123, jobName=opensearch, manifest=tests/data/opensearch-1.3.0.yml})
promoteRepos.legacySCM(groovy.lang.Closure)
promoteRepos.library({identifier=jenkins@main, retriever=null})
promoteRepos.readYaml({file=tests/data/opensearch-1.3.0.yml})
InputManifest.asBoolean()
promoteRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
promoteRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
promoteRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
promoteRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pulling Prod yum)
promoteRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress)
promoteRepos.sh(
set -e
set +x

echo "Pulling 1.3.0 rpm"
cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm

ls -l
)
promoteRepos.println(Yum Repo Starts)
promoteRepos.sh(
set -e
set +x

cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum
rm -vf repodata/repomd.xml.asc

echo "Update repo metadata"
createrepo --update .

# Rename .xml to .pom for signing
# Please do not add .xml to signer filter
# As maven have many .xml and we do not want to sign them
# This is an outlier case for yum repo only
mv -v repodata/repomd.xml repodata/repomd.pom

echo "Complete metadata update, awaiting signing repomd.xml"
cd -
)
promoteRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.sig, platform=linux})
signArtifacts.echo(PGP or Windows Signature Signing)
signArtifacts.fileExists(/tmp/workspace/sign.sh)
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE})
signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID})
signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET})
signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET})
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure)
signArtifacts.sh(
#!/bin/bash
set +x
export ROLE=SIGNER_CLIENT_ROLE
export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID
export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET
export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET

/tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype .sig --platform linux
)
promoteRepos.sh(
set -e
set +x

cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/

ls -l

mv -v repomd.pom repomd.xml
mv -v repomd.pom.sig repomd.xml.sig

# This step is required as yum only accept .asc and signing workflow only support .sig
cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc

rm -vf repomd.xml.sig

ls -l

cd -
)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pushing Prod yum)
promoteRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ --no-progress)
PromoteRepos_Jenkinsfile.stage(promote apt repo, groovy.lang.Closure)
PromoteRepos_Jenkinsfile.script(groovy.lang.Closure)
PromoteRepos_Jenkinsfile.promoteRepos({distributionRepoType=apt, buildNumber=123, jobName=opensearch, manifest=tests/data/opensearch-1.3.0.yml})
promoteRepos.legacySCM(groovy.lang.Closure)
promoteRepos.library({identifier=jenkins@main, retriever=null})
promoteRepos.readYaml({file=tests/data/opensearch-1.3.0.yml})
InputManifest.asBoolean()
promoteRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
promoteRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
promoteRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
promoteRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pulling Prod apt)
promoteRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ --no-progress)
promoteRepos.sh(
set -e
set +x

echo "Pulling 1.3.0 deb"
cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/deb/dist/opensearch/opensearch-1.3.0-linux-x64.deb
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/deb/dist/opensearch/opensearch-1.3.0-linux-arm64.deb

ls -l
)
promoteRepos.println(Apt Repo Starts)
promoteRepos.string({credentialsId=jenkins-rpm-signing-account-number, variable=RPM_SIGNING_ACCOUNT_NUMBER})
promoteRepos.string({credentialsId=jenkins-rpm-signing-passphrase-secrets-arn, variable=RPM_SIGNING_PASSPHRASE_SECRETS_ARN})
promoteRepos.string({credentialsId=jenkins-rpm-signing-secret-key-secrets-arn, variable=RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN})
promoteRepos.string({credentialsId=jenkins-rpm-signing-key-id, variable=RPM_SIGNING_KEY_ID})
promoteRepos.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure)
promoteRepos.echo(APT Sign Repo)
promoteRepos.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure)
promoteRepos.sh(#!/bin/bash
set -e
set +x

ARTIFACT_PATH="/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt"

echo "------------------------------------------------------------------------"
echo "Check Utility Versions"
gpg_version_requirement="2.2.0"
aptly_version_requirement="1.5.0"

gpg_version_check=`gpg --version | head -n 1 | grep -oE '[0-9.]+'`
gpg_version_check_final=`echo $gpg_version_check $gpg_version_requirement | tr ' ' '
' | sort -V | head -n 1`
aptly_version_check=`aptly version | head -n 1 | grep -oE '[0-9.]+'`
aptly_version_check_final=`echo $aptly_version_check $aptly_version_requirement | tr ' ' '
' | sort -V | head -n 1`

echo -e "gpg_version_requirement gpg_version_check"
echo -e "$gpg_version_requirement $gpg_version_check"
echo -e "aptly_version_requirement aptly_version_check"
echo -e "$aptly_version_requirement $aptly_version_check"

if [[ $gpg_version_requirement = $gpg_version_check_final ]] && [[ $aptly_version_requirement = $aptly_version_check_final ]]; then
echo "Utility version is equal or greater than set limit, continue."
else
echo "Utility version is lower than set limit, exit 1"
exit 1
fi

export GPG_TTY=`tty`

echo "------------------------------------------------------------------------"
echo "Import OpenSearch keys"
aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_PASSPHRASE_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode > passphrase
aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase -

echo "------------------------------------------------------------------------"
echo "Start Signing Apt"
rm -rf ~/.aptly
mkdir $ARTIFACT_PATH/base
find $ARTIFACT_PATH -type f -name "*.deb" | xargs -I {} mv -v {} $ARTIFACT_PATH/base
aptly repo create -distribution=stable -component=main opensearch
aptly repo add opensearch $ARTIFACT_PATH/base
aptly repo show -with-packages opensearch
aptly snapshot create opensearch-1.x from repo opensearch
aptly publish snapshot -batch=true -passphrase-file=passphrase opensearch-1.x
rm -v passphrase
rm -rf $ARTIFACT_PATH/*
cp -rvp ~/.aptly/public/* $ARTIFACT_PATH/
ls $ARTIFACT_PATH

)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pushing Prod apt)
promoteRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ --no-progress)
Loading