Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add apt repo and the artifacts promotion setups #136

Merged
merged 4 commits into from
Feb 22, 2023
Merged

Add apt repo and the artifacts promotion setups #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cda8700
Add apt repo and the artifacts promotion setups
peterzhuamazon Feb 8, 2023
2cfaec1
Add Apt Support Test Cases Now
peterzhuamazon Feb 9, 2023
e4ba68a
Update test cases
peterzhuamazon Feb 16, 2023
06ac499
Merge remote-tracking branch 'upstream/main' into apt-repo-automation
peterzhuamazon Feb 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions tests/jenkins/TestPromoteRepos.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package jenkins.tests

import org.junit.Before
import org.junit.Test
import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
import static org.hamcrest.CoreMatchers.hasItem
import static org.hamcrest.CoreMatchers.hasItems
import static org.hamcrest.MatcherAssert.assertThat

class TestPromoteRepos extends BuildPipelineTest {

@Override
@Before
void setUp() {
this.registerLibTester(new PromoteReposLibTester('opensearch', '123', 'yum'))
this.registerLibTester(new PromoteReposLibTester('opensearch', '123', 'apt'))
super.setUp()

}

@Test
public void test() {
super.testPipeline("tests/jenkins/jobs/PromoteRepos_Jenkinsfile")
peterzhuamazon marked this conversation as resolved.
Show resolved Hide resolved
}

@Test
void 'yum verification'() {
runScript("tests/jenkins/jobs/PromoteRepos_Jenkinsfile")
assertThat(getShellCommands('sh', 'curl'), hasItems('\n set -e\n set +x\n\n echo \"Pulling 1.3.0 rpm\"\n cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm\n\n ls -l\n '))
assertThat(getShellCommands('sh', 'aws'), hasItems('aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress'))
assertThat(getShellCommands('signArtifacts', ''), hasItems('{artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.asc, platform=linux}'))
assertThat(getShellCommands('sh', 'repomd.pom.asc'), hasItems('\n set -e\n set +x\n \n cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/\n \n ls -l\n \n mv -v repomd.pom repomd.xml\n mv -v repomd.pom.asc repomd.xml.asc\n \n ls -l\n \n cd -\n '))
}

@Test
void 'apt verification'() {
runScript("tests/jenkins/jobs/PromoteRepos_Jenkinsfile")
assertThat(getShellCommands('sh', 'curl'), hasItems('\n set -e\n set +x\n\n echo \"Pulling 1.3.0 deb\"\n cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/deb/dist/opensearch/opensearch-1.3.0-linux-x64.deb\n curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/deb/dist/opensearch/opensearch-1.3.0-linux-arm64.deb\n\n ls -l\n '))
assertThat(getShellCommands('sh', 'aws'), hasItems('aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ --no-progress'))
assertThat(getShellCommands('sh', 'aptly'), hasItems('#!/bin/bash\n\n echo \"Start Signing Apt\"\n rm -rf ~/.aptly\n mkdir $ARTIFACT_PATH/base\n find $ARTIFACT_PATH -type f -name \"*.deb\" | xargs -I {} mv -v {} $ARTIFACT_PATH/base\n aptly repo create -distribution=stable -component=main opensearch\n aptly repo add opensearch $ARTIFACT_PATH/base\n aptly repo show -with-packages opensearch\n aptly snapshot create opensearch-1.x from repo opensearch\n aptly publish snapshot -batch=true -passphrase-file=passphrase opensearch-1.x\n echo \"------------------------------------------------------------------------\"\n echo \"Clean up gpg\"\n gpg --batch --yes --delete-secret-keys RPM_SIGNING_KEY_ID\n gpg --batch --yes --delete-keys RPM_SIGNING_KEY_ID\n rm -v passphrase\n echo \"------------------------------------------------------------------------\"\n rm -rf $ARTIFACT_PATH/*\n cp -rvp ~/.aptly/public/* $ARTIFACT_PATH/\n ls $ARTIFACT_PATH\n\n '))
}

def getShellCommands(methodName, searchString) {
def shCommands = helper.callStack.findAll { call ->
call.methodName == methodName
}.collect { call ->
callArgsToString(call)
}.findAll { command ->
command.contains(searchString)
}
return shCommands
}
}
2 changes: 1 addition & 1 deletion tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
signArtifacts.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure)
signArtifacts.echo(RPM Add Sign)
signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure)
signArtifacts.sh(
signArtifacts.sh(#!/bin/bash
set -e
set +x

Expand Down
17 changes: 15 additions & 2 deletions .../jenkins/jobs/PromoteYumRepos_Jenkinsfile → tests/jenkins/jobs/PromoteRepos_Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,25 @@
pipeline {
agent none
stages {
stage('promote') {
stage('promote yum repo') {
steps {
script {
promoteYumRepos(
promoteRepos(
jobName: "opensearch",
buildNumber: "123",
distributionRepoType: "yum",
manifest: "tests/data/opensearch-1.3.0.yml"
)
}
}
}
stage('promote apt repo') {
steps {
script {
promoteRepos(
jobName: "opensearch",
buildNumber: "123",
distributionRepoType: "apt",
manifest: "tests/data/opensearch-1.3.0.yml"
)
}
Expand Down
185 changes: 185 additions & 0 deletions tests/jenkins/jobs/PromoteRepos_Jenkinsfile.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
PromoteRepos_Jenkinsfile.run()
PromoteRepos_Jenkinsfile.pipeline(groovy.lang.Closure)
PromoteRepos_Jenkinsfile.echo(Executing on agent [label:none])
PromoteRepos_Jenkinsfile.stage(promote yum repo, groovy.lang.Closure)
PromoteRepos_Jenkinsfile.script(groovy.lang.Closure)
PromoteRepos_Jenkinsfile.promoteRepos({jobName=opensearch, buildNumber=123, distributionRepoType=yum, manifest=tests/data/opensearch-1.3.0.yml})
promoteRepos.legacySCM(groovy.lang.Closure)
promoteRepos.library({identifier=jenkins@main, retriever=null})
promoteRepos.readYaml({file=tests/data/opensearch-1.3.0.yml})
InputManifest.asBoolean()
promoteRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
promoteRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
promoteRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
promoteRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pulling Prod yum)
promoteRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress)
promoteRepos.sh(
set -e
set +x

echo "Pulling 1.3.0 rpm"
cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm

ls -l
)
promoteRepos.println(Yum Repo Starts)
promoteRepos.sh(
set -e
set +x

cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum
rm -vf repodata/repomd.xml.asc

echo "Update repo metadata"
createrepo --update .

# Rename .xml to .pom for signing
# Please do not add .xml to signer filter
# As maven have many .xml and we do not want to sign them
# This is an outlier case for yum repo only
mv -v repodata/repomd.xml repodata/repomd.pom

echo "Complete metadata update, awaiting signing repomd.xml"
cd -
)
promoteRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.asc, platform=linux})
signArtifacts.echo(PGP or Windows Signature Signing)
signArtifacts.fileExists(/tmp/workspace/sign.sh)
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE})
signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID})
signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET})
signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET})
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure)
signArtifacts.sh(
#!/bin/bash
set +x
export ROLE=SIGNER_CLIENT_ROLE
export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID
export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET
export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET

/tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype .asc --platform linux
)
promoteRepos.sh(
set -e
set +x

cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/

ls -l

mv -v repomd.pom repomd.xml
mv -v repomd.pom.asc repomd.xml.asc

ls -l

cd -
)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pushing Prod yum)
promoteRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ --no-progress)
PromoteRepos_Jenkinsfile.stage(promote apt repo, groovy.lang.Closure)
PromoteRepos_Jenkinsfile.script(groovy.lang.Closure)
PromoteRepos_Jenkinsfile.promoteRepos({jobName=opensearch, buildNumber=123, distributionRepoType=apt, manifest=tests/data/opensearch-1.3.0.yml})
promoteRepos.legacySCM(groovy.lang.Closure)
promoteRepos.library({identifier=jenkins@main, retriever=null})
promoteRepos.readYaml({file=tests/data/opensearch-1.3.0.yml})
InputManifest.asBoolean()
promoteRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
promoteRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
promoteRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
promoteRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pulling Prod apt)
promoteRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ --no-progress)
promoteRepos.sh(
set -e
set +x

echo "Pulling 1.3.0 deb"
cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/deb/dist/opensearch/opensearch-1.3.0-linux-x64.deb
curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/deb/dist/opensearch/opensearch-1.3.0-linux-arm64.deb

ls -l
)
promoteRepos.println(Apt Repo Starts)
promoteRepos.sh(#!/bin/bash
set -e
set +x

ARTIFACT_PATH="/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt"

echo "------------------------------------------------------------------------"
echo "Check Utility Versions"
gpg_version_requirement="2.2.0"
aptly_version_requirement="1.5.0"

gpg_version_check=`gpg --version | head -n 1 | grep -oE '[0-9.]+'`
gpg_version_check_final=`echo $gpg_version_check $gpg_version_requirement | tr ' ' '
' | sort -V | head -n 1`
aptly_version_check=`aptly version | head -n 1 | grep -oE '[0-9.]+'`
aptly_version_check_final=`echo $aptly_version_check $aptly_version_requirement | tr ' ' '
' | sort -V | head -n 1`

echo -e "gpg_version_requirement gpg_version_check"
echo -e "$gpg_version_requirement $gpg_version_check"
echo -e "aptly_version_requirement aptly_version_check"
echo -e "$aptly_version_requirement $aptly_version_check"

if [[ $gpg_version_requirement = $gpg_version_check_final ]] && [[ $aptly_version_requirement = $aptly_version_check_final ]]; then
echo "Utility version is equal or greater than set limit, continue."
else
echo "Utility version is lower than set limit, exit 1"
exit 1
fi

)
promoteRepos.string({credentialsId=jenkins-rpm-signing-account-number, variable=RPM_SIGNING_ACCOUNT_NUMBER})
promoteRepos.string({credentialsId=jenkins-rpm-signing-passphrase-secrets-arn, variable=RPM_SIGNING_PASSPHRASE_SECRETS_ARN})
promoteRepos.string({credentialsId=jenkins-rpm-signing-secret-key-secrets-arn, variable=RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN})
promoteRepos.string({credentialsId=jenkins-rpm-signing-key-id, variable=RPM_SIGNING_KEY_ID})
promoteRepos.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure)
promoteRepos.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure)
promoteRepos.sh(#!/bin/bash

export GPG_TTY=`tty`

echo "------------------------------------------------------------------------"
echo "Import OpenSearch keys"
aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_PASSPHRASE_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode > passphrase
aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase -

echo "------------------------------------------------------------------------"
)
promoteRepos.sh(#!/bin/bash

echo "Start Signing Apt"
rm -rf ~/.aptly
mkdir $ARTIFACT_PATH/base
find $ARTIFACT_PATH -type f -name "*.deb" | xargs -I {} mv -v {} $ARTIFACT_PATH/base
aptly repo create -distribution=stable -component=main opensearch
aptly repo add opensearch $ARTIFACT_PATH/base
aptly repo show -with-packages opensearch
aptly snapshot create opensearch-1.x from repo opensearch
aptly publish snapshot -batch=true -passphrase-file=passphrase opensearch-1.x
echo "------------------------------------------------------------------------"
echo "Clean up gpg"
gpg --batch --yes --delete-secret-keys RPM_SIGNING_KEY_ID
gpg --batch --yes --delete-keys RPM_SIGNING_KEY_ID
rm -v passphrase
echo "------------------------------------------------------------------------"
rm -rf $ARTIFACT_PATH/*
cp -rvp ~/.aptly/public/* $ARTIFACT_PATH/
ls $ARTIFACT_PATH

)
promoteRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteRepos.println(Pushing Prod apt)
promoteRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/apt/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/apt/ --no-progress)
Loading