Skip to content

Commit

Permalink
Upload signature alongside snapshots artifacts and checksums (#561)
Browse files Browse the repository at this point in the history 
Signed-off-by: Peter Zhu <[email protected]>
  • Loading branch information
@peterzhuamazon
peterzhuamazon authored Nov 7, 2024
1 parent ac5825d commit 0196bcb
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 54 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ jacocoTestReport {
}
}

String version = '7.3.2'
String version = '7.3.3'

task updateVersion {
doLast {
Expand Down
46 changes: 24 additions & 22 deletions tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,36 +13,38 @@
uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml)
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
BuildManifest.asBoolean()
uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
bbb
ccc
})
createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
bbb
ccc
})
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.sh(
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml
sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
)
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.sh(
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sig tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig
cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml
sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
)
uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadMinSnapshotsToS3.echo(Upload min snapshots)
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml})
66 changes: 35 additions & 31 deletions vars/uploadMinSnapshotsToS3.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,47 +46,51 @@ void call(Map args = [:]) {
String dstDir = "snapshots/core/${productName}/${version}"
String baseName = "${productName}-min-${version}-${platform}-${architecture}"
argsMap = [:]
argsMap['sigtype'] = '.sig'

// Setup core plugins (Tar x64 only)
String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
boolean corePluginDirExists = fileExists(corePluginDir)
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Create .sha512 for Core Plugins Snapshots")
argsMap['artifactPath'] = corePluginDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}
}

// Create checksums
echo('Create .sha512 for Min Snapshots Artifacts')
argsMap['artifactPath'] = srcDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}

echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")

String sedCmd = "sed"
if (platform == "darwin") {
sedCmd = "gsed"
}

sh """
cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml
${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
"""
withCredentials([
string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'),
string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),
string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) {

// Setup core plugins snapshots with .sha512 and .sig (Tar x64 only)
String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
boolean corePluginDirExists = fileExists(corePluginDir)
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Create .sha512 for Core Plugins Snapshots")
argsMap['artifactPath'] = corePluginDir
for (Closure action : fileActions) {
action(argsMap)
}
}

// Setup min snapshots with .sha512 and .sig (All distributions)
echo('Create .sha512 for Min Snapshots Artifacts')
argsMap['artifactPath'] = srcDir
for (Closure action : fileActions) {
action(argsMap)
}

echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")

String sedCmd = "sed"
if (platform == "darwin") {
sedCmd = "gsed"
}

sh """
cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
cp -v ${srcDir}/${baseName}.${extension}.sig ${srcDir}/${baseName}-latest.${extension}.sig
cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml
${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
"""
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
// min artifacts
echo("Upload min snapshots")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sig", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sig")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml")
// core plugins
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
Expand Down

0 comments on commit 0196bcb

Please sign in to comment.