Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upload signature alongside snapshots artifacts and checksums #561

Merged
merged 4 commits into from
Nov 7, 2024
Merged

Upload signature alongside snapshots artifacts and checksums #561

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7361dac
Upload .sig alongside snapshots artifacts and checksums
peterzhuamazon Nov 6, 2024
1d92630
Update test cases
peterzhuamazon Nov 6, 2024
98b8ff0
Update test cases
peterzhuamazon Nov 6, 2024
89679b5
Update test cases
peterzhuamazon Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ jacocoTestReport {
}
}

String version = '7.3.2'
String version = '7.3.3'

task updateVersion {
doLast {
Expand Down
46 changes: 24 additions & 22 deletions tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,36 +13,38 @@
uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml)
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
BuildManifest.asBoolean()
uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
bbb
ccc
})
createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz)
createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true})
createSha512Checksums.sh({script=basename ccc, returnStdout=true})
createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb
bbb
ccc
})
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.sh(
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml
sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
)
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.sh(
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sig tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig
cp -v tests/data/tar/builds/opensearch/dist/../manifest.yml tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml
sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
)
uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadMinSnapshotsToS3.echo(Upload min snapshots)
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sig})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml})
66 changes: 35 additions & 31 deletions vars/uploadMinSnapshotsToS3.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,47 +46,51 @@ void call(Map args = [:]) {
String dstDir = "snapshots/core/${productName}/${version}"
String baseName = "${productName}-min-${version}-${platform}-${architecture}"
argsMap = [:]
argsMap['sigtype'] = '.sig'

// Setup core plugins (Tar x64 only)
String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
boolean corePluginDirExists = fileExists(corePluginDir)
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Create .sha512 for Core Plugins Snapshots")
argsMap['artifactPath'] = corePluginDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}
}

// Create checksums
echo('Create .sha512 for Min Snapshots Artifacts')
argsMap['artifactPath'] = srcDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}

echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")

String sedCmd = "sed"
if (platform == "darwin") {
sedCmd = "gsed"
}

sh """
cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml
${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
"""
withCredentials([
string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'),
string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),
string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) {

// Setup core plugins snapshots with .sha512 and .sig (Tar x64 only)
String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
boolean corePluginDirExists = fileExists(corePluginDir)
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Create .sha512 for Core Plugins Snapshots")
argsMap['artifactPath'] = corePluginDir
for (Closure action : fileActions) {
action(argsMap)
}
}

// Setup min snapshots with .sha512 and .sig (All distributions)
echo('Create .sha512 for Min Snapshots Artifacts')
argsMap['artifactPath'] = srcDir
for (Closure action : fileActions) {
action(argsMap)
}

echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")

String sedCmd = "sed"
if (platform == "darwin") {
sedCmd = "gsed"
}

sh """
cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
cp -v ${srcDir}/${baseName}.${extension}.sig ${srcDir}/${baseName}-latest.${extension}.sig
cp -v ${srcDir}/../manifest.yml ${srcDir}/${baseName}-latest.${extension}.build-manifest.yml
${sedCmd} -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
"""
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
// min artifacts
echo("Upload min snapshots")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sig", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sig")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml")
// core plugins
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
Expand Down
Loading