From b75458858f861d5b1e2e3e5fd5f923865d7eea28 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 27 Jan 2022 20:04:48 -0500 Subject: [PATCH 1/8] Adding checksum for latest snapshots artifacts Signed-off-by: Peter Zhu --- .../opensearch/distribution-build.jenkinsfile | 8 +++- .../jenkins/TestUploadMinSnapshotsToS3.groovy | 22 +++------ tests/jenkins/data/opensearch-1.3.0.yml | 1 + tests/jenkins/data/opensearch-1.3.0.yml.lock | 1 + ...ile => uploadMinSnapshotsToS3_Jenkinsfile} | 5 +- .../uploadMinSnapshotsToS3_Jenkinsfile.txt | 37 +++++++++++++++ .../jobs/uploadSnapshotsToS3_Jenkinsfile.txt | 15 ------ .../UploadMinSnapshotsToS3LibTester.groovy | 46 +++++++++++++++++++ vars/uploadMinSnapshotsToS3.groovy | 35 ++++++++++++-- 9 files changed, 132 insertions(+), 38 deletions(-) rename tests/jenkins/jobs/{uploadSnapshotsToS3_Jenkinsfile => uploadMinSnapshotsToS3_Jenkinsfile} (52%) create mode 100644 tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt delete mode 100644 tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile.txt create mode 100644 tests/jenkins/lib-testers/UploadMinSnapshotsToS3LibTester.groovy diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 3305264a6d..020811a80f 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -53,7 +53,9 @@ pipeline { echo "Skipping publishing snapshots, builds/opensearch/maven does not exist." } echo("Uploading min snapshots to S3") - uploadMinSnapshotsToS3() + uploadMinSnapshotsToS3( + fileActions: [createSha512Checksums()] + ) } } post { @@ -78,7 +80,9 @@ pipeline { snapshot: true ) echo("Uploading min snapshots to S3") - uploadMinSnapshotsToS3() + uploadMinSnapshotsToS3( + fileActions: [createSha512Checksums()] + ) } } post { diff --git a/tests/jenkins/TestUploadMinSnapshotsToS3.groovy b/tests/jenkins/TestUploadMinSnapshotsToS3.groovy index d014188cb9..7e65adee0a 100644 --- a/tests/jenkins/TestUploadMinSnapshotsToS3.groovy +++ b/tests/jenkins/TestUploadMinSnapshotsToS3.groovy @@ -8,31 +8,21 @@ package jenkins.tests +import com.kenai.jffi.Closure +import jenkins.tests.BuildPipelineTest import org.junit.* -import java.util.* class TestUploadMinSnapshotsToS3 extends BuildPipelineTest { @Before void setUp() { + List fileActions = ['createSha512Checksums'] + this.registerLibTester(new UploadMinSnapshotsToS3LibTester( fileActions, 'tests/jenkins/data/opensearch-1.3.0.yml' )) super.setUp() - - binding.setVariable('WORKSPACE', 'tests/data/') - - binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'dummy_role') - binding.setVariable('AWS_ACCOUNT_ARTIFACT', '1234') - binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'dummy_bucket') - - helper.registerAllowedMethod("s3Upload", [Map]) - helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> - closure.delegate = delegate - return helper.callClosure(closure) - }) } @Test public void test() { - super.testPipeline("tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile") + super.testPipeline("tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile") } -} - +} \ No newline at end of file diff --git a/tests/jenkins/data/opensearch-1.3.0.yml b/tests/jenkins/data/opensearch-1.3.0.yml index 33f0c074b3..bf6381e28c 100644 --- a/tests/jenkins/data/opensearch-1.3.0.yml +++ b/tests/jenkins/data/opensearch-1.3.0.yml @@ -3,6 +3,7 @@ schema-version: '1.0' build: name: OpenSearch version: 1.3.0 + id: '717' ci: image: name: opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028 diff --git a/tests/jenkins/data/opensearch-1.3.0.yml.lock b/tests/jenkins/data/opensearch-1.3.0.yml.lock index 46d5a6ec82..dbe6787400 100644 --- a/tests/jenkins/data/opensearch-1.3.0.yml.lock +++ b/tests/jenkins/data/opensearch-1.3.0.yml.lock @@ -2,6 +2,7 @@ build: architecture: x64 name: OpenSearch version: 1.3.0 + id: '717' ci: image: args: -e JAVA_HOME=/opt/java/openjdk-11 diff --git a/tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile similarity index 52% rename from tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile rename to tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile index ee76d51e05..31274dde3c 100644 --- a/tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile @@ -1,3 +1,5 @@ +lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm)) + pipeline { agent none stages { @@ -5,7 +7,8 @@ pipeline { steps { script { uploadMinSnapshotsToS3( - manifest: "tests/jenkins/data/opensearch-1.3.0.yml" + fileActions: [createSha512Checksums()], + manifest: 'tests/jenkins/data/opensearch-1.3.0.yml' ) } } diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt new file mode 100644 index 0000000000..c240bb9309 --- /dev/null +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -0,0 +1,37 @@ + uploadMinSnapshotsToS3_Jenkinsfile.run() + uploadMinSnapshotsToS3_Jenkinsfile.legacySCM(groovy.lang.Closure) + uploadMinSnapshotsToS3_Jenkinsfile.library({identifier=jenkins@20211123, retriever=null}) + uploadMinSnapshotsToS3_Jenkinsfile.pipeline(groovy.lang.Closure) + uploadMinSnapshotsToS3_Jenkinsfile.echo(Executing on agent [label:none]) + uploadMinSnapshotsToS3_Jenkinsfile.stage(uploadMinSnapshotsToS3, groovy.lang.Closure) + uploadMinSnapshotsToS3_Jenkinsfile.script(groovy.lang.Closure) + uploadMinSnapshotsToS3_Jenkinsfile.createSha512Checksums() + uploadMinSnapshotsToS3_Jenkinsfile.uploadMinSnapshotsToS3({fileActions=[groovy.lang.Closure], manifest=tests/jenkins/data/opensearch-1.3.0.yml}) + uploadMinSnapshotsToS3.legacySCM(groovy.lang.Closure) + uploadMinSnapshotsToS3.library({identifier=jenkins@20211123, retriever=null}) + uploadMinSnapshotsToS3.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) + InputManifest.asBoolean() + uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/builds/opensearch/manifest.yml) + uploadMinSnapshotsToS3.readYaml({file=tests/data/builds/opensearch/manifest.yml}) + BuildManifest.asBoolean() + uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts) + createSha512Checksums.sh({script=find tests/data/builds/opensearch/dist -type f, returnStdout=true}) + createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz) + createSha512Checksums.sh({script=sha512sum opensearch-min-1.3.0-linux-x64.tar.gz, returnStdout=true}) + createSha512Checksums.sh({script=basename ccc, returnStdout=true}) + createSha512Checksums.writeFile({file=opensearch-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb +bbb +ccc +}) + createSha512Checksums.echo(Creating sha for opensearch-dashboards-min-1.3.0-linux-x64.tar.gz) + createSha512Checksums.sh({script=sha512sum opensearch-dashboards-min-1.3.0-linux-x64.tar.gz, returnStdout=true}) + createSha512Checksums.sh({script=basename ccc, returnStdout=true}) + createSha512Checksums.writeFile({file=opensearch-dashboards-min-1.3.0-linux-x64.tar.gz.sha512, text=bbb +bbb +ccc +}) + uploadMinSnapshotsToS3.withAWS({role=dummy_role, roleAccount=1234, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512}) diff --git a/tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile.txt deleted file mode 100644 index 3b74aa1f34..0000000000 --- a/tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile.txt +++ /dev/null @@ -1,15 +0,0 @@ - uploadSnapshotsToS3_Jenkinsfile.run() - uploadSnapshotsToS3_Jenkinsfile.pipeline(groovy.lang.Closure) - uploadSnapshotsToS3_Jenkinsfile.echo(Executing on agent [label:none]) - uploadSnapshotsToS3_Jenkinsfile.stage(uploadMinSnapshotsToS3, groovy.lang.Closure) - uploadSnapshotsToS3_Jenkinsfile.script(groovy.lang.Closure) - uploadSnapshotsToS3_Jenkinsfile.uploadMinSnapshotsToS3({manifest=tests/jenkins/data/opensearch-1.3.0.yml}) - uploadMinSnapshotsToS3.legacySCM(groovy.lang.Closure) - uploadMinSnapshotsToS3.library({identifier=jenkins@20211123, retriever=null}) - uploadMinSnapshotsToS3.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) - InputManifest.asBoolean() - uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data//builds/opensearch/manifest.yml) - uploadMinSnapshotsToS3.readYaml({file=tests/data//builds/opensearch/manifest.yml}) - BuildManifest.asBoolean() - uploadMinSnapshotsToS3.withAWS({role=dummy_role, roleAccount=1234, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - uploadMinSnapshotsToS3.s3Upload({file=tests/data//builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) diff --git a/tests/jenkins/lib-testers/UploadMinSnapshotsToS3LibTester.groovy b/tests/jenkins/lib-testers/UploadMinSnapshotsToS3LibTester.groovy new file mode 100644 index 0000000000..9f67b5c7bd --- /dev/null +++ b/tests/jenkins/lib-testers/UploadMinSnapshotsToS3LibTester.groovy @@ -0,0 +1,46 @@ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat + +class UploadMinSnapshotsToS3LibTester extends LibFunctionTester { + + private List fileActions + private String manifest + + public UploadMinSnapshotsToS3LibTester(fileActions, manifest){ + this.fileActions = fileActions + this.manifest = manifest + } + + void parameterInvariantsAssertions(call){ + assertThat(call.args.fileActions.first(), notNullValue()) + assertThat(call.args.manifest.first(), notNullValue()) + } + + boolean expectedParametersMatcher(call) { + boolean actionMatch = call.args.fileActions.size() == 0 + for (actionCall in call.args.fileActions) { + actionMatch = this.fileActions.any( actionThis -> actionCall.toString().contains(actionThis.toString()) ) + } + return actionMatch && call.args.manifest.first().toString().equals(this.manifest) + } + + String libFunctionName(){ + return 'uploadMinSnapshotsToS3' + } + + void configure(helper, binding){ + binding.setVariable('WORKSPACE', 'tests/data') + binding.setVariable('productName', 'opensearch') + binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'dummy_role') + binding.setVariable('AWS_ACCOUNT_ARTIFACT', '1234') + binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'dummy_bucket') + helper.registerAllowedMethod("s3Upload", [Map]) + helper.addShMock('find tests/data/builds/opensearch/dist -type f') { script -> + return [stdout: "opensearch-min-1.3.0-linux-x64.tar.gz opensearch-dashboards-min-1.3.0-linux-x64.tar.gz", exitValue: 0] + } + helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + } +} diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index a97be36fa8..7b827bdfa8 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -1,15 +1,42 @@ void call(Map args = [:]) { + def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) + List fileActions = args.fileActions ?: [] String manifest = args.manifest ?: "manifests/${INPUT_MANIFEST}" + def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest)) echo("Retreving build manifest from: $WORKSPACE/builds/${inputManifest.build.getFilename()}/manifest.yml") - def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/builds/${inputManifest.build.getFilename()}/manifest.yml")) + + productName = inputManifest.build.getFilename() + def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/builds/${productName}/manifest.yml")) version = buildManifest.build.version architecture = buildManifest.build.architecture platform = buildManifest.build.platform - minArtifactSourceName = "opensearch-min-${version}-${platform}-${architecture}.tar.gz" - minArtifactName = "opensearch-min-${version}-${platform}-${architecture}-latest.tar.gz" + id = buildManifest.build.id + + // Setup src & dst variables for artifacts + srcDir = "${WORKSPACE}/builds/${productName}/dist" + dstDir = "test/others/core/${productName}/${version}" + baseName = "${productName}-min-${version}-${platform}-${architecture}" + tarSrcName = "${baseName}.tar.gz" + tarDstNameId = "${baseName}-${id}.tar.gz" + tarDstNameLatest = "${baseName}-latest.tar.gz" + shaSrcName = "${tarSrcName}.sha512" + shaDstNameId = "${tarDstNameId}.sha512" + shaDstNameLatest = "${tarDstNameLatest}.sha512" + + // Create checksums + echo("Create .sha512 for Min Snapshots Artifacts") + argsMap = [:] + argsMap['artifactPath'] = srcDir + for (Closure action : fileActions) { // running createSha512Checksums() + action(argsMap) + } + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - s3Upload(file: "$WORKSPACE/builds/opensearch/dist/${minArtifactSourceName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "snapshots/core/opensearch/${version}/${minArtifactName}") + s3Upload(file: "${srcDir}/${tarSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${tarDstNameId}") + s3Upload(file: "${srcDir}/${tarSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${tarDstNameLatest}") + s3Upload(file: "${srcDir}/${shaSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${shaDstNameId}") + s3Upload(file: "${srcDir}/${shaSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${shaDstNameLatest}") } } From 55e0158ad2f631b5604de0be58ea5c7a582cecfb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 31 Jan 2022 14:32:53 -0500 Subject: [PATCH 2/8] Change to snapshots dir name in path Signed-off-by: Peter Zhu --- vars/uploadMinSnapshotsToS3.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index 7b827bdfa8..e368042d5c 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -16,7 +16,7 @@ void call(Map args = [:]) { // Setup src & dst variables for artifacts srcDir = "${WORKSPACE}/builds/${productName}/dist" - dstDir = "test/others/core/${productName}/${version}" + dstDir = "snapshots/core/${productName}/${version}" baseName = "${productName}-min-${version}-${platform}-${architecture}" tarSrcName = "${baseName}.tar.gz" tarDstNameId = "${baseName}-${id}.tar.gz" From 50e76409c2a57399927a12682bef9f5f52492b84 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 31 Jan 2022 15:01:41 -0500 Subject: [PATCH 3/8] Update test stack with new location Signed-off-by: Peter Zhu --- tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index c240bb9309..c3650a689d 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -31,7 +31,7 @@ bbb ccc }) uploadMinSnapshotsToS3.withAWS({role=dummy_role, roleAccount=1234, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=test/others/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512}) From 19d8127156f3939584382617a904e07b820d65f8 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 31 Jan 2022 15:34:34 -0500 Subject: [PATCH 4/8] Simplify the naming in artifacts variables Signed-off-by: Peter Zhu --- vars/uploadMinSnapshotsToS3.groovy | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index e368042d5c..b01641d77b 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -18,12 +18,6 @@ void call(Map args = [:]) { srcDir = "${WORKSPACE}/builds/${productName}/dist" dstDir = "snapshots/core/${productName}/${version}" baseName = "${productName}-min-${version}-${platform}-${architecture}" - tarSrcName = "${baseName}.tar.gz" - tarDstNameId = "${baseName}-${id}.tar.gz" - tarDstNameLatest = "${baseName}-latest.tar.gz" - shaSrcName = "${tarSrcName}.sha512" - shaDstNameId = "${tarDstNameId}.sha512" - shaDstNameLatest = "${tarDstNameLatest}.sha512" // Create checksums echo("Create .sha512 for Min Snapshots Artifacts") @@ -34,9 +28,9 @@ void call(Map args = [:]) { } withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - s3Upload(file: "${srcDir}/${tarSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${tarDstNameId}") - s3Upload(file: "${srcDir}/${tarSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${tarDstNameLatest}") - s3Upload(file: "${srcDir}/${shaSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${shaDstNameId}") - s3Upload(file: "${srcDir}/${shaSrcName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${shaDstNameLatest}") + s3Upload(file: "${srcDir}/${baseName}.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-${id}.tar.gz") + s3Upload(file: "${srcDir}/${baseName}.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz") + s3Upload(file: "${srcDir}/${baseName}.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-${id}.tar.gz.sha512") + s3Upload(file: "${srcDir}/${baseName}.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz.sha512") } } From fe24362aea0aaaf6888bc8a55f4ed3f16181b442 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 1 Feb 2022 17:09:51 -0500 Subject: [PATCH 5/8] Rename internal filename of sha512 so checksum pass Signed-off-by: Peter Zhu --- vars/uploadMinSnapshotsToS3.groovy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index b01641d77b..bb141e64dc 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -28,9 +28,14 @@ void call(Map args = [:]) { } withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + sh """ + cp ${srcDir}/${baseName}.tar.gz.sha512 ${srcDir}/${baseName}-${id}.tar.gz.sha512 ${srcDir}/${baseName}-latest.tar.gz.sha512 + sed -i "s/.tar.gz.sha512/-${id}.tar.gz.sha512/g" ${srcDir}/${baseName}-${id}.tar.gz.sha512 + sed -i "s/.tar.gz.sha512/-latest.tar.gz.sha512/g" ${srcDir}/${baseName}-latest.tar.gz.sha512 + """ s3Upload(file: "${srcDir}/${baseName}.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-${id}.tar.gz") s3Upload(file: "${srcDir}/${baseName}.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz") - s3Upload(file: "${srcDir}/${baseName}.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-${id}.tar.gz.sha512") - s3Upload(file: "${srcDir}/${baseName}.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz.sha512") + s3Upload(file: "${srcDir}/${baseName}-${id}.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-${id}.tar.gz.sha512") + s3Upload(file: "${srcDir}/${baseName}-latest.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz.sha512") } } From 28317ac86bdbcdd93d356e100d2db56b1ba06757 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 1 Feb 2022 17:35:39 -0500 Subject: [PATCH 6/8] Update tests Signed-off-by: Peter Zhu --- .../jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index c3650a689d..58f799f73f 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -31,7 +31,12 @@ bbb ccc }) uploadMinSnapshotsToS3.withAWS({role=dummy_role, roleAccount=1234, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadMinSnapshotsToS3.sh( + cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512 tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 + sed -i "s/.tar.gz.sha512/-29.tar.gz.sha512/g" tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512 + sed -i "s/.tar.gz.sha512/-latest.tar.gz.sha512/g" tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 + ) uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512}) From f01c2a3b9a08b5049ea741b34141d6900f1b7ec2 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 1 Feb 2022 18:21:08 -0500 Subject: [PATCH 7/8] Remove id number Signed-off-by: Peter Zhu --- .../jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt | 10 ++++------ vars/uploadMinSnapshotsToS3.groovy | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index 58f799f73f..5a15bfbc83 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -32,11 +32,9 @@ ccc }) uploadMinSnapshotsToS3.withAWS({role=dummy_role, roleAccount=1234, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadMinSnapshotsToS3.sh( - cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512 tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 - sed -i "s/.tar.gz.sha512/-29.tar.gz.sha512/g" tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512 - sed -i "s/.tar.gz.sha512/-latest.tar.gz.sha512/g" tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 + cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz + cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 + sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 ) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) - uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-29.tar.gz.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512}) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index bb141e64dc..7e7bbb23b1 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -29,13 +29,11 @@ void call(Map args = [:]) { withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { sh """ - cp ${srcDir}/${baseName}.tar.gz.sha512 ${srcDir}/${baseName}-${id}.tar.gz.sha512 ${srcDir}/${baseName}-latest.tar.gz.sha512 - sed -i "s/.tar.gz.sha512/-${id}.tar.gz.sha512/g" ${srcDir}/${baseName}-${id}.tar.gz.sha512 - sed -i "s/.tar.gz.sha512/-latest.tar.gz.sha512/g" ${srcDir}/${baseName}-latest.tar.gz.sha512 + cp ${srcDir}/${baseName}.tar.gz ${srcDir}/${baseName}-latest.tar.gz + cp ${srcDir}/${baseName}.tar.gz.sha512 ${srcDir}/${baseName}-latest.tar.gz.sha512 + sed -i "s/.tar.gz/-latest.tar.gz/g" ${srcDir}/${baseName}-latest.tar.gz.sha512 """ - s3Upload(file: "${srcDir}/${baseName}.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-${id}.tar.gz") - s3Upload(file: "${srcDir}/${baseName}.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz") - s3Upload(file: "${srcDir}/${baseName}-${id}.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-${id}.tar.gz.sha512") + s3Upload(file: "${srcDir}/${baseName}-latest.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz") s3Upload(file: "${srcDir}/${baseName}-latest.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz.sha512") } } From c91b5a37512c38b8ebcbc71b266a83a8ca40629e Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 1 Feb 2022 18:39:14 -0500 Subject: [PATCH 8/8] Move sh block outside of withAWS block Signed-off-by: Peter Zhu --- .../jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt | 10 +++++----- vars/uploadMinSnapshotsToS3.groovy | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt index 5a15bfbc83..1a1badc1ab 100644 --- a/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt @@ -30,11 +30,11 @@ ccc bbb ccc }) + uploadMinSnapshotsToS3.sh( + cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz + cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 + sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 + ) uploadMinSnapshotsToS3.withAWS({role=dummy_role, roleAccount=1234, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - uploadMinSnapshotsToS3.sh( - cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz - cp tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 - sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512 - ) uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz}) uploadMinSnapshotsToS3.s3Upload({file=tests/data/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512, bucket=dummy_bucket, path=snapshots/core/opensearch/1.2.2-SNAPSHOT/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512}) diff --git a/vars/uploadMinSnapshotsToS3.groovy b/vars/uploadMinSnapshotsToS3.groovy index 7e7bbb23b1..ecf7eb5b8c 100644 --- a/vars/uploadMinSnapshotsToS3.groovy +++ b/vars/uploadMinSnapshotsToS3.groovy @@ -27,12 +27,12 @@ void call(Map args = [:]) { action(argsMap) } + sh """ + cp ${srcDir}/${baseName}.tar.gz ${srcDir}/${baseName}-latest.tar.gz + cp ${srcDir}/${baseName}.tar.gz.sha512 ${srcDir}/${baseName}-latest.tar.gz.sha512 + sed -i "s/.tar.gz/-latest.tar.gz/g" ${srcDir}/${baseName}-latest.tar.gz.sha512 + """ withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - sh """ - cp ${srcDir}/${baseName}.tar.gz ${srcDir}/${baseName}-latest.tar.gz - cp ${srcDir}/${baseName}.tar.gz.sha512 ${srcDir}/${baseName}-latest.tar.gz.sha512 - sed -i "s/.tar.gz/-latest.tar.gz/g" ${srcDir}/${baseName}-latest.tar.gz.sha512 - """ s3Upload(file: "${srcDir}/${baseName}-latest.tar.gz", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz") s3Upload(file: "${srcDir}/${baseName}-latest.tar.gz.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.tar.gz.sha512") }