Skip to content

Commit

Permalink
Add core plugins snapshots publishing alongside min artifacts
Browse files Browse the repository at this point in the history 
Signed-off-by: Peter Zhu <[email protected]>
  • Loading branch information
@peterzhuamazon
peterzhuamazon committed Oct 31, 2024
1 parent d6e0d54 commit 6101a1b
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 4 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ jacocoTestReport {
}
}

String version = '7.3.1'
String version = '7.3.2'

task updateVersion {
doLast {
Expand Down
5 changes: 4 additions & 1 deletion tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@
uploadMinSnapshotsToS3.library({identifier=jenkins@main, retriever=null})
uploadMinSnapshotsToS3.readYaml({file=tests/data/opensearch-1.3.0.yml})
InputManifest.asBoolean()
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml)
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
BuildManifest.asBoolean()
uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
Expand All @@ -28,7 +30,7 @@ ccc
bbb
ccc
})
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.sh(
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
Expand All @@ -40,6 +42,7 @@ ccc
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadMinSnapshotsToS3.echo(Upload min snapshots)
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml})
39 changes: 37 additions & 2 deletions vars/uploadMinSnapshotsToS3.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ void call(Map args = [:]) {

def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest))
String productName = inputManifest.build.getFilename()
String version_plain = inputManifest.build.version
String qualifier = inputManifest.build.qualifier ? '-' + inputManifest.build.qualifier : ''
String revision = version_plain + qualifier + '-SNAPSHOT'
def buildManifestYamlOnly = readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")
echo("Retreving build manifest from: $WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")

def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml"))
Expand All @@ -41,16 +45,27 @@ void call(Map args = [:]) {
String srcDir = "${WORKSPACE}/${distribution}/builds/${productName}/dist".replace("\\", "/")
String dstDir = "snapshots/core/${productName}/${version}"
String baseName = "${productName}-min-${version}-${platform}-${architecture}"
argsMap = [:]

// Setup core plugins (Tar x64 only)
String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
boolean corePluginDirExists = fileExists(corePluginDir)
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Create .sha512 for Core Plugins Snapshots")
argsMap['artifactPath'] = corePluginDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}
}

// Create checksums
echo('Create .sha512 for Min Snapshots Artifacts')
argsMap = [:]
argsMap['artifactPath'] = srcDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}

echo("Start copying files: version-${version} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")
echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")

String sedCmd = "sed"
if (platform == "darwin") {
Expand All @@ -68,9 +83,29 @@ void call(Map args = [:]) {
string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),
string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) {
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
// min artifacts
echo("Upload min snapshots")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml")
// core plugins
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Upload core-plugins snapshots")
List<String> corePluginList = buildManifestYamlOnly.components.artifacts.'core-plugins'[0]
echo("corePluginList: ${corePluginList}")
for (String pluginSubPath : corePluginList) {
String pluginSubFolder = pluginSubPath.split('/')[0]
String pluginNameWithExt = pluginSubPath.split('/')[1]
String pluginName = pluginNameWithExt.replace('-' + revision + '.zip', '')
String pluginFullPath = ['plugins', pluginName, revision].join('/')
s3Upload(
bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}",
path: "snapshots/${pluginFullPath}/",
workingDir: "${corePluginDir}/",
includePathPattern: "**/${pluginName}*"
)
}
}
}
}
}

0 comments on commit 6101a1b

Please sign in to comment.