Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 7.x] Add core plugins snapshots publishing alongside min snapshots #560

Merged
merged 1 commit into from
Nov 5, 2024
+41 −3
Merged

[Backport 7.x] Add core plugins snapshots publishing alongside min snapshots #560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@
uploadMinSnapshotsToS3.library({[email protected], retriever=null})
uploadMinSnapshotsToS3.readYaml({file=tests/data/opensearch-1.3.0.yml})
InputManifest.asBoolean()
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
uploadMinSnapshotsToS3.echo(Retreving build manifest from: tests/data/tar/builds/opensearch/manifest.yml)
uploadMinSnapshotsToS3.readYaml({file=tests/data/tar/builds/opensearch/manifest.yml})
BuildManifest.asBoolean()
uploadMinSnapshotsToS3.fileExists(tests/data/tar/builds/opensearch/core-plugins)
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find tests/data/tar/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Creating sha for opensearch-min-1.3.0-linux-x64.tar.gz)
Expand All @@ -28,7 +30,7 @@ ccc
bbb
ccc
})
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.echo(Start copying files: version-1.3.0 revision-1.3.0-SNAPSHOT architecture-x64 platform-linux buildid-29 distribution-tar extension-tar.gz)
uploadMinSnapshotsToS3.sh(
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512
Expand All @@ -40,6 +42,7 @@ ccc
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadMinSnapshotsToS3.echo(Upload min snapshots)
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.sha512})
uploadMinSnapshotsToS3.s3Upload({file=tests/data/tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/1.3.0/opensearch-min-1.3.0-linux-x64-latest.tar.gz.build-manifest.yml})
39 changes: 37 additions & 2 deletions vars/uploadMinSnapshotsToS3.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ void call(Map args = [:]) {

def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest))
String productName = inputManifest.build.getFilename()
String version_plain = inputManifest.build.version
String qualifier = inputManifest.build.qualifier ? '-' + inputManifest.build.qualifier : ''
String revision = version_plain + qualifier + '-SNAPSHOT'
def buildManifestYamlOnly = readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")
echo("Retreving build manifest from: $WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml")

def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/${distribution_arg}/builds/${productName}/manifest.yml"))
Expand All @@ -41,16 +45,27 @@ void call(Map args = [:]) {
String srcDir = "${WORKSPACE}/${distribution}/builds/${productName}/dist".replace("\\", "/")
String dstDir = "snapshots/core/${productName}/${version}"
String baseName = "${productName}-min-${version}-${platform}-${architecture}"
argsMap = [:]

// Setup core plugins (Tar x64 only)
String corePluginDir = "${WORKSPACE}/${distribution}/builds/${productName}/core-plugins".replace("\\", "/")
boolean corePluginDirExists = fileExists(corePluginDir)
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Create .sha512 for Core Plugins Snapshots")
argsMap['artifactPath'] = corePluginDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}
}

// Create checksums
echo('Create .sha512 for Min Snapshots Artifacts')
argsMap = [:]
argsMap['artifactPath'] = srcDir
for (Closure action : fileActions) { // running createSha512Checksums()
action(argsMap)
}

echo("Start copying files: version-${version} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")
echo("Start copying files: version-${version} revision-${revision} architecture-${architecture} platform-${platform} buildid-${id} distribution-${distribution} extension-${extension}")

String sedCmd = "sed"
if (platform == "darwin") {
Expand All @@ -68,9 +83,29 @@ void call(Map args = [:]) {
string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),
string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) {
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
// min artifacts
echo("Upload min snapshots")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.sha512", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.sha512")
s3Upload(file: "${srcDir}/${baseName}-latest.${extension}.build-manifest.yml", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${dstDir}/${baseName}-latest.${extension}.build-manifest.yml")
// core plugins
if (architecture == "x64" && platform == "linux" && distribution == "tar" && corePluginDirExists) {
echo("Upload core-plugins snapshots")
List<String> corePluginList = buildManifestYamlOnly.components.artifacts.'core-plugins'[0]
echo("corePluginList: ${corePluginList}")
for (String pluginSubPath : corePluginList) {
String pluginSubFolder = pluginSubPath.split('/')[0]
String pluginNameWithExt = pluginSubPath.split('/')[1]
String pluginName = pluginNameWithExt.replace('-' + revision + '.zip', '')
String pluginFullPath = ['plugins', pluginName, revision].join('/')
s3Upload(
bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}",
path: "snapshots/${pluginFullPath}/",
workingDir: "${corePluginDir}/",
includePathPattern: "**/${pluginName}*"
)
}
}
}
}
}
Loading