diff --git a/README.md b/README.md index 3d1fd5645..78d4abe67 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ lib = library(identifier: 'jenkins@', retriever: modernSCM([ | [publishToPyPi.groovy](./vars/publishToPyPi.groovy) | A library to publish artifacts to [PyPi registry](https://pypi.org/) with [OpenSearch](https://pypi.org/user/OpenSearch/) as the maintainer. This library takes care of signing the artifacts before publishing. You can use [PublishToPyPiLibTester](./tests/jenkins/lib-testers/PublishToPyPiLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToPyPi_Jenkinsfile). | | [publishToRubyGems.groovy](./vars/publishToRubyGems.groovy) | A library to publish gems to [rubygems.org](https://rubygems.org/) with [opensearchproject](https://rubygems.org/profiles/opensearchproject) as the owner. Please note that this library expects the gems to be pre-signed. You can use [PublishToRubyGemsLibTester](./tests/jenkins/lib-testers/PublishToRubyGemsLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToRubyGems_JenkinsFile). | | [publishToMaven.groovy](./vars/publishToMaven.groovy) | A library to sign and deploy opensearch maven artifacts to sonatype staging repository, it also has an optional parameter `autoPublish` to auto-release artifacts from staging repo to prod without manual intervention. You can use [PublishToMavenLibTester](./tests/jenkins/lib-testers/PublishToMavenLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToMaven_JenkinsFile). | -| [publishToNuget.groovy](./vars/publishToNuget.groovy) | A library to build, sign and publish dotnet artifacts to [Nuget Gallery](https://www.nuget.org/). Please check if the [default docker](https://github.com/opensearch-project/opensearch-build/blob/main/docker/ci/dockerfiles/current/release.centos.clients.x64.arm64.dockerfile) file contains the required dotnet sdk. You can use [PublishToNugetLibTester](./tests/jenkins/lib-testers/PublishToNugetLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToNuget_Jenkinsfile). +| [publishToNuget.groovy](./vars/publishToNuget.groovy) | A library to build, sign and publish dotnet artifacts to [Nuget Gallery](https://www.nuget.org/). Please check if the [default docker](https://github.com/opensearch-project/opensearch-build/blob/main/docker/ci/dockerfiles/current/release.centos.clients.x64.arm64.dockerfile) file contains the required dotnet sdk. You can use [PublishToNugetLibTester](./tests/jenkins/lib-testers/PublishToNugetLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToNuget_Jenkinsfile). +| [publishToArtifactsProdBucket.groovy](./vars/publishToArtifactsProdBucket.groovy) | This library signs and uploads the artifacts to production S3 bucket which points to artifacts.opensearch.org. Please make sure the role that you use to upload exists and has the right permission. For artifacts of different types like macos, linux and windows, call this lib for each artifact with different signing parameters. You can use [PublishToArtifactsProdBucketLibTester](./tests/jenkins/lib-testers/PublishToArtifactsProdBucketLibTester.groovy) to add tests in your repository. See how to use the lib in your [jenkinsFile](./tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile). ## Contributing diff --git a/RELEASING.md b/RELEASING.md index 142b982ab..6a5f08416 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -23,5 +23,5 @@ This respository, as other in this organization follows semantic versioning. The release process includes a [maintainer](MAINTAINERS.md) voluntering for the release. They need to follow the below steps: * Changing the version number in [build.gradle](https://github.com/opensearch-project/opensearch-build-libraries/blob/main/build.gradle#L123). -* This trigger the [version increment workflow](.github/workflows/version-increment.yml) and creates a version increment PR across the 1.x branch. [Example](https://github.com/gaiksaya/opensearch-build-libraries-1/pull/1) +* This triggers the [version increment workflow](.github/workflows/version-increment.yml) and creates a version increment PR across the 1.x branch. [Example](https://github.com/gaiksaya/opensearch-build-libraries-1/pull/1) * Once merged, the maintainer needs to push a tag based on 1.x which creates a release on GitHub via [release.yml](./.github/workflows/release.yml) workflow. \ No newline at end of file diff --git a/build.gradle b/build.gradle index 512e60412..90dfb516a 100644 --- a/build.gradle +++ b/build.gradle @@ -120,7 +120,7 @@ jacocoTestReport { } } -String version = '1.5.6' +String version = '1.6.0' task updateVersion { doLast { diff --git a/tests/jenkins/TestPublishToArtifactsProdBucket.groovy b/tests/jenkins/TestPublishToArtifactsProdBucket.groovy new file mode 100644 index 000000000..1f4378c74 --- /dev/null +++ b/tests/jenkins/TestPublishToArtifactsProdBucket.groovy @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package jenkins.tests + +import jenkins.tests.BuildPipelineTest +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat +import org.junit.Before +import org.junit.Test + +class TestPublishToArtifactsProdBucket extends BuildPipelineTest { + @Override + @Before + void setUp() { + + this.registerLibTester(new PublishToArtifactsProdBucketLibTester('test-role', 'the-windows-msi.msi', 'msi/', 'windows', 'null', true)) + this.registerLibTester(new PublishToArtifactsProdBucketLibTester('test-role-2', 'reporting-cli-2.3.0.tg.gz', 'reporting-cli/')) + super.setUp() + } + + @Test + public void test() { + super.testPipeline('tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile') + } + + @Test + void 'verify signing_with_defaults'(){ + runScript('tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile') + assertThat(getShellCommands('sh', 'sign.sh'), hasItem('\n #!/bin/bash\n set +x\n export ROLE=SIGNER_CLIENT_ROLE\n export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID\n export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET\n export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET\n\n /tmp/workspace/opensearch-build/sign.sh reporting-cli-2.3.0.tg.gz --platform linux --sigtype .sig\n ')) + } + + @Test + void 'verify_signing_with_args'(){ + runScript('tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile') + assertThat(getShellCommands('sh', 'sign.sh'), hasItem("\n #!/bin/bash\n set +x\n export ROLE=SIGNER_WINDOWS_ROLE\n export EXTERNAL_ID=SIGNER_WINDOWS_EXTERNAL_ID\n export UNSIGNED_BUCKET=SIGNER_WINDOWS_UNSIGNED_BUCKET\n export SIGNED_BUCKET=SIGNER_WINDOWS_SIGNED_BUCKET\n export PROFILE_IDENTIFIER=SIGNER_WINDOWS_PROFILE_IDENTIFIER\n export PLATFORM_IDENTIFIER=SIGNER_WINDOWS_PLATFORM_IDENTIFIER\n\n /tmp/workspace/opensearch-build/sign.sh the-windows-msi.msi --platform windows --sigtype null --overwrite \n ")) + } + + @Test + void 'verifyS3uploads'(){ + runScript('tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile') + assertThat(getShellCommands('s3Upload', ''), hasItems('{file=the-windows-msi.msi, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=msi/}', '{file=reporting-cli-2.3.0.tg.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=reporting-cli/}')) + } + def getShellCommands(methodName, searchString) { + def shCommands = helper.callStack.findAll { call -> + call.methodName == methodName + }.collect { call -> + callArgsToString(call) + }.findAll { command -> + command.contains(searchString) + } + return shCommands + } +} diff --git a/tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile b/tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile new file mode 100644 index 000000000..ccb6fd1bc --- /dev/null +++ b/tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('publishToartifactsDotOrg') { + steps { + script { + publishToArtifactsProdBucket( + assumedRoleName: 'test-role', + source: 'the-windows-msi.msi', + destination: 'msi/', + signingPlatform: 'windows', + sigType: 'null', + sigOverwrite: true + ) + + publishToArtifactsProdBucket( + assumedRoleName: 'test-role-2', + source: 'reporting-cli-2.3.0.tg.gz', + destination: 'reporting-cli/' + ) + } + } + } + } +} diff --git a/tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile.txt b/tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile.txt new file mode 100644 index 000000000..0592902a2 --- /dev/null +++ b/tests/jenkins/jobs/PublishToArtifactsProdBucket_Jenkinsfile.txt @@ -0,0 +1,69 @@ + PublishToArtifactsProdBucket_Jenkinsfile.run() + PublishToArtifactsProdBucket_Jenkinsfile.pipeline(groovy.lang.Closure) + PublishToArtifactsProdBucket_Jenkinsfile.echo(Executing on agent [label:none]) + PublishToArtifactsProdBucket_Jenkinsfile.stage(publishToartifactsDotOrg, groovy.lang.Closure) + PublishToArtifactsProdBucket_Jenkinsfile.script(groovy.lang.Closure) + PublishToArtifactsProdBucket_Jenkinsfile.publishToArtifactsProdBucket({assumedRoleName=test-role, source=the-windows-msi.msi, destination=msi/, signingPlatform=windows, sigType=null, sigOverwrite=true}) + publishToArtifactsProdBucket.legacySCM(groovy.lang.Closure) + publishToArtifactsProdBucket.library({identifier=jenkins@main, retriever=null}) + publishToArtifactsProdBucket.signArtifacts({artifactPath=the-windows-msi.msi, platform=windows, sigtype=null, overwrite=true}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(/tmp/workspace/sign.sh) + signArtifacts.dir(opensearch-build, groovy.lang.Closure) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) + signArtifacts.string({credentialsId=jenkins-signer-windows-role, variable=SIGNER_WINDOWS_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-windows-external-id, variable=SIGNER_WINDOWS_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-windows-unsigned-bucket, variable=SIGNER_WINDOWS_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-windows-signed-bucket, variable=SIGNER_WINDOWS_SIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-windows-profile-identifier, variable=SIGNER_WINDOWS_PROFILE_IDENTIFIER}) + signArtifacts.string({credentialsId=jenkins-signer-windows-platform-identifier, variable=SIGNER_WINDOWS_PLATFORM_IDENTIFIER}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_WINDOWS_ROLE, SIGNER_WINDOWS_EXTERNAL_ID, SIGNER_WINDOWS_UNSIGNED_BUCKET, SIGNER_WINDOWS_SIGNED_BUCKET, SIGNER_WINDOWS_PROFILE_IDENTIFIER, SIGNER_WINDOWS_PLATFORM_IDENTIFIER], groovy.lang.Closure) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_WINDOWS_ROLE + export EXTERNAL_ID=SIGNER_WINDOWS_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_WINDOWS_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_WINDOWS_SIGNED_BUCKET + export PROFILE_IDENTIFIER=SIGNER_WINDOWS_PROFILE_IDENTIFIER + export PLATFORM_IDENTIFIER=SIGNER_WINDOWS_PLATFORM_IDENTIFIER + + /tmp/workspace/opensearch-build/sign.sh the-windows-msi.msi --platform windows --sigtype null --overwrite + ) + publishToArtifactsProdBucket.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + publishToArtifactsProdBucket.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + publishToArtifactsProdBucket.withCredentials([AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + publishToArtifactsProdBucket.withAWS({role=test-role, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + publishToArtifactsProdBucket.s3Upload({file=the-windows-msi.msi, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=msi/}) + PublishToArtifactsProdBucket_Jenkinsfile.publishToArtifactsProdBucket({assumedRoleName=test-role-2, source=reporting-cli-2.3.0.tg.gz, destination=reporting-cli/}) + publishToArtifactsProdBucket.legacySCM(groovy.lang.Closure) + publishToArtifactsProdBucket.library({identifier=jenkins@main, retriever=null}) + publishToArtifactsProdBucket.signArtifacts({artifactPath=reporting-cli-2.3.0.tg.gz, platform=linux, sigtype=.sig, overwrite=false}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(/tmp/workspace/sign.sh) + signArtifacts.dir(opensearch-build, groovy.lang.Closure) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + + /tmp/workspace/opensearch-build/sign.sh reporting-cli-2.3.0.tg.gz --platform linux --sigtype .sig + ) + publishToArtifactsProdBucket.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + publishToArtifactsProdBucket.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + publishToArtifactsProdBucket.withCredentials([AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + publishToArtifactsProdBucket.withAWS({role=test-role-2, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + publishToArtifactsProdBucket.s3Upload({file=reporting-cli-2.3.0.tg.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=reporting-cli/}) diff --git a/tests/jenkins/lib-testers/PublishToArtifactsProdBucketLibTester.groovy b/tests/jenkins/lib-testers/PublishToArtifactsProdBucketLibTester.groovy new file mode 100644 index 000000000..630969005 --- /dev/null +++ b/tests/jenkins/lib-testers/PublishToArtifactsProdBucketLibTester.groovy @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat + +class PublishToArtifactsProdBucketLibTester extends LibFunctionTester { + + private String assumedRoleName + private String source + private String destination + private String signingPlatform + private String sigtype + private boolean overwrite + + public PublishToArtifactsProdBucketLibTester(assumedRoleName, source, destination, signingPlatform, sigtype, overwrite) { + this.assumedRoleName = assumedRoleName + this.source = source + this.destination = destination + this.sigtype = sigtype + this.signingPlatform = signingPlatform + this.overwrite = overwrite + } + + public PublishToArtifactsProdBucketLibTester(assumedRoleName, source, destination){ + this.assumedRoleName = assumedRoleName + this.source = source + this.destination = destination + } + + void configure(helper, binding){ + helper.registerAllowedMethod("s3Upload", [Map]) + binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') + helper.registerAllowedMethod('git', [Map]) + helper.registerAllowedMethod('withCredentials', [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('withAWS', [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + } + + void parameterInvariantsAssertions(call){ + assertThat(call.args.assumedRoleName.first(), notNullValue()) + assertThat(call.args.source.first(), notNullValue()) + assertThat(call.args.destination.first(), notNullValue()) + } + + boolean expectedParametersMatcher(call) { + return call.args.assumedRoleName.first().toString().equals(this.assumedRoleName) + && call.args.source.first().toString().equals(this.source) + && call.args.destination.first().toString().equals(this.destination) + } + + String libFunctionName() { + return 'publishToArtifactsProdBucket' + } +} diff --git a/vars/publishToArtifactsProdBucket.groovy b/vars/publishToArtifactsProdBucket.groovy new file mode 100644 index 000000000..2d38e4994 --- /dev/null +++ b/vars/publishToArtifactsProdBucket.groovy @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + + /** + * Library to sign and upload artifacts to artifacts.opensearch.org +@param Map[assumedRoleName] - IAM role to be assumed for uploading artifacts +@param Map[source] - Path to yml or artifact file. +@param Map[destination] - Artifact type in the manifest, [type] is required for signing yml. +@param Map[signingPlatform] - The distribution platform for signing. Defaults to linux +@param Map[sigType] - signature type. Defaults to '.sig' +@param Map[sigOverwrite] - Allow output artifacts to overwrite the existing artifacts. Defaults to false + */ +void call(Map args = [:]) { + lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) + println('Signing the artifacts') + signArtifacts( + artifactPath: args.source, + platform: args.signingPlatform ?: 'linux', + sigtype: args.sigType ?: '.sig', + overwrite: args.sigOverwrite ?: false + ) + println('Uploading the artifacts') + withCredentials([ + string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), + string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) { + withAWS(role: "${args.assumedRoleName}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + s3Upload(file: "${args.source}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "${args.destination}") + } + } +} \ No newline at end of file