Add library to publish artifacts to PyPi (#36)

Signed-off-by: Sayali Gaikawad <[email protected]>
(cherry picked from commit 75b24ef)

build.gradle

@@ -119,3 +119,17 @@ jacocoTestReport {
         xml.required = true
     }
 }
+
+String version = '1.3.0'
+
+task updateVersion {
+    doLast {
+        println "Setting version to ${version} in all libraries"
+        ant.replaceregexp(match:'jenkins@main', replace: 'jenkins@' + version, flags: 'g') {
+            fileset(dir: projectDir) {
+                include (name: "vars/**")
+                include (name: "tests/**")
+            }
+        }
+    }
+}

tests/jenkins/TestPublishToPyPi.groovy

@@ -0,0 +1,68 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins.tests
+
+import jenkins.tests.BuildPipelineTest
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.MatcherAssert.assertThat
+import org.junit.Before
+import org.junit.Test
+
+class TestPublishToPyPi extends BuildPipelineTest {
+
+    @Test
+    void setUp() {
+        this.registerLibTester(new PublishToPyPiLibTester())
+        super.setUp()
+        super.testPipeline('tests/jenkins/jobs/PublishToPyPi_Jenkinsfile')
+    }
+
+    @Test
+    void 'verify default run'(){
+        runScript('tests/jenkins/jobs/PublishToPyPi_Jenkinsfile')
+
+        def twineCommands = getCommands('sh', 'twine')
+        assertThat(twineCommands, hasItem(
+            'twine upload -r pypi dist/*'
+        ))
+
+        def signing = getCommands('signArtifacts', '')
+        def signing_sh = getCommands('sh', 'sign.sh')
+        assertThat(signing, hasItem('{artifactPath=dist, sigtype=.asc, platform=linux}'))
+        assertThat(signing_sh, hasItem('\n                   #!/bin/bash\n                   set +x\n                   export ROLE=SIGNER_CLIENT_ROLE\n                   export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID\n                   export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET\n                   export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET\n\n                   /tmp/workspace/sign.sh dist --sigtype=.asc --platform=linux\n               '))
+    }
+
+    @Test
+    void 'verify custom dir'(){
+        runScript('tests/jenkins/jobs/PublishToPyPiWithDir_Jenkinsfile')
+
+        def twineCommands = getCommands('sh', 'twine')
+        assertThat(twineCommands, hasItem(
+            'twine upload -r pypi test/*'
+        ))
+        def signing = getCommands('signArtifacts', '')
+        assertThat(signing, hasItem('{artifactPath=test, sigtype=.asc, platform=linux}'))
+
+        def signing_sh = getCommands('sh', 'sign.sh')
+        assertThat(signing_sh, hasItem('\n                   #!/bin/bash\n                   set +x\n                   export ROLE=SIGNER_CLIENT_ROLE\n                   export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID\n                   export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET\n                   export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET\n\n                   /tmp/workspace/sign.sh test --sigtype=.asc --platform=linux\n               '))
+
+    }
+    def getCommands(method, text) {
+        def shCommands = helper.callStack.findAll { call ->
+            call.methodName == method
+        }.collect { call ->
+            callArgsToString(call)
+        }.findAll { command ->
+            command.contains(text)
+        }
+        return shCommands
+    }
+}

tests/jenkins/jobs/PublishToPyPiWithDir_Jenkinsfile

@@ -0,0 +1,21 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('publishToPyPi') {
+            steps {
+                script {
+                    publishToPyPi(artifactsPath: 'test')
+                }
+            }
+        }
+    }
+}

tests/jenkins/jobs/PublishToPyPiWithDir_Jenkinsfile.txt

@@ -0,0 +1,34 @@
+   PublishToPyPiWithDir_Jenkinsfile.run()
+      PublishToPyPiWithDir_Jenkinsfile.pipeline(groovy.lang.Closure)
+         PublishToPyPiWithDir_Jenkinsfile.echo(Executing on agent [label:none])
+         PublishToPyPiWithDir_Jenkinsfile.stage(publishToPyPi, groovy.lang.Closure)
+            PublishToPyPiWithDir_Jenkinsfile.script(groovy.lang.Closure)
+               PublishToPyPiWithDir_Jenkinsfile.publishToPyPi({artifactsPath=test})
+                  publishToPyPi.legacySCM(groovy.lang.Closure)
+                  publishToPyPi.library({identifier=jenkins@main, retriever=null})
+                  publishToPyPi.sh(echo args to test)
+                  publishToPyPi.sh(echo release test)
+                  publishToPyPi.signArtifacts({artifactPath=test, sigtype=.asc, platform=linux})
+                     signArtifacts.echo(PGP or Windows Signature Signing)
+                     signArtifacts.fileExists(/tmp/workspace/sign.sh)
+                     signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
+                     signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
+                     signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET})
+                     signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure)
+                        signArtifacts.sh(
+                   #!/bin/bash
+                   set +x
+                   export ROLE=SIGNER_CLIENT_ROLE
+                   export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID
+                   export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET
+                   export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET
+
+                   /tmp/workspace/sign.sh test --sigtype=.asc --platform=linux
+               )
+                  publishToPyPi.usernamePassword({credentialsId=jenkins-opensearch-pypi-username, usernameVariable=TWINE_USERNAME, passwordVariable=TWINE_PASSWORD})
+                  publishToPyPi.withCredentials([[TWINE_USERNAME, TWINE_PASSWORD]], groovy.lang.Closure)
+                     publishToPyPi.sh(twine upload -r pypi test)

tests/jenkins/jobs/PublishToPyPi_Jenkinsfile

@@ -0,0 +1,21 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('publishToPyPi') {
+            steps {
+                script {
+                    publishToPyPi()
+                }
+            }
+        }
+    }
+}

tests/jenkins/jobs/PublishToPyPi_Jenkinsfile.txt

@@ -0,0 +1,32 @@
+   PublishToPyPi_Jenkinsfile.run()
+      PublishToPyPi_Jenkinsfile.pipeline(groovy.lang.Closure)
+         PublishToPyPi_Jenkinsfile.echo(Executing on agent [label:none])
+         PublishToPyPi_Jenkinsfile.stage(publishToPyPi, groovy.lang.Closure)
+            PublishToPyPi_Jenkinsfile.script(groovy.lang.Closure)
+               PublishToPyPi_Jenkinsfile.publishToPyPi()
+                  publishToPyPi.legacySCM(groovy.lang.Closure)
+                  publishToPyPi.library({identifier=jenkins@main, retriever=null})
+                  publishToPyPi.signArtifacts({artifactPath=dist, sigtype=.asc, platform=linux})
+                     signArtifacts.echo(PGP or Windows Signature Signing)
+                     signArtifacts.fileExists(/tmp/workspace/sign.sh)
+                     signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
+                     signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
+                     signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET})
+                     signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET})
+                     signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure)
+                        signArtifacts.sh(
+                   #!/bin/bash
+                   set +x
+                   export ROLE=SIGNER_CLIENT_ROLE
+                   export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID
+                   export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET
+                   export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET
+
+                   /tmp/workspace/sign.sh dist --sigtype=.asc --platform=linux
+               )
+                  publishToPyPi.usernamePassword({credentialsId=jenkins-opensearch-pypi-username, usernameVariable=TWINE_USERNAME, passwordVariable=TWINE_PASSWORD})
+                  publishToPyPi.withCredentials([[TWINE_USERNAME, TWINE_PASSWORD]], groovy.lang.Closure)
+                     publishToPyPi.sh(twine upload -r pypi dist/*)

tests/jenkins/lib-testers/PublishToPyPiLibTester.groovy

@@ -0,0 +1,45 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+import static org.hamcrest.CoreMatchers.notNullValue
+import static org.hamcrest.MatcherAssert.assertThat
+import static org.hamcrest.CoreMatchers.NullValue
+
+class PublishToPyPiLibTester extends LibFunctionTester {
+
+    private String artifactsPath = 'dist'
+
+    public PublishToPyPiLibTester(){}
+    public PublishToPyPiLibTester(String artifactsPath){
+        this.artifactsPath = artifactsPath
+    }
+
+    void configure(helper, binding){
+        binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
+        helper.registerAllowedMethod("git", [Map])
+        helper.registerAllowedMethod("withCredentials", [Map, Closure], { args, closure ->
+            closure.delegate = delegate
+            return helper.callClosure(closure)
+        })
+    }
+    void parameterInvariantsAssertions(call){
+        assertThat(call.args.artifactsPath.toString(), notNullValue())
+    }
+
+    boolean expectedParametersMatcher(call) {
+        if (call.args.artifactsPath.isEmpty()) {
+            return (this.artifactsPath.equals('dist'))
+        }
+        return (call.args.artifactsPath.first().toString().equals(this.artifactsPath))
+    }
+
+    String libFunctionName(){
+        return 'publishToPyPi'
+    }
+
+}

vars/publishToPyPi.groovy

@@ -0,0 +1,27 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Library to publish artifacts to PyPi registry with OpenSearch as maintainer
+@param Map args = [:] args A map of the following parameters
+@param args.artifactsPath <optional> - The directory containing distribution files to upload to the repository. Defaults to 'dist/*'
+*/
+void call(Map args = [:]) {
+    lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm))
+    String releaseArtifactsDir = args.artifactsPath ?: 'dist'
+
+    signArtifacts(
+        artifactPath: releaseArtifactsDir,
+        sigtype: '.asc',
+        platform: 'linux'
+    )
+
+    withCredentials([usernamePassword(credentialsId: 'jenkins-opensearch-pypi-username', usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) {
+            sh """twine upload -r pypi ${releaseArtifactsDir}/*"""
+    }
+}