Added ability to publish to PyPI

We now use twine and can publish our package to PyPI!

Twine is required to properly push metadata to PyPI irrelevant of the
underlying python versions

Upstream: pypi/warehouse#3889

Dockerfile.test

@@ -5,6 +5,7 @@ ENV INSTALL_DIR=/opt/conjur-api-python3
 RUN apk add --no-cache bash \
                        binutils \
                        build-base \
+                       git \
                        python3 \
                        python3-dev
 

Jenkinsfile

@@ -39,6 +39,18 @@ pipeline {
         }
       }
     }
+
+    // Only publish to PyPI if the HEAD is
+    // tagged with the same version as in __version__.py
+    stage('Publish to PyPI') {
+      steps {
+        sh 'summon -e testing ./bin/publish'
+      }
+
+      when {
+        branch "add-policy-change-output-support"
+      }
+    }
   }
 
   post {

bin/needs_publishing

@@ -0,0 +1,47 @@
+#!/bin/bash -e
+
+PYPI_URL="${1-https://pypi.org}"
+echo "Using $PYPI_URL as the publish endpoint"
+
+package_name=$(python3 ./setup.py --name)
+package_name=Conjur
+
+echo "Determining if publishing is needed for package '$package_name'..."
+
+local_version=$(python3 ./setup.py --version)
+echo "Local version: $local_version"
+
+published_version=$(curl -Ls $PYPI_URL/pypi/$package_name/json | jq -r '.info.version' || echo "<not published>")
+echo "Published version on $PYPI_URL: $published_version"
+if [[ "$local_version" == "$current_published_version" ]]; then
+  echo "WARN: Local version of $package_name ($local_version) is the same as published version ($published_version)!"
+  echo "WARN: Skipping publishing!"
+  exit 1
+fi
+
+echo "Published version ($published_version) does not match local version ($local_version)!"
+echo "Checking current git tag to see if it matches this commit..."
+
+# Jenkins git plugin is broken and always fetches with `--no-tags`
+# (or `--tags`, neither of which is what you want), so tags end up
+# not being fetched. Try to fix that.
+# (Unfortunately this fetches all remote heads, so we may have to find
+# another solution for bigger repos.)
+git fetch -q
+
+tag_commit=$(git rev-list -n 1 "v$local_version" 2>/dev/null || true)
+echo "Tag v$local_version commit: '$tag_commit'"
+
+head_commit=$(git rev-parse HEAD)
+echo "Head commit: '$head_commit'"
+
+if [[ "$head_commit" != "$tag_commit" ]]; then
+  echo "WARN: HEAD commit '$head_commit' does not match version commit '$tag_commit'!"
+  echo "WARN: Skipping publishing!"
+  exit 1
+fi
+
+echo "We are on a tagged commit that matches our declared version!"
+echo "*** Publishing is needed! ***"
+
+exit 0

bin/publish

@@ -0,0 +1,43 @@
+#!/bin/bash -e
+
+CURRENT_DIR=$(pwd)
+
+$CURRENT_DIR/bin/build_container
+
+REQUIRED_VARS=( "TWINE_REPOSITORY_URL"
+                "TWINE_USERNAME"
+                "TWINE_PASSWORD" )
+
+# Sanity check
+for required_var in "${REQUIRED_VARS[@]}"; do
+  if [[ "${!required_var}" == "" ]]; then
+    echo "ERROR: '$required_var' not set!"
+    exit 1
+  fi
+done
+
+rm -rf $CURRENT_DIR/dist/
+docker run --rm \
+           -t \
+           -e TWINE_REPOSITORY_URL \
+           -e TWINE_USERNAME \
+           -e TWINE_PASSWORD \
+           -v "$(pwd):/opt/conjur-api-python3" \
+           conjur-api-python3-test bash -exc "
+               echo 'Installing new versions of pip and wheel...'
+               /usr/bin/env pip3 install --upgrade pip wheel
+
+               if ! ./bin/needs_publishing \$TWINE_REPOSITORY_URL; then
+                 echo 'WARN: Publishing skipped!'
+                 exit 0
+               fi
+
+               echo 'Building distributable package...'
+               /usr/bin/env python3 setup.py sdist bdist_wheel
+
+               echo 'Testing artifacts in dist/*'
+               /usr/bin/env twine check dist/*
+
+               echo 'Publishing package to \$TWINE_REPOSITORY_URL using account '\$TWINE_USERNAME'...'
+               /usr/bin/env twine upload dist/*
+           "

requirements.txt

@@ -5,6 +5,7 @@ pylint>=2.3.1
 PyInstaller>=3.4
 PyYAML>=3.13
 requests>=2.21.0
+twine>=1.13.0
 
 # https://github.com/kennethreitz/requests/issues/5065
 urllib3>=1.24.2,<1.25

secrets.yml

@@ -0,0 +1,11 @@
+production:
+  TWINE_REPOSITORY_URL: !var ecosystems/pypi/users/endpoint
+  TWINE_USERNAME: !var ecosystems/pypi/users/conjur/username
+  TWINE_PASSWORD: !var ecosystems/pypi/users/conjur/password
+
+# https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives
+# NOTE: Sometimes, test PyPI wipes their DB so re-registration will be needed
+testing:
+  TWINE_REPOSITORY_URL: !var ecosystems/pypi/test-users/endpoint
+  TWINE_USERNAME: !var ecosystems/pypi/test-users/conjur/username
+  TWINE_PASSWORD: !var ecosystems/pypi/test-users/conjur/password