Automate PyPI uploads on release (#60)

* Add changes to automate PyPI uploads

* Update python version to 3.8

* Update docker image to base2

* Fix bug

* Bump cache version

* Remove .whl file

* Fix lint error

* Remove line deleting .whl

* Remove version.py

* Use base image now new image pushed

Co-authored-by: Adam Gleave <[email protected]>

.circleci/config.yml

@@ -51,7 +51,7 @@ commands:
       # released that you want to upgrade to, without mandating the newer version in setup.py.
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "setup.py" }}
+            - v2-dependencies-{{ checksum "setup.py" }}
 
       # Create virtual environment and install dependencies using `ci/build_venv.sh`.
       # `mujoco_py` needs a MuJoCo key, so download that first.
@@ -64,7 +64,7 @@ commands:
       - save_cache:
           paths:
             - /venv
-          key: v1-dependencies-{{ checksum "setup.py" }}
+          key: v2-dependencies-{{ checksum "setup.py" }}
 
       # Install seals.
       # Note we install the source distribution, not in developer mode (`pip install -e`).
@@ -144,7 +144,7 @@ jobs:
             # later, and CircleCI then parses this to pretty-print results.
             # --shard-id and --num-shards are used to split tests across parallel executors using `pytest-shard`.
             # -n uses `pytest-xdist` to parallelize tests within a single instance.
-            Xdummy-entrypoint.py pytest --cov=/venv/lib/python3.7/site-packages/seals --cov=tests \
+            Xdummy-entrypoint.py pytest --cov=/venv/lib/python3.8/site-packages/seals --cov=tests \
                    --junitxml=/tmp/test-reports/junit.xml \
                     -n ${NUM_CPUS} -vv tests/
             # Following two lines rewrite paths from venv/ to src/, based on `coverage:paths` in `setup.cfg`

.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,52 @@
+# Adapted from https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+
+name: Publish seals distributions 📦 to PyPI and TestPyPI
+
+on: push
+
+jobs:
+  build-n-publish:
+    name: Build and publish seals distributions 📦 to PyPI and TestPyPI
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        # Fetch tags needed by setuptools_scm to infer version number
+        # See https://github.com/pypa/setuptools_scm/issues/414
+        fetch-depth: 0
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+
+    - name: Install pypa/build
+      run: >-
+        python -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: >-
+        python -m
+        build
+        --sdist
+        --wheel
+        --outdir dist/
+        .
+
+    # Publish new distribution to Test PyPi on every push.
+    # This ensures the workflow stays healthy, and will also serve
+    # as a source of alpha builds.
+    - name: Publish distribution 📦 to Test PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+
+    # Publish new distribution to production PyPi on releases.
+    - name: Publish distribution 📦 to PyPI
+      if: startsWith(github.ref, 'refs/tags/v')
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}

Dockerfile

@@ -16,8 +16,8 @@ RUN    apt-get update -q \
     net-tools \
     parallel \
     patchelf \
-    python3.7 \
-    python3.7-dev \
+    python3.8 \
+    python3.8-dev \
     python3-pip \
     rsync \
     software-properties-common \

ci/build_venv.sh

@@ -7,6 +7,6 @@ if [[ ${venv} == "" ]]; then
 	venv="venv"
 fi
 
-virtualenv -p python3.7 ${venv}
+virtualenv -p python3.8 ${venv}
 source ${venv}/bin/activate
 pip install .[cpu,docs,mujoco,test]

docs/conf.py

@@ -4,7 +4,7 @@
 # list see the documentation:
 # http://www.sphinx-doc.org/en/master/config
 
-import seals
+from importlib import metadata
 
 # -- Project information -----------------------------------------------------
 
@@ -13,7 +13,7 @@
 author = "Center for Human-Compatible AI"
 
 # The full version, including alpha/beta/rc tags
-release = seals.__version__
+version = metadata.version("seals")
 
 
 # -- General configuration ---------------------------------------------------

pyproject.toml

@@ -1,8 +1,13 @@
+[build-system]
+requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
 [tool.black]
-target-version = ["py37"]
+target-version = ["py38"]
 
 [[tool.mypy.overrides]]
 module = [
-    "gym.*"
+    "gym.*",
+    "setuptools_scm.*",
 ]
 ignore_missing_imports = true

readthedocs.yml

@@ -6,7 +6,7 @@ sphinx:
 formats: all
 
 python:
-  version: 3.7
+  version: 3.8
   install:
     - method: pip
       path: .

setup.cfg

@@ -38,7 +38,7 @@ inputs =
 	src/
 	tests/
 	setup.py
-python_version = 3.7
+python_version >= 3.8
 
 [tool:pytest]
 markers =

setup.py

@@ -1,24 +1,83 @@
 """setup.py for seals project."""
 
-import os
-import sys
+from typing import TYPE_CHECKING
 
 from setuptools import find_packages, setup  # type:ignore
 
+if TYPE_CHECKING:
+    from setuptools_scm.version import ScmVersion
 
-def get_version() -> str:
-    """Load version from version.py.
 
-    Changes system path internally to avoid missing dependencies breaking imports.
+def get_version(version: "ScmVersion") -> str:
+    """Generates the version string for the package.
+
+    This function replaces the default version format used by setuptools_scm
+    to allow development builds to be versioned using the git commit hash
+    instead of the number of commits since the last release, which leads to
+    duplicate version identifiers when using multiple branches
+    (see https://github.com/HumanCompatibleAI/imitation/issues/500).
+    The version has the following format:
+    {version}[.dev{build}]
+    where build is the shortened commit hash converted to base 10.
+
+    Args:
+        version: The version object given by setuptools_scm, calculated
+            from the git repository.
+
+    Returns:
+        The formatted version string to use for the package.
     """
-    sys.path.insert(
-        0,
-        os.path.join(os.path.dirname(__file__), "src", "seals"),
-    )
-    from version import VERSION  # type:ignore
+    # We import setuptools_scm here because it is only installed after the module
+    # is loaded and the setup function is called.
+    from setuptools_scm import version as scm_version
+
+    if version.node:
+        # By default node corresponds to the short commit hash when using git,
+        # plus a "g" prefix. We remove the "g" prefix from the commit hash which
+        # is added by setuptools_scm by default ("g" for git vs. mercurial etc.)
+        # because letters are not valid for version identifiers in PEP 440.
+        # We also convert from hexadecimal to base 10 for the same reason.
+        version.node = str(int(version.node.lstrip("g"), 16))
+    if version.exact:
+        # an exact version is when the current commit is tagged with a version.
+        return version.format_with("{tag}")
+    else:
+        # the current commit is not tagged with a version, so we guess
+        # what the "next" version will be (this can be disabled but is the
+        # default behavior of setuptools_scm so it has been left in).
+        return version.format_next_version(
+            scm_version.guess_next_version,
+            fmt="{guessed}.dev{node}",
+        )
+
 
-    del sys.path[0]
-    return VERSION
+def get_local_version(version: "ScmVersion", time_format="%Y%m%d") -> str:
+    """Generates the local version string for the package.
+
+    By default, when commits are made on top of a release version, setuptools_scm
+    sets the version to be {version}.dev{distance}+{node} where {distance} is the number
+    of commits since the last release and {node} is the short commit hash.
+    This function replaces the default version format used by setuptools_scm
+    so that committed changes away from a release version are not considered
+    local versions but dev versions instead (by using the format
+    {version}.dev{node} instead. This is so that we can push test releases
+    to TestPyPI (it does not accept local versions).
+    Local versions are still present if there are uncommitted changes (if the tree
+    is dirty), in which case the current date is added to the version.
+
+    Args:
+        version: The version object given by setuptools_scm, calculated
+            from the git repository.
+        time_format: The format to use for the date.
+
+    Returns:
+        The formatted local version string to use for the package.
+    """
+    return version.format_choice(
+        "",
+        "+d{time:{time_format}}",
+        time_format=time_format,
+    )
 
 
 def get_readme() -> str:
@@ -56,6 +115,7 @@ def get_readme() -> str:
     "pytype",
     "stable-baselines3>=0.9.0",
     "pyglet>=1.4.0",
+    "setuptools_scm~=7.0.5",
     *ATARI_REQUIRE,
 ]
 DOCS_REQUIRE = [
@@ -68,12 +128,12 @@ def get_readme() -> str:
 
 setup(
     name="seals",
-    version=get_version(),
+    use_scm_version={"local_scheme": get_local_version, "version_scheme": get_version},
     description="Suite of Environments for Algorithms that Learn Specifications",
     long_description=get_readme(),
     long_description_content_type="text/markdown",
     author="Center for Human-Compatible AI",
-    python_requires=">=3.7.0",
+    python_requires=">=3.8.0",
     packages=find_packages("src"),
     package_dir={"": "src"},
     package_data={"seals": ["py.typed"]},
@@ -97,8 +157,9 @@ def get_readme() -> str:
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "License :: OSI Approved :: MIT License",

src/seals/__init__.py

@@ -1,10 +1,17 @@
 """Benchmark environments for reward modeling and imitation."""
 
+from importlib import metadata
+
 import gym
 
 from seals import atari, util
 import seals.diagnostics  # noqa: F401
-from seals.version import VERSION as __version__  # noqa: F401
+
+try:
+    __version__ = metadata.version("seals")
+except metadata.PackageNotFoundError:  # pragma: no cover
+    # package is not installed
+    pass
 
 # Classic control
 

src/seals/diagnostics/noisy_obs.py

@@ -31,9 +31,14 @@ def __init__(self, *, size: int = 5, noise_length: int = 20):
             state_space=spaces.MultiDiscrete([size, size]),
             action_space=spaces.Discrete(5),
             observation_space=spaces.Box(
-                low=np.concatenate(([0, 0], np.full(self._noise_length, -np.inf))),
+                low=np.concatenate(
+                    (np.array([0, 0]), np.full(self._noise_length, -np.inf)),
+                ),
                 high=np.concatenate(
-                    ([size - 1, size - 1], np.full(self._noise_length, np.inf)),
+                    (
+                        np.array([size - 1, size - 1]),
+                        np.full(self._noise_length, np.inf),
+                    ),
                 ),
                 dtype=np.float32,
             ),