Draft: Build and release wheel packages

.github/compose/ci.docker-compose.yml

@@ -85,3 +85,27 @@ services:
   mypy:
     <<: *tests-template
     command: tox -e mypy
+
+  # Test that each BuildStream+BuildBox wheel package can install and run.
+  # on the PyPA official 'manylinux' images that define the base ABI for
+  # Python binary packages.
+
+  wheels-manylinux_2_28-cp37:
+    <<: *tests-template
+    image: quay.io/pypa/manylinux_2_28_x86_64
+    command: .github/wheel-helpers/test-wheel-manylinux.sh cp37-cp37m-manylinux_2_28_x86_64 /opt/python/cp37-cp37m/bin/python3
+
+  wheels-manylinux_2_28-cp38:
+    <<: *tests-template
+    image: quay.io/pypa/manylinux_2_28_x86_64
+    command: .github/wheel-helpers/test-wheel-manylinux.sh cp38-cp38-manylinux_2_28_x86_64 /opt/python/cp38-cp38/bin/python3
+
+  wheels-manylinux_2_28-cp39:
+    <<: *tests-template
+    image: quay.io/pypa/manylinux_2_28_x86_64
+    command: .github/wheel-helpers/test-wheel-manylinux.sh cp39-cp39-manylinux_2_28_x86_64 /opt/python/cp39-cp39/bin/python3
+
+  wheels-manylinux_2_28-cp310:
+    <<: *tests-template
+    image: quay.io/pypa/manylinux_2_28_x86_64
+    command: .github/wheel-helpers/test-wheel-manylinux.sh cp310-cp310-manylinux_2_28_x86_64 /opt/python/cp310-cp310/bin/python3

.github/wheel-helpers/fetch-latest-buildbox-release.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Download latest release binaries of BuildBox. These are statically linked
+# binaries produced by the buildbox-integration GitLab project, which we
+# bundle into BuildStream wheel packages.
+
+set -eux
+
+wget https://gitlab.com/buildgrid/buildbox/buildbox-integration/-/releases/permalink/latest/downloads/binaries.tgz
+
+mkdir -p src/buildstream/subprojects/buildbox
+tar --extract --file ./binaries.tgz --directory src/buildstream/subprojects/buildbox
+
+cd src/buildstream/subprojects/buildbox
+rm buildbox-run
+mv buildbox-run-bubblewrap buildbox-run

.github/wheel-helpers/test-wheel-manylinux.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# Script to test that a generated BuildStream+BuildBox wheel package is
+# functional in the PyPA "manylinux" container images.
+#
+# The test is run via `run-ci.sh` which in turn uses `docker-compose` to
+# execute this script.
+
+set -eux
+
+COMPATIBILITY_TAGS=$1
+PYTHON=$2
+
+dnf install -y bubblewrap
+
+"$PYTHON" -m venv /tmp/venv
+/tmp/venv/bin/pip3 install ./wheelhouse/BuildStream-*-$COMPATIBILITY_TAGS.whl buildstream-plugins
+
+cd doc/examples/autotools
+/tmp/venv/bin/bst build hello.bst

.github/workflows/ci.yml

@@ -83,7 +83,7 @@ jobs:
         run: |
           ${GITHUB_WORKSPACE}/.github/run-ci.sh --service ${{ matrix.test-name }}
 
-  docs:
+  build_docs:
     runs-on: ubuntu-18.04
     steps:
       - name: Check out repository
@@ -108,3 +108,55 @@ jobs:
         with:
           name: docs
           path: doc/build/html
+
+  build_wheels:
+    name: Build Python wheel packages on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Fetch latest BuildBox release
+        run: ${GITHUB_WORKSPACE}/.github/wheel-helpers/fetch-latest-buildbox-release.sh
+
+      - name: Build wheels
+        run: pipx run cibuildwheel==2.8.1
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: ./wheelhouse/*.whl
+
+  test_wheels:
+    name: "Test Python packages: ${{ matrix.test-name }}"
+    needs: [build_wheels]
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        # The names here should map to a valid service defined in
+        # "../compose/ci.docker-compose.yml"
+        test-name:
+          - wheels-manylinux_2_28-cp37
+          - wheels-manylinux_2_28-cp38
+          - wheels-manylinux_2_28-cp39
+          - wheels-manylinux_2_28-cp310
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: ./wheelhouse
+
+      - name: Run tests with Docker Compose
+        run: |
+          ${GITHUB_WORKSPACE}/.github/run-ci.sh ${{ matrix.test-name }}

.github/workflows/merge.yml

@@ -6,8 +6,8 @@ on:
     - master
 
 jobs:
-  build:
-    name: Build documentation
+  build_docs:
+    name: "Build documentation tarball"
     runs-on: ubuntu-18.04
     steps:
     - name: Checkout code
@@ -42,7 +42,7 @@ jobs:
           doc/build/html
           docs.tgz
 
-  publish:
+  publish_docs:
     needs: build
     runs-on: ubuntu-18.04
     steps:

.github/workflows/release.yml

@@ -1,13 +1,13 @@
-name: Upload Release Asset
+name: Release actions
 
 on:
   push:
     tags:
     - '*.*.*'
 
 jobs:
-  build:
-    name: Upload Release Asset
+  build_docs:
+    name: "Build documentation tarball"
     runs-on: ubuntu-18.04
     steps:
       - name: Checkout code
@@ -32,9 +32,114 @@ jobs:
 
           tar -C doc/build/html -zcf docs.tgz .
 
+      - uses: actions/upload-artifact@v3
+        with:
+          name: docs
+          path: docs.tgz
+
+  build_sdist:
+    name: "Build Python source distribution tarball"
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Build sdist
+      run: pipx run build --sdist
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: sdist
+        path: dist/*.tar.gz
+
+  build_wheels:
+    name: Build Python wheel packages on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Fetch latest BuildBox release
+        run: ${GITHUB_WORKSPACE}/.github/wheel-helpers/fetch-latest-buildbox-release.sh
+
+      - name: Build wheels
+        run: pipx run cibuildwheel==2.8.1
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: ./wheelhouse/*.whl
+
+  test_wheels:
+    name: Test Python wheel packages on ${{ matrix.os }}
+    needs: [build_wheels]
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        # The names here should map to a valid service defined in
+        # "../compose/ci.docker-compose.yml"
+        test-name:
+          - wheels-manylinux_2_28-cp37
+          - wheels-manylinux_2_28-cp38
+          - wheels-manylinux_2_28-cp39
+          - wheels-manylinux_2_28-cp310
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: ./wheelhouse
+
+      - name: Run tests with Docker Compose
+        run: |
+          ${GITHUB_WORKSPACE}/.github/run-ci.sh ${{ matrix.test-name }}
+
+  upload_github_release:
+    name: Upload GitHub release assets
+    needs: [build_docs, build_sdist, build_wheels, test_wheels]
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: docs
+
       - name: Upload release assets
         run: |
           tag_name="${GITHUB_REF##*/}"
           hub release create -a "docs.tgz" -m "$tag_name" "$tag_name"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  upload_pypi_release:
+    name: Upload PyPI release assets
+    needs: [build_docs, build_sdist, build_wheels, test_wheels]
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: sdist
+          path: dist
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+      - name: Upload to PyPI
+        run: |
+          pipx run twine upload --repository pypi --username __token__ --password "${{ secrets.PYPI_TOKEN }}" dist/*

.pylintrc

@@ -80,14 +80,14 @@ confidence=
 #    pylint to also check for those things.
 #
 
-disable=#####################################
+disable=
+        #####################################
         # Messages that are of no use to us #
         #####################################
         ,
 	consider-using-f-string,
         fixme,
         missing-docstring,
-        no-self-use,
         no-else-return,
         protected-access,
         too-few-public-methods,
@@ -130,7 +130,6 @@ disable=#####################################
         # Formatting-related messages, enforced by Black #
         ##################################################
 
-        bad-continuation,
         line-too-long,
         superfluous-parens,
 
@@ -430,13 +429,6 @@ max-line-length=119
 # Maximum number of lines in a module
 max-module-lines=1000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
 single-line-class-stmt=no

MANIFEST.in

@@ -47,3 +47,6 @@ include versioneer.py
 
 # setuptools.build_meta don't include setup.py by default. Add it
 include setup.py
+
+# bundled binaries should only be in the bdist packages
+recursive-exclude src/buildstream/subprojects *

doc/source/hacking/making_releases.rst

@@ -173,6 +173,11 @@ Release process
      git tag -s 1.1.1
      git push origin 1.1.1
 
+  This will trigger the "Release actions" workflow which also takes care of:
+
+    * uploading Github release artifacts
+    * uploading Python source and binary packages to PyPI
+
 * Upload the release tarball
 
   First get yourself into a clean repository state, ensure that you
@@ -213,16 +218,9 @@ Post-release activities
 Once the release has been published, there are some activities
 which need to be done to ensure everything is up to date.
 
-* If this is a stable release, then the tarball should also be
-  uploaded to PyPI.
-
-  Make sure you have ``twine`` installed and upload the tarball
-  like so:
-
-  .. code:: shell
-
-     pip3 install --user twine
-     twine upload -r pypi dist/BuildStream-1.0.1.tar.gz
+* Check that the release was successfully uploaded to PyPI. If
+  it is an unstable release, make sure the version on PyPI has
+  the correct "dev0" postfix so to avoid being treated as stable.
 
 * Update the topic line in the #buildstream IRC channel if needed
 

pyproject.toml

@@ -4,7 +4,8 @@ requires = [
     "setuptools>=36.6.0",
     # In order to build wheels, and as required by PEP 517
     "wheel",
-    "Cython"
+    "Cython",
+    "packaging",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -24,3 +25,26 @@ exclude = '''
   | src/buildstream/_protos
 )
 '''
+
+[tool.cibuildwheel]
+build-frontend = "build"
+environment = { BST_BUNDLE_BUILDBOX = "1" }
+
+# The BuildBox binaries produced in buildbox-integration are linked against GLIBC 2.28
+# from Debian 10. See: https://gitlab.com/BuildGrid/buildbox/buildbox-integration.
+#
+# The PyPA manylinux_2_28 platform tag identifies that our wheel will run on any x86_64
+# OS with GLIBC >= 2.28. Following this setting, `cibuildwheel` builds the packages in
+# the corresponding manylinux_2_28 container image. See: https://github.com/pypa/manylinux
+manylinux-x86_64-image = "manylinux_2_28"
+
+skip = [
+  # BuildStream supports Python >= 3.7
+  "cp36-*",
+  # PyPy may work, but nobody is testing it so avoid distributing prebuilt binaries.
+  "pp*",
+  # Skipping this niche archicture ~halves overall build time.
+  "*_i686",
+  # The prebuilt BuildBox binaries link against GLibc so will work on manylinux but not musllinux
+  "*-musllinux_*",
+]