Add support and partial CI coverage for Python 3.12

There's a test run in test_python.yml that is non-trivial to get working with Python 3.12 due to some refactoring of our Docker images that would be needed. But this change updates everything else to add coverage for Python 3.12. The main changes necessary to get the builds working were to upgrade some Pip packages via requirements.txt, including in a patch to `rules_fuzzing` that I plan to upstream soon. I also had to take an explicit dependency on `setuptools`. I removed tox.ini, since it was outdated and we have not been actively maintaining it. PiperOrigin-RevId: 580548224
mkruskal-google · Nov 8, 2023 · 24fef03 · 24fef03
1 parent 2bcfbd8
commit 24fef03
Show file tree

Hide file tree

Showing 10 changed files with 22 additions and 36 deletions.
diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml
@@ -17,7 +17,8 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         type: [ Pure, C++]
-        version: ["3.8", "3.9", "3.10", "3.11" ]
+        # TODO: b/309627662 - Add coverage for Python 3.12.
+        version: ["3.8", "3.9", "3.10", "3.11"]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
@@ -55,7 +56,7 @@ jobs:
       matrix:
         type: [ Pure, C++]
         # TODO Consider expanding this set of versions.
-        version: [ "3.11" ]
+        version: [ "3.12" ]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test

diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml
@@ -100,7 +100,6 @@ jobs:
         with:
           cache: pip
           cache-dependency-path: 'python/requirements.txt'
-          python-version: '3.11' # 3.12 doesn't have setuptools
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/bazel@v2
         with:
@@ -171,23 +170,24 @@ jobs:
           # coverage.
           - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'binary' }
           - { os: macos-11, python-version: "3.8", architecture: x64, type: 'binary' }
-          - { os: ubuntu-latest, python-version: "3.11", architecture: x64, type: 'binary' }
-          - { os: macos-12, python-version: "3.11", architecture: x64, type: 'binary' }
+          - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'binary' }
+          - { os: macos-12, python-version: "3.12", architecture: x64, type: 'binary' }
           - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'source' }
           - { os: macos-11, python-version: "3.8", architecture: x64, type: 'source' }
-          - { os: ubuntu-latest, python-version: "3.11", architecture: x64, type: 'source' }
-          - { os: macos-12, python-version: "3.11", architecture: x64, type: 'source' }
+          - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'source' }
+          - { os: macos-12, python-version: "3.12", architecture: x64, type: 'source' }
 
-          # Windows uses the full API up until Python 3.10, so each of these
-          # jobs tests a distinct binary wheel.
+          # Windows uses the full API up until Python 3.10.
           - { os: windows-2019, python-version: "3.8", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.9", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.10", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.11", architecture: x86, type: 'binary' }
+          - { os: windows-2019, python-version: "3.12", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.8", architecture: x64, type: 'binary' }
           - { os: windows-2019, python-version: "3.9", architecture: x64, type: 'binary' }
           - { os: windows-2019, python-version: "3.10", architecture: x64, type: 'binary' }
           - { os: windows-2019, python-version: "3.11", architecture: x64, type: 'binary' }
+          - { os: windows-2019, python-version: "3.12", architecture: x64, type: 'binary' }
     runs-on: ${{ matrix.os }}
     if: ${{ github.event_name != 'pull_request_target' }}
     defaults:
@@ -248,7 +248,7 @@ jobs:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.12"]
     runs-on: ubuntu-latest
     if: ${{ github.event_name != 'pull_request_target' }}
     steps:

diff --git a/python/build_targets.bzl b/python/build_targets.bzl
@@ -437,7 +437,6 @@ def build_targets(name):
             "README.md",
             "google/__init__.py",
             "setup.cfg",
-            "tox.ini",
         ],
         strip_prefix = "",
         visibility = ["//python/dist:__pkg__"],
@@ -463,7 +462,6 @@ def build_targets(name):
             "python_version_test.py",
             "setup.cfg",
             "setup.py",
-            "tox.ini",
         ],
         strip_prefix = strip_prefix.from_root(""),
         visibility = ["//pkg:__pkg__"],

diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel
@@ -5,6 +5,7 @@
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
 
+load("@pip_deps//:requirements.bzl", "requirement")
 load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
 load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@rules_python//python:packaging.bzl", "py_wheel")
@@ -281,6 +282,7 @@ genrule(
     srcs = [":source_tarball"],
     outs = ["protobuf-%s.tar.gz" % PROTOBUF_PYTHON_VERSION],
     cmd = """
+        export PYTHONPATH=$$PWD/external/pip_deps_setuptools/site-packages
         set -eux
         tar -xzvf $(location :source_tarball)
         cd protobuf/
@@ -292,6 +294,7 @@ genrule(
         "@system_python//:none": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
+    tools = [requirement("setuptools")],
 )
 
 py_wheel(
@@ -310,6 +313,7 @@ py_wheel(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     distribution = "protobuf",
     extra_distinfo_files = {
@@ -365,6 +369,7 @@ py_wheel(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     distribution = "protobuf",
     extra_distinfo_files = {

diff --git a/python/dist/setup.py b/python/dist/setup.py
@@ -58,6 +58,7 @@ def GetVersion():
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
     ],
     namespace_packages=['google'],
     packages=find_packages(),

diff --git a/python/protobuf_distutils/setup.py b/python/protobuf_distutils/setup.py
@@ -34,6 +34,7 @@
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Topic :: Software Development :: Code Generators',
     ],
     description=(

diff --git a/python/requirements.txt b/python/requirements.txt
@@ -1 +1,2 @@
-numpy<=1.24.4
+numpy<=1.26.1
+setuptools<=68.2.2
diff --git a/python/setup.py b/python/setup.py
@@ -389,7 +389,6 @@ def HasLibraryDirsOpt():
     ])
     os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
 
-  # Keep this list of dependencies in sync with tox.ini.
   install_requires = []
 
   setup(
@@ -415,6 +414,7 @@ def HasLibraryDirsOpt():
           'Programming Language :: Python :: 3.9',
           'Programming Language :: Python :: 3.10',
           'Programming Language :: Python :: 3.11',
+          'Programming Language :: Python :: 3.12',
       ],
       namespace_packages=['google'],
       packages=find_packages(

diff --git a/python/tox.ini b/python/tox.ini
diff --git a/third_party/rules_fuzzing.patch b/third_party/rules_fuzzing.patch
@@ -22,4 +22,4 @@ index 01482d4..4b36f4f 100644
 -absl-py==0.11.0 --hash=sha256:b3d9eb5119ff6e0a0125f6dabf2f9fae02f8acae7be70576002fac27235611c5
 -six==1.15.0 --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
 +absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3
-+six==1.15.0
++six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254