Add support for Python 3.12 (#4998)

Co-authored-by: Ken Odegard <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/tests.yml

@@ -69,7 +69,7 @@ jobs:
       fail-fast: false
       matrix:
         # test all lower versions (w/ stable conda) and upper version (w/ canary conda)
-        python-version: ['3.9', '3.10']
+        python-version: ['3.9', '3.10', '3.11']
         conda-version: [release]
         test-type: [serial, parallel]
         include:
@@ -81,10 +81,10 @@ jobs:
             conda-version: 22.11.0
             test-type: parallel
           # maximum Python/conda combo
-          - python-version: '3.11'
+          - python-version: '3.12'
             conda-version: canary
             test-type: serial
-          - python-version: '3.11'
+          - python-version: '3.12'
             conda-version: canary
             test-type: parallel
     env:
@@ -173,10 +173,10 @@ jobs:
         conda-version: [release]
         test-type: [serial, parallel]
         include:
-          - python-version: '3.11'
+          - python-version: '3.12'
             conda-version: canary
             test-type: serial
-          - python-version: '3.11'
+          - python-version: '3.12'
             conda-version: canary
             test-type: parallel
     env:
@@ -270,10 +270,10 @@ jobs:
         conda-version: [release]
         test-type: [serial, parallel]
         include:
-          - python-version: '3.11'
+          - python-version: '3.12'
             conda-version: canary
             test-type: serial
-          - python-version: '3.11'
+          - python-version: '3.12'
             conda-version: canary
             test-type: parallel
     env:
@@ -426,10 +426,10 @@ jobs:
           clean: true
           fetch-depth: 0
 
-      # Explicitly use Python 3.11 since each of the OSes has a different default Python
+      # Explicitly use Python 3.12 since each of the OSes has a different default Python
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Detect label
         shell: python

conda_build/_load_setup_py_data.py

@@ -90,6 +90,8 @@ def setup(**kw):
         del sys.modules["versioneer"]
 
     try:
+        # numpy.distutils deprecated in Python 3.12+
+        # see https://numpy.org/doc/stable/reference/distutils_status_migration.html
         import numpy.distutils.core
 
         numpy_setup = numpy.distutils.core.setup

conda_build/skeletons/pypi.py

@@ -558,7 +558,7 @@ def add_parser(repos):
         action="store",
         default=default_python,
         help="""Version of Python to use to run setup.py. Default is %(default)s.""",
-        choices=["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"],
+        choices=["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"],
     )
 
     pypi.add_argument(
@@ -1371,39 +1371,40 @@ def run_setuppy(src_dir, temp_dir, python_version, extra_specs, config, setup_op
     with open(patch, "wb") as f:
         f.write(DISTUTILS_PATCH.format(temp_dir.replace("\\", "\\\\")).encode("utf-8"))
 
-    if exists(join(stdlib_dir, "distutils", "core.py-copy")):
-        rm_rf(join(stdlib_dir, "distutils", "core.py"))
-        copy2(
-            join(stdlib_dir, "distutils", "core.py-copy"),
-            join(stdlib_dir, "distutils", "core.py"),
-        )
-        # Avoid race conditions. Invalidate the cache.
-        rm_rf(
-            join(
-                stdlib_dir,
-                "distutils",
-                "__pycache__",
-                f"core.cpython-{sys.version_info[0]}{sys.version_info[1]}.pyc",
+    # distutils deprecated in Python 3.10+, removed in Python 3.12+
+    distutils = join(stdlib_dir, "distutils")
+    if isdir(distutils):
+        if exists(join(distutils, "core.py-copy")):
+            rm_rf(join(distutils, "core.py"))
+            copy2(
+                join(distutils, "core.py-copy"),
+                join(distutils, "core.py"),
             )
-        )
-        rm_rf(
-            join(
-                stdlib_dir,
-                "distutils",
-                "__pycache__",
-                f"core.cpython-{sys.version_info[0]}{sys.version_info[1]}.pyo",
+            # Avoid race conditions. Invalidate the cache.
+            rm_rf(
+                join(
+                    distutils,
+                    "__pycache__",
+                    f"core.cpython-{sys.version_info[0]}{sys.version_info[1]}.pyc",
+                )
             )
-        )
-    else:
-        copy2(
-            join(stdlib_dir, "distutils", "core.py"),
-            join(stdlib_dir, "distutils", "core.py-copy"),
-        )
-    apply_patch(join(stdlib_dir, "distutils"), patch, config=config)
+            rm_rf(
+                join(
+                    distutils,
+                    "__pycache__",
+                    f"core.cpython-{sys.version_info[0]}{sys.version_info[1]}.pyo",
+                )
+            )
+        else:
+            copy2(
+                join(distutils, "core.py"),
+                join(distutils, "core.py-copy"),
+            )
+        apply_patch(distutils, patch, config=config)
 
-    vendored = join(stdlib_dir, "site-packages", "setuptools", "_distutils")
-    if os.path.isdir(vendored):
-        apply_patch(vendored, patch, config=config)
+    setuptools = join(stdlib_dir, "site-packages", "setuptools", "_distutils")
+    if isdir(setuptools):
+        apply_patch(setuptools, patch, config=config)
 
     # Save PYTHONPATH for later
     env = os.environ.copy()

conda_build/variants.py

@@ -18,14 +18,24 @@
 
 DEFAULT_VARIANTS = {
     "python": f"{sys.version_info.major}.{sys.version_info.minor}",
-    "numpy": "1.22",
+    "numpy": {
+        # (python): numpy_version,  # range of versions built for given python
+        (3, 8): "1.22",  # 1.19-1.24
+        (3, 9): "1.22",  # 1.19-1.26
+        (3, 10): "1.22",  # 1.21-1.26
+        (3, 11): "1.23",  # 1.23-1.26
+        (3, 12): "1.26",  # 1.26-
+    }.get(sys.version_info[:2], "1.26"),
     # this one actually needs to be pretty specific.  The reason is that cpan skeleton uses the
     #    version to say what's in their standard library.
     "perl": "5.26.2",
     "lua": "5",
     "r_base": "3.4" if on_win else "3.5",
     "cpu_optimization_target": "nocona",
-    "pin_run_as_build": OrderedDict(python=OrderedDict(min_pin="x.x", max_pin="x.x")),
+    "pin_run_as_build": {
+        "python": {"min_pin": "x.x", "max_pin": "x.x"},
+        "r-base": {"min_pin": "x.x", "max_pin": "x.x"},
+    },
     "ignore_version": [],
     "ignore_build_only_deps": ["python", "numpy"],
     "extend_keys": [
@@ -37,11 +47,6 @@
     "cran_mirror": "https://cran.r-project.org",
 }
 
-# set this outside the initialization because of the dash in the key
-DEFAULT_VARIANTS["pin_run_as_build"]["r-base"] = OrderedDict(
-    min_pin="x.x", max_pin="x.x"
-)
-
 # map python version to default compiler on windows, to match upstream python
 #    This mapping only sets the "native" compiler, and can be overridden by specifying a compiler
 #    in the conda-build variant configuration

pyproject.toml

@@ -22,6 +22,7 @@ classifiers = [
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy"
 ]

recipe/conda_build_config.yaml

@@ -1,5 +1,6 @@
 python:
-  - 3.8
-  - 3.9
-  - 3.10
-  - 3.11
+  - "3.8"
+  - "3.9"
+  - "3.10"
+  - "3.11"
+  - "3.12"

tests/cli/test_main_build.py

@@ -1,33 +1,29 @@
 # Copyright (C) 2014 Anaconda, Inc
 # SPDX-License-Identifier: BSD-3-Clause
+from __future__ import annotations
+
 import os
 import re
 from pathlib import Path
 
 import pytest
+from pytest import FixtureRequest, MonkeyPatch
+from pytest_mock import MockerFixture
 
-import conda_build
 from conda_build import api
 from conda_build.cli import main_build, main_render
-from conda_build.conda_interface import (
-    TemporaryDirectory,
-    cc_conda_build,
-    context,
-    reset_context,
+from conda_build.conda_interface import TemporaryDirectory
+from conda_build.config import (
+    Config,
+    zstd_compression_level_default,
 )
-from conda_build.config import Config, zstd_compression_level_default
 from conda_build.exceptions import DependencyNeedsBuildingError
+from conda_build.metadata import MetaData
+from conda_build.os_utils.external import find_executable
 from conda_build.utils import get_build_folders, on_win, package_has_file
 
 from ..utils import metadata_dir
-
-
-def _reset_config(search_path=None):
-    reset_context(search_path)
-    cc_conda_build.clear()
-    cc_conda_build.update(
-        context.conda_build if hasattr(context, "conda_build") else {}
-    )
+from ..utils import reset_config as _reset_config
 
 
 @pytest.mark.sanity
@@ -266,25 +262,42 @@ def test_purge_all(testing_workdir, testing_metadata):
 
 
 @pytest.mark.serial
-def test_no_force_upload(mocker, testing_workdir, testing_metadata, request):
-    with open(os.path.join(testing_workdir, ".condarc"), "w") as f:
-        f.write("anaconda_upload: True\n")
-        f.write("conda_build:\n")
-        f.write("    force_upload: False\n")
-    del testing_metadata.meta["test"]
-    api.output_yaml(testing_metadata, "meta.yaml")
-    args = ["--no-force-upload", testing_workdir]
-    call = mocker.patch.object(conda_build.build.subprocess, "call")
+def test_no_force_upload(
+    mocker: MockerFixture,
+    monkeypatch: MonkeyPatch,
+    testing_workdir: str | os.PathLike | Path,
+    testing_metadata: MetaData,
+    request: FixtureRequest,
+):
+    # this is nearly identical to tests/test_api_build.py::test_no_force_upload
+    # only difference is this tests `conda_build.cli.main_build.execute`
     request.addfinalizer(_reset_config)
-    _reset_config([os.path.join(testing_workdir, ".condarc")])
-    main_build.execute(args)
+    call = mocker.patch("subprocess.call")
+    anaconda = find_executable("anaconda")
+
+    # render recipe
+    api.output_yaml(testing_metadata, "meta.yaml")
     pkg = api.get_output_file_path(testing_metadata)
-    assert call.called_once_with(["anaconda", "upload", pkg])
-    args = [testing_workdir]
-    with open(os.path.join(testing_workdir, ".condarc"), "w") as f:
-        f.write("anaconda_upload: True\n")
-    main_build.execute(args)
-    assert call.called_once_with(["anaconda", "upload", "--force", pkg])
+
+    # mock Config.set_keys to always set anaconda_upload to True
+    # conda's Context + conda_build's MetaData & Config objects interact in such an
+    # awful way that mocking these configurations is ugly and confusing, all of it
+    # needs major refactoring
+    set_keys = Config.set_keys  # store original method
+    monkeypatch.setattr(
+        Config,
+        "set_keys",
+        lambda self, **kwargs: set_keys(self, **{**kwargs, "anaconda_upload": True}),
+    )
+
+    # check for normal upload
+    main_build.execute(["--no-force-upload", testing_workdir])
+    call.assert_called_once_with([anaconda, "upload", *pkg])
+    call.reset_mock()
+
+    # check for force upload
+    main_build.execute([testing_workdir])
+    call.assert_called_once_with([anaconda, "upload", "--force", *pkg])
 
 
 @pytest.mark.slow

tests/cli/test_main_skeleton.py

@@ -34,20 +34,26 @@ def test_skeleton_pypi_arguments_work(testing_workdir):
 
     https://github.com/conda/conda-build/pull/1384
     """
-    args = ["pypi", "msumastro", "--version=1.1.6", "--pin-numpy"]
+    args = ["pypi", "fasttext", "--version=0.9.2", "--pin-numpy"]
     main_skeleton.execute(args)
-    assert os.path.isdir("msumastro")
+    assert os.path.isdir("fasttext")
 
     # Deliberately bypass metadata reading in conda build to get as
     # close to the "ground truth" as possible.
-    with open(os.path.join("msumastro", "meta.yaml")) as f:
+    with open(os.path.join("fasttext", "meta.yaml")) as f:
         assert f.read().count("numpy x.x") == 2
 
-    args = ["pypi", "photutils", "--version=0.2.2", "--setup-options=--offline"]
+    args = [
+        "pypi",
+        "photutils",
+        "--version=1.10.0",
+        "--setup-options=--offline",
+        "--extra-specs=extension-helpers",
+    ]
     main_skeleton.execute(args)
     assert os.path.isdir("photutils")
     # Check that the setup option occurs in bld.bat and build.sh.
 
     m = api.render("photutils")[0][0]
     assert "--offline" in m.meta["build"]["script"]
-    assert m.version() == "0.2.2"
+    assert m.version() == "1.10.0"

tests/requirements.txt

@@ -1,7 +1,7 @@
-anaconda-client
 beautifulsoup4
 chardet
 conda >=22.11.0
+conda-forge::anaconda-client
 conda-index
 conda-package-handling >=1.3
 conda-verify

tests/test-recipes/metadata/source_setup_py_data/bld.bat

@@ -6,6 +6,6 @@ if errorlevel 1 exit 1
 for /f "delims=" %%i in ('git describe') do set gitdesc=%%i
 if errorlevel 1 exit 1
 echo "%gitdesc%"
-if not "%gitdesc%"=="1.21.0" exit 1
+if not "%gitdesc%"=="1.22.0" exit 1
 echo "%PKG_VERSION%"
-if not "%PKG_VERSION%"=="1.21.0" exit 1
+if not "%PKG_VERSION%"=="1.22.0" exit 1

tests/test-recipes/metadata/source_setup_py_data/build.sh

@@ -3,6 +3,6 @@
 # Ensure we are in a git repo
 [ -d .git ]
 git describe
-[ "$(git describe)" = 1.21.0 ]
+[ "$(git describe)" = 1.22.0 ]
 echo "\$PKG_VERSION = $PKG_VERSION"
-[ "${PKG_VERSION}" = 1.21.0 ]
+[ "${PKG_VERSION}" = 1.22.0 ]

tests/test-recipes/metadata/source_setup_py_data/meta.yaml

@@ -12,7 +12,7 @@ package:
 
 source:
   git_url: {{ environ.get('CONDA_BUILD_TEST_RECIPE_PATH') }}
-  git_tag: 1.21.0
+  git_tag: 1.22.0
 
 build:
   entry_points:

tests/test-recipes/published_code/building_jinja2_setup_py_data/meta.yaml

@@ -8,7 +8,7 @@ package:
 # Example assumes that this folder has setup.py in it
 source:
   git_url: {{ environ.get('CONDA_BUILD_TEST_RECIPE_PATH') }}
-  git_tag: 1.21.0
+  git_tag: 1.22.0
 
 requirements:
   build: