From c7ef2753c6684fcb03260075e8449a1b59ef2965 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 18 May 2024 19:06:18 -0400 Subject: [PATCH 01/29] feat: add Pyodide support Signed-off-by: Henry Schreiner Co-authored-by: Hood Chatham Co-authored-by: Matthieu Darbois tests: fix two merge issues Signed-off-by: Henry Schreiner fix: include schema Signed-off-by: Henry Schreiner --- .github/workflows/test.yml | 36 ++ README.md | 1 + bin/generate_schema.py | 1 + cibuildwheel/__main__.py | 7 +- cibuildwheel/architecture.py | 8 + cibuildwheel/logger.py | 1 + cibuildwheel/pyodide.py | 384 ++++++++++++++++++ cibuildwheel/resources/build-platforms.toml | 5 + .../resources/cibuildwheel.schema.json | 45 ++ cibuildwheel/resources/defaults.toml | 2 + cibuildwheel/typing.py | 4 +- cibuildwheel/util.py | 16 + cibuildwheel/windows.py | 7 +- docs/options.md | 33 +- test/conftest.py | 2 + test/test_abi_variants.py | 2 + test/test_before_test.py | 21 +- ...e_wheel.py => test_custom_repair_wheel.py} | 24 +- test/test_emscripten.py | 48 +++ test/test_environment.py | 15 +- test/test_projects/c.py | 6 +- test/test_testing.py | 2 +- test/utils.py | 12 +- 23 files changed, 645 insertions(+), 37 deletions(-) create mode 100644 cibuildwheel/pyodide.py rename test/{test_same_wheel.py => test_custom_repair_wheel.py} (51%) create mode 100644 test/test_emscripten.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98df08c88..7d1588dbc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,7 @@ jobs: needs: lint runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-13, macos-14] python_version: ['3.12'] @@ -148,3 +149,38 @@ jobs: - name: Run the emulation tests run: | pytest --run-emulation test/test_emulation.py + + test-pyodide: + name: Test cibuildwheel building pyodide wheels + needs: lint + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + name: Install Python 3.12 + with: + python-version: '3.12' + + + - name: Install dependencies + run: | + python -m pip install ".[test]" + + - name: Generate a sample project + run: | + python -m test.test_projects test.test_0_basic.basic_project sample_proj + + - name: Run a sample build (GitHub Action) + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse + env: + CIBW_PLATFORM: pyodide + + - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' + run: | + python ./bin/run_tests.py + env: + CIBW_PLATFORM: pyodide diff --git a/README.md b/README.md index 4fc21ea54..43276864b 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ What does it do? - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs your library's tests against the wheel-installed version of your library +- Can also build pyodide wheels for web deployment (experimental, not supported on PyPI yet) with `--platform pyodide` See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) if you need to build unsupported versions of Python, such as Python 2. diff --git a/bin/generate_schema.py b/bin/generate_schema.py index 231edd909..a1ac23c3e 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -280,6 +280,7 @@ def as_object(d: dict[str, Any]) -> dict[str, Any]: "linux": as_object(non_global_options), "windows": as_object(not_linux), "macos": as_object(not_linux), + "pyodide": as_object(not_linux), } oses["linux"]["properties"]["repair-wheel-command"] = { diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 9e0f75b8e..64be4551c 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -15,6 +15,7 @@ import cibuildwheel import cibuildwheel.linux import cibuildwheel.macos +import cibuildwheel.pyodide import cibuildwheel.util import cibuildwheel.windows from cibuildwheel._compat.typing import assert_never @@ -45,7 +46,7 @@ def main() -> None: parser.add_argument( "--platform", - choices=["auto", "linux", "macos", "windows"], + choices=["auto", "linux", "macos", "windows", "pyodide"], default=None, help=""" Platform to build for. Use this option to override the @@ -176,6 +177,8 @@ def _compute_platform_only(only: str) -> PlatformName: return "macos" if "win_" in only or "win32" in only: return "windows" + if "pyodide_" in only: + return "pyodide" print( f"Invalid --only='{only}', must be a build selector with a known platform", file=sys.stderr, @@ -246,6 +249,8 @@ def get_platform_module(platform: PlatformName) -> PlatformModule: return cibuildwheel.windows if platform == "macos": return cibuildwheel.macos + if platform == "pyodide": + return cibuildwheel.pyodide assert_never(platform) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 8515621fd..a3820508a 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -15,6 +15,7 @@ "linux": "Linux", "macos": "macOS", "windows": "Windows", + "pyodide": "Pyodide", } ARCH_SYNONYMS: Final[list[dict[PlatformName, str | None]]] = [ @@ -46,6 +47,9 @@ class Architecture(Enum): AMD64 = "AMD64" ARM64 = "ARM64" + # WebAssembly + wasm32 = "wasm32" + # Allow this to be sorted def __lt__(self, other: Architecture) -> bool: return self.value < other.value @@ -75,6 +79,9 @@ def parse_config(config: str, platform: PlatformName) -> set[Architecture]: def auto_archs(platform: PlatformName) -> set[Architecture]: native_machine = platform_module.machine() + if platform == "pyodide": + return {Architecture.wasm32} + # Cross-platform support. Used for --print-build-identifiers or docker builds. host_platform: PlatformName = ( "windows" @@ -120,6 +127,7 @@ def all_archs(platform: PlatformName) -> set[Architecture]: }, "macos": {Architecture.x86_64, Architecture.arm64, Architecture.universal2}, "windows": {Architecture.x86, Architecture.AMD64, Architecture.ARM64}, + "pyodide": {Architecture.wasm32}, } return all_archs_map[platform] diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 0aa5263f5..c9b0d5714 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -34,6 +34,7 @@ "macosx_x86_64": "macOS x86_64", "macosx_universal2": "macOS Universal 2 - x86_64 and arm64", "macosx_arm64": "macOS arm64 - Apple Silicon", + "pyodide_wasm32": "Pyodide v0.23.x", } diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py new file mode 100644 index 000000000..d17cae5b4 --- /dev/null +++ b/cibuildwheel/pyodide.py @@ -0,0 +1,384 @@ +from __future__ import annotations + +import contextlib +import os +import shutil +import sys +from collections.abc import Sequence, Set +from dataclasses import dataclass +from pathlib import Path + +from filelock import FileLock + +from .architecture import Architecture +from .environment import ParsedEnvironment +from .logger import log +from .options import Options +from .typing import PathOrStr +from .util import ( + CIBW_CACHE_PATH, + AlreadyBuiltWheelError, + BuildFrontendConfig, + BuildSelector, + NonPlatformWheelError, + call, + download, + extract_zip, + find_compatible_wheel, + get_pip_version, + prepare_command, + read_python_configs, + shell, + split_config_settings, + test_fail_cwd_file, + virtualenv, +) + + +@dataclass(frozen=True) +class PythonConfiguration: + version: str + identifier: str + pyodide_version: str + emscripten_version: str + + +def install_emscripten(tmp: Path, version: str) -> Path: + url = "https://github.com/emscripten-core/emsdk/archive/main.zip" + installation_path = CIBW_CACHE_PATH / ("emsdk-" + version) + emsdk_path = installation_path / "emsdk-main/emsdk" + emcc_path = installation_path / "emsdk-main/upstream/emscripten/emcc" + with FileLock(str(installation_path) + ".lock"): + if installation_path.exists(): + return emcc_path + emsdk_zip = tmp / "emsdk.zip" + download(url, emsdk_zip) + installation_path.mkdir() + extract_zip(emsdk_zip, installation_path) + call(emsdk_path, "install", version) + call(emsdk_path, "activate", version) + + return emcc_path + + +def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str: + xbuildenv_cache_dir = CIBW_CACHE_PATH / f"pyodide-xbuildenv-{pyodide_version}" + pyodide_root = xbuildenv_cache_dir / ".pyodide-xbuildenv/xbuildenv/pyodide-root/" + if pyodide_root.exists(): + return str(pyodide_root) + + xbuildenv_cache_dir.mkdir(exist_ok=True) + # We don't want to mutate env but we need to delete any existing + # PYODIDE_ROOT so copy it first. + env = dict(env) + env.pop("PYODIDE_ROOT", None) + call( + "pyodide", + "xbuildenv", + "install", + "--download", + env=env, + cwd=xbuildenv_cache_dir, + ) + return str(pyodide_root) + + +def get_base_python(identifier: str) -> Path: + implementation_id = identifier.split("-")[0] + majorminor = implementation_id[len("cp") :] + major_minor = f"{majorminor[0]}.{majorminor[1:]}" + python_name = f"python{major_minor}" + which_python = shutil.which(python_name) + if which_python is None: + print( + f"Error: CPython {major_minor} is not installed.", + file=sys.stderr, + ) + raise SystemExit(1) + return Path(which_python) + + +def setup_python( + tmp: Path, + python_configuration: PythonConfiguration, + dependency_constraint_flags: Sequence[PathOrStr], + environment: ParsedEnvironment, +) -> dict[str, str]: + pyodide_version = python_configuration.pyodide_version + base_python = get_base_python(python_configuration.identifier) + + log.step("Setting up build environment...") + venv_path = tmp / "venv" + env = virtualenv(base_python, venv_path, dependency_constraint_flags) + venv_bin_path = venv_path / "bin" + assert venv_bin_path.exists() + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + # upgrade pip to the version matching our constraints + # if necessary, reinstall it to ensure that it's available on PATH as 'pip' + call( + "python", + "-m", + "pip", + "install", + "--upgrade", + "pip", + *dependency_constraint_flags, + env=env, + cwd=venv_path, + ) + + env = environment.as_dictionary(prev_environment=env) + if "HOME" not in env: + # Workaround for https://github.com/pyodide/pyodide/pull/3744 + env["HOME"] = "" + + # check what pip version we're on + assert (venv_bin_path / "pip").exists() + call("which", "pip", env=env) + call("pip", "--version", env=env) + which_pip = call("which", "pip", env=env, capture_stdout=True).strip() + if which_pip != str(venv_bin_path / "pip"): + print( + "cibuildwheel: pip available on PATH doesn't match our venv instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) + + # check what Python version we're on + call("which", "python", env=env) + call("python", "--version", env=env) + which_python = call("which", "python", env=env, capture_stdout=True).strip() + if which_python != str(venv_bin_path / "python"): + print( + "cibuildwheel: python available on PATH doesn't match our venv instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", + file=sys.stderr, + ) + sys.exit(1) + + log.step("Installing build tools...") + call( + "pip", + "install", + "--upgrade", + "auditwheel-emscripten", + "build[virtualenv]", + f"pyodide-build=={pyodide_version}", + *dependency_constraint_flags, + env=env, + ) + + log.step("Installing emscripten...") + emcc_path = install_emscripten(tmp, python_configuration.emscripten_version) + + env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) + + log.step("Installing Pyodide xbuildenv...") + env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version) + + return env + + +def get_python_configurations( + build_selector: BuildSelector, + architectures: Set[Architecture], # noqa: ARG001 +) -> list[PythonConfiguration]: + full_python_configs = read_python_configs("pyodide") + + python_configurations = [PythonConfiguration(**item) for item in full_python_configs] + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + return python_configurations + + +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + + if not python_configurations: + return + + try: + before_all_options_identifier = python_configurations[0].identifier + before_all_options = options.build_options(before_all_options_identifier) + + if before_all_options.before_all: + log.step("Running before_all...") + env = before_all_options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command( + before_all_options.before_all, project=".", package=before_all_options.package_dir + ) + shell(before_all_prepared, env=env) + + built_wheels: list[Path] = [] + + for config in python_configurations: + build_options = options.build_options(config.identifier) + build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + + if build_frontend.name == "pip": + print("The pyodide platform doesn't support pip frontend", file=sys.stderr) + sys.exit(1) + log.build_start(config.identifier) + + identifier_tmp_dir = tmp_path / config.identifier + built_wheel_dir = identifier_tmp_dir / "built_wheel" + repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + identifier_tmp_dir.mkdir() + built_wheel_dir.mkdir() + repaired_wheel_dir.mkdir() + + dependency_constraint_flags: Sequence[PathOrStr] = [] + if build_options.dependency_constraints: + constraints_path = build_options.dependency_constraints.get_for_python_version( + config.version + ) + dependency_constraint_flags = ["-c", constraints_path] + + env = setup_python( + identifier_tmp_dir / "build", + config, + dependency_constraint_flags, + build_options.environment, + ) + # The Pyodide command line runner mounts all directories in the host + # filesystem into the Pyodide file system, except for the custom + # file systems /dev, /lib, /proc, and /tmp. Mounting the mount + # points for alternate file systems causes some mysterious failure + # of the process (it just quits without any clear error). + # + # Because of this, by default Pyodide can't see anything under /tmp. + # This environment variable tells it also to mount our temp + # directory. + oldmounts = "" + extra_mounts = [str(identifier_tmp_dir)] + if str(Path(".").resolve()).startswith("/tmp"): + extra_mounts.append(str(Path(".").resolve())) + + if "_PYODIDE_EXTRA_MOUNTS" in env: + oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" + env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + ) + built_wheel = compatible_wheel + else: + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, project=".", package=build_options.package_dir + ) + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + + extra_flags = split_config_settings(build_options.config_settings, "build") + extra_flags += build_frontend.args + + if not 0 <= build_options.build_verbosity < 2: + msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." + log.warning(msg) + + build_env = env.copy() + if build_options.dependency_constraints: + build_env["PIP_CONSTRAINT"] = str(constraints_path) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + "pyodide", + "build", + build_options.package_dir, + f"--outdir={built_wheel_dir}", + *extra_flags, + env=build_env, + ) + built_wheel = next(built_wheel_dir.glob("*.whl")) + + if built_wheel.name.endswith("none-any.whl"): + raise NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + ) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) + + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise AlreadyBuiltWheelError(repaired_wheel.name) + + if build_options.test_command and build_options.test_selector(config.identifier): + log.step("Testing wheel...") + + venv_dir = identifier_tmp_dir / "venv-test" + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + + # --no-download?? + call("pyodide", "venv", venv_dir, env=env) + + virtualenv_env = env.copy() + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(venv_dir / "bin"), + virtualenv_env["PATH"], + ] + ) + + # check that we are using the Python from the virtual environment + call("which", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call( + "pip", + "install", + f"{repaired_wheel}{build_options.test_extras}", + env=virtualenv_env, + ) + + # test the wheel + if build_options.test_requires: + call("pip", "install", *build_options.test_requires, env=virtualenv_env) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path(".").resolve(), + package=build_options.package_dir.resolve(), + ) + + test_cwd = identifier_tmp_dir / "test_cwd" + test_cwd.mkdir(exist_ok=True) + (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text()) + + shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) + + # we're all done here; move it to output (overwrite existing) + if compatible_wheel is None: + with contextlib.suppress(FileNotFoundError): + (build_options.output_dir / repaired_wheel.name).unlink() + + shutil.move(str(repaired_wheel), build_options.output_dir) + built_wheels.append(build_options.output_dir / repaired_wheel.name) + finally: + pass diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 2f0393e11..a7a9bce1a 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -166,3 +166,8 @@ python_configurations = [ { identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" }, { identifier = "pp310-win_amd64", version = "3.10", arch = "64", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.16-win64.zip" }, ] + +[pyodide] +python_configurations = [ + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a4", emscripten_version = "3.1.52" }, +] diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index d0ba3ff60..088b5d542 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -765,6 +765,51 @@ "$ref": "#/properties/test-requires" } } + }, + "pyodide": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "repair-wheel-command": { + "$ref": "#/properties/repair-wheel-command" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + } + } } } } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 85e82f2f7..984af48ea 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -46,3 +46,5 @@ repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" [tool.cibuildwheel.windows] + +[tool.cibuildwheel.pyodide] diff --git a/cibuildwheel/typing.py b/cibuildwheel/typing.py index 3fe480aa6..61e1c96f9 100644 --- a/cibuildwheel/typing.py +++ b/cibuildwheel/typing.py @@ -22,8 +22,8 @@ PathOrStr = Union[str, "os.PathLike[str]"] -PlatformName = Literal["linux", "macos", "windows"] -PLATFORMS: Final[set[PlatformName]] = {"linux", "macos", "windows"} +PlatformName = Literal["linux", "macos", "windows", "pyodide"] +PLATFORMS: Final[set[PlatformName]] = {"linux", "macos", "windows", "pyodide"} class GenericPythonConfiguration(Protocol): diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 72c937d71..c3e8b00c5 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -21,6 +21,7 @@ from pathlib import Path, PurePath from time import sleep from typing import Any, ClassVar, Final, Literal, TextIO, TypeVar +from zipfile import ZipFile import bracex import certifi @@ -334,6 +335,21 @@ def download(url: str, dest: Path) -> None: sleep(3) +def extract_zip(zip_src: Path, dest: Path) -> None: + with ZipFile(zip_src) as zip_: + for zinfo in zip_.filelist: + zip_.extract(zinfo, dest) + + # Set permissions to the same values as they were set in the archive + # We have to do this manually due to + # https://github.com/python/cpython/issues/59999 + # But some files in the zipfile seem to have external_attr with 0 + # permissions. In that case just use the default value??? + permissions = (zinfo.external_attr >> 16) & 0o777 + if permissions != 0: + dest.joinpath(zinfo.filename).chmod(permissions) + + class DependencyConstraints: def __init__(self, base_file_path: Path): assert base_file_path.exists() diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f2ddc0127..ab5bc4645 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -11,7 +11,6 @@ from dataclasses import dataclass from functools import lru_cache from pathlib import Path -from zipfile import ZipFile from filelock import FileLock from packaging.version import Version @@ -31,6 +30,7 @@ NonPlatformWheelError, call, download, + extract_zip, find_compatible_wheel, get_build_verbosity_extra_flags, get_pip_version, @@ -96,11 +96,6 @@ def get_python_configurations( return python_configurations -def extract_zip(zip_src: Path, dest: Path) -> None: - with ZipFile(zip_src) as zip_: - zip_.extractall(dest) - - @lru_cache(maxsize=None) def _ensure_nuget() -> Path: nuget = CIBW_CACHE_PATH / "nuget.exe" diff --git a/docs/options.md b/docs/options.md index 7872ec5ef..c0afd021f 100644 --- a/docs/options.md +++ b/docs/options.md @@ -91,9 +91,9 @@ You can configure cibuildwheel with a config file, such as `pyproject.toml`. Options have the same names as the environment variable overrides, but are placed in `[tool.cibuildwheel]` and are lower case, with dashes, following common [TOML][] practice. Anything placed in subsections `linux`, `windows`, -or `macos` will only affect those platforms. Lists can be used instead of -strings for items that are naturally a list. Multiline strings also work just -like in the environment variables. Environment variables will take +`macos`, or `pyodide` will only affect those platforms. Lists can be used +instead of strings for items that are naturally a list. Multiline strings also +work just like in the environment variables. Environment variables will take precedence if defined. The example above using environment variables could have been written like this: @@ -247,7 +247,7 @@ environment variables will completely override any TOML configuration. > Override the auto-detected target platform -Options: `auto` `linux` `macos` `windows` +Options: `auto` `linux` `macos` `windows` `pyodide` Default: `auto` @@ -255,6 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. +- For `pyodide` you need to be on an x86-64 linux runner and run cibuildwheel from Python 3.11. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. @@ -474,6 +475,7 @@ Options: - Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` - macOS: `x86_64` `arm64` `universal2` - Windows: `AMD64` `x86` `ARM64` +- Pyodide: `wasm32` - `auto`: The default archs for your machine - see the table below. - `auto64`: Just the 64-bit auto archs - `auto32`: Just the 32-bit auto archs @@ -684,7 +686,7 @@ a table of items, including arrays. single values. Platform-specific environment variables also available:
-`CIBW_CONFIG_SETTINGS_MACOS` | `CIBW_CONFIG_SETTINGS_WINDOWS` | `CIBW_CONFIG_SETTINGS_LINUX` +`CIBW_CONFIG_SETTINGS_MACOS` | `CIBW_CONFIG_SETTINGS_WINDOWS` | `CIBW_CONFIG_SETTINGS_LINUX` | `CIBW_CONFIG_SETTINGS_PYODIDE` #### Examples @@ -715,7 +717,7 @@ You can use `$PATH` syntax to insert other variables, or the `$(pwd)` syntax to To specify more than one environment variable, separate the assignments by spaces. Platform-specific environment variables are also available:
-`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` +`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` | `CIBW_ENVIRONMENT_PYODIDE` #### Examples @@ -847,7 +849,7 @@ On linux, overriding it triggers a new container launch. It cannot be overridden on macOS and Windows. Platform-specific environment variables also available:
-`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` +`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` | `CIBW_BEFORE_ALL_PYODIDE` !!! note @@ -912,7 +914,7 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` + `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` | `CIBW_BEFORE_BUILD_PYODIDE` #### Examples @@ -993,6 +995,7 @@ Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on macOS: `'delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'` - on Windows: `''` +- on Pyodide: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. The command is run on each built wheel (except for pure Python ones) before testing it. @@ -1006,7 +1009,7 @@ The following placeholders must be used inside the command and will be replaced The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`. Platform-specific environment variables are also available:
-`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` +`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` | `CIBW_REPAIR_WHEEL_COMMAND_PYODIDE` !!! tip cibuildwheel doesn't yet ship a default repair command for Windows. @@ -1281,7 +1284,7 @@ here and it will be used instead. `./constraints.txt` if that's not found. Platform-specific environment variables are also available:
-`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` +`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` | `CIBW_DEPENDENCY_VERSIONS_PYODIDE` !!! note This option does not affect the tools used on the Linux build - those versions @@ -1339,7 +1342,7 @@ not be installed after building. The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
-`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` +`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_PYODIDE` #### Examples @@ -1402,7 +1405,7 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` + `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` | `CIBW_BEFORE_TEST_PYODIDE` #### Examples @@ -1462,7 +1465,7 @@ Platform-specific environment variables are also available:
Space-separated list of dependencies required for running the tests. Platform-specific environment variables are also available:
-`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` +`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` | `CIBW_TEST_REQUIRES_PYODIDE` #### Examples @@ -1502,7 +1505,7 @@ tests. This can be used to avoid having to redefine test dependencies in `setup.cfg` or `setup.py`. Platform-specific environment variables are also available:
-`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` +`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` | `CIBW_TEST_EXTRAS_PYODIDE` #### Examples @@ -1581,7 +1584,7 @@ export CIBW_DEBUG_KEEP_CONTAINER=TRUE A number from 1 to 3 to increase the level of verbosity (corresponding to invoking pip with `-v`, `-vv`, and `-vvv`), between -1 and -3 (`-q`, `-qq`, and `-qqq`), or just 0 (default verbosity). These flags are useful while debugging a build when the output of the actual build invoked by `pip wheel` is required. Has no effect on the `build` backend, which produces verbose output by default. Platform-specific environment variables are also available:
-`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` +`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` | `CIBW_BUILD_VERBOSITY_PYODIDE` #### Examples diff --git a/test/conftest.py b/test/conftest.py index 06a4f2424..7881c9b67 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -28,6 +28,8 @@ def pytest_addoption(parser) -> None: params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"] ) def build_frontend_env(request) -> dict[str, str]: + if platform == "pyodide": + pytest.skip("Can't use pip as build frontend for pyodide platform") return request.param # type: ignore[no-any-return] diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index 02c75b0e6..cda49054b 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -44,6 +44,7 @@ def get_tag(self): limited_api_project.files["pyproject.toml"] = pyproject_toml +@utils.skip_if_pyodide(reason="No abi3, no py38") def test_abi3(tmp_path): project_dir = tmp_path / "project" limited_api_project.generate(project_dir) @@ -173,6 +174,7 @@ def test(): ctypes_project.files["pyproject.toml"] = pyproject_toml +@utils.skip_if_pyodide(reason="Doesn't work for some reason") def test_abi_none(tmp_path, capfd): project_dir = tmp_path / "project" ctypes_project.generate(project_dir) diff --git a/test/test_before_test.py b/test/test_before_test.py index 4108a392a..206888315 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from . import test_projects, utils before_test_project = test_projects.new_c_project() @@ -40,11 +42,19 @@ def test(tmp_path): test_project_dir = project_dir / "dependency" test_projects.new_c_project().generate(test_project_dir) - before_test = ( - """python -c "import os, sys; open('{project}/pythonversion_bt.txt', 'w').write(sys.version)" && """ - """python -c "import os, sys; open('{project}/pythonprefix_bt.txt', 'w').write(sys.prefix)" && """ - """python -m pip install {project}/dependency""" - ) + before_test_steps = [ + '''python -c "import os, sys; open('{project}/pythonversion_bt.txt', 'w').write(sys.version)"''', + '''python -c "import os, sys; open('{project}/pythonprefix_bt.txt', 'w').write(sys.prefix)"''', + ] + + if utils.platform == "pyodide": + before_test_steps.extend( + ["pyodide build {project}/dependency", "pip install --find-links dist/ spam"] + ) + else: + before_test_steps.append("python -m pip install {project}/dependency") + + before_test = " && ".join(before_test_steps) # build the wheels actual_wheels = utils.cibuildwheel_run( @@ -58,6 +68,7 @@ def test(tmp_path): # mac/linux. "CIBW_TEST_COMMAND": "false || pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test", + "_PYODIDE_EXTRA_MOUNTS": "/tmp/my-tmp-dir/", }, ) diff --git a/test/test_same_wheel.py b/test/test_custom_repair_wheel.py similarity index 51% rename from test/test_same_wheel.py rename to test/test_custom_repair_wheel.py index 51db2d88b..51d5dcc21 100644 --- a/test/test_same_wheel.py +++ b/test/test_custom_repair_wheel.py @@ -1,6 +1,7 @@ from __future__ import annotations import subprocess +from contextlib import nullcontext as does_not_raise from test import test_projects import pytest @@ -15,6 +16,7 @@ wheel = Path(sys.argv[1]) dest_dir = Path(sys.argv[2]) +print("reparing", wheel, dest_dir) platform = wheel.stem.split("-")[-1] name = f"spam-0.1.0-py2-none-{platform}.whl" dest = dest_dir / name @@ -30,8 +32,14 @@ def test(tmp_path, capfd): project_dir = tmp_path / "project" basic_project.generate(project_dir) - with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run( + num_builds = len(utils.cibuildwheel_get_build_identifiers(project_dir)) + if num_builds > 1: + expectation = pytest.raises(subprocess.CalledProcessError) + else: + expectation = does_not_raise() + + with expectation: + result = utils.cibuildwheel_run( project_dir, add_env={ "CIBW_REPAIR_WHEEL_COMMAND": "python repair.py {wheel} {dest_dir}", @@ -39,4 +47,14 @@ def test(tmp_path, capfd): ) captured = capfd.readouterr() - assert "Build failed because a wheel named" in captured.err + if num_builds > 1: + assert "Build failed because a wheel named" in captured.err + else: + # We only produced one wheel (currently Pyodide) + # check that it has the right name + # + # As far as I can tell, this is the only full test coverage for + # CIBW_REPAIR_WHEEL_COMMAND so this is useful even in the case when no + # error is raised + assert "spam-0.1.0-py2-none-emscripten" in captured.out + assert result[0].startswith("spam-0.1.0-py2-none-") diff --git a/test/test_emscripten.py b/test/test_emscripten.py new file mode 100644 index 000000000..d84f8eecb --- /dev/null +++ b/test/test_emscripten.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import platform +import shutil +import textwrap + +import pytest + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() + + +@pytest.mark.parametrize("use_pyproject_toml", [True, False]) +def test_pyodide_build(tmp_path, use_pyproject_toml): + if platform.machine() == "arm64": + pytest.skip("emsdk doesn't work correctly on arm64") + + if not shutil.which("python3.12"): + pytest.skip("Python 3.12 not installed") + + if use_pyproject_toml: + basic_project.files["pyproject.toml"] = textwrap.dedent( + """ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + """ + ) + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_args=["--platform", "pyodide"], + ) + + # check that the expected wheels are produced + expected_wheels = [ + "spam-0.1.0-cp312-cp312-emscripten_3_1_52_wasm32.whl", + ] + + print("actual_wheels", actual_wheels) + print("expected_wheels", expected_wheels) + + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_environment.py b/test/test_environment.py index b4a0120a5..9070e7460 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -93,10 +93,21 @@ def test_overridden_path(tmp_path, capfd): assert len(os.listdir(output_dir)) == 0 captured = capfd.readouterr() - assert "python available on PATH doesn't match our installed instance" in captured.err + assert "python available on PATH doesn't match our installed instance" in captured.err.replace( + "venv", "installed" + ) -@pytest.mark.parametrize("build_frontend", ["pip", "build"]) +@pytest.mark.parametrize( + "build_frontend", + [ + pytest.param( + "pip", + marks=pytest.mark.skipif(utils.platform == "pyodide", reason="No pip for pyodide"), + ), + "build", + ], +) def test_overridden_pip_constraint(tmp_path, build_frontend): """ Verify that users can use PIP_CONSTRAINT to specify a specific version of diff --git a/test/test_projects/c.py b/test/test_projects/c.py index 62252a620..027d1ee47 100644 --- a/test/test_projects/c.py +++ b/test/test_projects/c.py @@ -42,15 +42,19 @@ """ SETUP_PY_TEMPLATE = r""" +import os import sys + from setuptools import setup, Extension {{ setup_py_add }} libraries = [] -if sys.platform.startswith('linux'): +# Emscripten fails if you pass -lc... +if sys.platform.startswith('linux') and "emscripten" not in os.environ.get("_PYTHON_HOST_PLATFORM", ""): libraries.extend(['m', 'c']) + setup( ext_modules=[Extension( 'spam', diff --git a/test/test_testing.py b/test/test_testing.py index 03f6afb00..0a89ec31e 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -66,7 +66,7 @@ def test_uname(self): # See #336 for more info. bits = struct.calcsize("P") * 8 if bits == 32: - self.assertEqual(platform.machine(), "i686") + self.assertIn(platform.machine(), ["i686", "wasm32"]) ''' diff --git a/test/utils.py b/test/utils.py index 76c8163b8..dcf7fd54f 100644 --- a/test/utils.py +++ b/test/utils.py @@ -13,6 +13,8 @@ from tempfile import TemporaryDirectory from typing import Final +import pytest + from cibuildwheel.util import CIBW_CACHE_PATH SINGLE_PYTHON_VERSION: Final[tuple[int, int]] = (3, 12) @@ -231,6 +233,11 @@ def expected_wheels( wheels = [] + if platform == "pyodide": + python_abi_tag = "cp311-cp311" + platform_tag = "emscripten_3_1_32_wasm32" + return [f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl"] + for python_abi_tag in python_abi_tags: platform_tags = [] @@ -278,7 +285,6 @@ def expected_wheels( platform_tags.append( f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2', ) - else: msg = f"Unsupported platform {platform!r}" raise Exception(msg) @@ -301,6 +307,10 @@ def get_macos_version(): return tuple(map(int, version_str.split(".")[:2])) +def skip_if_pyodide(reason): + return pytest.mark.skipif(platform == "pyodide", reason=reason) + + def arch_name_for_linux(arch: str): """ Archs have different names on different platforms, but it's useful to be From 0b285d0ce7026722b919aaef66ad1f74193945eb Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 19 May 2024 15:42:07 -0400 Subject: [PATCH 02/29] Try to fix xbuildenv path [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Try to install pyodide-build from main branch Try again Try again Update constraints file Try again Remove unused variable Drop constraints Remove --download option Fix xbuildenv install Try again Try again [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/pyodide.py | 15 ++++++++------- noxfile.py | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index d17cae5b4..33941ffa3 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -62,8 +62,11 @@ def install_emscripten(tmp: Path, version: str) -> Path: def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str: - xbuildenv_cache_dir = CIBW_CACHE_PATH / f"pyodide-xbuildenv-{pyodide_version}" - pyodide_root = xbuildenv_cache_dir / ".pyodide-xbuildenv/xbuildenv/pyodide-root/" + xbuildenv_cache_dir = CIBW_CACHE_PATH + pyodide_root = ( + xbuildenv_cache_dir + / f".pyodide-xbuildenv-0.26.0.dev0/{pyodide_version}/xbuildenv/pyodide-root" + ) if pyodide_root.exists(): return str(pyodide_root) @@ -76,7 +79,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str: "pyodide", "xbuildenv", "install", - "--download", + pyodide_version, env=env, cwd=xbuildenv_cache_dir, ) @@ -104,12 +107,11 @@ def setup_python( dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, ) -> dict[str, str]: - pyodide_version = python_configuration.pyodide_version base_python = get_base_python(python_configuration.identifier) log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(base_python, venv_path, dependency_constraint_flags) + env = virtualenv(base_python, venv_path, []) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" @@ -123,7 +125,6 @@ def setup_python( "install", "--upgrade", "pip", - *dependency_constraint_flags, env=env, cwd=venv_path, ) @@ -163,7 +164,7 @@ def setup_python( "--upgrade", "auditwheel-emscripten", "build[virtualenv]", - f"pyodide-build=={pyodide_version}", + "pyodide-build", *dependency_constraint_flags, env=env, ) diff --git a/noxfile.py b/noxfile.py index f12f50066..fff498537 100644 --- a/noxfile.py +++ b/noxfile.py @@ -73,7 +73,7 @@ def update_constraints(session: nox.Session) -> None: if session.venv_backend != "uv": session.install("uv>=0.1.23") - for minor_version in range(7, 14): + for minor_version in [12]: python_version = f"3.{minor_version}" env = os.environ.copy() # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to @@ -83,6 +83,7 @@ def update_constraints(session: nox.Session) -> None: "uv", "pip", "compile", + "--prerelease=allow", f"--python-version={python_version}", "--upgrade", "cibuildwheel/resources/constraints.in", From b2ad069eb8911f544340614d2e13e21062c4947b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 20 May 2024 07:01:25 -0400 Subject: [PATCH 03/29] fix: remove pinning on pyodide Signed-off-by: Henry Schreiner --- cibuildwheel/pyodide.py | 4 ++-- noxfile.py | 3 +-- test/utils.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 33941ffa3..198af4d53 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -111,7 +111,7 @@ def setup_python( log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(base_python, venv_path, []) + env = virtualenv(python_configuration.version, base_python, venv_path, []) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" @@ -164,7 +164,7 @@ def setup_python( "--upgrade", "auditwheel-emscripten", "build[virtualenv]", - "pyodide-build", + "git+https://github.com/pyodide/pyodide.git@main#subdirectory=pyodide-build", *dependency_constraint_flags, env=env, ) diff --git a/noxfile.py b/noxfile.py index fff498537..f12f50066 100644 --- a/noxfile.py +++ b/noxfile.py @@ -73,7 +73,7 @@ def update_constraints(session: nox.Session) -> None: if session.venv_backend != "uv": session.install("uv>=0.1.23") - for minor_version in [12]: + for minor_version in range(7, 14): python_version = f"3.{minor_version}" env = os.environ.copy() # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to @@ -83,7 +83,6 @@ def update_constraints(session: nox.Session) -> None: "uv", "pip", "compile", - "--prerelease=allow", f"--python-version={python_version}", "--upgrade", "cibuildwheel/resources/constraints.in", diff --git a/test/utils.py b/test/utils.py index dcf7fd54f..8c5a55638 100644 --- a/test/utils.py +++ b/test/utils.py @@ -234,8 +234,8 @@ def expected_wheels( wheels = [] if platform == "pyodide": - python_abi_tag = "cp311-cp311" - platform_tag = "emscripten_3_1_32_wasm32" + python_abi_tag = "cp312-cp312" + platform_tag = "emscripten_3_1_52_wasm32" return [f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl"] for python_abi_tag in python_abi_tags: From 1109010b57366c32a91ce8b88aef5c278741ee01 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 21 May 2024 16:26:06 -0400 Subject: [PATCH 04/29] Update for Pyodide 0.26.0a5 --- cibuildwheel/logger.py | 2 +- cibuildwheel/resources/build-platforms.toml | 2 +- test/test_custom_repair_wheel.py | 4 ++-- test/test_emscripten.py | 2 +- test/utils.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index c9b0d5714..d679f57ae 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -34,7 +34,7 @@ "macosx_x86_64": "macOS x86_64", "macosx_universal2": "macOS Universal 2 - x86_64 and arm64", "macosx_arm64": "macOS arm64 - Apple Silicon", - "pyodide_wasm32": "Pyodide v0.23.x", + "pyodide_wasm32": "Pyodide", } diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index a7a9bce1a..c69318b67 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -169,5 +169,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a4", emscripten_version = "3.1.52" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a5", emscripten_version = "3.1.58" }, ] diff --git a/test/test_custom_repair_wheel.py b/test/test_custom_repair_wheel.py index 51d5dcc21..60d502386 100644 --- a/test/test_custom_repair_wheel.py +++ b/test/test_custom_repair_wheel.py @@ -16,7 +16,7 @@ wheel = Path(sys.argv[1]) dest_dir = Path(sys.argv[2]) -print("reparing", wheel, dest_dir) +print("repairing", wheel, dest_dir) platform = wheel.stem.split("-")[-1] name = f"spam-0.1.0-py2-none-{platform}.whl" dest = dest_dir / name @@ -56,5 +56,5 @@ def test(tmp_path, capfd): # As far as I can tell, this is the only full test coverage for # CIBW_REPAIR_WHEEL_COMMAND so this is useful even in the case when no # error is raised - assert "spam-0.1.0-py2-none-emscripten" in captured.out + assert "spam-0.1.0-py2-none-pyodide" in captured.out assert result[0].startswith("spam-0.1.0-py2-none-") diff --git a/test/test_emscripten.py b/test/test_emscripten.py index d84f8eecb..509fb8f35 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -39,7 +39,7 @@ def test_pyodide_build(tmp_path, use_pyproject_toml): # check that the expected wheels are produced expected_wheels = [ - "spam-0.1.0-cp312-cp312-emscripten_3_1_52_wasm32.whl", + "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", ] print("actual_wheels", actual_wheels) diff --git a/test/utils.py b/test/utils.py index 8c5a55638..6d4545ebe 100644 --- a/test/utils.py +++ b/test/utils.py @@ -235,7 +235,7 @@ def expected_wheels( if platform == "pyodide": python_abi_tag = "cp312-cp312" - platform_tag = "emscripten_3_1_52_wasm32" + platform_tag = "pyodide_2024_0_wasm32" return [f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl"] for python_abi_tag in python_abi_tags: From 040f5ecdbfa3461cbcc01dc629f4295bdd2ec07a Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 21 May 2024 16:38:16 -0400 Subject: [PATCH 05/29] Install pyodide-build from pypi --- cibuildwheel/pyodide.py | 5 +++-- cibuildwheel/resources/build-platforms.toml | 2 +- test/test_before_test.py | 1 + test/test_build_frontend_args.py | 3 +++ test/test_environment.py | 1 + test/test_from_sdist.py | 4 ++++ test/test_pure_wheel.py | 1 + test/test_subdir_package.py | 1 + 8 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 198af4d53..e9e1f2ac0 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -65,7 +65,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str: xbuildenv_cache_dir = CIBW_CACHE_PATH pyodide_root = ( xbuildenv_cache_dir - / f".pyodide-xbuildenv-0.26.0.dev0/{pyodide_version}/xbuildenv/pyodide-root" + / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root" ) if pyodide_root.exists(): return str(pyodide_root) @@ -108,6 +108,7 @@ def setup_python( environment: ParsedEnvironment, ) -> dict[str, str]: base_python = get_base_python(python_configuration.identifier) + pyodide_version = python_configuration.pyodide_version log.step("Setting up build environment...") venv_path = tmp / "venv" @@ -164,7 +165,7 @@ def setup_python( "--upgrade", "auditwheel-emscripten", "build[virtualenv]", - "git+https://github.com/pyodide/pyodide.git@main#subdirectory=pyodide-build", + f"pyodide-build=={pyodide_version}", *dependency_constraint_flags, env=env, ) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index c69318b67..afca3d2fb 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -169,5 +169,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a5", emscripten_version = "3.1.58" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a6", emscripten_version = "3.1.58" }, ] diff --git a/test/test_before_test.py b/test/test_before_test.py index 206888315..5420e8807 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -36,6 +36,7 @@ def test_prefix(self): """ +@utils.skip_if_pyodide(reason="TODO: fix!") def test(tmp_path): project_dir = tmp_path / "project" before_test_project.generate(project_dir) diff --git a/test/test_build_frontend_args.py b/test/test_build_frontend_args.py index 4480a0875..4429ac6c4 100644 --- a/test/test_build_frontend_args.py +++ b/test/test_build_frontend_args.py @@ -6,6 +6,9 @@ from .test_projects.c import new_c_project +@utils.skip_if_pyodide( + reason="pyodide build -h doesn't print help text https://github.com/pyodide/pyodide/issues/4783" +) @pytest.mark.parametrize("frontend_name", ["pip", "build"]) def test_build_frontend_args(tmp_path, capfd, frontend_name): project = new_c_project() diff --git a/test/test_environment.py b/test/test_environment.py index 9070e7460..30e67f2e1 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -98,6 +98,7 @@ def test_overridden_path(tmp_path, capfd): ) +@utils.skip_if_pyodide(reason="TODO: fix!") @pytest.mark.parametrize( "build_frontend", [ diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py index c8a0d6a3e..76fd86907 100644 --- a/test/test_from_sdist.py +++ b/test/test_from_sdist.py @@ -55,6 +55,7 @@ def cibuildwheel_from_sdist_run(sdist_path, add_env=None, config_file=None): # tests +@utils.skip_if_pyodide(reason="TODO: fix!") def test_simple(tmp_path): basic_project = test_projects.new_c_project() @@ -83,6 +84,7 @@ def test_simple(tmp_path): assert set(actual_wheels) == set(expected_wheels) +@utils.skip_if_pyodide(reason="TODO: fix!") def test_external_config_file_argument(tmp_path, capfd): basic_project = test_projects.new_c_project() @@ -114,6 +116,7 @@ def test_external_config_file_argument(tmp_path, capfd): assert "test log statement from before-all" in captured.out +@utils.skip_if_pyodide(reason="TODO: fix!") def test_config_in_pyproject_toml(tmp_path, capfd): # make a project with a pyproject.toml project = test_projects.new_c_project() @@ -141,6 +144,7 @@ def test_config_in_pyproject_toml(tmp_path, capfd): assert "test log statement from before-build 8419" in captured.out +@utils.skip_if_pyodide(reason="Doesn't work") def test_internal_config_file_argument(tmp_path, capfd): # make a project with a config file inside project = test_projects.new_c_project( diff --git a/test/test_pure_wheel.py b/test/test_pure_wheel.py index 998f967f3..b3cfce3e7 100644 --- a/test/test_pure_wheel.py +++ b/test/test_pure_wheel.py @@ -24,6 +24,7 @@ def a_function(): """ +@utils.skip_if_pyodide(reason="Doesn't work") def test(tmp_path, capfd): # this test checks that if a pure wheel is generated, the build should # fail. diff --git a/test/test_subdir_package.py b/test/test_subdir_package.py index 665339701..8334f68c7 100644 --- a/test/test_subdir_package.py +++ b/test/test_subdir_package.py @@ -33,6 +33,7 @@ """ +@utils.skip_if_pyodide(reason="Doesn't work") def test(capfd, tmp_path): project_dir = tmp_path / "project" subdir_package_project.generate(project_dir) From 53490d141314f3ff5e22738c0b438c68acea44f3 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 24 May 2024 11:46:56 -0400 Subject: [PATCH 06/29] Update docs/options.md Co-authored-by: Henry Schreiner --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index c0afd021f..409062b97 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` you need to be on an x86-64 linux runner and run cibuildwheel from Python 3.11. +- For `pyodide` you need to be on an x86-64 linux runner and run cibuildwheel from Python 3.12. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From b9635688a303103bca941da6dade163387f3b8d7 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 24 May 2024 11:54:13 -0400 Subject: [PATCH 07/29] Unxfail things that look like they were just a version mismatch --- bin/run_tests.py | 1 - test/test_from_sdist.py | 4 ---- test/test_subdir_package.py | 1 - 3 files changed, 6 deletions(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 4f4ae7ed3..a64d431f2 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -43,7 +43,6 @@ "-m", "pytest", f"--numprocesses={args.num_processes}", - "-x", "--durations", "0", "--timeout=2400", diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py index 76fd86907..c8a0d6a3e 100644 --- a/test/test_from_sdist.py +++ b/test/test_from_sdist.py @@ -55,7 +55,6 @@ def cibuildwheel_from_sdist_run(sdist_path, add_env=None, config_file=None): # tests -@utils.skip_if_pyodide(reason="TODO: fix!") def test_simple(tmp_path): basic_project = test_projects.new_c_project() @@ -84,7 +83,6 @@ def test_simple(tmp_path): assert set(actual_wheels) == set(expected_wheels) -@utils.skip_if_pyodide(reason="TODO: fix!") def test_external_config_file_argument(tmp_path, capfd): basic_project = test_projects.new_c_project() @@ -116,7 +114,6 @@ def test_external_config_file_argument(tmp_path, capfd): assert "test log statement from before-all" in captured.out -@utils.skip_if_pyodide(reason="TODO: fix!") def test_config_in_pyproject_toml(tmp_path, capfd): # make a project with a pyproject.toml project = test_projects.new_c_project() @@ -144,7 +141,6 @@ def test_config_in_pyproject_toml(tmp_path, capfd): assert "test log statement from before-build 8419" in captured.out -@utils.skip_if_pyodide(reason="Doesn't work") def test_internal_config_file_argument(tmp_path, capfd): # make a project with a config file inside project = test_projects.new_c_project( diff --git a/test/test_subdir_package.py b/test/test_subdir_package.py index 8334f68c7..665339701 100644 --- a/test/test_subdir_package.py +++ b/test/test_subdir_package.py @@ -33,7 +33,6 @@ """ -@utils.skip_if_pyodide(reason="Doesn't work") def test(capfd, tmp_path): project_dir = tmp_path / "project" subdir_package_project.generate(project_dir) From a4b2aeb9127d6a6cd68f261fb31525efb29bb3f2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 24 May 2024 11:58:08 -0400 Subject: [PATCH 08/29] refactor: add constraints for pyodide Signed-off-by: Henry Schreiner --- cibuildwheel/pyodide.py | 5 +- .../resources/constraints-pyodide312.txt | 123 ++++++++++++++++++ cibuildwheel/util.py | 6 +- noxfile.py | 34 ++++- 4 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 cibuildwheel/resources/constraints-pyodide312.txt diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index e9e1f2ac0..f152c8d0d 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -108,7 +108,6 @@ def setup_python( environment: ParsedEnvironment, ) -> dict[str, str]: base_python = get_base_python(python_configuration.identifier) - pyodide_version = python_configuration.pyodide_version log.step("Setting up build environment...") venv_path = tmp / "venv" @@ -165,7 +164,7 @@ def setup_python( "--upgrade", "auditwheel-emscripten", "build[virtualenv]", - f"pyodide-build=={pyodide_version}", + "pyodide-build", *dependency_constraint_flags, env=env, ) @@ -233,7 +232,7 @@ def build(options: Options, tmp_path: Path) -> None: dependency_constraint_flags: Sequence[PathOrStr] = [] if build_options.dependency_constraints: constraints_path = build_options.dependency_constraints.get_for_python_version( - config.version + config.version, variant="pyodide" ) dependency_constraint_flags = ["-c", constraints_path] diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt new file mode 100644 index 000000000..878bb8ee5 --- /dev/null +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -0,0 +1,123 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +annotated-types==0.7.0 + # via pydantic +anyio==4.3.0 + # via httpx +auditwheel-emscripten==0.0.14 + # via + # -r .nox/update_constraints/tmp/constraints-pyodide.in + # pyodide-build +build==1.2.1 + # via + # -r .nox/update_constraints/tmp/constraints-pyodide.in + # pyodide-build +certifi==2024.2.2 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via typer +cloudpickle==3.0.0 + # via loky +cmake==3.29.3 + # via pyodide-build +distlib==0.3.8 + # via virtualenv +filelock==3.14.0 + # via virtualenv +h11==0.14.0 + # via httpcore +httpcore==1.0.5 + # via httpx +httpx==0.27.0 + # via unearth +idna==3.7 + # via + # anyio + # httpx + # requests +leb128==1.0.7 + # via auditwheel-emscripten +loky==3.4.1 + # via pyodide-build +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +packaging==24.0 + # via + # auditwheel-emscripten + # build + # pyodide-build + # unearth +platformdirs==4.2.2 + # via virtualenv +pydantic==2.7.1 + # via + # pyodide-build + # pyodide-lock +pydantic-core==2.18.2 + # via pydantic +pygments==2.18.0 + # via rich +pyodide-build==0.26.0a6 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in +pyodide-cli==0.2.3 + # via + # auditwheel-emscripten + # pyodide-build +pyodide-lock==0.1.0a6 + # via pyodide-build +pyproject-hooks==1.1.0 + # via build +pyyaml==6.0.1 + # via pyodide-build +requests==2.32.2 + # via pyodide-build +resolvelib==1.0.1 + # via pyodide-build +rich==13.7.1 + # via + # pyodide-build + # pyodide-cli + # typer +ruamel-yaml==0.18.6 + # via pyodide-build +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +shellingham==1.5.4 + # via typer +sniffio==1.3.1 + # via + # anyio + # httpx +typer==0.12.3 + # via + # auditwheel-emscripten + # pyodide-build + # pyodide-cli +types-requests==2.32.0.20240523 + # via pyodide-build +typing-extensions==4.12.0 + # via + # pydantic + # pydantic-core + # typer +unearth==0.15.2 + # via pyodide-build +urllib3==2.2.1 + # via + # requests + # types-requests +virtualenv==20.26.2 + # via + # build + # pyodide-build +wheel==0.43.0 + # via + # auditwheel-emscripten + # pyodide-build diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index c3e8b00c5..b8475f752 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -359,12 +359,14 @@ def __init__(self, base_file_path: Path): def with_defaults() -> DependencyConstraints: return DependencyConstraints(base_file_path=resources_dir / "constraints.txt") - def get_for_python_version(self, version: str) -> Path: + def get_for_python_version( + self, version: str, *, variant: Literal["python", "pyodide"] = "python" + ) -> Path: version_parts = version.split(".") # try to find a version-specific dependency file e.g. if # ./constraints.txt is the base, look for ./constraints-python36.txt - specific_stem = self.base_file_path.stem + f"-python{version_parts[0]}{version_parts[1]}" + specific_stem = self.base_file_path.stem + f"-{variant}{version_parts[0]}{version_parts[1]}" specific_name = specific_stem + self.base_file_path.suffix specific_file_path = self.base_file_path.with_name(specific_name) diff --git a/noxfile.py b/noxfile.py index f12f50066..51a6df421 100644 --- a/noxfile.py +++ b/noxfile.py @@ -70,6 +70,8 @@ def update_constraints(session: nox.Session) -> None: Update the dependencies inplace. """ + resources = Path("cibuildwheel/resources") + if session.venv_backend != "uv": session.install("uv>=0.1.23") @@ -79,22 +81,44 @@ def update_constraints(session: nox.Session) -> None: # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to # regenerate the constraints files env["UV_CUSTOM_COMPILE_COMMAND"] = f"nox -s {session.name}" + output_file = resources / f"constraints-python{python_version.replace('.', '')}.txt" session.run( "uv", "pip", "compile", f"--python-version={python_version}", "--upgrade", - "cibuildwheel/resources/constraints.in", - f"--output-file=cibuildwheel/resources/constraints-python{python_version.replace('.', '')}.txt", + resources / "constraints.in", + f"--output-file={output_file}", env=env, ) - RESOURCES = DIR / "cibuildwheel" / "resources" + shutil.copyfile( - RESOURCES / "constraints-python312.txt", - RESOURCES / "constraints.txt", + resources / "constraints-python312.txt", + resources / "constraints.txt", ) + build_platforms = nox.project.load_toml(resources / "build-platforms.toml") + pyodides = build_platforms["pyodide"]["python_configurations"] + for pyodide in pyodides: + python_version = ".".join(pyodide["version"].split(".")[:2]) + pyodide_version = pyodide["pyodide_version"] + output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" + tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" + tmp_file.write_text( + f"auditwheel-emscripten\nbuild[virtualenv]\npyodide-build=={pyodide_version}" + ) + session.run( + "uv", + "pip", + "compile", + f"--python-version={python_version}", + "--upgrade", + tmp_file, + f"--output-file={output_file}", + env=env, + ) + @nox.session def update_pins(session: nox.Session) -> None: From acd81fdc372a02ff68d215cd77ca774105cf3deb Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 24 May 2024 15:21:56 -0400 Subject: [PATCH 09/29] chore: minor cleanup Signed-off-by: Henry Schreiner --- README.md | 32 ++++++++++++++++---------------- bin/run_tests.py | 1 + 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 43276864b..999962bc4 100644 --- a/README.md +++ b/README.md @@ -22,30 +22,30 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult What does it do? ---------------- -| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | -|----------------|----|-----|-----|-----|-----|----|-----|----|-----|-----| -| CPython 3.6 | ✅ | N/A | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.7 | ✅ | N/A | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.12 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.13³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | -| PyPy 3.7 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | -| PyPy 3.8 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | -| PyPy 3.9 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | -| PyPy 3.10 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | +| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | Pyodide | +|----------------|----|-----|-----|-----|-----|----|-----|----|-----|-----|-----| +| CPython 3.6 | ✅ | N/A | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | N/A | +| CPython 3.7 | ✅ | N/A | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | N/A | +| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | N/A | +| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | N/A | +| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | N/A | +| CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | N/A | +| CPython 3.12 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁴ | +| CPython 3.13³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | N/A | +| PyPy 3.7 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | +| PyPy 3.8 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | +| PyPy 3.9 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | +| PyPy 3.10 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | ¹ PyPy is only supported for manylinux wheels.
² Windows arm64 support is experimental.
-³ CPython 3.13 is available using the [CIBW_PRERELEASE_PYTHONS](https://cibuildwheel.pypa.io/en/stable/options/#prerelease-pythons) option.
+³ CPython 3.13 is available using the [`CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.pypa.io/en/stable/options/#prerelease-pythons) option. Free-threaded mode requires opt-in, not yet available on macOS.
+⁴ Experimental, not yet supported on PyPI, but can be used directly in web deployment. Use `--platform pyodide` to build.
- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs your library's tests against the wheel-installed version of your library -- Can also build pyodide wheels for web deployment (experimental, not supported on PyPI yet) with `--platform pyodide` See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) if you need to build unsupported versions of Python, such as Python 2. diff --git a/bin/run_tests.py b/bin/run_tests.py index a64d431f2..4f4ae7ed3 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -43,6 +43,7 @@ "-m", "pytest", f"--numprocesses={args.num_processes}", + "-x", "--durations", "0", "--timeout=2400", From ed2d09dc8ab927bf8d97f3ec0b427d3e772b08a4 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 24 May 2024 15:26:53 -0400 Subject: [PATCH 10/29] Apply suggestions from code review --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d1588dbc..d3528e7ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,6 @@ jobs: needs: lint runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-13, macos-14] python_version: ['3.12'] @@ -162,7 +161,6 @@ jobs: with: python-version: '3.12' - - name: Install dependencies run: | python -m pip install ".[test]" From 99831ecbcd065cc9df77b449b86426de1ab4fec0 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Sat, 25 May 2024 10:13:01 +0200 Subject: [PATCH 11/29] Apply suggestion from code review --- .github/workflows/test.yml | 4 ++-- docs/options.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3528e7ad..23b25b1ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,8 +155,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 180 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Install Python 3.12 with: python-version: '3.12' diff --git a/docs/options.md b/docs/options.md index 409062b97..483225627 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` you need to be on an x86-64 linux runner and run cibuildwheel from Python 3.12. +- For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From eabc33e454473831d3c916f73e17a5832c2f07e7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 26 May 2024 00:35:27 -0400 Subject: [PATCH 12/29] refactor: minor touchup Signed-off-by: Henry Schreiner --- cibuildwheel/pyodide.py | 16 ++++++++-------- .../resources/constraints-pyodide312.txt | 6 +++--- docs/options.md | 9 +++++++++ docs/setup.md | 7 +++++++ noxfile.py | 4 +--- test/test_before_test.py | 3 +-- test/test_custom_repair_wheel.py | 1 - 7 files changed, 29 insertions(+), 17 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index f152c8d0d..58f0dae7b 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -44,11 +44,13 @@ class PythonConfiguration: def install_emscripten(tmp: Path, version: str) -> Path: - url = "https://github.com/emscripten-core/emsdk/archive/main.zip" - installation_path = CIBW_CACHE_PATH / ("emsdk-" + version) - emsdk_path = installation_path / "emsdk-main/emsdk" - emcc_path = installation_path / "emsdk-main/upstream/emscripten/emcc" - with FileLock(str(installation_path) + ".lock"): + # We don't need to match the emsdk version to the version we install, but + # we do for stability + url = f"https://github.com/emscripten-core/emsdk/archive/refs/tags/{version}.zip" + installation_path = CIBW_CACHE_PATH / f"emsdk-{version}" + emsdk_path = installation_path / f"emsdk-{version}/emsdk" + emcc_path = installation_path / f"emsdk-{version}/upstream/emscripten/emcc" + with FileLock(f"{installation_path}.lock"): if installation_path.exists(): return emcc_path emsdk_zip = tmp / "emsdk.zip" @@ -125,14 +127,12 @@ def setup_python( "install", "--upgrade", "pip", + *dependency_constraint_flags, env=env, cwd=venv_path, ) env = environment.as_dictionary(prev_environment=env) - if "HOME" not in env: - # Workaround for https://github.com/pyodide/pyodide/pull/3744 - env["HOME"] = "" # check what pip version we're on assert (venv_bin_path / "pip").exists() diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 878bb8ee5..cc5de2c73 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -5,9 +5,7 @@ annotated-types==0.7.0 anyio==4.3.0 # via httpx auditwheel-emscripten==0.0.14 - # via - # -r .nox/update_constraints/tmp/constraints-pyodide.in - # pyodide-build + # via pyodide-build build==1.2.1 # via # -r .nox/update_constraints/tmp/constraints-pyodide.in @@ -54,6 +52,8 @@ packaging==24.0 # build # pyodide-build # unearth +pip==24.0 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.2.2 # via virtualenv pydantic==2.7.1 diff --git a/docs/options.md b/docs/options.md index 483225627..8eaa60943 100644 --- a/docs/options.md +++ b/docs/options.md @@ -310,6 +310,9 @@ For CPython, the minimally supported macOS version is 10.9; for PyPy 3.7, macOS Windows arm64 platform support is experimental. +For an experimental WebAssembly build with `--platform pyodide`, +`cp312-pyodide_wasm32` is the only platform identifier. + See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7. #### Examples @@ -1020,6 +1023,12 @@ Platform-specific environment variables are also available:
[Delvewheel]: https://github.com/adang1345/delvewheel +!!! tip + When using `--platform pyodide`, `pyodide build` is used to do the build, + which already uses `auditwheel-emscripten` to repair the wheel, so the default + repair command is empty. If there is a way to do this in two steps in the future, + this could change. + #### Examples !!! tab examples "Environment variables" diff --git a/docs/setup.md b/docs/setup.md index b89dd5268..99f2052e9 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -107,6 +107,13 @@ You can override the cache folder using the ``CIBW_CACHE_PATH`` environment vari Download link: https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg ``` +### Pyodide (WebAssembly) builds (experimental) + +Pre-requisite: you need to have a matching host version of Python (unlike all +other cibuildwheel platforms) and Node 20. Linux host highly recommended; macOS hosts may +work and Windows hosts will not work. + +You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). ## Configure a CI service diff --git a/noxfile.py b/noxfile.py index 51a6df421..9c1d80c9d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -105,9 +105,7 @@ def update_constraints(session: nox.Session) -> None: pyodide_version = pyodide["pyodide_version"] output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" - tmp_file.write_text( - f"auditwheel-emscripten\nbuild[virtualenv]\npyodide-build=={pyodide_version}" - ) + tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_version}") session.run( "uv", "pip", diff --git a/test/test_before_test.py b/test/test_before_test.py index 5420e8807..d7ca3251a 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -36,7 +36,6 @@ def test_prefix(self): """ -@utils.skip_if_pyodide(reason="TODO: fix!") def test(tmp_path): project_dir = tmp_path / "project" before_test_project.generate(project_dir) @@ -67,7 +66,7 @@ def test(tmp_path): "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || pytest {project}/test", + "CIBW_TEST_COMMAND": "false || python -m pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test", "_PYODIDE_EXTRA_MOUNTS": "/tmp/my-tmp-dir/", }, diff --git a/test/test_custom_repair_wheel.py b/test/test_custom_repair_wheel.py index 60d502386..72c3bce9e 100644 --- a/test/test_custom_repair_wheel.py +++ b/test/test_custom_repair_wheel.py @@ -16,7 +16,6 @@ wheel = Path(sys.argv[1]) dest_dir = Path(sys.argv[2]) -print("repairing", wheel, dest_dir) platform = wheel.stem.split("-")[-1] name = f"spam-0.1.0-py2-none-{platform}.whl" dest = dest_dir / name From 202174a184d60e6759c490eb1d7120f792c59056 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 26 May 2024 01:32:00 -0400 Subject: [PATCH 13/29] ci: xfail the pyodide test Signed-off-by: Henry Schreiner --- test/test_before_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_before_test.py b/test/test_before_test.py index d7ca3251a..e151a71d7 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -36,6 +36,7 @@ def test_prefix(self): """ +@pytest.mark.xfail(utils.platform == "pyodide", reason="TODO: venv error on pyodide!", strict=True) def test(tmp_path): project_dir = tmp_path / "project" before_test_project.generate(project_dir) From 013e8111aa769c1aceb0e318a7ab656f150ea947 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 26 May 2024 12:39:43 +0200 Subject: [PATCH 14/29] review: use a pinned version of node --- bin/update_nodejs.py | 148 ++++++++++++++++++++ cibuildwheel/architecture.py | 18 ++- cibuildwheel/pyodide.py | 13 +- cibuildwheel/resources/build-platforms.toml | 2 +- cibuildwheel/resources/nodejs.toml | 2 + cibuildwheel/util.py | 41 ++++++ noxfile.py | 1 + 7 files changed, 216 insertions(+), 9 deletions(-) create mode 100755 bin/update_nodejs.py create mode 100644 cibuildwheel/resources/nodejs.toml diff --git a/bin/update_nodejs.py b/bin/update_nodejs.py new file mode 100755 index 000000000..c2cb1c0f0 --- /dev/null +++ b/bin/update_nodejs.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import difflib +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import click +import packaging.specifiers +import requests +import rich +from packaging.version import InvalidVersion, Version +from rich.logging import RichHandler +from rich.syntax import Syntax + +from cibuildwheel._compat import tomllib + +log = logging.getLogger("cibw") + +# Looking up the dir instead of using utils.resources_dir +# since we want to write to it. +DIR: Final[Path] = Path(__file__).parent.parent.resolve() +RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources" + +NODEJS_DIST: Final[str] = "https://nodejs.org/dist/" +NODEJS_INDEX: Final[str] = f"{NODEJS_DIST}index.json" + + +@dataclass(frozen=True, order=True) +class VersionTuple: + version: Version + version_string: str + + +def parse_nodejs_index() -> list[VersionTuple]: + versions: list[VersionTuple] = [] + response = requests.get(NODEJS_INDEX) + response.raise_for_status() + versions_info = response.json() + for version_info in versions_info: + version_string = version_info.get("version", "???") + if not version_info.get("lts", False): + log.debug("Ignoring non LTS release %r", version_string) + continue + if "linux-x64" not in version_info.get("files", []): + log.warning( + "Ignoring release %r which does not include a linux-x64 binary", version_string + ) + continue + try: + version = Version(version_string) + if version.is_devrelease: + log.info("Ignoring development release %r", str(version)) + continue + if version.is_prerelease: + log.info("Ignoring pre-release %r", str(version)) + continue + versions.append(VersionTuple(version, version_string)) + except InvalidVersion: + log.warning("Ignoring release %r", version_string) + versions.sort(reverse=True) + return versions + + +@click.command() +@click.option("--force", is_flag=True) +@click.option( + "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False) +) +def update_nodejs(force: bool, level: str) -> None: + logging.basicConfig( + level="INFO", + format="%(message)s", + datefmt="[%X]", + handlers=[RichHandler(rich_tracebacks=True, markup=True)], + ) + log.setLevel(level) + + toml_file_path = RESOURCES_DIR / "nodejs.toml" + + original_toml = toml_file_path.read_text() + with toml_file_path.open("rb") as f: + nodejs_data = tomllib.load(f) + + nodejs_data.pop("url") + + major_versions = [VersionTuple(Version(key), key) for key in nodejs_data] + major_versions.sort(reverse=True) + + versions = parse_nodejs_index() + + # update existing versions, 1 per LTS + for major_version in major_versions: + current = Version(nodejs_data[major_version.version_string]) + specifier = packaging.specifiers.SpecifierSet( + specifiers=f"=={major_version.version.major}.*" + ) + for version in versions: + if specifier.contains(version.version) and version.version > current: + nodejs_data[major_version.version_string] = version.version_string + break + + # check for a new major LTS to insert + if versions and versions[0].version.major > major_versions[0].version.major: + major_versions.insert( + 0, + VersionTuple(Version(str(versions[0].version.major)), f"v{versions[0].version.major}"), + ) + nodejs_data[major_versions[0].version_string] = versions[0].version_string + + versions_toml = "\n".join( + f'{major_version.version_string} = "{nodejs_data[major_version.version_string]}"' + for major_version in major_versions + ) + result_toml = f'url = "{NODEJS_DIST}"\n{versions_toml}\n' + + rich.print() # spacer + + if original_toml == result_toml: + rich.print("[green]Check complete, nodejs version unchanged.") + return + + rich.print("nodejs version updated.") + rich.print("Changes:") + rich.print() + + toml_relpath = toml_file_path.relative_to(DIR).as_posix() + diff_lines = difflib.unified_diff( + original_toml.splitlines(keepends=True), + result_toml.splitlines(keepends=True), + fromfile=toml_relpath, + tofile=toml_relpath, + ) + rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light")) + rich.print() + + if force: + toml_file_path.write_text(result_toml) + rich.print("[green]TOML file updated.") + else: + rich.print("[yellow]File left unchanged. Use --force flag to update.") + + +if __name__ == "__main__": + update_nodejs() diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index a3820508a..75def59c5 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -76,11 +76,9 @@ def parse_config(config: str, platform: PlatformName) -> set[Architecture]: return result @staticmethod - def auto_archs(platform: PlatformName) -> set[Architecture]: - native_machine = platform_module.machine() - + def native_arch(platform: PlatformName) -> Architecture | None: if platform == "pyodide": - return {Architecture.wasm32} + return Architecture.wasm32 # Cross-platform support. Used for --print-build-identifiers or docker builds. host_platform: PlatformName = ( @@ -89,6 +87,7 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: else ("macos" if sys.platform.startswith("darwin") else "linux") ) + native_machine = platform_module.machine() native_architecture = Architecture(native_machine) # we might need to rename the native arch to the machine we're running @@ -100,11 +99,18 @@ def auto_archs(platform: PlatformName) -> set[Architecture]: if synonym is None: # can't build anything on this platform - return set() + return None native_architecture = Architecture(synonym) - result = {native_architecture} + return native_architecture + + @staticmethod + def auto_archs(platform: PlatformName) -> set[Architecture]: + native_arch = Architecture.native_arch(platform) + if native_arch is None: + return set() # can't build anything on this platform + result = {native_arch} if platform == "linux" and Architecture.x86_64 in result: # x86_64 machines can run i686 containers diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 58f0dae7b..edec10150 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -23,6 +23,7 @@ NonPlatformWheelError, call, download, + ensure_node, extract_zip, find_compatible_wheel, get_pip_version, @@ -41,6 +42,7 @@ class PythonConfiguration: identifier: str pyodide_version: str emscripten_version: str + node_version: str def install_emscripten(tmp: Path, version: str) -> Path: @@ -325,10 +327,17 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. + virtualenv_env = env.copy() + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(ensure_node(config.node_version)), + virtualenv_env["PATH"], + ] + ) + # --no-download?? - call("pyodide", "venv", venv_dir, env=env) + call("pyodide", "venv", venv_dir, env=virtualenv_env) - virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( [ str(venv_dir / "bin"), diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index afca3d2fb..103144896 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -169,5 +169,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a6", emscripten_version = "3.1.58" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a6", emscripten_version = "3.1.58", node_version = "v20" }, ] diff --git a/cibuildwheel/resources/nodejs.toml b/cibuildwheel/resources/nodejs.toml new file mode 100644 index 000000000..83651ed0d --- /dev/null +++ b/cibuildwheel/resources/nodejs.toml @@ -0,0 +1,2 @@ +url = "https://nodejs.org/dist/" +v20 = "v20.13.1" diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index b8475f752..7ac842cf7 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -9,6 +9,7 @@ import ssl import subprocess import sys +import tarfile import textwrap import time import typing @@ -19,6 +20,7 @@ from enum import Enum from functools import cached_property, lru_cache from pathlib import Path, PurePath +from tempfile import TemporaryDirectory from time import sleep from typing import Any, ClassVar, Final, Literal, TextIO, TypeVar from zipfile import ZipFile @@ -33,6 +35,7 @@ from platformdirs import user_cache_path from ._compat import tomllib +from .architecture import Architecture from .typing import PathOrStr, PlatformName __all__ = [ @@ -350,6 +353,12 @@ def extract_zip(zip_src: Path, dest: Path) -> None: dest.joinpath(zinfo.filename).chmod(permissions) +def extract_tar(tar_src: Path, dest: Path) -> None: + with tarfile.open(tar_src) as tar_: + tar_.extraction_filter = getattr(tarfile, "tar_filter", (lambda member, _: member)) + tar_.extractall(dest) + + class DependencyConstraints: def __init__(self, base_file_path: Path): assert base_file_path.exists() @@ -552,6 +561,38 @@ def get_pip_version(env: Mapping[str, str]) -> str: return pip_version +@lru_cache(maxsize=None) +def ensure_node(major_version: str) -> Path: + input_file = resources_dir / "nodejs.toml" + with input_file.open("rb") as f: + loaded_file = tomllib.load(f) + version = str(loaded_file[major_version]) + base_url = str(loaded_file["url"]) + ext = "zip" if IS_WIN else "tar.xz" + platform = "win" if IS_WIN else ("darwin" if sys.platform.startswith("darwin") else "linux") + linux_arch = Architecture.native_arch("linux") + assert linux_arch is not None + arch = {"x86_64": "x64", "i686": "x86", "aarch64": "arm64"}.get( + linux_arch.value, linux_arch.value + ) + name = f"node-{version}-{platform}-{arch}" + path = CIBW_CACHE_PATH / name + with FileLock(str(path) + ".lock"): + if not path.exists(): + url = f"{base_url}{version}/{name}.{ext}" + with TemporaryDirectory() as tmp_path: + archive = Path(tmp_path) / f"{name}.{ext}" + download(url, archive) + if ext == "zip": + extract_zip(archive, path.parent) + else: + extract_tar(archive, path.parent) + assert path.exists() + if not IS_WIN: + return path / "bin" + return path + + @lru_cache(maxsize=None) def _ensure_virtualenv(version: str) -> Path: version_parts = version.split(".") diff --git a/noxfile.py b/noxfile.py index 9c1d80c9d..0c2057b6d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -127,6 +127,7 @@ def update_pins(session: nox.Session) -> None: session.run("python", "bin/update_pythons.py", "--force") session.run("python", "bin/update_docker.py") session.run("python", "bin/update_virtualenv.py", "--force") + session.run("python", "bin/update_nodejs.py", "--force") @nox.session(reuse_venv=True) From 4ac060d60361a36b2f33cbff3d16b87ee8357558 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 26 May 2024 18:10:02 +0200 Subject: [PATCH 15/29] fix tests --- cibuildwheel/pyodide.py | 6 +++++- test/test_abi_variants.py | 34 +++++++++++++++++++++----------- test/test_before_build.py | 12 +++++++---- test/test_before_test.py | 4 ---- test/test_build_frontend_args.py | 19 +++++++++++++++--- test/test_environment.py | 3 +-- test/test_pure_wheel.py | 6 +++++- test/test_testing.py | 4 ++-- test/utils.py | 5 ++++- 9 files changed, 64 insertions(+), 29 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index edec10150..1ce9111e0 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -288,7 +288,11 @@ def build(options: Options, tmp_path: Path) -> None: build_env = env.copy() if build_options.dependency_constraints: - build_env["PIP_CONSTRAINT"] = str(constraints_path) + our_constraints = str(constraints_path) + user_constraints = build_env.get("PIP_CONSTRAINT") + build_env["PIP_CONSTRAINT"] = " ".join( + c for c in [our_constraints, user_constraints] if c + ) build_env["VIRTUALENV_PIP"] = get_pip_version(env) call( "pyodide", diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index cda49054b..71fbc5c97 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -44,27 +44,36 @@ def get_tag(self): limited_api_project.files["pyproject.toml"] = pyproject_toml -@utils.skip_if_pyodide(reason="No abi3, no py38") def test_abi3(tmp_path): project_dir = tmp_path / "project" limited_api_project.generate(project_dir) + single_python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) + # build the wheels actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ # free_threaded and PyPy do not have a Py_LIMITED_API equivalent, just build one of those # also limit the number of builds for test performance reasons - "CIBW_BUILD": "cp39-* cp310-* pp310-* cp311-* cp313t-*" + "CIBW_BUILD": f"cp39-* cp310-* pp310-* {single_python_tag}-* cp313t-*" }, ) # check that the expected wheels are produced - expected_wheels = [ - w.replace("cp310-cp310", "cp310-abi3") - for w in utils.expected_wheels("spam", "0.1.0") - if "-cp39" in w or "-cp310" in w or "-pp310" in w or "-cp313t" in w - ] + expected_wheels = utils.expected_wheels("spam", "0.1.0") + if utils.platform == "pyodide": + # there's only 1 possible configuration for pyodide, the single_python_tag one + expected_wheels = [ + w.replace(f"{single_python_tag}-{single_python_tag}", f"{single_python_tag}-abi3") + for w in expected_wheels + ] + else: + expected_wheels = [ + w.replace("cp310-cp310", "cp310-abi3") + for w in expected_wheels + if "-cp39" in w or "-cp310" in w or "-pp310" in w or "-cp313t" in w + ] assert set(actual_wheels) == set(expected_wheels) @@ -174,7 +183,6 @@ def test(): ctypes_project.files["pyproject.toml"] = pyproject_toml -@utils.skip_if_pyodide(reason="Doesn't work for some reason") def test_abi_none(tmp_path, capfd): project_dir = tmp_path / "project" ctypes_project.generate(project_dir) @@ -184,9 +192,9 @@ def test_abi_none(tmp_path, capfd): project_dir, add_env={ "CIBW_TEST_REQUIRES": "pytest", - "CIBW_TEST_COMMAND": "pytest {project}/test", + "CIBW_TEST_COMMAND": "python -m pytest {project}/test", # limit the number of builds for test performance reasons - "CIBW_BUILD": "cp38-* cp310-* cp313t-* pp310-*", + "CIBW_BUILD": "cp38-* cp{}{}-* cp313t-* pp310-*".format(*utils.SINGLE_PYTHON_VERSION), }, ) @@ -197,4 +205,8 @@ def test_abi_none(tmp_path, capfd): # check that each wheel was built once, and reused captured = capfd.readouterr() assert "Building wheel..." in captured.out - assert "Found previously built wheel" in captured.out + if utils.platform == "pyodide": + # there's only 1 possible configuration for pyodide, we won't see the message expected on following builds + assert "Found previously built wheel" not in captured.out + else: + assert "Found previously built wheel" in captured.out diff --git a/test/test_before_build.py b/test/test_before_build.py index b7a9b67ba..3aea640cb 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -7,9 +7,13 @@ from . import test_projects, utils +# pyodide does not support building without isolation, need to check the base_prefix +SYS_PREFIX = f"sys.{'base_' if utils.platform == 'pyodide' else ''}prefix" + + project_with_before_build_asserts = test_projects.new_c_project( setup_py_add=textwrap.dedent( - r""" + rf""" import os # assert that the Python version as written to pythonversion_bb.txt in the CIBW_BEFORE_BUILD step @@ -24,11 +28,11 @@ with open('pythonprefix_bb.txt') as f: stored_prefix = f.read() print('stored_prefix', stored_prefix) - print('sys.prefix', sys.prefix) + print('{SYS_PREFIX}', {SYS_PREFIX}) # Works around path-comparison bugs caused by short-paths on Windows e.g. # vssadm~1 instead of vssadministrator - assert os.path.samefile(stored_prefix, sys.prefix) + assert os.path.samefile(stored_prefix, {SYS_PREFIX}) """ ) ) @@ -40,7 +44,7 @@ def test(tmp_path): before_build = ( """python -c "import sys; open('{project}/pythonversion_bb.txt', 'w').write(sys.version)" && """ - '''python -c "import sys; open('{project}/pythonprefix_bb.txt', 'w').write(sys.prefix)"''' + f'''python -c "import sys; open('{{project}}/pythonprefix_bb.txt', 'w').write({SYS_PREFIX})"''' ) # build the wheels diff --git a/test/test_before_test.py b/test/test_before_test.py index e151a71d7..8ed33c063 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -1,7 +1,5 @@ from __future__ import annotations -import pytest - from . import test_projects, utils before_test_project = test_projects.new_c_project() @@ -36,7 +34,6 @@ def test_prefix(self): """ -@pytest.mark.xfail(utils.platform == "pyodide", reason="TODO: venv error on pyodide!", strict=True) def test(tmp_path): project_dir = tmp_path / "project" before_test_project.generate(project_dir) @@ -69,7 +66,6 @@ def test(tmp_path): # mac/linux. "CIBW_TEST_COMMAND": "false || python -m pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test", - "_PYODIDE_EXTRA_MOUNTS": "/tmp/my-tmp-dir/", }, ) diff --git a/test/test_build_frontend_args.py b/test/test_build_frontend_args.py index 4429ac6c4..6de83cbe2 100644 --- a/test/test_build_frontend_args.py +++ b/test/test_build_frontend_args.py @@ -6,10 +6,23 @@ from .test_projects.c import new_c_project -@utils.skip_if_pyodide( - reason="pyodide build -h doesn't print help text https://github.com/pyodide/pyodide/issues/4783" +@pytest.mark.parametrize( + "frontend_name", + [ + pytest.param( + "pip", + marks=pytest.mark.skipif(utils.platform == "pyodide", reason="No pip for pyodide"), + ), + pytest.param( + "build", + marks=pytest.mark.xfail( + condition=utils.platform == "pyodide", + reason="pyodide build -h doesn't print help text https://github.com/pyodide/pyodide/issues/4783", + strict=True, + ), + ), + ], ) -@pytest.mark.parametrize("frontend_name", ["pip", "build"]) def test_build_frontend_args(tmp_path, capfd, frontend_name): project = new_c_project() project_dir = tmp_path / "project" diff --git a/test/test_environment.py b/test/test_environment.py index 30e67f2e1..aa5c5df5c 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -98,7 +98,6 @@ def test_overridden_path(tmp_path, capfd): ) -@utils.skip_if_pyodide(reason="TODO: fix!") @pytest.mark.parametrize( "build_frontend", [ @@ -121,7 +120,7 @@ def test_overridden_pip_constraint(tmp_path, build_frontend): setup_py_add=textwrap.dedent( """ import pytz - assert pytz.__version__ == "2022.4" + assert pytz.__version__ == "2022.4", f"{pytz.__version__!r} != '2022.4'" """ ) ) diff --git a/test/test_pure_wheel.py b/test/test_pure_wheel.py index b3cfce3e7..27a41c835 100644 --- a/test/test_pure_wheel.py +++ b/test/test_pure_wheel.py @@ -24,7 +24,11 @@ def a_function(): """ -@utils.skip_if_pyodide(reason="Doesn't work") +@pytest.mark.xfail( + condition=utils.platform == "pyodide", + reason="pyodide build re-tag platform 'none' as 'pyodide_2024_0_wasm32'", + strict=True, +) def test(tmp_path, capfd): # this test checks that if a pure wheel is generated, the build should # fail. diff --git a/test/test_testing.py b/test/test_testing.py index 0a89ec31e..a8777f1fa 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -81,7 +81,7 @@ def test(tmp_path): "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || pytest {project}/test", + "CIBW_TEST_COMMAND": "false || python -m pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", }, ) @@ -102,7 +102,7 @@ def test_extras_require(tmp_path): "CIBW_TEST_EXTRAS": "test", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || pytest {project}/test", + "CIBW_TEST_COMMAND": "false || python -m pytest {project}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", }, single_python=True, diff --git a/test/utils.py b/test/utils.py index 6d4545ebe..e662d3e47 100644 --- a/test/utils.py +++ b/test/utils.py @@ -185,6 +185,8 @@ def expected_wheels( if musllinux_versions is None: musllinux_versions = ["musllinux_1_2"] + if platform == "pyodide" and python_abi_tags is None: + python_abi_tags = ["cp312-cp312"] if python_abi_tags is None: python_abi_tags = [ "cp36-cp36m", @@ -234,7 +236,8 @@ def expected_wheels( wheels = [] if platform == "pyodide": - python_abi_tag = "cp312-cp312" + assert len(python_abi_tags) == 1 + python_abi_tag = python_abi_tags[0] platform_tag = "pyodide_2024_0_wasm32" return [f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl"] From 9239f251c3ef7730485427dd826ae7a48f3d6d76 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 26 May 2024 19:22:18 +0200 Subject: [PATCH 16/29] review: error out on Windows --- cibuildwheel/__main__.py | 5 +++++ unit_test/main_tests/main_platform_test.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 64be4551c..ad354d1e6 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -256,6 +256,11 @@ def get_platform_module(platform: PlatformName) -> PlatformModule: def build_in_directory(args: CommandLineArguments) -> None: platform: PlatformName = _compute_platform(args) + if platform == "pyodide" and sys.platform == "win32": + msg = "cibuildwheel: Building for pyodide is not supported on Windows" + print(msg, file=sys.stderr) + sys.exit(2) + options = compute_options(platform=platform, command_line_arguments=args, env=os.environ) package_dir = options.globals.package_dir diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 0fc0f3d71..ca943112e 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -264,3 +264,16 @@ def test_only_overrides_env_vars(monkeypatch, intercepted_build_args, envvar_nam assert options.globals.build_selector.skip_config == "" assert options.platform == "linux" assert options.globals.architectures == Architecture.all_archs("linux") + + +def test_pyodide_on_windows(monkeypatch, capsys): + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(sys, "argv", [*sys.argv, "--only", "cp312-pyodide_wasm32"]) + + with pytest.raises(SystemExit) as exit: + main() + + _, err = capsys.readouterr() + + assert exit.value.code == 2 + assert "cibuildwheel: Building for pyodide is not supported on Windows" in err From 437352755fd615f315d5355cebc547c4b2d3f5c3 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 26 May 2024 20:33:37 +0200 Subject: [PATCH 17/29] test: check node & test on macos arm64 --- test/test_emscripten.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 509fb8f35..0c62f5059 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -1,20 +1,44 @@ from __future__ import annotations -import platform import shutil +import sys import textwrap import pytest +from cibuildwheel.util import CIBW_CACHE_PATH + from . import test_projects, utils basic_project = test_projects.new_c_project() +basic_project.files["check_node.py"] = r""" +import sys +import shutil +from pathlib import Path +from pyodide.code import run_js + + +def check_node(): + node = shutil.which("node") + assert node is not None, "node is None" + node_path = Path(node).resolve(strict=True) + cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) + assert cibw_cache_path in node_path.parents, f"{cibw_cache_path} not a parent of {node_path}" + node_js = run_js("globalThis.process.execPath") + assert node_js is not None, "node_js is None" + node_js_path = Path(node_js).resolve(strict=True) + assert node_js_path == node_path, f"{node_js_path} != {node_path}" + + +if __name__ == "__main__": + check_node() +""" @pytest.mark.parametrize("use_pyproject_toml", [True, False]) def test_pyodide_build(tmp_path, use_pyproject_toml): - if platform.machine() == "arm64": - pytest.skip("emsdk doesn't work correctly on arm64") + if sys.platform == "win32": + pytest.skip("emsdk doesn't work correctly on Windows") if not shutil.which("python3.12"): pytest.skip("Python 3.12 not installed") @@ -31,10 +55,16 @@ def test_pyodide_build(tmp_path, use_pyproject_toml): project_dir = tmp_path / "project" basic_project.generate(project_dir) + # check for node in 1 case only to reduce CI load + add_env = {} + if use_pyproject_toml: + add_env["CIBW_TEST_COMMAND"] = f"python {{project}}/check_node.py {CIBW_CACHE_PATH}" + # build the wheels actual_wheels = utils.cibuildwheel_run( project_dir, add_args=["--platform", "pyodide"], + add_env=add_env, ) # check that the expected wheels are produced From c90b018d95e7ed5e46b6dd0fa9f1a41cc76e430f Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 26 May 2024 20:59:11 +0200 Subject: [PATCH 18/29] chore: minor cleanup --- test/test_build_frontend_args.py | 5 +---- test/test_emscripten.py | 8 +++++++- test/test_environment.py | 5 +---- test/utils.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/test_build_frontend_args.py b/test/test_build_frontend_args.py index 6de83cbe2..c4387fdb5 100644 --- a/test/test_build_frontend_args.py +++ b/test/test_build_frontend_args.py @@ -9,10 +9,7 @@ @pytest.mark.parametrize( "frontend_name", [ - pytest.param( - "pip", - marks=pytest.mark.skipif(utils.platform == "pyodide", reason="No pip for pyodide"), - ), + pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")), pytest.param( "build", marks=pytest.mark.xfail( diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 0c62f5059..9e96791bd 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -19,14 +19,20 @@ def check_node(): + # cibuildwheel adds a pinned node version to the PATH + # check it's in the PATH then, check it's the one that runs pyoodide + cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) + # find the node executable in PATH node = shutil.which("node") assert node is not None, "node is None" node_path = Path(node).resolve(strict=True) - cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) + # it shall be in cibuildwheel cache assert cibw_cache_path in node_path.parents, f"{cibw_cache_path} not a parent of {node_path}" + # find the path to the node executable that runs pyodide node_js = run_js("globalThis.process.execPath") assert node_js is not None, "node_js is None" node_js_path = Path(node_js).resolve(strict=True) + # it shall be the one pinned by cibuildwheel assert node_js_path == node_path, f"{node_js_path} != {node_path}" diff --git a/test/test_environment.py b/test/test_environment.py index aa5c5df5c..9b741cebf 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -101,10 +101,7 @@ def test_overridden_path(tmp_path, capfd): @pytest.mark.parametrize( "build_frontend", [ - pytest.param( - "pip", - marks=pytest.mark.skipif(utils.platform == "pyodide", reason="No pip for pyodide"), - ), + pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")), "build", ], ) diff --git a/test/utils.py b/test/utils.py index e662d3e47..955987aaa 100644 --- a/test/utils.py +++ b/test/utils.py @@ -310,7 +310,7 @@ def get_macos_version(): return tuple(map(int, version_str.split(".")[:2])) -def skip_if_pyodide(reason): +def skip_if_pyodide(reason: str): return pytest.mark.skipif(platform == "pyodide", reason=reason) From 809427edecfb0a34c7938c14db162ba7d86a5726 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 26 May 2024 14:10:47 -0700 Subject: [PATCH 19/29] Add reference to emscripten libc issue --- test/test_projects/c.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_projects/c.py b/test/test_projects/c.py index 027d1ee47..eee128f7f 100644 --- a/test/test_projects/c.py +++ b/test/test_projects/c.py @@ -51,6 +51,7 @@ libraries = [] # Emscripten fails if you pass -lc... +# See: https://github.com/emscripten-core/emscripten/issues/16680 if sys.platform.startswith('linux') and "emscripten" not in os.environ.get("_PYTHON_HOST_PLATFORM", ""): libraries.extend(['m', 'c']) From b1b73179f1722c202922348d5512881297deaf1b Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 27 May 2024 10:38:37 +0200 Subject: [PATCH 20/29] Apply suggestion from code review --- docs/setup.md | 2 +- test/test_pure_wheel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/setup.md b/docs/setup.md index 99f2052e9..1a98d32b8 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -110,7 +110,7 @@ You can override the cache folder using the ``CIBW_CACHE_PATH`` environment vari ### Pyodide (WebAssembly) builds (experimental) Pre-requisite: you need to have a matching host version of Python (unlike all -other cibuildwheel platforms) and Node 20. Linux host highly recommended; macOS hosts may +other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work and Windows hosts will not work. You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). diff --git a/test/test_pure_wheel.py b/test/test_pure_wheel.py index 27a41c835..503697231 100644 --- a/test/test_pure_wheel.py +++ b/test/test_pure_wheel.py @@ -26,7 +26,7 @@ def a_function(): @pytest.mark.xfail( condition=utils.platform == "pyodide", - reason="pyodide build re-tag platform 'none' as 'pyodide_2024_0_wasm32'", + reason="pyodide build re-tag platform as 'pyodide_2024_0_wasm32', https://github.com/pyodide/pyodide/pull/4803", strict=True, ) def test(tmp_path, capfd): From 560126fa25c9998e3bc0a7b2358be017e41ecd57 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 27 May 2024 12:17:41 +0200 Subject: [PATCH 21/29] review: use a pinned pip in test virtual environment --- cibuildwheel/pyodide.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 1ce9111e0..244ad7b14 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -339,8 +339,14 @@ def build(options: Options, tmp_path: Path) -> None: ] ) - # --no-download?? - call("pyodide", "venv", venv_dir, env=virtualenv_env) + # pyodide venv uses virtualenv under the hood + # use the pip embeded with virtualenv & disable network updates + virtualenv_create_env = virtualenv_env.copy() + virtualenv_create_env["VIRTUALENV_PIP"] = "embed" + virtualenv_create_env["VIRTUALENV_DOWNLOAD"] = "0" + virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1" + + call("pyodide", "venv", venv_dir, env=virtualenv_create_env) virtualenv_env["PATH"] = os.pathsep.join( [ From 2fac554855ea5d4e06b947b204fb04710364ff4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 10:18:09 +0000 Subject: [PATCH 22/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 244ad7b14..ad9a041ed 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -340,7 +340,7 @@ def build(options: Options, tmp_path: Path) -> None: ) # pyodide venv uses virtualenv under the hood - # use the pip embeded with virtualenv & disable network updates + # use the pip embedded with virtualenv & disable network updates virtualenv_create_env = virtualenv_env.copy() virtualenv_create_env["VIRTUALENV_PIP"] = "embed" virtualenv_create_env["VIRTUALENV_DOWNLOAD"] = "0" From 43d6042ef88dbe4381cf904b3db7b4b88f48e2a4 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 27 May 2024 13:24:55 +0200 Subject: [PATCH 23/29] chore: rework test virtual environment seed packages --- cibuildwheel/pyodide.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index ad9a041ed..6de0bd6e7 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -244,6 +244,7 @@ def build(options: Options, tmp_path: Path) -> None: dependency_constraint_flags, build_options.environment, ) + pip_version = get_pip_version(env) # The Pyodide command line runner mounts all directories in the host # filesystem into the Pyodide file system, except for the custom # file systems /dev, /lib, /proc, and /tmp. Mounting the mount @@ -293,7 +294,7 @@ def build(options: Options, tmp_path: Path) -> None: build_env["PIP_CONSTRAINT"] = " ".join( c for c in [our_constraints, user_constraints] if c ) - build_env["VIRTUALENV_PIP"] = get_pip_version(env) + build_env["VIRTUALENV_PIP"] = pip_version call( "pyodide", "build", @@ -342,8 +343,7 @@ def build(options: Options, tmp_path: Path) -> None: # pyodide venv uses virtualenv under the hood # use the pip embedded with virtualenv & disable network updates virtualenv_create_env = virtualenv_env.copy() - virtualenv_create_env["VIRTUALENV_PIP"] = "embed" - virtualenv_create_env["VIRTUALENV_DOWNLOAD"] = "0" + virtualenv_create_env["VIRTUALENV_PIP"] = pip_version virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1" call("pyodide", "venv", venv_dir, env=virtualenv_create_env) From 4a8bf61e0efe8e4b3dab65c236504290efe5c8b2 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 27 May 2024 15:22:20 +0200 Subject: [PATCH 24/29] chore: workaround direct invocation of pytest This allows to still test direct invocation of `pytest` on most platforms (including pyodide on Linux) but falls back to `python -m pytest` when running pyodide on macOS. --- docs/setup.md | 2 +- test/test_abi_variants.py | 2 +- test/test_before_test.py | 2 +- test/test_testing.py | 4 ++-- test/utils.py | 7 +++++++ 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/setup.md b/docs/setup.md index 1a98d32b8..f67669eed 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -111,7 +111,7 @@ You can override the cache folder using the ``CIBW_CACHE_PATH`` environment vari Pre-requisite: you need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may -work and Windows hosts will not work. +work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index 71fbc5c97..a7879943d 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -192,7 +192,7 @@ def test_abi_none(tmp_path, capfd): project_dir, add_env={ "CIBW_TEST_REQUIRES": "pytest", - "CIBW_TEST_COMMAND": "python -m pytest {project}/test", + "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} {{project}}/test", # limit the number of builds for test performance reasons "CIBW_BUILD": "cp38-* cp{}{}-* cp313t-* pp310-*".format(*utils.SINGLE_PYTHON_VERSION), }, diff --git a/test/test_before_test.py b/test/test_before_test.py index 8ed33c063..08ef12def 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -64,7 +64,7 @@ def test(tmp_path): "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || python -m pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test", }, ) diff --git a/test/test_testing.py b/test/test_testing.py index a8777f1fa..2e6fe3c37 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -81,7 +81,7 @@ def test(tmp_path): "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || python -m pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", }, ) @@ -102,7 +102,7 @@ def test_extras_require(tmp_path): "CIBW_TEST_EXTRAS": "test", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || python -m pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", }, single_python=True, diff --git a/test/utils.py b/test/utils.py index 955987aaa..f90557c32 100644 --- a/test/utils.py +++ b/test/utils.py @@ -314,6 +314,13 @@ def skip_if_pyodide(reason: str): return pytest.mark.skipif(platform == "pyodide", reason=reason) +def invoke_pytest() -> str: + # see https://github.com/pyodide/pyodide/issues/4802 + if platform == "pyodide" and sys.platform.startswith("darwin"): + return "python -m pytest" + return "pytest" + + def arch_name_for_linux(arch: str): """ Archs have different names on different platforms, but it's useful to be From 7aade4171e91aa3beae7c11c4f548e31aa9792d2 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 27 May 2024 12:44:08 -0700 Subject: [PATCH 25/29] Use release version of pyodide --- cibuildwheel/resources/build-platforms.toml | 2 +- cibuildwheel/resources/constraints-pyodide312.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 103144896..da3f77663 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -169,5 +169,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0a6", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0", emscripten_version = "3.1.58", node_version = "v20" }, ] diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index cc5de2c73..08a9b9408 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -64,7 +64,7 @@ pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via rich -pyodide-build==0.26.0a6 +pyodide-build==0.26.0 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.2.3 # via From 9b0118962d1bd7922bc2fc53c2966375c496cc77 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 27 May 2024 21:47:53 +0200 Subject: [PATCH 26/29] fix: tests for 0.26.0 & parallel initialization of xbuildenv --- cibuildwheel/pyodide.py | 35 +++++++++---------- .../resources/constraints-pyodide312.txt | 4 +-- test/test_build_frontend_args.py | 11 ++---- test/test_pure_wheel.py | 5 --- 4 files changed, 22 insertions(+), 33 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 6de0bd6e7..02a036bc0 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -66,27 +66,26 @@ def install_emscripten(tmp: Path, version: str) -> Path: def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str: - xbuildenv_cache_dir = CIBW_CACHE_PATH pyodide_root = ( - xbuildenv_cache_dir + CIBW_CACHE_PATH / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root" ) - if pyodide_root.exists(): - return str(pyodide_root) - - xbuildenv_cache_dir.mkdir(exist_ok=True) - # We don't want to mutate env but we need to delete any existing - # PYODIDE_ROOT so copy it first. - env = dict(env) - env.pop("PYODIDE_ROOT", None) - call( - "pyodide", - "xbuildenv", - "install", - pyodide_version, - env=env, - cwd=xbuildenv_cache_dir, - ) + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): + if pyodide_root.exists(): + return str(pyodide_root) + + # We don't want to mutate env but we need to delete any existing + # PYODIDE_ROOT so copy it first. + env = dict(env) + env.pop("PYODIDE_ROOT", None) + call( + "pyodide", + "xbuildenv", + "install", + pyodide_version, + env=env, + cwd=CIBW_CACHE_PATH, + ) return str(pyodide_root) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 08a9b9408..fcb0693e0 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -2,7 +2,7 @@ # nox -s update_constraints annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.4.0 # via httpx auditwheel-emscripten==0.0.14 # via pyodide-build @@ -107,7 +107,7 @@ typing-extensions==4.12.0 # pydantic # pydantic-core # typer -unearth==0.15.2 +unearth==0.15.3 # via pyodide-build urllib3==2.2.1 # via diff --git a/test/test_build_frontend_args.py b/test/test_build_frontend_args.py index c4387fdb5..d8ca6bdd0 100644 --- a/test/test_build_frontend_args.py +++ b/test/test_build_frontend_args.py @@ -10,14 +10,7 @@ "frontend_name", [ pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")), - pytest.param( - "build", - marks=pytest.mark.xfail( - condition=utils.platform == "pyodide", - reason="pyodide build -h doesn't print help text https://github.com/pyodide/pyodide/issues/4783", - strict=True, - ), - ), + "build", ], ) def test_build_frontend_args(tmp_path, capfd, frontend_name): @@ -40,6 +33,8 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name): if frontend_name == "pip": assert "Usage:" in captured.out assert "Wheel Options:" in captured.out + elif utils.platform == "pyodide": + assert "Usage: pyodide build" in captured.out else: assert "usage:" in captured.out assert "A simple, correct Python build frontend." in captured.out diff --git a/test/test_pure_wheel.py b/test/test_pure_wheel.py index 503697231..998f967f3 100644 --- a/test/test_pure_wheel.py +++ b/test/test_pure_wheel.py @@ -24,11 +24,6 @@ def a_function(): """ -@pytest.mark.xfail( - condition=utils.platform == "pyodide", - reason="pyodide build re-tag platform as 'pyodide_2024_0_wasm32', https://github.com/pyodide/pyodide/pull/4803", - strict=True, -) def test(tmp_path, capfd): # this test checks that if a pure wheel is generated, the build should # fail. From 917646cffc96dbfc619c47d1c03a828f447bcdb7 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Mon, 27 May 2024 13:04:22 -0700 Subject: [PATCH 27/29] Debug CI --- bin/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 4f4ae7ed3..c088a8cdf 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -43,7 +43,7 @@ "-m", "pytest", f"--numprocesses={args.num_processes}", - "-x", + "-vv", "--durations", "0", "--timeout=2400", From f13a0b732bc299c86a4c8ffa1e0ab2af3c8902df Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 27 May 2024 23:02:37 +0200 Subject: [PATCH 28/29] fix: test/test_build_frontend_args.py --- test/test_build_frontend_args.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_build_frontend_args.py b/test/test_build_frontend_args.py index d8ca6bdd0..b508728e4 100644 --- a/test/test_build_frontend_args.py +++ b/test/test_build_frontend_args.py @@ -19,12 +19,11 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name): project.generate(project_dir) # the build will fail because the frontend is called with '-h' - it prints the help message + add_env = {"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"} + if utils.platform == "pyodide": + add_env["TERM"] = "dumb" # disable color / style with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run( - project_dir, - add_env={"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"}, - single_python=True, - ) + utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True) captured = capfd.readouterr() print(captured.out) From 6ea11f9636d66b04bebc8393c7c719457ba86910 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 27 May 2024 23:11:25 +0200 Subject: [PATCH 29/29] Revert "Debug CI" This reverts commit 917646cffc96dbfc619c47d1c03a828f447bcdb7. --- bin/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index c088a8cdf..4f4ae7ed3 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -43,7 +43,7 @@ "-m", "pytest", f"--numprocesses={args.num_processes}", - "-vv", + "-x", "--durations", "0", "--timeout=2400",