Skip to content

Commit

Permalink
fix: enforce minimum version of docker/podman (#1961)
Browse files Browse the repository at this point in the history
* fix: enforce minimum version of docker/podman

This allows to always pass `--platform` to the OCI engine
thus fixing issues with multiarch images.

* Allow older versions with warnings

* Upgrade docker on Travis CI

* fix: use `docker cp` instead of `tar`

* Enforce docker>=24.0

* move log to include container.copy_into

* fix: travis-ci, only update docker on aarch64

* skip test_multiarch_image on s390x / ppc64le

* skip flaky test

* chore: only install test deps on Travis CI

* fix: do not try to pull images tagged `cibw_local`

* use "--pull=never" for local images

* Use docker image inspect to check if an image needs to be pulled
  • Loading branch information
mayeut authored Sep 12, 2024
1 parent fd11286 commit 0787a44
Show file tree
Hide file tree
Showing 15 changed files with 299 additions and 119 deletions.
2 changes: 2 additions & 0 deletions .circleci/prepare.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ set -o xtrace

if [ "$(uname -s)" == "Darwin" ]; then
sudo softwareupdate --install-rosetta --agree-to-license
else
docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
fi

$PYTHON --version
Expand Down
2 changes: 2 additions & 0 deletions .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ linux_x86_task:
memory: 8G

install_pre_requirements_script:
- docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
- apt install -y python3-venv python-is-python3
<<: *RUN_TESTS

Expand All @@ -30,6 +31,7 @@ linux_aarch64_task:
memory: 4G

install_pre_requirements_script:
- docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
- apt install -y python3-venv python-is-python3
<<: *RUN_TESTS

Expand Down
8 changes: 5 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ jobs:
docker system prune -a -f
df -h
# for oci_container unit tests
- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v3

- name: Install dependencies
run: |
uv pip install --system ".[test]"
Expand Down Expand Up @@ -157,10 +162,7 @@ jobs:
run: python -m pip install ".[test,uv]"

- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v3
with:
platforms: all

- name: Run the emulation tests
run: pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py
Expand Down
1 change: 1 addition & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ linux:
PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code
script:
- curl -sSL https://get.docker.com/ | sh
- docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
- python -m pip install -e ".[dev]" pytest-custom-exit-code
- python ./bin/run_tests.py

Expand Down
11 changes: 10 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@ jobs:
group: edge
virt: vm
env: PYTHON=python
# docker is outdated in the arm64-graviton2 vm focal image (19.x)
# we need to upgrade to get >= 24.0
addons:
apt:
sources:
- sourceline: 'deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable'
packages:
- docker-ce docker-ce-cli containerd.io

- name: Linux | ppc64le | Python 3.9
python: 3.9
Expand Down Expand Up @@ -48,8 +56,9 @@ jobs:
env: PYTHON=python

install:
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all; fi
- $PYTHON -m pip install -U pip
- $PYTHON -m pip install -e ".[dev]" pytest-custom-exit-code
- $PYTHON -m pip install -e ".[test]" pytest-custom-exit-code

script: |
# travis_wait disable the output while waiting
Expand Down
3 changes: 3 additions & 0 deletions appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ init:
if (-not ($BRANCH -eq 'main' -or $BRANCH.ToLower().StartsWith('appveyor-'))) {
$env:PYTEST_ADDOPTS = '-k "unit_test or test_0_basic" --suppress-no-test-exit-code'
}
if ($IsLinux) {
docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
}
install:
- python -m pip install -U pip
Expand Down
1 change: 1 addition & 0 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ jobs:
inputs:
versionSpec: '3.8'
- bash: |
docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
python -m pip install -e ".[dev]"
python ./bin/run_tests.py
Expand Down
2 changes: 1 addition & 1 deletion bin/run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
# unit tests
unit_test_args = [sys.executable, "-m", "pytest", "unit_test"]

if sys.platform.startswith("linux"):
if sys.platform.startswith("linux") and os.environ.get("CIBW_PLATFORM", "linux") == "linux":
# run the docker unit tests only on Linux
unit_test_args += ["--run-docker"]

Expand Down
4 changes: 3 additions & 1 deletion cibuildwheel/architecture.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,9 @@ def parse_config(config: str, platform: PlatformName) -> set[Architecture]:
if arch_str == "auto":
result |= Architecture.auto_archs(platform=platform)
elif arch_str == "native":
result.add(Architecture(platform_module.machine()))
native_arch = Architecture.native_arch(platform=platform)
if native_arch:
result.add(native_arch)
elif arch_str == "all":
result |= Architecture.all_archs(platform=platform)
elif arch_str == "auto64":
Expand Down
4 changes: 4 additions & 0 deletions cibuildwheel/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,7 @@ def __init__(self, wheel_name: str) -> None:
)
super().__init__(message)
self.return_code = 6


class OCIEngineTooOldError(FatalError):
return_code = 7
17 changes: 13 additions & 4 deletions cibuildwheel/linux.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from ._compat.typing import assert_never
from .architecture import Architecture
from .logger import log
from .oci_container import OCIContainer, OCIContainerEngineConfig
from .oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform
from .options import BuildOptions, Options
from .typing import PathOrStr
from .util import (
Expand All @@ -29,6 +29,14 @@
unwrap,
)

ARCHITECTURE_OCI_PLATFORM_MAP = {
Architecture.x86_64: OCIPlatform.AMD64,
Architecture.i686: OCIPlatform.i386,
Architecture.aarch64: OCIPlatform.ARM64,
Architecture.ppc64le: OCIPlatform.PPC64LE,
Architecture.s390x: OCIPlatform.S390X,
}


@dataclass(frozen=True)
class PythonConfiguration:
Expand Down Expand Up @@ -196,6 +204,8 @@ def build_in_container(

dependency_constraint_flags: list[PathOrStr] = []

log.step("Setting up build environment...")

if build_options.dependency_constraints:
constraints_file = build_options.dependency_constraints.get_for_python_version(
config.version
Expand All @@ -205,8 +215,6 @@ def build_in_container(
container.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ["-c", container_constraints_file]

log.step("Setting up build environment...")

env = container.get_environment()
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
env["PIP_ROOT_USER_ACTION"] = "ignore"
Expand Down Expand Up @@ -446,10 +454,11 @@ def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
log.step(f"Starting container image {build_step.container_image}...")

print(f"info: This container will host the build for {', '.join(ids_to_build)}...")
architecture = Architecture(build_step.platform_tag.split("_", 1)[1])

with OCIContainer(
image=build_step.container_image,
enforce_32_bit=build_step.platform_tag.endswith("i686"),
oci_platform=ARCHITECTURE_OCI_PLATFORM_MAP[architecture],
cwd=container_project_path,
engine=build_step.container_engine,
) as container:
Expand Down
Loading

0 comments on commit 0787a44

Please sign in to comment.