build binary wheels

But don't publish them yet

.circleci/config.yml

@@ -0,0 +1,104 @@
+version: 2.1
+
+parameters:
+  REF:
+    type: string
+    default: ''
+    description: Optional tag to build
+
+jobs:
+  arm-wheels:
+    parameters:
+      build:
+        type: string
+      image:
+        type: string
+
+    machine:
+      image: ubuntu-2204:current
+    resource_class: arm.medium  # two vCPUs
+
+    environment:
+      CIBW_ARCHS: "aarch64"
+      CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>"
+      CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>"
+      CIBW_BUILD: "<< parameters.build >>"
+
+    steps:
+      - checkout
+      - when:
+          condition: << pipeline.parameters.REF >>
+          steps:
+          - run:
+              name: Checkout branch/tag << pipeline.parameters.REF >>
+              command: |
+                echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists"
+                git checkout << pipeline.parameters.REF >> || true
+                git pull origin << pipeline.parameters.REF >> || true
+      - run:
+          name: install cibuildwheel and other build reqs
+          command: |
+            python3 -m pip install --upgrade pip setuptools setuptools_scm[toml]
+            python3 -m pip install -rcibw-requirements.txt
+
+      - run:
+          name: pip freeze
+          command: |
+            python3 -m pip freeze
+
+      - run:
+          name: list wheels
+          command: |
+            python3 -m cibuildwheel . --print-build-identifiers
+
+      - run:
+          name: cibuildwheel
+          command: |
+            python3 -m cibuildwheel .
+
+      - store_test_results:
+          path: test-results/
+
+      - store_artifacts:
+          path: wheelhouse/
+
+      # - when:
+      #     condition:
+      #       or:
+      #         - matches:
+      #             pattern: ".+"
+      #             value: "<< pipeline.git.tag >>"
+      #         - << pipeline.parameters.REF >>
+      #     steps:
+      #       - run:
+      #           environment:
+      #             TWINE_NONINTERACTIVE: "1"
+      #           command: |
+      #             python3 -m pip install twine
+      #             python3 -m twine upload --verbose --skip-existing wheelhouse/*
+
+workflows:
+  wheels: # This is the name of the workflow, feel free to change it to better match your workflow.
+    # Inside the workflow, you define the jobs you want to run.
+    jobs:
+      - arm-wheels:
+          name: arm-wheels-manylinux_2_28
+          filters:
+            tags:
+              only: /.*/
+          build: "*manylinux*"
+          image: quay.io/pypa/manylinux_2_28_aarch64
+      - arm-wheels:
+          name: arm-wheels-musllinux_1_1
+          filters:
+            tags:
+              only: /.*/
+          build: "*musllinux*"
+          image: quay.io/pypa/musllinux_1_1_aarch64
+      - arm-wheels:
+          name: arm-wheels-musllinux_1_2
+          filters:
+            tags:
+              only: /.*/
+          build: "*musllinux*"
+          image: quay.io/pypa/musllinux_1_2_aarch64

.github/workflows/ci-tests.yml

@@ -44,11 +44,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Singularity
+      - name: Set up Singularity and environment-modules
         if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
         run: |
           wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
-          sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules
 
       - name: Give the test runner user a name to make provenance happy.
         if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
@@ -132,10 +132,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Singularity
+      - name: Set up Singularity and environment-modules
         run: |
           wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb
-          sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules
 
       - name: Give the test runner user a name to make provenance happy.
         run: sudo usermod -c 'CI Runner' "$(whoami)"
@@ -180,11 +180,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Singularity
+      - name: Set up Singularity and environment-modules
         if: ${{ matrix.container == 'singularity' }}
         run: |
           wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
-          sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules
 
       - name: Singularity cache
         if: ${{ matrix.container == 'singularity' }}
@@ -229,10 +229,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Singularity
+      - name: Set up Singularity and environment-modules
         run: |
           wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
-          sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
+          sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules
 
       - name: Set up Python
         uses: actions/setup-python@v5

.github/workflows/wheels.yml

@@ -0,0 +1,130 @@
+name: Python package build and publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch: {}
+  repository_dispatch: {}
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: wheels-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_wheels:
+    name: ${{ matrix.image }} wheels
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        include:
+          - image: manylinux_2_28_x86_64
+            build: "*manylinux*"
+          - image: musllinux_1_1_x86_64
+            build: "*musllinux*"
+          - image: musllinux_1_2_x86_64
+            build: "*musllinux*"
+
+    steps:
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name != 'repository_dispatch' }}
+        with:
+          fetch-depth: 0  # slow, but gets all the tags
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name == 'repository_dispatch' }}
+        with:
+          fetch-depth: 0  # slow, but gets all the tags
+          ref: ${{ github.event.client_payload.ref }}
+
+      # - name: Set up QEMU
+      #   if: runner.os == 'Linux'
+      #   uses: docker/setup-qemu-action@v2
+      #   with:
+      #     platforms: all
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: ${{ matrix.build }}
+          CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }}
+          CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }}
+          # configure cibuildwheel to build native 64-bit archs ('auto64'), and some
+          # emulated ones
+          # Linux arm64 wheels are built on circleci
+          CIBW_ARCHS_LINUX: auto64 # ppc64le s390x
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-${{ matrix.image }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name != 'repository_dispatch' }}
+        with:
+          fetch-depth: 0  # slow, but gets all the tags
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name == 'repository_dispatch' }}
+        with:
+          fetch-depth: 0  # slow, but gets all the tags
+          ref: ${{ github.event.client_payload.ref }}
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-source
+          path: dist/*.tar.gz
+
+  build_wheels_macos:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # macos-13 is an intel runner, macos-14 is apple silicon
+        os: [macos-13, macos-14]
+    steps:
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name != 'repository_dispatch' }}
+        with:
+          fetch-depth: 0  # slow, but gets all the tags
+      - uses: actions/checkout@v4
+        if: ${{ github.event_name == 'repository_dispatch' }}
+        with:
+          fetch-depth: 0  # slow, but gets all the tags
+          ref: ${{ github.event.client_payload.ref }}
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+
+  # upload_pypi:
+  #   needs: [build_wheels, build_sdist]
+  #   runs-on: ubuntu-24.04
+  #   environment: deploy
+  #   permissions:
+  #     id-token: write
+  #   if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true)
+  #   steps:
+  #     - uses: actions/download-artifact@v4
+  #       with:
+  #         # unpacks default artifact into dist/
+  #         pattern: artifact-*
+  #         merge-multiple: true
+  #         path: dist
+
+  #     - name: Publish package distributions to PyPI
+  #       uses: pypa/gh-action-pypi-publish@release/v1
+  #       with:
+  #         skip-existing: true

MANIFEST.in

@@ -19,6 +19,7 @@ include tests/reloc/dir2/*
 include tests/checker_wf/*
 include tests/subgraph/*
 include tests/input_deps/*
+recursive-include tests/test_deps_env
 include tests/trs/*
 include tests/wf/generator/*
 include cwltool/py.typed

cibw-requirements.txt

@@ -0,0 +1 @@
+cibuildwheel==2.21.3

cwltool/software_requirements.py

@@ -50,6 +50,8 @@ class DependenciesConfiguration:
 
     def __init__(self, args: argparse.Namespace) -> None:
         """Initialize."""
+        self.tool_dependency_dir: Optional[str] = None
+        self.dependency_resolvers_config_file: Optional[str] = None
         conf_file = getattr(args, "beta_dependency_resolvers_configuration", None)
         tool_dependency_dir = getattr(args, "beta_dependencies_directory", None)
         conda_dependencies = getattr(args, "beta_conda_dependencies", None)

pyproject.toml

@@ -17,6 +17,19 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools_scm]
 write_to = "cwltool/_version.py"
 
+[tool.cibuildwheel]
+test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool"
+test-requires = "-r test-requirements.txt"
+test-extras = "deps"
+skip = "pp*"
+#      ^ skip building wheels on PyPy (any version)
+build-verbosity = 1
+environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" }
+
+# Install system library
+[tool.cibuildwheel.linux]
+before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-devel libxslt-devel nodejs environment-modules || apt-get install -y --no-install-recommends libxml2-dev libxslt-dev nodejs environment-modules"
+
 [tool.black]
 line-length = 100
 target-version = [ "py39" ]

test-requirements.txt

@@ -3,7 +3,7 @@ pytest>= 6.2,< 8.4
 pytest-xdist>=3.2.0 # for the worksteal scheduler
 psutil # enhances pytest-xdist to allow "-n logical"
 pytest-httpserver
-pytest-retry;python_version>'3.9'
+pytest-retry;python_version>='3.9'
 mock>=2.0.0
 pytest-mock>=1.10.0
 pytest-cov

tests/test_dependencies.py

@@ -119,15 +119,15 @@ def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement."""
     wflow = get_data("tests/random_lines.cwl")
     job = get_data("tests/random_lines_job.json")
-    monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles"))
+    monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles"))
     error_code, _, stderr = get_main_output(
         [
             "--outdir",
             str(tmp_path / "out"),
-            "--beta-dependency-resolvers-configuration",
             "--beta-dependencies-directory",
             str(tmp_path / "deps"),
-            "tests/test_deps_env_modules_resolvers_conf.yml",
+            "--beta-dependency-resolvers-configuration",
+            get_data("tests/test_deps_env_modules_resolvers_conf.yml"),
             "--debug",
             wflow,
             job,
@@ -145,7 +145,7 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
 
     Do so by  by running `env` as the tool and parsing its output.
     """
-    monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles"))
+    monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles"))
     tool_env = get_tool_env(
         tmp_path,
         [
@@ -155,6 +155,6 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
         get_data("tests/env_with_software_req.yml"),
     )
 
-    assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space "
+    assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space ", tool_env
     tool_path = tool_env["PATH"].split(":")
     assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path