linux.yml

name: linux

on:
  push:
    branches: [main]
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  FORCE_COLOR: 1

jobs:
  pythonbuild:
    if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }}
    needs:
      - generate-matrix
    runs-on: depot-ubuntu-22.04
    steps:
      - name: Install System Dependencies
        run: |
          sudo apt update
          sudo apt install -y --no-install-recommends libssl-dev pkg-config

      - uses: actions/checkout@v4

      - name: Emit rustc version
        run: |
          rustc --version > .rustc-version

      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-pythonbuild-${{ hashFiles('Cargo.lock', '.rustc-version') }}

      - name: Build
        run: |
          cargo build --release

      - name: Upload pythonbuild Executable
        uses: actions/upload-artifact@v4
        with:
          name: pythonbuild
          path: target/release/pythonbuild

  image:
    if: ${{ needs.generate-matrix.outputs.any_builds == 'true' }}
    needs:
      - generate-matrix
    strategy:
      fail-fast: false
      matrix:
        image:
          - build
          - build.cross
          - build.cross-riscv64
          - gcc
          - xcb
          - xcb.cross
          - xcb.cross-riscv64
    name: ${{ matrix.image }}
    runs-on: depot-ubuntu-22.04
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Generate Dockerfiles
        run: |
          ./build-linux.py --make-target empty
          repo_name=$(echo "${GITHUB_REPOSITORY,,}" | sed 's|\.|_|g')
          git_ref_name=$(echo "${GITHUB_REF_NAME,,}" | sed 's|[^a-z0-9_-]|_|g')
          echo "REPO_NAME=${repo_name}" >> "${GITHUB_ENV}"
          echo "GIT_REF_NAME=${git_ref_name}" >> "${GITHUB_ENV}"

      - name: Build Image
        id: build-image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: build/${{ matrix.image }}.Dockerfile
          labels: org.opencontainers.image.source=https://github.com/${{ env.REPO_NAME }}
          # Cache from/to the current branch of the current repo as the primary cache key.
          # Cache from the default branch of the current repo so branches can have cache hits.
          # Cache from the default branch of the canonical repo so forks can have cache hits.
          # Ignore errors on cache writes so CI of forks works without a valid GHCR config.
          cache-from: |
            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }}
            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-main
            type=registry,ref=ghcr.io/indygreg/python-build-standalone:${{ matrix.image }}-main
          cache-to: |
            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }},ignore-error=true
          outputs: |
            type=docker,dest=build/image-${{ matrix.image }}.tar

      - name: Compress Image
        run: |
          echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.image }}
          zstd -v -T0 -6 --rm build/image-*.tar

      - name: Upload Docker Image
        uses: actions/upload-artifact@v4
        with:
          name: image-${{ matrix.image }}
          path: build/image-*

  generate-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix-0: ${{ steps.set-matrix.outputs.matrix-0 }}
      matrix-1: ${{ steps.set-matrix.outputs.matrix-1 }}
      any_builds: ${{ steps.set-matrix.outputs.any_builds }}
      pythonbuild_changed: ${{ steps.changed.outputs.pythonbuild_any_changed }}
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: astral-sh/setup-uv@v4

      - name: Get pull request labels
        id: get-labels
        run: |
          # Convert GitHub labels array to comma-separated string
          LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
          echo "labels=$LABELS" >> $GITHUB_OUTPUT

      - name: Generate build matrix
        id: set-matrix
        run: |
          uv run ci-matrix.py \
            --platform linux \
            --labels '${{ steps.get-labels.outputs.labels }}' \
            --max-shards 2 \
            > matrix.json

          echo "matrix-0=$(jq -c '.["0"]' matrix.json)" >> $GITHUB_OUTPUT
          echo "matrix-1=$(jq -c '.["1"]' matrix.json)" >> $GITHUB_OUTPUT

          # Display the matrix for debugging too
          cat matrix.json | jq

          if jq -e '.["0"].include | length > 0' matrix.json > /dev/null; then
            # Build matrix has entries
            echo "any_builds=true" >> $GITHUB_OUTPUT
          else
            # Build matrix is empty
            echo "any_builds=false" >> $GITHUB_OUTPUT
          fi

      - uses: tj-actions/changed-files@v45
        id: changed
        with:
          files_yaml: |
            pythonbuild:
              - "src/*.rs"

  build-0:
    needs:
      - generate-matrix
      - pythonbuild
      - image
    # Permissions used for actions/attest-build-provenance
    permissions:
      id-token: write
      attestations: write
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-0) }}
      fail-fast: false
    name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Download pythonbuild
        uses: actions/download-artifact@v4
        with:
          name: pythonbuild
          path: build

      - name: Download images
        uses: actions/download-artifact@v4
        with:
          pattern: image-*
          path: build
          merge-multiple: true

      - name: Load Docker Images
        run: |
          for f in build/image-*.tar.zst; do
            echo "decompressing $f"
            zstd -d --rm ${f}
          done

          for f in build/image-*.tar; do
            echo "loading $f"
            docker load --input $f
          done

      - name: Build
        if: ${{ ! matrix.dry-run }}
        run: |
          # Do empty target so all generated files are touched.
          ./build-linux.py --make-target empty

          # Touch mtimes of all images so they are newer than autogenerated files above.
          touch build/image-*

          ./build-linux.py --target-triple ${{ matrix.target_triple }} --python cpython-${{ matrix.python }} --options ${{ matrix.build_options }}

      - name: Validate Distribution
        if: ${{ ! matrix.dry-run }}
        run: |
          chmod +x build/pythonbuild

          if [ "${{ matrix.run }}" == "true" ]; then
            EXTRA_ARGS="--run"
          fi

          build/pythonbuild validate-distribution ${EXTRA_ARGS} dist/*.tar.zst

      - name: Upload Distribution
        if: ${{ ! matrix.dry-run }}
        uses: actions/upload-artifact@v4
        with:
          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}
          path: dist/*

  build-1:
    needs:
      - generate-matrix
      - pythonbuild
      - image
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-1) }}
      fail-fast: false
    name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Download pythonbuild
        uses: actions/download-artifact@v4
        with:
          name: pythonbuild
          path: build

      - name: Download images
        uses: actions/download-artifact@v4
        with:
          pattern: image-*
          path: build
          merge-multiple: true

      - name: Load Docker Images
        run: |
          for f in build/image-*.tar.zst; do
            echo "decompressing $f"
            zstd -d --rm ${f}
          done

          for f in build/image-*.tar; do
            echo "loading $f"
            docker load --input $f
          done

      - name: Build
        if: ${{ ! matrix.dry-run }}
        run: |
          # Do empty target so all generated files are touched.
          ./build-linux.py --make-target empty

          # Touch mtimes of all images so they are newer than autogenerated files above.
          touch build/image-*

          ./build-linux.py --target-triple ${{ matrix.target_triple }} --python cpython-${{ matrix.python }} --options ${{ matrix.build_options }}

      - name: Validate Distribution
        if: ${{ ! matrix.dry-run }}
        run: |
          chmod +x build/pythonbuild

          if [ "${{ matrix.run }}" == "true" ]; then
            EXTRA_ARGS="--run"
          fi

          build/pythonbuild validate-distribution ${EXTRA_ARGS} dist/*.tar.zst

      - name: Generate attestations
        uses: actions/attest-build-provenance@v2
        if: ${{ github.ref == 'refs/heads/main' }}
        with:
          subject-path: dist/*

      - name: Upload Distribution
        if: ${{ ! matrix.dry-run }}
        uses: actions/upload-artifact@v4
        with:
          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}
          path: dist/*