[3.12] pythongh-111062: CI: Move OS test jobs to reusable workflows (p…

…ythongh-111568) CI: Move OS test jobs to reusable workflows Co-authored-by: Donghee Na <[email protected]>
miss-islington · Nov 16, 2023 · 427cf61 · 427cf61
1 parent 77f59bf
commit 427cf61
Show file tree

Hide file tree

Showing 4 changed files with 184 additions and 147 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -108,7 +108,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
       - name: Install dependencies
         run: |
@@ -208,159 +208,32 @@ jobs:
         if: github.event_name == 'pull_request'  # $GITHUB_EVENT_NAME
         run: make check-c-globals
 
-  build_win32:
-    name: 'Windows (x86)'
-    runs-on: windows-latest
-    timeout-minutes: 60
+  build_windows:
+    name: 'Windows'
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
-    env:
-       IncludeUwp: 'true'
-    steps:
-    - uses: actions/checkout@v4
-    - name: Build CPython
-      run: .\PCbuild\build.bat -e -d -p Win32
-    - name: Display build info
-      run: .\python.bat -m test.pythoninfo
-    - name: Tests
-      run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
-
-  build_win_amd64:
-    name: 'Windows (x64)'
-    runs-on: windows-latest
-    timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
-    env:
-       IncludeUwp: 'true'
-    steps:
-    - uses: actions/checkout@v4
-    - name: Register MSVC problem matcher
-      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
-    - name: Build CPython
-      run: .\PCbuild\build.bat -e -d -p x64
-    - name: Display build info
-      run: .\python.bat -m test.pythoninfo
-    - name: Tests
-      run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
-
-  build_win_arm64:
-    name: 'Windows (arm64)'
-    runs-on: windows-latest
-    timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
-    env:
-       IncludeUwp: 'true'
-    steps:
-    - uses: actions/checkout@v4
-    - name: Register MSVC problem matcher
-      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
-    - name: Build CPython
-      run: .\PCbuild\build.bat -e -d -p arm64
+    uses: ./.github/workflows/reusable-windows.yml
 
   build_macos:
     name: 'macOS'
-    runs-on: macos-latest
-    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
-    env:
-      HOMEBREW_NO_ANALYTICS: 1
-      HOMEBREW_NO_AUTO_UPDATE: 1
-      HOMEBREW_NO_INSTALL_CLEANUP: 1
-      PYTHONSTRICTEXTENSIONBUILD: 1
-    steps:
-    - uses: actions/checkout@v4
-    - name: Restore config.cache
-      uses: actions/cache@v3
-      with:
-        path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
-    - name: Install Homebrew dependencies
-      run: brew install pkg-config [email protected] xz gdbm tcl-tk
-    - name: Configure CPython
-      run: |
-        GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
-        GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \
-        ./configure \
-          --config-cache \
-          --with-pydebug \
-          --prefix=/opt/python-dev \
-          --with-openssl="$(brew --prefix [email protected])"
-    - name: Build CPython
-      run: make -j4
-    - name: Display build info
-      run: make pythoninfo
-    - name: Tests
-      run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+    uses: ./.github/workflows/reusable-macos.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
 
   build_ubuntu:
     name: 'Ubuntu'
-    runs-on: ubuntu-20.04
-    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
-    env:
-      OPENSSL_VER: 3.0.11
-      PYTHONSTRICTEXTENSIONBUILD: 1
-    steps:
-    - uses: actions/checkout@v4
-    - name: Register gcc problem matcher
-      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
-    - name: Install dependencies
-      run: sudo ./.github/workflows/posix-deps-apt.sh
-    - name: Configure OpenSSL env vars
-      run: |
-        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
-        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
-        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
-    - name: 'Restore OpenSSL build'
-      id: cache-openssl
-      uses: actions/cache@v3
-      with:
-        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
-    - name: Install OpenSSL
-      if: steps.cache-openssl.outputs.cache-hit != 'true'
-      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
-    - name: Add ccache to PATH
-      run: |
-        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
-    - name: Configure ccache action
-      uses: hendrikmuhs/[email protected]
-    - name: Setup directory envs for out-of-tree builds
-      run: |
-        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
-        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
-    - name: Create directories for read-only out-of-tree builds
-      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
-    - name: Bind mount sources read-only
-      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
-    - name: Restore config.cache
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
-    - name: Configure CPython out-of-tree
-      working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: |
+    uses: ./.github/workflows/reusable-ubuntu.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      options: |
         ../cpython-ro-srcdir/configure \
           --config-cache \
           --with-pydebug \
           --with-openssl=$OPENSSL_DIR
-    - name: Build CPython out-of-tree
-      working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: make -j4
-    - name: Display build info
-      working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: make pythoninfo
-    - name: Remount sources writable for tests
-      # some tests write to srcdir, lack of pyc files slows down testing
-      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
-    - name: Tests
-      working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
@@ -429,7 +302,7 @@ jobs:
     - uses: actions/checkout@v4
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
-    - name: Install Dependencies
+    - name: Install dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure OpenSSL env vars
       run: |
@@ -586,12 +459,10 @@ jobs:
     - check_source  # Transitive dependency, needed to access `run_tests` value
     - check-docs
     - check_generated_files
-    - build_win32
-    - build_win_amd64
-    - build_win_arm64
     - build_macos
     - build_ubuntu
     - build_ubuntu_ssltests
+    - build_windows
     - test_hypothesis
     - build_asan
 
@@ -604,8 +475,6 @@ jobs:
         allowed-failures: >-
           build_macos,
           build_ubuntu_ssltests,
-          build_win32,
-          build_win_arm64,
           test_hypothesis,
         allowed-skips: >-
           ${{
@@ -619,12 +488,10 @@ jobs:
             needs.check_source.outputs.run_tests != 'true'
             && '
             check_generated_files,
-            build_win32,
-            build_win_amd64,
-            build_win_arm64,
             build_macos,
             build_ubuntu,
             build_ubuntu_ssltests,
+            build_windows,
             build_asan,
             '
             || ''

diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
@@ -0,0 +1,46 @@
+on:
+  workflow_call:
+    inputs:
+      config_hash:
+        required: true
+        type: string
+      free-threaded:
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  build_macos:
+    name: 'build and test'
+    runs-on: macos-latest
+    timeout-minutes: 60
+    env:
+      HOMEBREW_NO_ANALYTICS: 1
+      HOMEBREW_NO_AUTO_UPDATE: 1
+      HOMEBREW_NO_INSTALL_CLEANUP: 1
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v4
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
+    - name: Install Homebrew dependencies
+      run: brew install pkg-config [email protected] xz gdbm tcl-tk
+    - name: Configure CPython
+      run: |
+        GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
+        GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \
+        ./configure \
+          --config-cache \
+          --with-pydebug \
+          ${{ inputs.free-threaded && '--disable-gil' || '' }} \
+          --prefix=/opt/python-dev \
+          --with-openssl="$(brew --prefix [email protected])"
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: Tests
+      run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml
@@ -0,0 +1,71 @@
+on:
+  workflow_call:
+    inputs:
+      config_hash:
+        required: true
+        type: string
+      options:
+        required: true
+        type: string
+
+jobs:
+  build_ubuntu_reusable:
+    name: 'build and test'
+    timeout-minutes: 60
+    runs-on: ubuntu-20.04
+    env:
+      OPENSSL_VER: 3.0.11
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v4
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v3
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ${{ inputs.options }}
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make -j4
+    - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+    - name: Tests
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml
@@ -0,0 +1,53 @@
+on:
+  workflow_call:
+    inputs:
+      free-threaded:
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  build_win32:
+    name: 'build and test (x86)'
+    runs-on: windows-latest
+    timeout-minutes: 60
+    env:
+      IncludeUwp: 'true'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build CPython
+      run: .\PCbuild\build.bat -e -d -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }}
+    - name: Display build info
+      run: .\python.bat -m test.pythoninfo
+    - name: Tests
+      run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+
+  build_win_amd64:
+    name: 'build and test (x64)'
+    runs-on: windows-latest
+    timeout-minutes: 60
+    env:
+       IncludeUwp: 'true'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Register MSVC problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
+    - name: Build CPython
+      run: .\PCbuild\build.bat -e -d -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }}
+    - name: Display build info
+      run: .\python.bat -m test.pythoninfo
+    - name: Tests
+      run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+
+  build_win_arm64:
+    name: 'build (arm64)'
+    runs-on: windows-latest
+    timeout-minutes: 60
+    env:
+      IncludeUwp: 'true'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Register MSVC problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
+    - name: Build CPython
+      run: .\PCbuild\build.bat -e -d -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }}