From 3210d52148c25d4e4876d16723281f2b9ad72343 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Tue, 23 May 2023 09:49:10 -0500 Subject: [PATCH] Add more linux x86 test CI shards (#19099) This is an attempt to stabilize CI while we also re-configure a remote caching solution. It appears to both speed up and stabilize CI. --- .github/workflows/test-cron.yaml | 493 +++++++++++++++++- .github/workflows/test.yaml | 493 +++++++++++++++++- .../bin/generate_github_workflows.py | 13 +- 3 files changed, 972 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test-cron.yaml b/.github/workflows/test-cron.yaml index 995e630fd2d..fb308ce82c9 100644 --- a/.github/workflows/test-cron.yaml +++ b/.github/workflows/test-cron.yaml @@ -415,7 +415,7 @@ jobs: test_python_linux_x86_64_0: env: {} if: github.repository_owner == 'pantsbuild' - name: Test Python (Linux-x86_64) Shard 0/3 + name: Test Python (Linux-x86_64) Shard 0/10 needs: bootstrap_pants_linux_x86_64 runs-on: - ubuntu-20.04 @@ -462,8 +462,8 @@ jobs: uses: actions/download-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - - name: Run Python test shard 0/3 - run: './pants test --shard=0/3 :: + - name: Run Python test shard 0/10 + run: './pants test --shard=0/10 :: ' - continue-on-error: true @@ -471,7 +471,7 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-0_3-Linux-x86_64 + name: pants-log-python-test-0_10-Linux-x86_64 path: .pants.d/pants.log strategy: matrix: @@ -482,7 +482,7 @@ jobs: test_python_linux_x86_64_1: env: {} if: github.repository_owner == 'pantsbuild' - name: Test Python (Linux-x86_64) Shard 1/3 + name: Test Python (Linux-x86_64) Shard 1/10 needs: bootstrap_pants_linux_x86_64 runs-on: - ubuntu-20.04 @@ -529,8 +529,8 @@ jobs: uses: actions/download-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - - name: Run Python test shard 1/3 - run: './pants test --shard=1/3 :: + - name: Run Python test shard 1/10 + run: './pants test --shard=1/10 :: ' - continue-on-error: true @@ -538,7 +538,7 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-1_3-Linux-x86_64 + name: pants-log-python-test-1_10-Linux-x86_64 path: .pants.d/pants.log strategy: matrix: @@ -549,7 +549,7 @@ jobs: test_python_linux_x86_64_2: env: {} if: github.repository_owner == 'pantsbuild' - name: Test Python (Linux-x86_64) Shard 2/3 + name: Test Python (Linux-x86_64) Shard 2/10 needs: bootstrap_pants_linux_x86_64 runs-on: - ubuntu-20.04 @@ -596,8 +596,8 @@ jobs: uses: actions/download-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - - name: Run Python test shard 2/3 - run: './pants test --shard=2/3 :: + - name: Run Python test shard 2/10 + run: './pants test --shard=2/10 :: ' - continue-on-error: true @@ -605,7 +605,476 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-2_3-Linux-x86_64 + name: pants-log-python-test-2_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + timeout-minutes: 90 + test_python_linux_x86_64_3: + env: {} + if: github.repository_owner == 'pantsbuild' + name: Test Python (Linux-x86_64) Shard 3/10 + needs: bootstrap_pants_linux_x86_64 + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift + -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 3/10 + run: './pants test --shard=3/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-3_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + timeout-minutes: 90 + test_python_linux_x86_64_4: + env: {} + if: github.repository_owner == 'pantsbuild' + name: Test Python (Linux-x86_64) Shard 4/10 + needs: bootstrap_pants_linux_x86_64 + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift + -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 4/10 + run: './pants test --shard=4/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-4_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + timeout-minutes: 90 + test_python_linux_x86_64_5: + env: {} + if: github.repository_owner == 'pantsbuild' + name: Test Python (Linux-x86_64) Shard 5/10 + needs: bootstrap_pants_linux_x86_64 + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift + -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 5/10 + run: './pants test --shard=5/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-5_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + timeout-minutes: 90 + test_python_linux_x86_64_6: + env: {} + if: github.repository_owner == 'pantsbuild' + name: Test Python (Linux-x86_64) Shard 6/10 + needs: bootstrap_pants_linux_x86_64 + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift + -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 6/10 + run: './pants test --shard=6/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-6_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + timeout-minutes: 90 + test_python_linux_x86_64_7: + env: {} + if: github.repository_owner == 'pantsbuild' + name: Test Python (Linux-x86_64) Shard 7/10 + needs: bootstrap_pants_linux_x86_64 + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift + -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 7/10 + run: './pants test --shard=7/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-7_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + timeout-minutes: 90 + test_python_linux_x86_64_8: + env: {} + if: github.repository_owner == 'pantsbuild' + name: Test Python (Linux-x86_64) Shard 8/10 + needs: bootstrap_pants_linux_x86_64 + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift + -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 8/10 + run: './pants test --shard=8/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-8_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.8' + - '3.9' + timeout-minutes: 90 + test_python_linux_x86_64_9: + env: {} + if: github.repository_owner == 'pantsbuild' + name: Test Python (Linux-x86_64) Shard 9/10 + needs: bootstrap_pants_linux_x86_64 + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift + -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version + }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 9/10 + run: './pants test --shard=9/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-9_10-Linux-x86_64 path: .pants.d/pants.log strategy: matrix: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 768f4ed4158..1b1309e2c7b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -666,6 +666,13 @@ jobs: - test_python_linux_x86_64_0 - test_python_linux_x86_64_1 - test_python_linux_x86_64_2 + - test_python_linux_x86_64_3 + - test_python_linux_x86_64_4 + - test_python_linux_x86_64_5 + - test_python_linux_x86_64_6 + - test_python_linux_x86_64_7 + - test_python_linux_x86_64_8 + - test_python_linux_x86_64_9 - test_python_macos11_x86_64 outputs: merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }} @@ -725,7 +732,7 @@ jobs: test_python_linux_x86_64_0: env: {} if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') - name: Test Python (Linux-x86_64) Shard 0/3 + name: Test Python (Linux-x86_64) Shard 0/10 needs: - bootstrap_pants_linux_x86_64 - classify_changes @@ -772,8 +779,8 @@ jobs: uses: actions/download-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - - name: Run Python test shard 0/3 - run: './pants test --shard=0/3 :: + - name: Run Python test shard 0/10 + run: './pants test --shard=0/10 :: ' - continue-on-error: true @@ -781,7 +788,7 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-0_3-Linux-x86_64 + name: pants-log-python-test-0_10-Linux-x86_64 path: .pants.d/pants.log strategy: matrix: @@ -791,7 +798,7 @@ jobs: test_python_linux_x86_64_1: env: {} if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') - name: Test Python (Linux-x86_64) Shard 1/3 + name: Test Python (Linux-x86_64) Shard 1/10 needs: - bootstrap_pants_linux_x86_64 - classify_changes @@ -838,8 +845,8 @@ jobs: uses: actions/download-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - - name: Run Python test shard 1/3 - run: './pants test --shard=1/3 :: + - name: Run Python test shard 1/10 + run: './pants test --shard=1/10 :: ' - continue-on-error: true @@ -847,7 +854,7 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-1_3-Linux-x86_64 + name: pants-log-python-test-1_10-Linux-x86_64 path: .pants.d/pants.log strategy: matrix: @@ -857,7 +864,7 @@ jobs: test_python_linux_x86_64_2: env: {} if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') - name: Test Python (Linux-x86_64) Shard 2/3 + name: Test Python (Linux-x86_64) Shard 2/10 needs: - bootstrap_pants_linux_x86_64 - classify_changes @@ -904,8 +911,8 @@ jobs: uses: actions/download-artifact@v3 with: name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 - - name: Run Python test shard 2/3 - run: './pants test --shard=2/3 :: + - name: Run Python test shard 2/10 + run: './pants test --shard=2/10 :: ' - continue-on-error: true @@ -913,7 +920,469 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-2_3-Linux-x86_64 + name: pants-log-python-test-2_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.7' + timeout-minutes: 90 + test_python_linux_x86_64_3: + env: {} + if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') + name: Test Python (Linux-x86_64) Shard 3/10 + needs: + - bootstrap_pants_linux_x86_64 + - classify_changes + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 3/10 + run: './pants test --shard=3/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-3_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.7' + timeout-minutes: 90 + test_python_linux_x86_64_4: + env: {} + if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') + name: Test Python (Linux-x86_64) Shard 4/10 + needs: + - bootstrap_pants_linux_x86_64 + - classify_changes + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 4/10 + run: './pants test --shard=4/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-4_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.7' + timeout-minutes: 90 + test_python_linux_x86_64_5: + env: {} + if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') + name: Test Python (Linux-x86_64) Shard 5/10 + needs: + - bootstrap_pants_linux_x86_64 + - classify_changes + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 5/10 + run: './pants test --shard=5/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-5_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.7' + timeout-minutes: 90 + test_python_linux_x86_64_6: + env: {} + if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') + name: Test Python (Linux-x86_64) Shard 6/10 + needs: + - bootstrap_pants_linux_x86_64 + - classify_changes + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 6/10 + run: './pants test --shard=6/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-6_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.7' + timeout-minutes: 90 + test_python_linux_x86_64_7: + env: {} + if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') + name: Test Python (Linux-x86_64) Shard 7/10 + needs: + - bootstrap_pants_linux_x86_64 + - classify_changes + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 7/10 + run: './pants test --shard=7/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-7_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.7' + timeout-minutes: 90 + test_python_linux_x86_64_8: + env: {} + if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') + name: Test Python (Linux-x86_64) Shard 8/10 + needs: + - bootstrap_pants_linux_x86_64 + - classify_changes + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 8/10 + run: './pants test --shard=8/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-8_10-Linux-x86_64 + path: .pants.d/pants.log + strategy: + matrix: + python-version: + - '3.7' + timeout-minutes: 90 + test_python_linux_x86_64_9: + env: {} + if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.docs_only != 'true') + name: Test Python (Linux-x86_64) Shard 9/10 + needs: + - bootstrap_pants_linux_x86_64 + - classify_changes + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Install AdoptJDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - if: runner.os == 'Linux' + name: Download Apache `thrift` binary (Linux) + run: 'mkdir -p "${HOME}/.thrift" + + curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" + + chmod +x "${HOME}/.thrift/thrift" + + echo "${HOME}/.thrift" >> $GITHUB_PATH + + ' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Tell Pants to use Python ${{ matrix.python-version }} + run: 'echo "PY=python${{ matrix.python-version }}" >> $GITHUB_ENV + + echo "PANTS_PYTHON_INTERPRETER_CONSTRAINTS=[''==${{ matrix.python-version }}.*'']" >> $GITHUB_ENV + + ' + - name: Expose Pythons + uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe + - name: Download native binaries + uses: actions/download-artifact@v3 + with: + name: native_binaries.${{ matrix.python-version }}.Linux-x86_64 + - name: Run Python test shard 9/10 + run: './pants test --shard=9/10 :: + + ' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-python-test-9_10-Linux-x86_64 path: .pants.d/pants.log strategy: matrix: diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index 36947beea89..c628df96b3d 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -652,9 +652,16 @@ def test_python_linux(shard: str) -> dict[str, Any]: helper.job_name("bootstrap_pants"): bootstrap_jobs( helper, python_versions, validate_ci_config=True, rust_testing=RustTesting.ALL ), - f"{shard_name_prefix}_0": test_python_linux("0/3"), - f"{shard_name_prefix}_1": test_python_linux("1/3"), - f"{shard_name_prefix}_2": test_python_linux("2/3"), + f"{shard_name_prefix}_0": test_python_linux("0/10"), + f"{shard_name_prefix}_1": test_python_linux("1/10"), + f"{shard_name_prefix}_2": test_python_linux("2/10"), + f"{shard_name_prefix}_3": test_python_linux("3/10"), + f"{shard_name_prefix}_4": test_python_linux("4/10"), + f"{shard_name_prefix}_5": test_python_linux("5/10"), + f"{shard_name_prefix}_6": test_python_linux("6/10"), + f"{shard_name_prefix}_7": test_python_linux("7/10"), + f"{shard_name_prefix}_8": test_python_linux("8/10"), + f"{shard_name_prefix}_9": test_python_linux("9/10"), } return jobs