ci: Switch PR checks for linux-64 and osx-64 to GitHub Actions (bioco…

…nda#51720) * ci: Switch PR checks for linux-64 and osx-64 to GitHub Actions * try removing pr:none
KennthShang · Oct 28, 2024 · 9a03cf0 · 9a03cf0
1 parent 3f0cfa7
commit 9a03cf0
Show file tree

Hide file tree

Showing 4 changed files with 180 additions and 179 deletions.
diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml
@@ -39,6 +39,7 @@ jobs:
           set -e
           eval "$(conda shell.bash hook)"
           conda activate bioconda
+          source common.sh
           echo '============'
           conda info --all
           conda config --show-sources
@@ -93,15 +94,15 @@ jobs:
 
           eval "$(conda shell.bash hook)"
           conda activate bioconda
+          source common.sh
           if [ -z "$GITHUB_BASE_REF" ] ; then
               export GITHUB_BASE_REF="master"
           fi
           git fetch origin "$GITHUB_BASE_REF"
-          docker pull quay.io/dpryan79/mulled_container:latest
+
           bioconda-utils build recipes config.yml \
             --docker --mulled-test \
             --git-range origin/"$GITHUB_BASE_REF" HEAD
-          docker rmi quay.io/dpryan79/mulled_container:latest
 
       - name: Prepare artifacts
         run: |
@@ -149,6 +150,7 @@ jobs:
     strategy:
       fail-fast: true
       max-parallel: 4
+    # Limited concurrency for osx, so first make sure linux can pass
     needs: build-linux
     steps:
       - uses: actions/checkout@v4
@@ -178,19 +180,20 @@ jobs:
           set -e
           eval "$(conda shell.bash hook)"
           conda activate bioconda
+          source common.sh
+          # Sets up OSX SDK
+          run_conda_forge_build_setup
 
           # Clean up lingering build artifacts
           for n in linux-64 osx-64 noarch; do
               rm -f /opt/mambaforge/envs/bioconda/conda-bld/$n/*.tar.bz2
           done
 
-          # The SDK isn't actually cached, so reinstall it
-          run_conda_forge_build_setup
-
           if [ -z "$GITHUB_BASE_REF" ] ; then
               export GITHUB_BASE_REF="master"
           fi
           git fetch origin "$GITHUB_BASE_REF"
+
           bioconda-utils build recipes config.yml \
             --git-range origin/"$GITHUB_BASE_REF" HEAD
 
@@ -213,84 +216,85 @@ jobs:
           path: |
             /tmp/artifacts/packages
 
-  build_and_test-osx-arm64:
-    name: OSX-ARM64 Tests
-    runs-on: macOS-14 # M1
-    strategy:
-      fail-fast: true
-      max-parallel: 4
-    needs: build-linux
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      # bail if there's no osx-arm64 recipes
-      - name: Check for Additional Platforms
-        id: additional_platforms
-        run: |
-          result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}")
-          if [[ ${result} != "build" ]]
-          then
-              echo "No recipes using this platform, skipping rest of job."
-              echo "skip_build=true" >> $GITHUB_OUTPUT
-          fi
-
-      - name: set path
-        run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH
-
-      - name: Fetch conda install script
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        run: |
-          wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh
-
-      - name: Set up bioconda-utils
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        run: bash install-and-set-up-conda.sh
-
-      - name: Configure conda
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        run: bash configure-conda.sh
-
-      - name: Build and Test
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        env:
-          # Mimic circleci
-          OSTYPE: "darwin"
-          CI: "true"
-        run: |
-          set -xe
-          eval "$(conda shell.bash hook)"
-          conda activate bioconda
-          source common.sh
-          # Sets up OSX SDK
-          run_conda_forge_build_setup
-
-          if [ -z "$GITHUB_BASE_REF" ] ; then
-              export GITHUB_BASE_REF="master"
-          fi
-          git fetch origin "$GITHUB_BASE_REF"
-
-          bioconda-utils build recipes config.yml \
-            --lint --git-range origin/"$GITHUB_BASE_REF" HEAD
-
-      - name: Prepare artifacts
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        run: |
-              (
-              rm -rf /tmp/artifacts
-              mkdir -p /tmp/artifacts/packages
-              cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0
-              find -name .cache | xargs rm -rf || true
-              for n in index.html channeldata.json osx-arm64 noarch; do
-                cp -rv $n /tmp/artifacts/packages || true
-              done
-              ) || true
-
-      - name: Archive packages
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        uses: actions/upload-artifact@v4
-        with:
-          name: osx-arm64-packages
-          path: |
-            /tmp/artifacts
+  # Disabled due to concurrency limits on GHA for OSX. osx-arm64 builds are on CircleCI.
+  # build_and_test-osx-arm64:
+  #   name: OSX-ARM64 Tests
+  #   runs-on: macOS-14 # M1
+  #   strategy:
+  #     fail-fast: true
+  #     max-parallel: 4
+  #   needs: build-linux
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+
+  #     # bail if there's no osx-arm64 recipes
+  #     - name: Check for Additional Platforms
+  #       id: additional_platforms
+  #       run: |
+  #         result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}")
+  #         if [[ ${result} != "build" ]]
+  #         then
+  #             echo "No recipes using this platform, skipping rest of job."
+  #             echo "skip_build=true" >> $GITHUB_OUTPUT
+  #         fi
+
+  #     - name: set path
+  #       run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH
+
+  #     - name: Fetch conda install script
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       run: |
+  #         wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh
+
+  #     - name: Set up bioconda-utils
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       run: bash install-and-set-up-conda.sh
+
+  #     - name: Configure conda
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       run: bash configure-conda.sh
+
+  #     - name: Build and Test
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       env:
+  #         # Mimic circleci
+  #         OSTYPE: "darwin"
+  #         CI: "true"
+  #       run: |
+  #         set -xe
+  #         eval "$(conda shell.bash hook)"
+  #         conda activate bioconda
+  #         source common.sh
+  #         # Sets up OSX SDK
+  #         run_conda_forge_build_setup
+
+  #         if [ -z "$GITHUB_BASE_REF" ] ; then
+  #             export GITHUB_BASE_REF="master"
+  #         fi
+  #         git fetch origin "$GITHUB_BASE_REF"
+
+  #         bioconda-utils build recipes config.yml \
+  #           --lint --git-range origin/"$GITHUB_BASE_REF" HEAD
+
+  #     - name: Prepare artifacts
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       run: |
+  #             (
+  #             rm -rf /tmp/artifacts
+  #             mkdir -p /tmp/artifacts/packages
+  #             cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0
+  #             find -name .cache | xargs rm -rf || true
+  #             for n in index.html channeldata.json osx-arm64 noarch; do
+  #               cp -rv $n /tmp/artifacts/packages || true
+  #             done
+  #             ) || true
+
+  #     - name: Archive packages
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: osx-arm64-packages
+  #         path: |
+  #           /tmp/artifacts
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -44,16 +44,12 @@ jobs:
           set -ex
           eval "$(conda shell.bash hook)"
           conda activate bioconda
-          docker pull quay.io/dpryan79/mulled_container:latest
-          # bioconda-utils handle-merged-pr recipes config.yml \
-          #   --repo bioconda/bioconda-recipes \
-          #   --git-range ${BUILD_SOURCEVERSION}~1 ${BUILD_SOURCEVERSION} \
-          #   --quay-upload-target biocontainers \
-          #   --fallback build
-          bioconda-utils build recipes config.yml \
+          bioconda-utils handle-merged-pr recipes config.yml \
+            --repo bioconda/bioconda-recipes \
             --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \
-            --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers
-          docker rmi quay.io/dpryan79/mulled_container:latest
+            --quay-upload-target biocontainers \
+            --fallback build \
+            --artifact-source github-actions
 
   build-osx-64:
     name: OSX-64 Upload
@@ -99,74 +95,75 @@ jobs:
           # Sets up OSX SDK
           run_conda_forge_build_setup
 
-          # bioconda-utils handle-merged-pr recipes config.yml \
-          #   --repo bioconda/bioconda-recipes \
-          #   --git-range ${BUILD_SOURCEVERSION}~1 ${BUILD_SOURCEVERSION} \
-          #   --fallback build
-          bioconda-utils build recipes config.yml \
-            --anaconda-upload --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA}
-
-  build_and_upload-osx-arm64:
-    name:  OSX-ARM64 Upload
-    if: github.repository == 'bioconda/bioconda-recipes'
-    runs-on: macOS-14 # M1
-    strategy:
-      fail-fast: false
-      max-parallel: 4
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      # bail if there's no osx-arm64 recipes
-      - name: Check for Additional Platforms
-        id: additional_platforms
-        run: |
-          result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}")
-          if [[ ${result} != "build" ]]
-          then
-              echo "No recipes using this platform, skipping rest of job."
-              echo "skip_build=true" >> $GITHUB_OUTPUT
-          fi
-
-      - name: set path
-        run: |
-          echo "/opt/mambaforge/bin" >> $GITHUB_PATH
-
-      - name: Fetch conda install script
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        run: |
-          wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh
-
-      - name: Set up bioconda-utils
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        run: bash install-and-set-up-conda.sh
-
-      - name: Configure conda
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        run: bash configure-conda.sh
-
-      - name: Build and Upload
-        if: steps.additional_platforms.outputs.skip_build != 'true'
-        env:
-          QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }}
-          QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }}
-          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
-          INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # Mimic circleci
-          OSTYPE: "darwin"
-          CI: "true"
-        run: |
-          set -xe
-          eval "$(conda shell.bash hook)"
-          conda activate bioconda
-          source common.sh
-          # Sets up OSX SDK
-          run_conda_forge_build_setup
-
           bioconda-utils handle-merged-pr recipes config.yml \
-            --repo bioconda/bioconda-recipes \
-            --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \
-            --fallback build \
-            --artifact-source github-actions
+          --repo bioconda/bioconda-recipes \
+          --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \
+          --fallback build \
+          --artifact-source github-actions
+
+
+  # Disabled due to concurrency limits on GHA for OSX. osx-arm64 builds are on CircleCI.
+  # build_and_upload-osx-arm64:
+  #   name:  OSX-ARM64 Upload
+  #   if: github.repository == 'bioconda/bioconda-recipes'
+  #   runs-on: macOS-14 # M1
+  #   strategy:
+  #     fail-fast: false
+  #     max-parallel: 4
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+
+  #     # bail if there's no osx-arm64 recipes
+  #     - name: Check for Additional Platforms
+  #       id: additional_platforms
+  #       run: |
+  #         result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}")
+  #         if [[ ${result} != "build" ]]
+  #         then
+  #             echo "No recipes using this platform, skipping rest of job."
+  #             echo "skip_build=true" >> $GITHUB_OUTPUT
+  #         fi
+
+  #     - name: set path
+  #       run: |
+  #         echo "/opt/mambaforge/bin" >> $GITHUB_PATH
+
+  #     - name: Fetch conda install script
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       run: |
+  #         wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh
+
+  #     - name: Set up bioconda-utils
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       run: bash install-and-set-up-conda.sh
+
+  #     - name: Configure conda
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       run: bash configure-conda.sh
+
+  #     - name: Build and Upload
+  #       if: steps.additional_platforms.outputs.skip_build != 'true'
+  #       env:
+  #         QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }}
+  #         QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }}
+  #         ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
+  #         INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }}
+  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  #         # Mimic circleci
+  #         OSTYPE: "darwin"
+  #         CI: "true"
+  #       run: |
+  #         set -xe
+  #         eval "$(conda shell.bash hook)"
+  #         conda activate bioconda
+  #         source common.sh
+  #         # Sets up OSX SDK
+  #         run_conda_forge_build_setup
+
+  #         bioconda-utils handle-merged-pr recipes config.yml \
+  #           --repo bioconda/bioconda-recipes \
+  #           --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \
+  #           --fallback build \
+  #           --artifact-source github-actions