From aee4ff56f2aaf87dccb9a07c7f00895d2f663f43 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 28 Oct 2024 11:29:59 -0400 Subject: [PATCH 1/2] ci: Switch PR checks for linux-64 and osx-64 to GitHub Actions --- .github/workflows/PR.yml | 176 ++++++++++++++++++----------------- .github/workflows/master.yml | 155 +++++++++++++++--------------- azure-pipeline-master.yml | 12 +-- azure-pipeline.yml | 17 ++-- 4 files changed, 181 insertions(+), 179 deletions(-) diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 82f17b7c4a736..53140212f477f 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -39,6 +39,7 @@ jobs: set -e eval "$(conda shell.bash hook)" conda activate bioconda + source common.sh echo '============' conda info --all conda config --show-sources @@ -93,15 +94,15 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda + source common.sh if [ -z "$GITHUB_BASE_REF" ] ; then export GITHUB_BASE_REF="master" fi git fetch origin "$GITHUB_BASE_REF" - docker pull quay.io/dpryan79/mulled_container:latest + bioconda-utils build recipes config.yml \ --docker --mulled-test \ --git-range origin/"$GITHUB_BASE_REF" HEAD - docker rmi quay.io/dpryan79/mulled_container:latest - name: Prepare artifacts run: | @@ -149,6 +150,7 @@ jobs: strategy: fail-fast: true max-parallel: 4 + # Limited concurrency for osx, so first make sure linux can pass needs: build-linux steps: - uses: actions/checkout@v4 @@ -178,19 +180,20 @@ jobs: set -e eval "$(conda shell.bash hook)" conda activate bioconda + source common.sh + # Sets up OSX SDK + run_conda_forge_build_setup # Clean up lingering build artifacts for n in linux-64 osx-64 noarch; do rm -f /opt/mambaforge/envs/bioconda/conda-bld/$n/*.tar.bz2 done - # The SDK isn't actually cached, so reinstall it - run_conda_forge_build_setup - if [ -z "$GITHUB_BASE_REF" ] ; then export GITHUB_BASE_REF="master" fi git fetch origin "$GITHUB_BASE_REF" + bioconda-utils build recipes config.yml \ --git-range origin/"$GITHUB_BASE_REF" HEAD @@ -213,84 +216,85 @@ jobs: path: | /tmp/artifacts/packages - build_and_test-osx-arm64: - name: OSX-ARM64 Tests - runs-on: macOS-14 # M1 - strategy: - fail-fast: true - max-parallel: 4 - needs: build-linux - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # bail if there's no osx-arm64 recipes - - name: Check for Additional Platforms - id: additional_platforms - run: | - result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") - if [[ ${result} != "build" ]] - then - echo "No recipes using this platform, skipping rest of job." - echo "skip_build=true" >> $GITHUB_OUTPUT - fi - - - name: set path - run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH - - - name: Fetch conda install script - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - - name: Set up bioconda-utils - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash install-and-set-up-conda.sh - - - name: Configure conda - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash configure-conda.sh - - - name: Build and Test - if: steps.additional_platforms.outputs.skip_build != 'true' - env: - # Mimic circleci - OSTYPE: "darwin" - CI: "true" - run: | - set -xe - eval "$(conda shell.bash hook)" - conda activate bioconda - source common.sh - # Sets up OSX SDK - run_conda_forge_build_setup - - if [ -z "$GITHUB_BASE_REF" ] ; then - export GITHUB_BASE_REF="master" - fi - git fetch origin "$GITHUB_BASE_REF" - - bioconda-utils build recipes config.yml \ - --lint --git-range origin/"$GITHUB_BASE_REF" HEAD - - - name: Prepare artifacts - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - ( - rm -rf /tmp/artifacts - mkdir -p /tmp/artifacts/packages - cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 - find -name .cache | xargs rm -rf || true - for n in index.html channeldata.json osx-arm64 noarch; do - cp -rv $n /tmp/artifacts/packages || true - done - ) || true - - - name: Archive packages - if: steps.additional_platforms.outputs.skip_build != 'true' - uses: actions/upload-artifact@v4 - with: - name: osx-arm64-packages - path: | - /tmp/artifacts + # Disabled due to concurrency limits on GHA for OSX. osx-arm64 builds are on CircleCI. + # build_and_test-osx-arm64: + # name: OSX-ARM64 Tests + # runs-on: macOS-14 # M1 + # strategy: + # fail-fast: true + # max-parallel: 4 + # needs: build-linux + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + + # # bail if there's no osx-arm64 recipes + # - name: Check for Additional Platforms + # id: additional_platforms + # run: | + # result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") + # if [[ ${result} != "build" ]] + # then + # echo "No recipes using this platform, skipping rest of job." + # echo "skip_build=true" >> $GITHUB_OUTPUT + # fi + + # - name: set path + # run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + # - name: Fetch conda install script + # if: steps.additional_platforms.outputs.skip_build != 'true' + # run: | + # wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + # - name: Set up bioconda-utils + # if: steps.additional_platforms.outputs.skip_build != 'true' + # run: bash install-and-set-up-conda.sh + + # - name: Configure conda + # if: steps.additional_platforms.outputs.skip_build != 'true' + # run: bash configure-conda.sh + + # - name: Build and Test + # if: steps.additional_platforms.outputs.skip_build != 'true' + # env: + # # Mimic circleci + # OSTYPE: "darwin" + # CI: "true" + # run: | + # set -xe + # eval "$(conda shell.bash hook)" + # conda activate bioconda + # source common.sh + # # Sets up OSX SDK + # run_conda_forge_build_setup + + # if [ -z "$GITHUB_BASE_REF" ] ; then + # export GITHUB_BASE_REF="master" + # fi + # git fetch origin "$GITHUB_BASE_REF" + + # bioconda-utils build recipes config.yml \ + # --lint --git-range origin/"$GITHUB_BASE_REF" HEAD + + # - name: Prepare artifacts + # if: steps.additional_platforms.outputs.skip_build != 'true' + # run: | + # ( + # rm -rf /tmp/artifacts + # mkdir -p /tmp/artifacts/packages + # cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 + # find -name .cache | xargs rm -rf || true + # for n in index.html channeldata.json osx-arm64 noarch; do + # cp -rv $n /tmp/artifacts/packages || true + # done + # ) || true + + # - name: Archive packages + # if: steps.additional_platforms.outputs.skip_build != 'true' + # uses: actions/upload-artifact@v4 + # with: + # name: osx-arm64-packages + # path: | + # /tmp/artifacts diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7e237b314da22..243e3a6ac1dd3 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -44,16 +44,12 @@ jobs: set -ex eval "$(conda shell.bash hook)" conda activate bioconda - docker pull quay.io/dpryan79/mulled_container:latest - # bioconda-utils handle-merged-pr recipes config.yml \ - # --repo bioconda/bioconda-recipes \ - # --git-range ${BUILD_SOURCEVERSION}~1 ${BUILD_SOURCEVERSION} \ - # --quay-upload-target biocontainers \ - # --fallback build - bioconda-utils build recipes config.yml \ + bioconda-utils handle-merged-pr recipes config.yml \ + --repo bioconda/bioconda-recipes \ --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ - --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers - docker rmi quay.io/dpryan79/mulled_container:latest + --quay-upload-target biocontainers \ + --fallback build \ + --artifact-source github-actions build-osx-64: name: OSX-64 Upload @@ -99,74 +95,75 @@ jobs: # Sets up OSX SDK run_conda_forge_build_setup - # bioconda-utils handle-merged-pr recipes config.yml \ - # --repo bioconda/bioconda-recipes \ - # --git-range ${BUILD_SOURCEVERSION}~1 ${BUILD_SOURCEVERSION} \ - # --fallback build - bioconda-utils build recipes config.yml \ - --anaconda-upload --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} - - build_and_upload-osx-arm64: - name: OSX-ARM64 Upload - if: github.repository == 'bioconda/bioconda-recipes' - runs-on: macOS-14 # M1 - strategy: - fail-fast: false - max-parallel: 4 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # bail if there's no osx-arm64 recipes - - name: Check for Additional Platforms - id: additional_platforms - run: | - result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") - if [[ ${result} != "build" ]] - then - echo "No recipes using this platform, skipping rest of job." - echo "skip_build=true" >> $GITHUB_OUTPUT - fi - - - name: set path - run: | - echo "/opt/mambaforge/bin" >> $GITHUB_PATH - - - name: Fetch conda install script - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - - name: Set up bioconda-utils - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash install-and-set-up-conda.sh - - - name: Configure conda - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash configure-conda.sh - - - name: Build and Upload - if: steps.additional_platforms.outputs.skip_build != 'true' - env: - QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }} - QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Mimic circleci - OSTYPE: "darwin" - CI: "true" - run: | - set -xe - eval "$(conda shell.bash hook)" - conda activate bioconda - source common.sh - # Sets up OSX SDK - run_conda_forge_build_setup - bioconda-utils handle-merged-pr recipes config.yml \ - --repo bioconda/bioconda-recipes \ - --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ - --fallback build \ - --artifact-source github-actions + --repo bioconda/bioconda-recipes \ + --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ + --fallback build \ + --artifact-source github-actions + + + # Disabled due to concurrency limits on GHA for OSX. osx-arm64 builds are on CircleCI. + # build_and_upload-osx-arm64: + # name: OSX-ARM64 Upload + # if: github.repository == 'bioconda/bioconda-recipes' + # runs-on: macOS-14 # M1 + # strategy: + # fail-fast: false + # max-parallel: 4 + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + + # # bail if there's no osx-arm64 recipes + # - name: Check for Additional Platforms + # id: additional_platforms + # run: | + # result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") + # if [[ ${result} != "build" ]] + # then + # echo "No recipes using this platform, skipping rest of job." + # echo "skip_build=true" >> $GITHUB_OUTPUT + # fi + + # - name: set path + # run: | + # echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + # - name: Fetch conda install script + # if: steps.additional_platforms.outputs.skip_build != 'true' + # run: | + # wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + # - name: Set up bioconda-utils + # if: steps.additional_platforms.outputs.skip_build != 'true' + # run: bash install-and-set-up-conda.sh + + # - name: Configure conda + # if: steps.additional_platforms.outputs.skip_build != 'true' + # run: bash configure-conda.sh + + # - name: Build and Upload + # if: steps.additional_platforms.outputs.skip_build != 'true' + # env: + # QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }} + # QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} + # ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + # INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # # Mimic circleci + # OSTYPE: "darwin" + # CI: "true" + # run: | + # set -xe + # eval "$(conda shell.bash hook)" + # conda activate bioconda + # source common.sh + # # Sets up OSX SDK + # run_conda_forge_build_setup + + # bioconda-utils handle-merged-pr recipes config.yml \ + # --repo bioconda/bioconda-recipes \ + # --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ + # --fallback build \ + # --artifact-source github-actions diff --git a/azure-pipeline-master.yml b/azure-pipeline-master.yml index 68f8bd157bc35..904519f9f3082 100644 --- a/azure-pipeline-master.yml +++ b/azure-pipeline-master.yml @@ -1,6 +1,8 @@ +# Disable Azure checks in favor of GitHub Actions due to memory issues pr: none -trigger: - - master +trigger: none +# trigger: +# - master jobs: - job: "build_and_push_linux" @@ -39,11 +41,7 @@ jobs: bioconda-utils handle-merged-pr recipes config.yml \ --repo bioconda/bioconda-recipes \ --git-range $(Build.SourceVersion)~1 $(Build.SourceVersion) \ - --quay-upload-target biocontainers \ - --fallback build - #bioconda-utils build recipes config.yml \ - # --git-range $(Build.SourceVersion)~1 $(Build.SourceVersion) \ - # --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers + --quay-upload-target biocontainers env: QUAY_LOGIN: $(QUAY_LOGIN) diff --git a/azure-pipeline.yml b/azure-pipeline.yml index 10df61e7f2a5e..f1d0863fc6fcc 100644 --- a/azure-pipeline.yml +++ b/azure-pipeline.yml @@ -1,10 +1,13 @@ -pr: - autoCancel: true -trigger: - branches: - exclude: - - master - - "*" +# Disable Azure checks in favor of GitHub Actions due to memory issues +pr: none +# pr: +# autoCancel: true +trigger: none +# trigger: + # branches: + # exclude: + # - master + # - "*" stages: - stage: "lint" From c6af4ecd26777841b08cf57698538a22d993c768 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 28 Oct 2024 11:52:13 -0400 Subject: [PATCH 2/2] try removing pr:none --- azure-pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipeline.yml b/azure-pipeline.yml index f1d0863fc6fcc..f39c53d4110f9 100644 --- a/azure-pipeline.yml +++ b/azure-pipeline.yml @@ -1,5 +1,4 @@ # Disable Azure checks in favor of GitHub Actions due to memory issues -pr: none # pr: # autoCancel: true trigger: none