Merge branch 'master' into patch-3

.circleci/config.yml

@@ -239,6 +239,52 @@ jobs: # a basic unit of work in a run
               --skiplist-leafs
             conda clean -y --all
 
+  nightly_build:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - 1e:85:74:42:35:5f:c5:a2:35:c2:ec:b7:80:c0:7c:40
+
+      - checkout
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt/
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      # For now, do not upload ARM images to biocontainers: --mulled-upload-target biocontainers
+      - run:
+          name: Build and push leftover packages
+          command: |
+            set -e
+            eval "$(conda shell.bash hook)"
+            conda activate bioconda
+            source common.sh
+
+            # build and push all leftover packages
+            bioconda-utils build recipes config.yml \
+                --docker --mulled-test  --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \
+                --anaconda-upload \
+                --prelint --exclude bioconda-repodata-patches
 
 workflows:
   build and test (ARM):
@@ -267,7 +313,6 @@ workflows:
                 - osx-arm64
                 - linux-aarch64
 
-
   Bulk branch (ARM):
     jobs:
       - bulk_build:
@@ -280,3 +325,21 @@ workflows:
                 # - osx-arm64 Bulk is on GitHub Actions
                 - linux-aarch64
               runner: [0, 1, 2, 3, 4, 5]
+
+  Nightly (ARM):
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only: master
+    jobs:
+      - nightly_build:
+          filters:
+            branches:
+              only: master
+          matrix:
+            parameters:
+              os:
+                # - osx-arm64 Nightly is on GitHub Actions
+                - linux-aarch64

.github/workflows/Bulk.yml

@@ -128,9 +128,9 @@ jobs:
     runs-on: macOS-14 # M1
     strategy:
       fail-fast: false
-      max-parallel: 4
+      max-parallel: 1 # GHA free plan allows 5 concurrent Mac runners total, we still need most on osx-64.
       matrix:
-        runner: [0, 1, 2, 3]
+        runner: [0]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -179,7 +179,7 @@ jobs:
           python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")'
           echo '============'
           bioconda-utils build recipes config.yml \
-            --worker-offset ${{ matrix.runner }} --n-workers 4 \
+            --worker-offset ${{ matrix.runner }} --n-workers 1 \
             --lint --anaconda-upload --record-build-failures --skiplist-leafs \
             --exclude bioconda-repodata-patches
           conda clean -y --all

.github/workflows/CommentResponder.yml

@@ -1,28 +1,14 @@
 name: CommentResponder
 on:
-  status:
-  # check_suite:
-  #   types:
-  #   - completed
   issue_comment:
     types:
     - created
 
-    # Runs too many times
-    # (
-    #   github.event_name == 'check_suite' &&
-    #   github.event.check_suite.conclusion == 'success'
-    # ) ||
-
 jobs:
   comment:
     runs-on: ubuntu-latest
     name: bioconda-bot comment
     if: >-
-      (
-        github.event_name == 'status' &&
-        github.event.state == 'success'
-      ) ||
       (
         github.event_name == 'issue_comment' &&
         github.event.issue.pull_request &&

.github/workflows/nightly.yml

@@ -1,76 +1,17 @@
 name: Nightly Uploader
 on:
   schedule:
-    - cron: '12 2 * * *'
+    - cron: "0 0 * * *"
 jobs:
-  build-linux:
-    name: Linux Upload
+  nightly-osx-arm:
+    name: Nightly OSX-ARM64 Builds
     if: github.repository == 'bioconda/bioconda-recipes'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      max-parallel: 13
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: set path
-        run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH
-
-      - name: Fetch conda install script
-        run: |
-          wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh
-
-      - name: Restore cache
-        id: cache
-        uses: actions/cache@v3
-        with:
-          path: /opt/mambaforge
-          key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }}
-
-      - name: Set up bioconda-utils
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: bash install-and-set-up-conda.sh
-
-      # This script can be used to reconfigure conda to use the right channel setup.
-      # This has to be done after the cache is restored, because
-      # the channel setup is not cached as it resides in the home directory.
-      # We could use a system-wide (and therefore cached) channel setup,
-      # but mamba does not support that at the time of implementation
-      # (it ignores settings made with --system).
-      - name: Configure conda
-        run: bash configure-conda.sh
-
-      - name: Build and upload
-        env:
-          QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }}
-          QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }}
-          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
-          INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # Mimic circleci
-          OSTYPE: "linux-gnu"
-          CI: "true"
-        run: |
-          set -ex
-          eval "$(conda shell.bash hook)"
-          conda activate bioconda
-          docker pull quay.io/dpryan79/mulled_container:latest
-          bioconda-utils build recipes config.yml \
-              --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers \
-              --prelint
-          docker rmi quay.io/dpryan79/mulled_container:latest
-
-  build-osx:
-    name: OSX Tests
-    if: github.repository == 'bioconda/bioconda-recipes'
-    runs-on: macos-13
+    runs-on: macOS-14 # M1
     strategy:
       fail-fast: false
       max-parallel: 4
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -81,27 +22,13 @@ jobs:
         run: |
           wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh
 
-      - name: Restore cache
-        id: cache
-        uses: actions/cache@v3
-        with:
-          path: /opt/mambaforge
-          key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }}
-
       - name: Set up bioconda-utils
-        if: steps.cache.outputs.cache-hit != 'true'
         run: bash install-and-set-up-conda.sh
 
-      # This script can be used to reconfigure conda to use the right channel setup.
-      # This has to be done after the cache is restored, because
-      # the channel setup is not cached as it resides in the home directory.
-      # We could use a system-wide (and therefore cached) channel setup,
-      # but mamba does not support that at the time of implementation
-      # (it ignores settings made with --system).
       - name: Configure conda
         run: bash configure-conda.sh
 
-      - name: Build and Test
+      - name: Build and push leftover packages
         env:
           QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }}
           QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }}
@@ -116,9 +43,11 @@ jobs:
           eval "$(conda shell.bash hook)"
           conda activate bioconda
 
-          # The SDK isn't actually cached, so reinstall it
+          source common.sh
+          # Sets up OSX SDK
           run_conda_forge_build_setup
 
+          # build and push all leftover packages
           bioconda-utils build recipes config.yml \
               --anaconda-upload \
-              --prelint
+              --prelint --exclude bioconda-repodata-patches

README.md

@@ -20,3 +20,12 @@ Please visit https://bioconda.github.io for details.
 
 Please visit the new docs at https://bioconda.github.io/contributor/index.html for details.
 
+## Nightly build status
+The nightly uploader jobs build any recipes that exist on master but were not successfully uploaded to the [bioconda channel](https://anaconda.org/bioconda). Any failure in the nightly build should be resolved with a PR for the affected recipe.
+
+| arch | build status |
+|------|--------------|
+| linux-64 | [![Nightly linux-64](https://dev.azure.com/bioconda/bioconda-recipes/_apis/build/status/Nightly%20uploader?branchName=master&jobName=build_and_push_linux&label=Nightly%20linux-64)](https://dev.azure.com/bioconda/bioconda-recipes/_build/latest?definitionId=4) |
+| osx-64 | [![Nightly osx-64](https://dev.azure.com/bioconda/bioconda-recipes/_apis/build/status/Nightly%20uploader?branchName=master&jobName=build_and_push_osx&label=Nightly%20osx-64)](https://dev.azure.com/bioconda/bioconda-recipes/_build/latest?definitionId=4) |
+| osx-arm64 | [![Nightly osx-arm64](https://github.com/bioconda/bioconda-recipes/actions/workflows/nightly.yml/badge.svg)](https://github.com/bioconda/bioconda-recipes/actions/workflows/nightly.yml) |
+| linux-aarch64 |[![CircleCI](https://dl.circleci.com/insights-snapshot/gh/bioconda/bioconda-recipes/master/Nightly%20(ARM)/badge.svg?window=24h)](https://app.circleci.com/insights/github/bioconda/bioconda-recipes/workflows/Nightly%20(ARM)/overview?branch=master&reporting-window=last-24-hours) |