upstream

.circleci/config.yml

@@ -20,9 +20,22 @@ variables:
     run:
       name: Setup bioconda-utils
       command: .circleci/setup.sh
+  setup_macos_sdk: &setup_macos_sdk
+    run:
+      name: Run run_conda_forge_build_setup
+      command: |
+        # download SDK and setup sysroot on macOS
+        #  primarily, run_conda_forge_build_setup does the following:
+        #   - download/setup SDK:
+        #     https://github.com/conda-forge/conda-forge-ci-setup-feedstock/blob/a1026adb523b6562c16329170e7e304a25ed4033/recipe/run_conda_forge_build_setup_osx#L21-L26
+        #   - add activation script to set MACOSX_DEPLOYMENT_TARGET and CONDA_BUILD_SYSROOT:
+        #     https://github.com/conda-forge/conda-forge-ci-setup-feedstock/blob/a1026adb523b6562c16329170e7e304a25ed4033/recipe/run_conda_forge_build_setup_osx#L60-L63
+        conda activate base
+        # use "CONFIG=" to avoid writing ./.ci_support/${CONFIG}.yaml which we don't need/use.
+        CONFIG= OSX_FORCE_SDK_DOWNLOAD=1 run_conda_forge_build_setup
   macos: &macos
     macos:
-      xcode: "8.3.3"
+      xcode: 9.4.1
     environment:
       LC_ALL: en_US.UTF-8
       LANG: en_US.UTF-8
@@ -51,7 +64,8 @@ variables:
       when: always
       no_output_timeout: 20m
       command: |
-        set +e  # Just to be safe. Don't want workflows to fail because of this.
+        (
+        # Run in subshell just to be safe. Don't want workflows to fail because of this.
         mkdir -p /tmp/artifacts/packages
         cd miniconda/conda-bld || exit 0
         find -name .cache | xargs rm -rf || true
@@ -61,11 +75,16 @@ variables:
         if command -V docker >/dev/null; then
           mkdir -p /tmp/artifacts/images
           cd /tmp/artifacts/images
-          docker image ls --format '{{.Repository}}:{{.Tag}}' | \
-            (grep biocontainers || true) | \
-            xargs -n1 -P4 bash -c 'docker save $1 | gzip -c > "${1##*/}.tar.gz"' --
+          docker image ls --format='{{.Repository}}:{{.Tag}}' | \
+            { grep biocontainers || true ; } | \
+            xargs -n1 -P4 bash -c '
+              test -n "${1+x}" || exit 0
+              echo "Start compressing docker image ${1} ..."
+              docker save "${1}" | gzip -c > "${1##*/}.tar.gz"
+              echo "Finished compressing docker image ${1} ."
+              ' --
         fi
-        true
+        ) || true
 
   store_artifacts: &store_artifacts
     store_artifacts:
@@ -130,9 +149,11 @@ jobs:
       - run:
           name: Building and testing
           command: |
+            docker pull quay.io/dpryan79/mulled_container:latest
             bioconda-utils build recipes config.yml \
               --docker --mulled-test \
               --git-range master HEAD
+            docker rmi quay.io/dpryan79/mulled_container:latest
       - *store_built_packages
       - *store_artifacts
 
@@ -145,6 +166,7 @@ jobs:
       - *restore_cache
       - *setup
       - *save_cache
+      - *setup_macos_sdk
       - run:
           name: Building and testing
           command: |
@@ -181,6 +203,7 @@ jobs:
       - *restore_cache
       - *setup
       - *save_cache
+      - *setup_macos_sdk
       # build only current commit (since we use squashed merging, this is safe)
       - run:
           name: Building, testing and uploading
@@ -216,6 +239,7 @@ jobs:
       - *restore_cache
       - *setup
       - *save_cache
+      - *setup_macos_sdk
       - run:
           name: Building, testing and uploading of all unpublished recipes
           command: |
@@ -300,21 +324,21 @@ workflows:
             branches:
               only:
                 - master
-  # workflow for bulk-updates
-  bioconda-upload-bulk-linux:
-    jobs:
-      - bulk-linux:
-          filters:
-            branches:
-              only:
-                - bulk
-  bioconda-upload-bulk-macos:
-    jobs:
-      - bulk-macos:
-          filters:
-            branches:
-              only:
-                - bulk
+  ## workflow for bulk-updates
+  #bioconda-upload-bulk-linux:
+  #  jobs:
+  #    - bulk-linux:
+  #        filters:
+  #          branches:
+  #            only:
+  #              - bulk
+  #bioconda-upload-bulk-macos:
+  #  jobs:
+  #    - bulk-macos:
+  #        filters:
+  #          branches:
+  #            only:
+  #              - bulk
   # nightly workflow to capture missed recipes
   bioconda-nightly-upload-linux:
      triggers:

.circleci/setup.sh

@@ -11,9 +11,17 @@ set -u
 # This file can be used to set BIOCONDA_UTILS_TAG and MINICONDA_VER.
 source .circleci/common.sh
 
+cat >> $BASH_ENV <<EOF
+
 # Set path
-echo "export PATH=\"$WORKSPACE/miniconda/bin:$PATH\"" >> $BASH_ENV
-source $BASH_ENV
+export PATH="${WORKSPACE}/miniconda/bin:${PATH}"
+
+if [ -f "${WORKSPACE}/miniconda/etc/profile.d/conda.sh" ] ; then
+    . "${WORKSPACE}/miniconda/etc/profile.d/conda.sh"
+fi
+EOF
+
+. $BASH_ENV
 
 # Make sure the CircleCI config is up to date.
 # add upstream as some semi-randomly named temporary remote to diff against
@@ -23,9 +31,21 @@ git fetch $UPSTREAM_REMOTE
 if ! git diff --quiet HEAD...$UPSTREAM_REMOTE/master -- .circleci/; then
     echo 'Your bioconda-recipes CI configuration is out of date.'
     echo 'Please update it to the latest version of the upstream master branch.'
-    echo 'You can do this, e.g., by running:'
+    echo ''
+    echo 'Have @BiocondaBot attempt to fix this by creating a comment on your PR:'
+    echo ''
+    echo '   @BiocondaBot update'
+    echo ''
+    echo 'Once the update commit has been created, update your local copy of'
+    echo 'your branch:'
+    echo ''
+    echo '  git pull'
+    echo ''
+    echo ''
+    echo 'You can also fix this manually, e.g., by running:'
     echo '  git fetch https://github.com/bioconda/bioconda-recipes.git master'
     echo '  git merge FETCH_HEAD'
+    echo ''
     exit 1
 fi
 git remote remove $UPSTREAM_REMOTE
@@ -48,23 +68,30 @@ if ! type bioconda-utils 2> /dev/null || [[ $BOOTSTRAP == "true" ]]; then
     fi
     curl -L -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-$MINICONDA_VER-$tag-x86_64.sh
     bash miniconda.sh -b -p $WORKSPACE/miniconda
+    . $BASH_ENV
+    conda activate base
 
     # step 2: setup channels
-    $WORKSPACE/miniconda/bin/conda config --system --add channels defaults
-    $WORKSPACE/miniconda/bin/conda config --system --add channels bioconda
-    $WORKSPACE/miniconda/bin/conda config --system --add channels conda-forge
+    conda config --system --add channels defaults
+    conda config --system --add channels bioconda
+    conda config --system --add channels conda-forge
 
     # step 3: install bioconda-utils
-    $WORKSPACE/miniconda/bin/conda install -y git pip --file https://raw.githubusercontent.com/bioconda/bioconda-utils/$BIOCONDA_UTILS_TAG/bioconda_utils/bioconda_utils-requirements.txt
-    $WORKSPACE/miniconda/bin/pip install git+https://github.com/bioconda/bioconda-utils.git@$BIOCONDA_UTILS_TAG
+    additional_packages='git pip'
+    if [[ $OSTYPE == darwin* ]]; then
+        # Pinned to 2.5.3 to make sure we don't get unexpected changes.
+        additional_packages="${additional_packages} conda-forge-ci-setup=2.5.3"
+    fi
+    conda install -y $additional_packages --file https://raw.githubusercontent.com/bioconda/bioconda-utils/$BIOCONDA_UTILS_TAG/bioconda_utils/bioconda_utils-requirements.txt
+    pip install git+https://github.com/bioconda/bioconda-utils.git@$BIOCONDA_UTILS_TAG
 
     # step 4: configure local channel
     mkdir -p $WORKSPACE/miniconda/conda-bld/{noarch,linux-64,osx-64}
-    $WORKSPACE/miniconda/bin/conda index $WORKSPACE/miniconda/conda-bld
-    $WORKSPACE/miniconda/bin/conda config --system --add channels file://$WORKSPACE/miniconda/conda-bld
+    conda index $WORKSPACE/miniconda/conda-bld
+    conda config --system --add channels file://$WORKSPACE/miniconda/conda-bld
 
     # step 5: cleanup
-    $WORKSPACE/miniconda/bin/conda clean -y --all
+    conda clean -y --all
     rm miniconda.sh
 fi
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,43 @@
-* [ ] I have read the [guidelines for bioconda recipes](https://bioconda.github.io/guidelines.html).
-* [ ] This PR adds a new recipe.
-* [ ] AFAIK, this recipe **is directly relevant to the biological sciences** (otherwise, please submit to the more general purpose [conda-forge channel](https://conda-forge.org/docs/)).
-* [ ] This PR updates an existing recipe.
-* [ ] This PR does something else (explain below).
+Describe your pull request here
+
+----
+
+Please read the [guidelines for Bioconda recipes](https://bioconda.github.io/contributor/guidelines.html) before opening a pull request (PR).
+
+* If this PR adds or updates a recipe, use "Add" or "Update" appropriately as the first word in its title.
+* New recipes not directly relevant to the biological sciences need to be submitted to the [conda-forge channel](https://conda-forge.org/docs/) instead of Bioconda.
+* PRs require reviews prior to being merged. Once your PR is passing tests and ready to be merged, please issue the `@BiocondaBot please add label` command.
+* Please post questions [on Gitter](https://gitter.im/bioconda/Lobby) or ping `@bioconda/core` in a comment.
+
+<details>
+  <summary>Please use the following BiocondaBot commands:</summary>
+
+Everyone has access to the following BiocondaBot commands, which can be given in a comment:
+
+<table>
+  <tr>
+    <td><code>@BiocondaBot please update</code></td>
+    <td>Merge the master branch into a PR.</td>
+  </tr>
+  <tr>
+    <td><code>@BiocondaBot please add label</code></td>
+    <td>Add the <code>please review & merge</code> label.</td>
+  </tr>
+  <tr>
+    <td><code>@BiocondaBot please fetch artifacts</code></td>
+    <td>Post links to CI-built packages/containers. <br />You can use this to test packages locally.</td>
+  </tr>
+</table>
+
+For members of the Bioconda project, the following command is also available:
+
+<table>
+  <tr>
+    <td><code>@BiocondaBot please merge</code></td>
+    <td>Upload built packages/containers and merge a PR. <br />Someone must approve a PR first! <br />This reduces CI build time by reusing built artifacts.</td>
+  </tr>
+</table>
+
+Also, the bot watches for comments from non-members that include `@bioconda/<team>` and will automatically re-post them to notify the addressed `<team>`.
+
+</details>

.github/welcome_new_contributor.md

@@ -1,4 +1,4 @@
-Welcome to Bioconda, @{{user}}! Thank you for opening your first pull request! As you're new to Bioconda, please read through our [documentation about contributing recipes](https://bioconda.github.io/contributing.html). If you need help, please ping one of the following teams:
+Welcome to Bioconda, @{{user}}! Thank you for opening your first pull request! As you're new to Bioconda, please read through our [documentation about contributing recipes](https://bioconda.github.io/contributor/guidelines.html). If you need help, please ping one of the following teams:
 
  - `@bioconda/haskel` for Haskell packages
  - `@bioconda/lib` for C/C++ packages

.github/workflows/Bulk.yml

@@ -0,0 +1,115 @@
+name: Bulk branch
+on:
+  push:
+    branches:
+      - bulk
+jobs:
+  build-linux:
+    name: Bulk Linux Builds
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 15
+      matrix:
+        runner: [0, 1, 2, 3, 4, 5, 6, 7]
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Fetch common.sh
+      run: |
+        wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/common.sh
+
+    - name: Restore cache
+      id: cache
+      uses: actions/cache@v1
+      with:
+        path: /home/runner/bioconda
+        key: ${{ runner.os }}-BULK-${{ hashFiles('**/common.sh') }}
+
+    - name: Setup Bioconda-utils
+      if: steps.cache.outputs.cache-hit != 'true'
+      uses: bioconda/bioconda-actions/bioconda_recipes_setup/@bulk
+
+    - name: Build and upload
+      env:
+        ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
+        INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }}
+        QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }}
+        # Mimic circleci
+        OSTYPE: "linux-gnu"
+        CI: "true"
+      run: |
+        . /home/runner/bioconda/etc/profile.d/conda.sh
+        conda activate base
+        echo '============'
+        conda info --all
+        conda config --show-sources
+        python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")'
+        echo '============'
+        bioconda-utils build recipes config.yml \
+          --worker-offset ${{ matrix.runner }} --n-workers 8 \
+          --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers
+        conda clean -y --all
+
+  build-osx:
+    name: Bulk OSX Builds
+    runs-on: macOS-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix:
+        runner: [0, 1, 2, 3, 4]
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Fetch common.sh
+      run: |
+        wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/common.sh
+
+    - name: Restore cache
+      id: cache
+      uses: actions/cache@v1
+      with:
+        path: /Users/runner/bioconda
+        key: ${{ runner.os }}--BULK-${{ hashFiles('**/common.sh') }}
+
+    - name: Setup Bioconda-utils
+      if: steps.cache.outputs.cache-hit != 'true'
+      uses: bioconda/bioconda-actions/bioconda_recipes_setup/@bulk
+
+    - name: OSX stuff
+      run: |
+        . /Users/runner/bioconda/etc/profile.d/conda.sh
+        conda activate base
+        # This is taken from conda-forge-ci-setup
+        curl -L -O https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.9.sdk.tar.xz
+        tar -xf MacOSX10.9.sdk.tar.xz -C /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs
+        # set minimum sdk version to our target
+        plutil -replace MinimumSDKVersion -string 10.9 $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist
+        plutil -replace DTSDKName -string macosx10.9internal $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist
+        export CONDA_PREFIX="${CONDA_PREFIX:-$(conda info --json | jq -r .root_prefix)}"
+        export CONDA_BUILD_SYSROOT="/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.9.sdk"
+        mkdir -p ${CONDA_PREFIX}/etc/conda/activate.d
+        echo "export CONDA_BUILD_SYSROOT=\"$CONDA_BUILD_SYSROOT\"" > "${CONDA_PREFIX}/etc/conda/activate.d/bioconda-ci-setup-activate.sh"
+        conda list
+
+    - name: Build and upload
+      env:
+        ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
+        INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }}
+        QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }}
+        # Mimic circleci
+        OSTYPE: "darwin"
+        CI: "true"
+      run: |
+        . /Users/runner/bioconda/etc/profile.d/conda.sh
+        conda activate base
+        echo '============'
+        conda info --all
+        conda config --show-sources
+        python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")'
+        echo '============'
+        bioconda-utils build recipes config.yml \
+          --worker-offset ${{ matrix.runner }} --n-workers 5 \
+          --anaconda-upload
+        conda clean -y --all