From da90958923931aa6508936caf11788d12a275791 Mon Sep 17 00:00:00 2001 From: regro-cf-autotick-bot <36490558+regro-cf-autotick-bot@users.noreply.github.com> Date: Wed, 26 May 2021 15:47:49 +0000 Subject: [PATCH 1/2] migration: OSXArm --- conda-forge.yml | 2 ++ recipe/meta.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/conda-forge.yml b/conda-forge.yml index a685653..9ea4417 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -1,2 +1,4 @@ +build_platform: {osx_arm64: osx_64} conda_forge_output_validation: true provider: {linux_ppc64le: azure} +test_on_native_only: true diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c8704f3..91c8957 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -15,6 +15,8 @@ build: requirements: build: + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] - {{ compiler("c") }} - {{ compiler('cxx') }} host: From 0d371600da4f404072543ccc23f8ba0cabf4dc2e Mon Sep 17 00:00:00 2001 From: regro-cf-autotick-bot <36490558+regro-cf-autotick-bot@users.noreply.github.com> Date: Wed, 26 May 2021 15:48:00 +0000 Subject: [PATCH 2/2] MNT: Re-rendered with conda-build 3.21.4, conda-smithy 3.10.1, and conda-forge-pinning 2021.05.25.06.47.12 --- .azure-pipelines/azure-pipelines-osx.yml | 6 ++ .ci_support/migrations/hdf51106.yaml | 7 -- .ci_support/migrations/python39.yaml | 26 ------- .../osx_arm64_python3.8.____cpython.yaml | 29 ++++++++ .../osx_arm64_python3.9.____cpython.yaml | 29 ++++++++ .scripts/build_steps.sh | 31 +++++++- .scripts/logging_utils.sh | 35 ++++++++++ .scripts/run_docker_build.sh | 19 +++-- .scripts/run_osx_build.sh | 70 ++++++++++++------- LICENSE.txt | 2 +- README.md | 19 ++++- build-locally.py | 27 ++++++- 12 files changed, 227 insertions(+), 73 deletions(-) delete mode 100644 .ci_support/migrations/hdf51106.yaml delete mode 100644 .ci_support/migrations/python39.yaml create mode 100644 .ci_support/osx_arm64_python3.8.____cpython.yaml create mode 100644 .ci_support/osx_arm64_python3.9.____cpython.yaml create mode 100644 .scripts/logging_utils.sh diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index 8aadc55..d3420bb 100755 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -20,6 +20,12 @@ jobs: osx_64_python3.9.____cpython: CONFIG: osx_64_python3.9.____cpython UPLOAD_PACKAGES: 'True' + osx_arm64_python3.8.____cpython: + CONFIG: osx_arm64_python3.8.____cpython + UPLOAD_PACKAGES: 'True' + osx_arm64_python3.9.____cpython: + CONFIG: osx_arm64_python3.9.____cpython + UPLOAD_PACKAGES: 'True' timeoutInMinutes: 360 steps: diff --git a/.ci_support/migrations/hdf51106.yaml b/.ci_support/migrations/hdf51106.yaml deleted file mode 100644 index dc5b3f1..0000000 --- a/.ci_support/migrations/hdf51106.yaml +++ /dev/null @@ -1,7 +0,0 @@ -__migrator: - build_number: 1 - kind: version - migration_number: 1 -hdf5: -- 1.10.6 -migrator_ts: 1587001869.689825 diff --git a/.ci_support/migrations/python39.yaml b/.ci_support/migrations/python39.yaml deleted file mode 100644 index 3eff5c7..0000000 --- a/.ci_support/migrations/python39.yaml +++ /dev/null @@ -1,26 +0,0 @@ -migrator_ts: 1602104489 -__migrator: - migration_number: 2 - operation: key_add - primary_key: python - ordering: - python: - - 3.6.* *_cpython - - 3.7.* *_cpython - - 3.8.* *_cpython - - 3.9.* *_cpython # new entry - - 3.6.* *_73_pypy - paused: false - pr_limit: 50 - exclude: - # this shouldn't attempt to modify the python feedstocks - - python - - pypy3.6 - - pypy-meta -python: - - 3.9.* *_cpython -# additional entries to add for zip_keys -numpy: - - 1.19 -python_impl: - - cpython diff --git a/.ci_support/osx_arm64_python3.8.____cpython.yaml b/.ci_support/osx_arm64_python3.8.____cpython.yaml new file mode 100644 index 0000000..c541452 --- /dev/null +++ b/.ci_support/osx_arm64_python3.8.____cpython.yaml @@ -0,0 +1,29 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge/label/rust_dev,conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +hdf5: +- 1.10.6 +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/.ci_support/osx_arm64_python3.9.____cpython.yaml b/.ci_support/osx_arm64_python3.9.____cpython.yaml new file mode 100644 index 0000000..33dd9df --- /dev/null +++ b/.ci_support/osx_arm64_python3.9.____cpython.yaml @@ -0,0 +1,29 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge/label/rust_dev,conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +hdf5: +- 1.10.6 +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index aa9727b..99baf86 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -6,8 +6,15 @@ # benefit from the improvement. set -xeuo pipefail -export PYTHONUNBUFFERED=1 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +source ${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh + + +( endgroup "Start Docker" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null + +export PYTHONUNBUFFERED=1 export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" @@ -18,8 +25,9 @@ conda-build: root-dir: ${FEEDSTOCK_ROOT}/build_artifacts CONDARC +BUILD_CMD=build -conda install --yes --quiet "conda-forge-ci-setup=3" conda-build pip -c conda-forge +conda install --yes --quiet "conda-forge-ci-setup=3" conda-build pip ${GET_BOA:-} -c conda-forge # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" @@ -29,6 +37,12 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" +if [[ "${HOST_PLATFORM}" != "${BUILD_PLATFORM}" ]] && [[ "${BUILD_WITH_CONDA_DEBUG:-0}" != 1 ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" +fi + + +( endgroup "Configuring conda" ) 2> /dev/null if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then @@ -37,17 +51,28 @@ if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then conda debug "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + # Drop into an interactive shell /bin/bash else - conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + conda $BUILD_CMD "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + ( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" + ( endgroup "Validating outputs" ) 2> /dev/null + + ( startgroup "Uploading packages" ) 2> /dev/null + if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" fi + + ( endgroup "Uploading packages" ) 2> /dev/null fi +( startgroup "Final checks" ) 2> /dev/null + touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh new file mode 100644 index 0000000..57bc95c --- /dev/null +++ b/.scripts/logging_utils.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Provide a unified interface for the different logging +# utilities CI providers offer. If unavailable, provide +# a compatible fallback (e.g. bare `echo xxxxxx`). + +function startgroup { + # Start a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[group]$1";; + travis ) + echo "$1" + echo -en 'travis_fold:start:'"${1// /}"'\\r';; + github_actions ) + echo "::group::$1";; + * ) + echo "$1";; + esac +} 2> /dev/null + +function endgroup { + # End a foldable group of log lines + # Pass a single argument, quoted + + case ${CI:-} in + azure ) + echo "##[endgroup]";; + travis ) + echo -en 'travis_fold:end:'"${1// /}"'\\r';; + github_actions ) + echo "::endgroup::";; + esac +} 2> /dev/null diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 36dacd6..244ccd1 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -5,6 +5,10 @@ # changes to this script, consider a proposal to conda-smithy so that other feedstocks can also # benefit from the improvement. +source .scripts/logging_utils.sh + +( startgroup "Configure Docker" ) 2> /dev/null + set -xeo pipefail THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" @@ -48,11 +52,11 @@ if [ -z "${DOCKER_IMAGE}" ]; then echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Trying to parse with coreutils" DOCKER_IMAGE=$(cat .ci_support/${CONFIG}.yaml | grep '^docker_image:$' -A 1 | tail -n 1 | cut -b 3-) if [ "${DOCKER_IMAGE}" = "" ]; then - echo "No docker_image entry found in ${CONFIG}. Falling back to condaforge/linux-anvil-comp7" - DOCKER_IMAGE="condaforge/linux-anvil-comp7" + echo "No docker_image entry found in ${CONFIG}. Falling back to quay.io/condaforge/linux-anvil-comp7" + DOCKER_IMAGE="quay.io/condaforge/linux-anvil-comp7" fi else - DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" + DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 quay.io/condaforge/linux-anvil-comp7 )" fi fi @@ -66,6 +70,10 @@ if [ -z "${CI}" ]; then DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" fi +( endgroup "Configure Docker" ) 2> /dev/null + +( startgroup "Start Docker" ) 2> /dev/null + export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ @@ -88,4 +96,7 @@ docker run ${DOCKER_RUN_ARGS} \ /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh # verify that the end of the script was reached -test -f "$DONE_CANARY" \ No newline at end of file +test -f "$DONE_CANARY" + +# This closes the last group opened in `build_steps.sh` +( endgroup "Final checks" ) 2> /dev/null \ No newline at end of file diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index c299b79..708b1e1 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -1,57 +1,73 @@ #!/usr/bin/env bash -set -x +source .scripts/logging_utils.sh + +set -xe + +MINIFORGE_HOME=${MINIFORGE_HOME:-${HOME}/miniforge3} + +( startgroup "Installing a fresh version of Miniforge" ) 2> /dev/null -echo -e "\n\nInstalling a fresh version of Miniforge." -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:start:install_miniforge\\r' -fi MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" -bash $MINIFORGE_FILE -b -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:end:install_miniforge\\r' -fi +bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME} -echo -e "\n\nConfiguring conda." -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:start:configure_conda\\r' -fi +( endgroup "Installing a fresh version of Miniforge" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null + +BUILD_CMD=build -source ${HOME}/miniforge3/etc/profile.d/conda.sh +source ${MINIFORGE_HOME}/etc/profile.d/conda.sh conda activate base echo -e "\n\nInstalling conda-forge-ci-setup=3 and conda-build." -conda install -n base --quiet --yes "conda-forge-ci-setup=3" conda-build pip +conda install -n base --quiet --yes "conda-forge-ci-setup=3" conda-build pip ${GET_BOA:-} echo -e "\n\nSetting up the condarc and mangling the compiler." setup_conda_rc ./ ./recipe ./.ci_support/${CONFIG}.yaml -mangle_compiler ./ ./recipe .ci_support/${CONFIG}.yaml -echo -e "\n\nMangling homebrew in the CI to avoid conflicts." -/usr/bin/sudo mangle_homebrew -/usr/bin/sudo -k +if [[ "${CI:-}" != "" ]]; then + mangle_compiler ./ ./recipe .ci_support/${CONFIG}.yaml +fi + +if [[ "${CI:-}" != "" ]]; then + echo -e "\n\nMangling homebrew in the CI to avoid conflicts." + /usr/bin/sudo mangle_homebrew + /usr/bin/sudo -k +else + echo -e "\n\nNot mangling homebrew as we are not running in CI" +fi echo -e "\n\nRunning the build setup script." source run_conda_forge_build_setup -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:end:configure_conda\\r' -fi -set -e +( endgroup "Configuring conda" ) 2> /dev/null + -echo -e "\n\nMaking the build clobber file and running the build." +echo -e "\n\nMaking the build clobber file" make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml -conda build ./recipe -m ./.ci_support/${CONFIG}.yaml --suppress-variables --clobber-file ./.ci_support/clobber_${CONFIG}.yaml ${EXTRA_CB_OPTIONS:-} +if [[ "${HOST_PLATFORM}" != "${BUILD_PLATFORM}" ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" +fi + +conda $BUILD_CMD ./recipe -m ./.ci_support/${CONFIG}.yaml --suppress-variables --clobber-file ./.ci_support/clobber_${CONFIG}.yaml ${EXTRA_CB_OPTIONS:-} +( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" +( endgroup "Validating outputs" ) 2> /dev/null + +( startgroup "Uploading packages" ) 2> /dev/null + if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - echo -e "\n\nUploading the packages." upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" ./ ./recipe ./.ci_support/${CONFIG}.yaml -fi \ No newline at end of file +fi + +( endgroup "Uploading packages" ) 2> /dev/null \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt index 5f30279..ed3f451 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ BSD 3-clause license -Copyright (c) 2015-2020, conda-forge contributors +Copyright (c) 2015-2021, conda-forge contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index a987176..8d81b5d 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,20 @@ Current build status variant + + osx_arm64_python3.8.____cpython + + + variant + + + + osx_arm64_python3.9.____cpython + + + variant + + win_64_python3.6.____cpython @@ -160,6 +174,7 @@ Installing `hdf5plugin` from the `conda-forge` channel can be achieved by adding ``` conda config --add channels conda-forge +conda config --set channel_priority strict ``` Once the `conda-forge` channel has been enabled, `hdf5plugin` can be installed with: @@ -230,9 +245,9 @@ build distinct package versions. In order to produce a uniquely identifiable distribution: * If the version of a package **is not** being increased, please add or increase - the [``build/number``](https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#build-number-and-string). + the [``build/number``](https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#build-number-and-string). * If the version of a package **is** being increased, please remember to return - the [``build/number``](https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#build-number-and-string) + the [``build/number``](https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#build-number-and-string) back to 0. Feedstock Maintainers diff --git a/build-locally.py b/build-locally.py index 3453cfe..00eeb34 100755 --- a/build-locally.py +++ b/build-locally.py @@ -7,6 +7,7 @@ import glob import subprocess from argparse import ArgumentParser +import platform def setup_environment(ns): @@ -16,6 +17,10 @@ def setup_environment(ns): os.environ["BUILD_WITH_CONDA_DEBUG"] = "1" if ns.output_id: os.environ["BUILD_OUTPUT_ID"] = ns.output_id + if "MINIFORGE_HOME" not in os.environ: + os.environ["MINIFORGE_HOME"] = os.path.join( + os.path.dirname(__file__), "miniforge3" + ) def run_docker_build(ns): @@ -23,6 +28,11 @@ def run_docker_build(ns): subprocess.check_call([script]) +def run_osx_build(ns): + script = ".scripts/run_osx_build.sh" + subprocess.check_call([script]) + + def verify_config(ns): valid_configs = { os.path.basename(f)[:-5] for f in glob.glob(".ci_support/*.yaml") @@ -46,10 +56,16 @@ def verify_config(ns): else: raise ValueError("config " + ns.config + " is not valid") # Remove the following, as implemented - if not ns.config.startswith("linux"): + if ns.config.startswith("win"): raise ValueError( - f"only Linux configs currently supported, got {ns.config}" + f"only Linux/macOS configs currently supported, got {ns.config}" ) + elif ns.config.startswith("osx") and platform.system() == "Darwin": + if "OSX_SDK_DIR" not in os.environ: + raise RuntimeError( + "Need OSX_SDK_DIR env variable set. Run 'export OSX_SDK_DIR=/opt'" + "to download the SDK automatically to '/opt/MacOSX.sdk'" + ) def main(args=None): @@ -68,7 +84,12 @@ def main(args=None): verify_config(ns) setup_environment(ns) - run_docker_build(ns) + if ns.config.startswith("linux") or ( + ns.config.startswith("osx") and platform.system() == "Linux" + ): + run_docker_build(ns) + elif ns.config.startswith("osx"): + run_osx_build(ns) if __name__ == "__main__":