From e8e485e34ad9659dff82af5339124bb169d02e8a Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 16:08:35 -0500 Subject: [PATCH 01/10] Add option to use mamba --- .github/workflows/test.yml | 14 ++++-- conda.bzl | 31 ++++++++++-- defs.bzl | 6 ++- docs/docs/usage/example.md | 4 ++ env.bzl | 42 +++++++++++++--- example/WORKSPACE | 14 ++++-- tests/miniconda/{ => conda}/.bazeliskrc | 0 tests/miniconda/{ => conda}/BUILD | 0 tests/miniconda/{ => conda}/WORKSPACE | 2 +- tests/miniconda/{ => conda}/environment.yml | 0 tests/miniconda/{ => conda}/test.py | 0 tests/miniconda/mamba/.bazeliskrc | 1 + tests/{miniforge => miniconda/mamba}/BUILD | 0 tests/miniconda/mamba/WORKSPACE | 49 ++++++++++++++++++ .../mamba}/environment.yml | 0 tests/{miniforge => miniconda/mamba}/test.py | 0 tests/miniforge/conda/BUILD | 7 +++ tests/miniforge/{ => conda}/WORKSPACE | 0 tests/miniforge/conda/environment.yml | 6 +++ tests/miniforge/conda/test.py | 11 ++++ tests/miniforge/mamba/BUILD | 7 +++ tests/miniforge/mamba/WORKSPACE | 50 +++++++++++++++++++ tests/miniforge/mamba/environment.yml | 6 +++ tests/miniforge/mamba/test.py | 11 ++++ 24 files changed, 239 insertions(+), 22 deletions(-) rename tests/miniconda/{ => conda}/.bazeliskrc (100%) rename tests/miniconda/{ => conda}/BUILD (100%) rename tests/miniconda/{ => conda}/WORKSPACE (97%) rename tests/miniconda/{ => conda}/environment.yml (100%) rename tests/miniconda/{ => conda}/test.py (100%) create mode 100644 tests/miniconda/mamba/.bazeliskrc rename tests/{miniforge => miniconda/mamba}/BUILD (100%) create mode 100644 tests/miniconda/mamba/WORKSPACE rename tests/{miniforge => miniconda/mamba}/environment.yml (100%) rename tests/{miniforge => miniconda/mamba}/test.py (100%) create mode 100644 tests/miniforge/conda/BUILD rename tests/miniforge/{ => conda}/WORKSPACE (100%) create mode 100644 tests/miniforge/conda/environment.yml create mode 100644 tests/miniforge/conda/test.py create mode 100644 tests/miniforge/mamba/BUILD create mode 100644 tests/miniforge/mamba/WORKSPACE create mode 100644 tests/miniforge/mamba/environment.yml create mode 100644 tests/miniforge/mamba/test.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06f7076..c47ac0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,9 +58,15 @@ jobs: with: path: ${{ env.BAZEL_CACHE_DIR }} key: ${{ runner.os }}-bazel-${{ env.CACHE_NUMBER }} - - name: Run miniconda tests - working-directory: tests/miniconda + - name: Test miniconda with conda + working-directory: tests/miniconda/conda run: bazel test --test_output=all --test_arg=--verbose --test_arg=-rA ... - - name: Run miniforge tests - working-directory: tests/miniforge + - name: Test miniconda with mamba + working-directory: tests/miniconda/mamba + run: bazel test --test_output=all --test_arg=--verbose --test_arg=-rA ... + - name: Test miniforge with conda + working-directory: tests/miniforge/conda + run: bazel test --test_output=all --test_arg=--verbose --test_arg=-rA ... + - name: Test miniforge with mamba + working-directory: tests/miniforge/mamba run: bazel test --test_output=all --test_arg=--verbose --test_arg=-rA ... diff --git a/conda.bzl b/conda.bzl index 375d236..40f0193 100644 --- a/conda.bzl +++ b/conda.bzl @@ -110,9 +110,21 @@ def _install_conda(rctx, installer): fail("Failure installing conda.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) return "{}/condabin/conda{}".format(rctx.attr.conda_dir, CONDA_EXT_MAP[os]) +def _install_mamba(rctx, executable): + rctx.report_progress("Installing mamba") + mamba_with_version = "mamba={}".format(rctx.attr.mamba_version) + + # `-n base` is necessary so that mamba is installed to the conda in the bazel cache. + # If we omit `-n base`, then mamba can get installed to the user's environment i.e. ~/anaconda3 which breaks + # the hermetic nature of the build. + args = [rctx.path(executable), "install", "-n", "base", "-c", "conda-forge", mamba_with_version, "-y"] + result = rctx.execute(args, quiet = rctx.attr.quiet, working_directory = rctx.attr.conda_dir, timeout = rctx.attr.timeout) + if result.return_code: + fail("Failure when installing mamba.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) + # use conda to update itself def _update_conda(rctx, executable): - conda_with_version = "conda={}".format(rctx.attr.version) + conda_with_version = "conda={}".format(rctx.attr.conda_version) args = [rctx.path(executable), "install", conda_with_version, "-y"] # update conda itself @@ -130,21 +142,30 @@ def _create_conda_build_file(rctx, executable): def _load_conda_impl(rctx): installer = _download_conda(rctx) - executable = _install_conda(rctx, installer) - _update_conda(rctx, executable) - _create_conda_build_file(rctx, executable) + conda_executable = _install_conda(rctx, installer) + _update_conda(rctx, conda_executable) + if rctx.attr.install_mamba: + _install_mamba(rctx, conda_executable) + _create_conda_build_file(rctx, conda_executable) load_conda_rule = repository_rule( _load_conda_impl, attrs = { "conda_dir": attr.string(mandatory = True), - "version": attr.string( + "conda_version": attr.string( mandatory = True, doc = "Conda version to install", ), "installer": attr.string( default = "miniconda", doc = 'Installer to use, either "miniconda" or "miniforge". Note that miniconda and miniforge have different OS/arch support.', + "install_mamba": attr.bool( + default = False, + doc = "False if mamba should not be installed", + ), + "mamba_version": attr.string( + mandatory = True, + doc = "Mamba version to install", ), "quiet": attr.bool( default = True, diff --git a/defs.bzl b/defs.bzl index 036c96f..d8623b7 100644 --- a/defs.bzl +++ b/defs.bzl @@ -9,14 +9,16 @@ DEFAULT_CONDA_VERSION = "4.10.3" DEFAULT_ENV_NAME = "my_env" DEFAULT_TOOLCHAIN_REPO_NAME = "conda_tools" DEFAULT_TOOLCHAIN_NAME = "python_toolchain" +DEFAULT_MAMBA_VERSION = "0.17.0" # download and install conda -def load_conda(version = DEFAULT_CONDA_VERSION, **kwargs): +def load_conda(conda_version = DEFAULT_CONDA_VERSION, mamba_version = DEFAULT_MAMBA_VERSION, **kwargs): maybe( load_conda_rule, CONDA_REPO_NAME, conda_dir = CONDA_DIR, - version = version, + conda_version = conda_version, + mamba_version = mamba_version, **kwargs ) diff --git a/docs/docs/usage/example.md b/docs/docs/usage/example.md index 18966f9..d53d4ca 100644 --- a/docs/docs/usage/example.md +++ b/docs/docs/usage/example.md @@ -29,10 +29,13 @@ http_archive( load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") +USE_MAMBA = False + load_conda( installer = "miniconda", # "miniconda" or "miniforge", defaults to "miniconda" quiet = False, # use True to hide conda output version = "4.10.3", # optional, defaults to 4.10.3 + install_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda ) conda_create( @@ -41,6 +44,7 @@ conda_create( clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//:environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output + use_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda ) register_toolchain( diff --git a/env.bzl b/env.bzl index e32884d..41302fb 100644 --- a/env.bzl +++ b/env.bzl @@ -13,6 +13,19 @@ py_runtime( ) """ +def _label_from_condabin(rctx, exe_name): + return Label("@{}//:{}/condabin/{}{}".format( + rctx.attr.conda_repo, + rctx.attr.conda_dir, + exe_name, + CONDA_EXT_MAP[get_os(rctx)], + )) + +def _user_chosen_executable(rctx): + if rctx.attr.use_mamba: + return _label_from_condabin(rctx, "mamba") + return _label_from_condabin(rctx, "conda") + # clean conda caches and unused packages def _clean(rctx, executable): rctx.report_progress("Cleaning up") @@ -23,18 +36,30 @@ def _clean(rctx, executable): if result.return_code: fail("Failure cleaning up.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) +def _create_empty_environment(rctx, executable, env_name): + rctx.report_progress("Creating empty conda environment, to be populated afterwards") + args = [rctx.path(executable), "create", "-y", "-p", "./{}".format(env_name)] + result = rctx.execute(args, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout) + if result.return_code: + fail("Failure creating empty environment.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) + +def _update_environment(rctx, executable, env_name, env_file): + rctx.report_progress("Updating empty conda environment to populate it") + args = [rctx.path(executable), "env", "update", "-f", env_file, "-p", "./{}".format(env_name)] + result = rctx.execute(args, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout) + if result.return_code: + fail("Failure updating environment.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) + # create new local conda environment from file def _create_environment(rctx, executable, env_name): rctx.report_progress("Creating conda environment") + executable = _user_chosen_executable(rctx) # path to env file as string env_file = str(rctx.path(rctx.attr.environment)) - args = [rctx.path(executable), "env", "create", "-f", env_file, "-p", "./{}".format(env_name)] - - result = rctx.execute(args, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout) - if result.return_code: - fail("Failure creating environment.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) + _create_empty_environment(rctx, executable, env_name) + _update_environment(rctx, executable, env_name, env_file) # check if python2 or python3 has been installed def _get_py_major(rctx, env_path, interpreter_path): @@ -59,8 +84,7 @@ def _create_env_build_file(rctx, env_name): ) def _conda_create_impl(rctx): - conda_label = Label("@{}//:{}/condabin/conda{}".format(rctx.attr.conda_repo, rctx.attr.conda_dir, CONDA_EXT_MAP[get_os(rctx)])) - executable = str(rctx.path(conda_label)) + executable = _user_chosen_executable(rctx) env_name = rctx.name _create_environment(rctx, executable, env_name) if rctx.attr.clean: @@ -81,6 +105,10 @@ conda_create_rule = repository_rule( default = True, doc = "False if conda output should be shown", ), + "use_mamba": attr.bool( + default = False, + doc = "True if mamba should be used", + ), "timeout": attr.int( default = EXECUTE_TIMEOUT, doc = "Timeout in seconds for each execute action", diff --git a/example/WORKSPACE b/example/WORKSPACE index ee56897..9649a91 100644 --- a/example/WORKSPACE +++ b/example/WORKSPACE @@ -60,28 +60,36 @@ http_archive( load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") +USE_MAMBA = True + +CLEAN = True + # download and install conda load_conda( + conda_version = "4.10.3", # optional, defaults to 4.10.3 + install_mamba = USE_MAMBA, + mamba_version = "0.17.0", # optional, defaults to 0.17.0 quiet = False, # use True to hide conda output - version = "4.10.3", # optional, defaults to 4.10.3 ) # create environment with python2 conda_create( name = "py2_env", timeout = 600, # each execute action can take up to 600 seconds - clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) + clean = CLEAN, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//third_party/conda:py2_environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output + use_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda ) # create environment with python3 conda_create( name = "py3_env", timeout = 600, # each execute action can take up to 600 seconds - clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) + clean = CLEAN, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//third_party/conda:py3_environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output + use_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda ) # register pythons from environment as toolchain diff --git a/tests/miniconda/.bazeliskrc b/tests/miniconda/conda/.bazeliskrc similarity index 100% rename from tests/miniconda/.bazeliskrc rename to tests/miniconda/conda/.bazeliskrc diff --git a/tests/miniconda/BUILD b/tests/miniconda/conda/BUILD similarity index 100% rename from tests/miniconda/BUILD rename to tests/miniconda/conda/BUILD diff --git a/tests/miniconda/WORKSPACE b/tests/miniconda/conda/WORKSPACE similarity index 97% rename from tests/miniconda/WORKSPACE rename to tests/miniconda/conda/WORKSPACE index 572854a..5912fd0 100644 --- a/tests/miniconda/WORKSPACE +++ b/tests/miniconda/conda/WORKSPACE @@ -25,7 +25,7 @@ http_archive( local_repository( name = "rules_conda", - path = "../../", + path = "../../../", ) load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") diff --git a/tests/miniconda/environment.yml b/tests/miniconda/conda/environment.yml similarity index 100% rename from tests/miniconda/environment.yml rename to tests/miniconda/conda/environment.yml diff --git a/tests/miniconda/test.py b/tests/miniconda/conda/test.py similarity index 100% rename from tests/miniconda/test.py rename to tests/miniconda/conda/test.py diff --git a/tests/miniconda/mamba/.bazeliskrc b/tests/miniconda/mamba/.bazeliskrc new file mode 100644 index 0000000..f81d3f7 --- /dev/null +++ b/tests/miniconda/mamba/.bazeliskrc @@ -0,0 +1 @@ +USE_BAZEL_VERSION=4.2.1 diff --git a/tests/miniforge/BUILD b/tests/miniconda/mamba/BUILD similarity index 100% rename from tests/miniforge/BUILD rename to tests/miniconda/mamba/BUILD diff --git a/tests/miniconda/mamba/WORKSPACE b/tests/miniconda/mamba/WORKSPACE new file mode 100644 index 0000000..ea85137 --- /dev/null +++ b/tests/miniconda/mamba/WORKSPACE @@ -0,0 +1,49 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +RULES_PYTHON_NAME = "rules_python" + +RULES_PYTHON_TAG = "0.4.0" + +RULES_PYTHON_SHA = "954aa89b491be4a083304a2cb838019c8b8c3720a7abb9c4cb81ac7a24230cea" + +RULES_PYTHON_REPO = "bazelbuild" + +RULES_PYTHON_ARCHIVE = "tar.gz" + +RULES_PYTHON_URL = "https://github.com/{repo}/{name}/releases/download/{tag}/{name}-{tag}.{archive}".format( + name = RULES_PYTHON_NAME, + archive = RULES_PYTHON_ARCHIVE, + repo = RULES_PYTHON_REPO, + tag = RULES_PYTHON_TAG, +) + +http_archive( + name = RULES_PYTHON_NAME, + sha256 = RULES_PYTHON_SHA, + url = RULES_PYTHON_URL, +) + +local_repository( + name = "rules_conda", + path = "../../../", +) + +load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") + +load_conda( + install_mamba = True, + quiet = False, +) + +conda_create( + name = "test_env", + timeout = 600, + clean = False, + environment = "@//:environment.yml", + quiet = False, + use_mamba = True, +) + +register_toolchain( + py3_env = "test_env", +) diff --git a/tests/miniforge/environment.yml b/tests/miniconda/mamba/environment.yml similarity index 100% rename from tests/miniforge/environment.yml rename to tests/miniconda/mamba/environment.yml diff --git a/tests/miniforge/test.py b/tests/miniconda/mamba/test.py similarity index 100% rename from tests/miniforge/test.py rename to tests/miniconda/mamba/test.py diff --git a/tests/miniforge/conda/BUILD b/tests/miniforge/conda/BUILD new file mode 100644 index 0000000..4e484ce --- /dev/null +++ b/tests/miniforge/conda/BUILD @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_test") + +py_test( + name = "test", + size = "small", + srcs = ["test.py"], +) diff --git a/tests/miniforge/WORKSPACE b/tests/miniforge/conda/WORKSPACE similarity index 100% rename from tests/miniforge/WORKSPACE rename to tests/miniforge/conda/WORKSPACE diff --git a/tests/miniforge/conda/environment.yml b/tests/miniforge/conda/environment.yml new file mode 100644 index 0000000..daf9b5e --- /dev/null +++ b/tests/miniforge/conda/environment.yml @@ -0,0 +1,6 @@ +channels: + - default + - conda-forge +dependencies: + - python==3.9.7 + - pytest==6.2.4 diff --git a/tests/miniforge/conda/test.py b/tests/miniforge/conda/test.py new file mode 100644 index 0000000..08f1efb --- /dev/null +++ b/tests/miniforge/conda/test.py @@ -0,0 +1,11 @@ +import sys + +import pytest + + +def test_correct_pytest_version(): + assert pytest.__version__ == "6.2.4" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__] + sys.argv[1:])) diff --git a/tests/miniforge/mamba/BUILD b/tests/miniforge/mamba/BUILD new file mode 100644 index 0000000..4e484ce --- /dev/null +++ b/tests/miniforge/mamba/BUILD @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_test") + +py_test( + name = "test", + size = "small", + srcs = ["test.py"], +) diff --git a/tests/miniforge/mamba/WORKSPACE b/tests/miniforge/mamba/WORKSPACE new file mode 100644 index 0000000..3043971 --- /dev/null +++ b/tests/miniforge/mamba/WORKSPACE @@ -0,0 +1,50 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +RULES_PYTHON_NAME = "rules_python" + +RULES_PYTHON_TAG = "0.4.0" + +RULES_PYTHON_SHA = "954aa89b491be4a083304a2cb838019c8b8c3720a7abb9c4cb81ac7a24230cea" + +RULES_PYTHON_REPO = "bazelbuild" + +RULES_PYTHON_ARCHIVE = "tar.gz" + +RULES_PYTHON_URL = "https://github.com/{repo}/{name}/releases/download/{tag}/{name}-{tag}.{archive}".format( + name = RULES_PYTHON_NAME, + archive = RULES_PYTHON_ARCHIVE, + repo = RULES_PYTHON_REPO, + tag = RULES_PYTHON_TAG, +) + +http_archive( + name = RULES_PYTHON_NAME, + sha256 = RULES_PYTHON_SHA, + url = RULES_PYTHON_URL, +) + +local_repository( + name = "rules_conda", + path = "../../", +) + +load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") + +load_conda( + installer = "miniforge", + install_mamba = True, + quiet = False, +) + +conda_create( + name = "test_env", + timeout = 600, + use_mamba = True, + clean = False, + environment = "@//:environment.yml", + quiet = False, +) + +register_toolchain( + py3_env = "test_env", +) diff --git a/tests/miniforge/mamba/environment.yml b/tests/miniforge/mamba/environment.yml new file mode 100644 index 0000000..daf9b5e --- /dev/null +++ b/tests/miniforge/mamba/environment.yml @@ -0,0 +1,6 @@ +channels: + - default + - conda-forge +dependencies: + - python==3.9.7 + - pytest==6.2.4 diff --git a/tests/miniforge/mamba/test.py b/tests/miniforge/mamba/test.py new file mode 100644 index 0000000..08f1efb --- /dev/null +++ b/tests/miniforge/mamba/test.py @@ -0,0 +1,11 @@ +import sys + +import pytest + + +def test_correct_pytest_version(): + assert pytest.__version__ == "6.2.4" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__] + sys.argv[1:])) From f06f0f84658310201e09fc2cbacd826aa61b140d Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 16:11:22 -0500 Subject: [PATCH 02/10] Use boolean literals in examples --- example/WORKSPACE | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/example/WORKSPACE b/example/WORKSPACE index 9649a91..5a81c07 100644 --- a/example/WORKSPACE +++ b/example/WORKSPACE @@ -60,14 +60,10 @@ http_archive( load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") -USE_MAMBA = True - -CLEAN = True - # download and install conda load_conda( conda_version = "4.10.3", # optional, defaults to 4.10.3 - install_mamba = USE_MAMBA, + install_mamba = False, # use True to install mamba, which a faster drop-in replacement for conda mamba_version = "0.17.0", # optional, defaults to 0.17.0 quiet = False, # use True to hide conda output ) @@ -76,20 +72,20 @@ load_conda( conda_create( name = "py2_env", timeout = 600, # each execute action can take up to 600 seconds - clean = CLEAN, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) + clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//third_party/conda:py2_environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output - use_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda + use_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda ) # create environment with python3 conda_create( name = "py3_env", timeout = 600, # each execute action can take up to 600 seconds - clean = CLEAN, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) + clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//third_party/conda:py3_environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output - use_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda + use_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda ) # register pythons from environment as toolchain From 8fb69cc921d826c4900523ece570ea43767e4c9e Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 16:20:41 -0500 Subject: [PATCH 03/10] update docs --- docs/docs/usage/api.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/docs/usage/api.md b/docs/docs/usage/api.md index ce6042f..35f4a97 100644 --- a/docs/docs/usage/api.md +++ b/docs/docs/usage/api.md @@ -8,12 +8,13 @@ **Parameters:** - | Name | Description | Default | - | ------------- | ------------------------------------------------------------------------------------------ | ---------------------- | - | `installer` | Which `conda` installer to download, either `miniconda` or `miniforge` | `miniconda` | - | `version` | Version of `conda` to download | `4.10.3` | - | `quiet` | `True` if `conda` output should be hidden | `True` | - | `timeout` | How many seconds each execute action can take | `3600` | + | Name | Description | Default | + | --------------- | ------------------------------------------------------------------------------------------ | ---------------------- | + | `installer` | Which `conda` installer to download, either `miniconda` or `miniforge` | `miniconda` | + | `install_mamba` | Whether to install mamba, which is a faster drop-in replacement for conda | `miniconda` | + | `version` | Version of `conda` to download | `4.10.3` | + | `quiet` | `True` if `conda` output should be hidden | `True` | + | `timeout` | How many seconds each execute action can take | `3600` | ## `conda_create` @@ -23,13 +24,14 @@ **Parameters:** - | Name | Description | Default | - | ------------- | ------------------------------------------------------------------------------------------ | ---------------------- | - | `environment` | label pointing to environment configuration file (typically named `environment.yml`) | | - | `name` | Name of the environment | `my_env` | - | `quiet` | `True` if `conda` output should be hidden | `True` | - | `timeout` | How many seconds each execute action can take | `3600` | - | `clean` | `True` if `conda` cache should be cleaned (less space taken, but slower subsequent builds) | `False` | + | Name | Description | Default | + | ------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------- | + | `environment` | label pointing to environment configuration file (typically named `environment.yml`) | | + | `name` | Name of the environment | `my_env` | + | `quiet` | `True` if `conda` output should be hidden | `True` | + | `timeout` | How many seconds each execute action can take | `3600` | + | `clean` | `True` if `conda` cache should be cleaned (less space taken, but slower subsequent builds) | `False` | + | `use_mamba` | Whether to use mamba to create the conda environment. If this is `True`, `install_mamba` must also be `True` | `False` | ## `register_toolchain` From 6358a78b3f76bf4f741e1639955c8f33b9944a40 Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 16:23:01 -0500 Subject: [PATCH 04/10] fix --- docs/docs/usage/example.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/docs/usage/example.md b/docs/docs/usage/example.md index d53d4ca..ff3ad76 100644 --- a/docs/docs/usage/example.md +++ b/docs/docs/usage/example.md @@ -29,13 +29,11 @@ http_archive( load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") -USE_MAMBA = False - load_conda( installer = "miniconda", # "miniconda" or "miniforge", defaults to "miniconda" quiet = False, # use True to hide conda output version = "4.10.3", # optional, defaults to 4.10.3 - install_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda + install_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda ) conda_create( @@ -44,7 +42,7 @@ conda_create( clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//:environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output - use_mamba = USE_MAMBA, # use True to use mamba, which a faster drop-in replacement for conda + use_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda ) register_toolchain( From f2f6644a21e9d98b46c7fd86936b86bb242822e8 Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 16:24:11 -0500 Subject: [PATCH 05/10] fix --- conda.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/conda.bzl b/conda.bzl index 40f0193..e003f37 100644 --- a/conda.bzl +++ b/conda.bzl @@ -159,6 +159,7 @@ load_conda_rule = repository_rule( "installer": attr.string( default = "miniconda", doc = 'Installer to use, either "miniconda" or "miniforge". Note that miniconda and miniforge have different OS/arch support.', + ), "install_mamba": attr.bool( default = False, doc = "False if mamba should not be installed", From 58bc9694700477a4e8dd2d444bf45ae71eaff858 Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 16:27:36 -0500 Subject: [PATCH 06/10] fix --- docs/docs/usage/api.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs/usage/api.md b/docs/docs/usage/api.md index 35f4a97..cf4d00d 100644 --- a/docs/docs/usage/api.md +++ b/docs/docs/usage/api.md @@ -11,7 +11,7 @@ | Name | Description | Default | | --------------- | ------------------------------------------------------------------------------------------ | ---------------------- | | `installer` | Which `conda` installer to download, either `miniconda` or `miniforge` | `miniconda` | - | `install_mamba` | Whether to install mamba, which is a faster drop-in replacement for conda | `miniconda` | + | `install_mamba` | Whether to install `mamba`, which is a faster drop-in replacement for `conda` | `miniconda` | | `version` | Version of `conda` to download | `4.10.3` | | `quiet` | `True` if `conda` output should be hidden | `True` | | `timeout` | How many seconds each execute action can take | `3600` | @@ -24,14 +24,14 @@ **Parameters:** - | Name | Description | Default | - | ------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------- | - | `environment` | label pointing to environment configuration file (typically named `environment.yml`) | | - | `name` | Name of the environment | `my_env` | - | `quiet` | `True` if `conda` output should be hidden | `True` | - | `timeout` | How many seconds each execute action can take | `3600` | - | `clean` | `True` if `conda` cache should be cleaned (less space taken, but slower subsequent builds) | `False` | - | `use_mamba` | Whether to use mamba to create the conda environment. If this is `True`, `install_mamba` must also be `True` | `False` | + | Name | Description | Default | + | ------------- | ---------------------------------------------------------------------------------------------------------------- | ---------------------- | + | `environment` | label pointing to environment configuration file (typically named `environment.yml`) | | + | `name` | Name of the environment | `my_env` | + | `quiet` | `True` if `conda` output should be hidden | `True` | + | `timeout` | How many seconds each execute action can take | `3600` | + | `clean` | `True` if `conda` cache should be cleaned (less space taken, but slower subsequent builds) | `False` | + | `use_mamba` | Whether to use `mamba` to create the `conda` environment. If this is `True`, `install_mamba` must also be `True` | `False` | ## `register_toolchain` From bc7b3f7f349d82f28e03240ded0abdb27229ca4a Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 16:33:23 -0500 Subject: [PATCH 07/10] fix --- tests/miniforge/conda/WORKSPACE | 2 +- tests/miniforge/mamba/WORKSPACE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/miniforge/conda/WORKSPACE b/tests/miniforge/conda/WORKSPACE index 613c7f9..1a08ea3 100644 --- a/tests/miniforge/conda/WORKSPACE +++ b/tests/miniforge/conda/WORKSPACE @@ -25,7 +25,7 @@ http_archive( local_repository( name = "rules_conda", - path = "../../", + path = "../../../", ) load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") diff --git a/tests/miniforge/mamba/WORKSPACE b/tests/miniforge/mamba/WORKSPACE index 3043971..99a6047 100644 --- a/tests/miniforge/mamba/WORKSPACE +++ b/tests/miniforge/mamba/WORKSPACE @@ -25,7 +25,7 @@ http_archive( local_repository( name = "rules_conda", - path = "../../", + path = "../../../", ) load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") From 7a2448da35153370f8deb12df6978090988597a5 Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 17:52:59 -0500 Subject: [PATCH 08/10] resolve executable earlier --- env.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/env.bzl b/env.bzl index 41302fb..2998139 100644 --- a/env.bzl +++ b/env.bzl @@ -30,7 +30,7 @@ def _user_chosen_executable(rctx): def _clean(rctx, executable): rctx.report_progress("Cleaning up") - args = [rctx.path(executable), "clean", "-a", "-y"] + args = [executable, "clean", "-a", "-y"] result = rctx.execute(args, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout) if result.return_code: @@ -38,14 +38,14 @@ def _clean(rctx, executable): def _create_empty_environment(rctx, executable, env_name): rctx.report_progress("Creating empty conda environment, to be populated afterwards") - args = [rctx.path(executable), "create", "-y", "-p", "./{}".format(env_name)] + args = [executable, "create", "-y", "-p", "./{}".format(env_name)] result = rctx.execute(args, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout) if result.return_code: fail("Failure creating empty environment.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) def _update_environment(rctx, executable, env_name, env_file): rctx.report_progress("Updating empty conda environment to populate it") - args = [rctx.path(executable), "env", "update", "-f", env_file, "-p", "./{}".format(env_name)] + args = [executable, "env", "update", "-f", env_file, "-p", "./{}".format(env_name)] result = rctx.execute(args, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout) if result.return_code: fail("Failure updating environment.\nstdout: {}\nstderr: {}".format(result.stdout, result.stderr)) @@ -54,7 +54,7 @@ def _update_environment(rctx, executable, env_name, env_file): def _create_environment(rctx, executable, env_name): rctx.report_progress("Creating conda environment") - executable = _user_chosen_executable(rctx) + executable = rctx.path(_user_chosen_executable(rctx)) # path to env file as string env_file = str(rctx.path(rctx.attr.environment)) From fa0c621df55339ad402e25fb47fd27f9610fbb1d Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 18:16:06 -0500 Subject: [PATCH 09/10] fix docs and formatting --- conda.bzl | 2 +- docs/docs/usage/api.md | 5 +++-- env.bzl | 1 + example/WORKSPACE | 6 +++--- tests/miniforge/mamba/WORKSPACE | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/conda.bzl b/conda.bzl index e003f37..423877a 100644 --- a/conda.bzl +++ b/conda.bzl @@ -159,7 +159,7 @@ load_conda_rule = repository_rule( "installer": attr.string( default = "miniconda", doc = 'Installer to use, either "miniconda" or "miniforge". Note that miniconda and miniforge have different OS/arch support.', - ), + ), "install_mamba": attr.bool( default = False, doc = "False if mamba should not be installed", diff --git a/docs/docs/usage/api.md b/docs/docs/usage/api.md index cf4d00d..e8701fc 100644 --- a/docs/docs/usage/api.md +++ b/docs/docs/usage/api.md @@ -10,9 +10,10 @@ | Name | Description | Default | | --------------- | ------------------------------------------------------------------------------------------ | ---------------------- | + | `conda_version` | Version of `conda` to download | `4.10.3` | | `installer` | Which `conda` installer to download, either `miniconda` or `miniforge` | `miniconda` | - | `install_mamba` | Whether to install `mamba`, which is a faster drop-in replacement for `conda` | `miniconda` | - | `version` | Version of `conda` to download | `4.10.3` | + | `install_mamba` | Whether to install `mamba`, which is a faster drop-in replacement for `conda` | `False` | + | `mamba_version` | Version of `conda` to install | `0.17.0` | | `quiet` | `True` if `conda` output should be hidden | `True` | | `timeout` | How many seconds each execute action can take | `3600` | diff --git a/env.bzl b/env.bzl index 2998139..53afe9a 100644 --- a/env.bzl +++ b/env.bzl @@ -55,6 +55,7 @@ def _create_environment(rctx, executable, env_name): rctx.report_progress("Creating conda environment") executable = rctx.path(_user_chosen_executable(rctx)) + # path to env file as string env_file = str(rctx.path(rctx.attr.environment)) diff --git a/example/WORKSPACE b/example/WORKSPACE index 5a81c07..e3d2fe5 100644 --- a/example/WORKSPACE +++ b/example/WORKSPACE @@ -63,7 +63,7 @@ load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchai # download and install conda load_conda( conda_version = "4.10.3", # optional, defaults to 4.10.3 - install_mamba = False, # use True to install mamba, which a faster drop-in replacement for conda + install_mamba = False, # use True to install mamba, which a faster drop-in replacement for conda mamba_version = "0.17.0", # optional, defaults to 0.17.0 quiet = False, # use True to hide conda output ) @@ -75,7 +75,7 @@ conda_create( clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//third_party/conda:py2_environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output - use_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda + use_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda ) # create environment with python3 @@ -85,7 +85,7 @@ conda_create( clean = False, # use True if you want to clean conda cache (less space taken, but slower subsequent builds) environment = "@//third_party/conda:py3_environment.yml", # label pointing to environment.yml file quiet = False, # use True to hide conda output - use_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda + use_mamba = False, # use True to use mamba, which a faster drop-in replacement for conda ) # register pythons from environment as toolchain diff --git a/tests/miniforge/mamba/WORKSPACE b/tests/miniforge/mamba/WORKSPACE index 99a6047..92b1239 100644 --- a/tests/miniforge/mamba/WORKSPACE +++ b/tests/miniforge/mamba/WORKSPACE @@ -31,18 +31,18 @@ local_repository( load("@rules_conda//:defs.bzl", "conda_create", "load_conda", "register_toolchain") load_conda( - installer = "miniforge", install_mamba = True, + installer = "miniforge", quiet = False, ) conda_create( name = "test_env", timeout = 600, - use_mamba = True, clean = False, environment = "@//:environment.yml", quiet = False, + use_mamba = True, ) register_toolchain( From ed7fdd8ea7cda81c247b62fd7b8ab53f811a80d7 Mon Sep 17 00:00:00 2001 From: Gabriel Dougherty Date: Thu, 11 Nov 2021 18:17:57 -0500 Subject: [PATCH 10/10] don't shadow executable and try w/o str() --- env.bzl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/env.bzl b/env.bzl index 53afe9a..da3c3bb 100644 --- a/env.bzl +++ b/env.bzl @@ -54,10 +54,8 @@ def _update_environment(rctx, executable, env_name, env_file): def _create_environment(rctx, executable, env_name): rctx.report_progress("Creating conda environment") - executable = rctx.path(_user_chosen_executable(rctx)) - # path to env file as string - env_file = str(rctx.path(rctx.attr.environment)) + env_file = rctx.path(rctx.attr.environment) _create_empty_environment(rctx, executable, env_name) _update_environment(rctx, executable, env_name, env_file)