From 30f031baf53cfa313633ece34024570487f08ae5 Mon Sep 17 00:00:00 2001 From: Sophia Castellarin Date: Wed, 4 Dec 2024 08:23:15 -0800 Subject: [PATCH] Setup pluggy with locking plugin (#965) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jaimergp --- .../_internal/action/__init__.py | 1 - .../_internal/action/generate_lockfile.py | 69 +---------- .../_internal/plugins/__init__.py | 3 + .../_internal/plugins/lock/__init__.py | 3 + .../plugins/lock/conda_lock/__init__.py | 3 + .../plugins/lock/conda_lock/conda_lock.py | 97 +++++++++++++++ .../conda_store_server/_internal/utils.py | 21 +++- .../_internal/worker/build.py | 37 +++--- conda-store-server/conda_store_server/app.py | 29 +++++ .../conda_store_server/exception.py | 19 +++ .../conda_store_server/plugins/__init__.py | 9 ++ .../conda_store_server/plugins/hookspec.py | 22 ++++ .../plugins/plugin_context.py | 64 ++++++++++ .../plugins/plugin_manager.py | 40 +++++++ .../plugins/types/__init__.py | 3 + .../conda_store_server/plugins/types/lock.py | 33 ++++++ .../conda_store_server/plugins/types/types.py | 21 ++++ conda-store-server/pyproject.toml | 1 + .../tests/_internal/action/test_actions.py | 110 ----------------- .../tests/_internal/plugins/__init__.py | 3 + .../tests/_internal/plugins/lock/__init__.py | 3 + .../_internal/plugins/lock/test_conda_lock.py | 111 ++++++++++++++++++ conda-store-server/tests/conftest.py | 9 ++ conda-store-server/tests/plugins/__init__.py | 3 + .../tests/plugins/test_plugin_context.py | 61 ++++++++++ .../tests/plugins/test_plugin_manager.py | 41 +++++++ conda-store-server/tests/test_app.py | 17 +++ .../references/configuration-options.md | 3 + recipe/meta.yaml | 1 + 29 files changed, 637 insertions(+), 200 deletions(-) create mode 100644 conda-store-server/conda_store_server/_internal/plugins/__init__.py create mode 100644 conda-store-server/conda_store_server/_internal/plugins/lock/__init__.py create mode 100644 conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/__init__.py create mode 100644 conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py create mode 100644 conda-store-server/conda_store_server/exception.py create mode 100644 conda-store-server/conda_store_server/plugins/__init__.py create mode 100644 conda-store-server/conda_store_server/plugins/hookspec.py create mode 100644 conda-store-server/conda_store_server/plugins/plugin_context.py create mode 100644 conda-store-server/conda_store_server/plugins/plugin_manager.py create mode 100644 conda-store-server/conda_store_server/plugins/types/__init__.py create mode 100644 conda-store-server/conda_store_server/plugins/types/lock.py create mode 100644 conda-store-server/conda_store_server/plugins/types/types.py create mode 100644 conda-store-server/tests/_internal/plugins/__init__.py create mode 100644 conda-store-server/tests/_internal/plugins/lock/__init__.py create mode 100644 conda-store-server/tests/_internal/plugins/lock/test_conda_lock.py create mode 100644 conda-store-server/tests/plugins/__init__.py create mode 100644 conda-store-server/tests/plugins/test_plugin_context.py create mode 100644 conda-store-server/tests/plugins/test_plugin_manager.py diff --git a/conda-store-server/conda_store_server/_internal/action/__init__.py b/conda-store-server/conda_store_server/_internal/action/__init__.py index cc85ed974..40c787c42 100644 --- a/conda-store-server/conda_store_server/_internal/action/__init__.py +++ b/conda-store-server/conda_store_server/_internal/action/__init__.py @@ -26,7 +26,6 @@ ) from conda_store_server._internal.action.generate_lockfile import ( # noqa action_save_lockfile, - action_solve_lockfile, ) from conda_store_server._internal.action.get_conda_prefix_stats import ( # noqa action_get_conda_prefix_stats, diff --git a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py index f4824ba15..1ba492a8f 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py @@ -3,76 +3,9 @@ # license that can be found in the LICENSE file. import json -import os import pathlib -import typing -import yaml -from conda_lock.conda_lock import run_lock - -from conda_store_server._internal import action, conda_utils, schema, utils -from conda_store_server._internal.action.utils import logged_command - - -@action.action -def action_solve_lockfile( - context, - conda_command: str, - specification: schema.CondaSpecification, - platforms: typing.List[str] = [conda_utils.conda_platform()], - # Avoids package compatibility issues, see: - # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html - conda_flags: str = "--strict-channel-priority", -): - environment_filename = pathlib.Path.cwd() / "environment.yaml" - lockfile_filename = pathlib.Path.cwd() / "conda-lock.yaml" - - with environment_filename.open("w") as f: - json.dump(specification.model_dump(), f) - - context.log.info( - "Note that the output of `conda config --show` displayed below only reflects " - "settings in the conda configuration file, which might be overridden by " - "variables required to be set by conda-store via the environment. Overridden " - f"settings: CONDA_FLAGS={conda_flags}" - ) - - # The info command can be used with either mamba or conda - logged_command(context, [conda_command, "info"]) - # The config command is not supported by mamba - logged_command(context, ["conda", "config", "--show"]) - logged_command(context, ["conda", "config", "--show-sources"]) - - # conda-lock ignores variables defined in the specification, so this code - # gets the value of CONDA_OVERRIDE_CUDA and passes it to conda-lock via - # the with_cuda parameter, see: - # https://github.com/conda-incubator/conda-store/issues/719 - # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html#overriding-detected-packages - # TODO: Support all variables once upstream fixes are made to conda-lock, - # see the discussion in issue 719. - if specification.variables is not None: - cuda_version = specification.variables.get("CONDA_OVERRIDE_CUDA") - else: - cuda_version = None - - # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch - try: - conda_flags_name = "CONDA_FLAGS" - print(f"{conda_flags_name}={conda_flags}") - os.environ[conda_flags_name] = conda_flags - - run_lock( - environment_files=[environment_filename], - platforms=platforms, - lockfile_path=lockfile_filename, - conda_exe=conda_command, - with_cuda=cuda_version, - ) - finally: - os.environ.pop(conda_flags_name, None) - - with lockfile_filename.open() as f: - return yaml.safe_load(f) +from conda_store_server._internal import action, schema, utils @action.action diff --git a/conda-store-server/conda_store_server/_internal/plugins/__init__.py b/conda-store-server/conda_store_server/_internal/plugins/__init__.py new file mode 100644 index 000000000..b559bd2ff --- /dev/null +++ b/conda-store-server/conda_store_server/_internal/plugins/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/conda-store-server/conda_store_server/_internal/plugins/lock/__init__.py b/conda-store-server/conda_store_server/_internal/plugins/lock/__init__.py new file mode 100644 index 000000000..b559bd2ff --- /dev/null +++ b/conda-store-server/conda_store_server/_internal/plugins/lock/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/__init__.py b/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/__init__.py new file mode 100644 index 000000000..b559bd2ff --- /dev/null +++ b/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py b/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py new file mode 100644 index 000000000..85399d4f6 --- /dev/null +++ b/conda-store-server/conda_store_server/_internal/plugins/lock/conda_lock/conda_lock.py @@ -0,0 +1,97 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import json +import os +import pathlib +import typing + +import yaml +from conda_lock.conda_lock import run_lock + +from conda_store_server._internal import conda_utils, schema, utils +from conda_store_server.plugins.hookspec import hookimpl +from conda_store_server.plugins.plugin_context import PluginContext +from conda_store_server.plugins.types import lock, types + + +class CondaLock(lock.LockPlugin): + def _conda_command(self, conda_store) -> str: + with conda_store.session_factory() as db: + settings = conda_store.get_settings(db=db) + return settings.conda_command + + def _conda_flags(self, conda_store) -> str: + return conda_store.conda_flags + + @utils.run_in_tempdir + def lock_environment( + self, + context: PluginContext, + spec: schema.CondaSpecification, + platforms: typing.List[str] = [conda_utils.conda_platform()], + ) -> str: + context.log.info("lock_environment entrypoint for conda-lock") + conda_command = self._conda_command(context.conda_store) + conda_flags = self._conda_flags(context.conda_store) + + environment_filename = pathlib.Path.cwd() / "environment.yaml" + lockfile_filename = pathlib.Path.cwd() / "conda-lock.yaml" + + with environment_filename.open("w") as f: + json.dump(spec.dict(), f) + + context.log.info( + "Note that the output of `conda config --show` displayed below only reflects " + "settings in the conda configuration file, which might be overridden by " + "variables required to be set by conda-store via the environment. Overridden " + f"settings: CONDA_FLAGS={conda_flags}" + ) + + # The info command can be used with either mamba or conda + context.run_command([conda_command, "info"]) + # The config command is not supported by mamba + context.run_command(["conda", "config", "--show"]) + context.run_command(["conda", "config", "--show-sources"]) + + # conda-lock ignores variables defined in the specification, so this code + # gets the value of CONDA_OVERRIDE_CUDA and passes it to conda-lock via + # the with_cuda parameter, see: + # https://github.com/conda-incubator/conda-store/issues/719 + # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html#overriding-detected-packages + # TODO: Support all variables once upstream fixes are made to conda-lock, + # see the discussion in issue 719. + if spec.variables is not None: + cuda_version = spec.variables.get("CONDA_OVERRIDE_CUDA") + else: + cuda_version = None + + # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch + try: + conda_flags_name = "CONDA_FLAGS" + print(f"{conda_flags_name}={conda_flags}") + os.environ[conda_flags_name] = conda_flags + + run_lock( + environment_files=[environment_filename], + platforms=platforms, + lockfile_path=lockfile_filename, + conda_exe=conda_command, + with_cuda=cuda_version, + ) + finally: + os.environ.pop(conda_flags_name, None) + + with lockfile_filename.open() as f: + return yaml.safe_load(f) + + +@hookimpl +def lock_plugins(): + """conda-lock locking plugin""" + yield types.TypeLockPlugin( + name="conda-lock", + synopsis="Generate a lockfile using conda-lock", + backend=CondaLock, + ) diff --git a/conda-store-server/conda_store_server/_internal/utils.py b/conda-store-server/conda_store_server/_internal/utils.py index 1ad717202..4e7e06fc8 100644 --- a/conda-store-server/conda_store_server/_internal/utils.py +++ b/conda-store-server/conda_store_server/_internal/utils.py @@ -3,6 +3,7 @@ # license that can be found in the LICENSE file. import contextlib +import functools import hashlib import json import os @@ -10,8 +11,9 @@ import re import subprocess import sys +import tempfile import time -from typing import AnyStr +from typing import AnyStr, Callable from filelock import FileLock @@ -187,3 +189,20 @@ def compile_arn_sql_like( re.sub(r"\*", "%", match.group(1)), re.sub(r"\*", "%", match.group(2)), ) + + +def run_in_tempdir(f: Callable): + @functools.wraps(f) + def wrapper(*args, **kwargs): + with contextlib.ExitStack() as stack: + # create a temporary directory + tmpdir = stack.enter_context(tempfile.TemporaryDirectory()) + + # enter temporary directory + stack.enter_context(chdir(tmpdir)) + + # run function and store result + result = f(*args, **kwargs) + return result + + return wrapper diff --git a/conda-store-server/conda_store_server/_internal/worker/build.py b/conda-store-server/conda_store_server/_internal/worker/build.py index 50d4be82a..6bb72eac9 100644 --- a/conda-store-server/conda_store_server/_internal/worker/build.py +++ b/conda-store-server/conda_store_server/_internal/worker/build.py @@ -18,6 +18,7 @@ from conda_store_server import api from conda_store_server._internal import action, conda_utils, orm, schema, utils +from conda_store_server.plugins import plugin_context class LoggedStream: @@ -222,19 +223,20 @@ def build_conda_environment(db: Session, conda_store, build): prefix="action_save_lockfile: ", ), ) + conda_lock_spec = context.result else: - context = action.action_solve_lockfile( - settings.conda_command, - specification=schema.CondaSpecification.parse_obj( - build.specification.spec - ), + lock_backend, locker = conda_store.lock_plugin() + conda_lock_spec = locker.lock_environment( + spec=schema.CondaSpecification.parse_obj(build.specification.spec), platforms=settings.conda_solve_platforms, - conda_flags=conda_store.conda_flags, - stdout=LoggedStream( - db=db, + context=plugin_context.PluginContext( conda_store=conda_store, - build=build, - prefix="action_solve_lockfile: ", + stdout=LoggedStream( + db=db, + conda_store=conda_store, + build=build, + prefix=f"plugin-{lock_backend}: ", + ), ), ) @@ -243,14 +245,12 @@ def build_conda_environment(db: Session, conda_store, build): build.id, build.conda_lock_key, json.dumps( - context.result, indent=4, cls=utils.CustomJSONEncoder + conda_lock_spec, indent=4, cls=utils.CustomJSONEncoder ).encode("utf-8"), content_type="application/json", artifact_type=schema.BuildArtifactType.LOCKFILE, ) - conda_lock_spec = context.result - context = action.action_fetch_and_extract_conda_packages( conda_lock_spec=conda_lock_spec, pkgs_dir=conda_utils.conda_root_package_dir(), @@ -335,18 +335,15 @@ def build_conda_environment(db: Session, conda_store, build): def solve_conda_environment(db: Session, conda_store, solve: orm.Solve): - settings = conda_store.get_settings(db=db) - solve.started_on = datetime.datetime.utcnow() db.commit() - context = action.action_solve_lockfile( - conda_command=settings.conda_command, - specification=schema.CondaSpecification.parse_obj(solve.specification.spec), + _, locker = conda_store.lock_plugin() + conda_lock_spec = locker.lock_environment( + context=plugin_context.PluginContext(conda_store=conda_store), + spec=schema.CondaSpecification.parse_obj(solve.specification.spec), platforms=[conda_utils.conda_platform()], - conda_flags=conda_store.conda_flags, ) - conda_lock_spec = context.result action.action_add_lockfile_packages( db=db, diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index 226f913ea..ebc095861 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -28,6 +28,8 @@ from conda_store_server import CONDA_STORE_DIR, BuildKey, api, registry, storage from conda_store_server._internal import conda_utils, environment, orm, schema, utils +from conda_store_server.plugins import hookspec, plugin_manager +from conda_store_server.plugins.types import lock def conda_store_validate_specification( @@ -192,6 +194,12 @@ def _check_build_key_version(self, proposal): config=True, ) + lock_backend = Unicode( + default_value="conda-lock", + allow_none=False, + config=True, + ) + pypi_default_packages = List( [], help="PyPi packages that included by default if none are included", @@ -471,6 +479,27 @@ def celery_app(self): self._celery_app.config_from_object(self.celery_config) return self._celery_app + @property + def plugin_manager(self): + """Creates a plugin manager (if it doesn't already exist) and registers all plugins""" + if hasattr(self, "_plugin_manager"): + return self._plugin_manager + + self._plugin_manager = plugin_manager.PluginManager(hookspec.spec_name) + self._plugin_manager.add_hookspecs(hookspec.CondaStoreSpecs) + + # Register all available plugins + self._plugin_manager.collect_plugins() + + return self._plugin_manager + + def lock_plugin(self) -> tuple[str, lock.LockPlugin]: + """Returns the configured lock plugin""" + # TODO: get configured lock plugin name from settings + lock_plugin = self.plugin_manager.get_lock_plugin(name=self.lock_backend) + locker = lock_plugin.backend() + return lock_plugin.name, locker + def ensure_settings(self, db: Session): """Ensure that conda-store traitlets settings are applied""" settings = schema.Settings( diff --git a/conda-store-server/conda_store_server/exception.py b/conda-store-server/conda_store_server/exception.py new file mode 100644 index 000000000..2ff754650 --- /dev/null +++ b/conda-store-server/conda_store_server/exception.py @@ -0,0 +1,19 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + + +class CondaStoreError(Exception): + pass + + +class CondaStorePluginNotFoundError(CondaStoreError): + """Exception raised by conda store when a specified plugin is not found + Attributes: + plugin -- plugin that was not found + available_plugins -- list of registered plugins + """ + + def __init__(self, plugin, available_plugins): + self.message = f"Plugin {plugin} was requested but not found! The following plugins are available: {', '.join(available_plugins)}" + super().__init__(self.message) diff --git a/conda-store-server/conda_store_server/plugins/__init__.py b/conda-store-server/conda_store_server/plugins/__init__.py new file mode 100644 index 000000000..0aa1e4cca --- /dev/null +++ b/conda-store-server/conda_store_server/plugins/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from conda_store_server._internal.plugins.lock.conda_lock import conda_lock + +BUILTIN_PLUGINS = [ + conda_lock, +] diff --git a/conda-store-server/conda_store_server/plugins/hookspec.py b/conda-store-server/conda_store_server/plugins/hookspec.py new file mode 100644 index 000000000..7af80fdd2 --- /dev/null +++ b/conda-store-server/conda_store_server/plugins/hookspec.py @@ -0,0 +1,22 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from collections.abc import Iterable + +import pluggy + +from conda_store_server.plugins.types.lock import LockPlugin + +spec_name = "conda-store" +hookspec = pluggy.HookspecMarker(spec_name) +hookimpl = pluggy.HookimplMarker(spec_name) + + +class CondaStoreSpecs: + """Conda Store hookspecs""" + + @hookspec + def lock_plugins(self) -> Iterable[LockPlugin]: + """Lock spec""" + yield from () diff --git a/conda-store-server/conda_store_server/plugins/plugin_context.py b/conda-store-server/conda_store_server/plugins/plugin_context.py new file mode 100644 index 000000000..4a80e443a --- /dev/null +++ b/conda-store-server/conda_store_server/plugins/plugin_context.py @@ -0,0 +1,64 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import io +import logging +import subprocess +import uuid + + +class PluginContext: + """The plugin context provides some useful attributes to a hook. + + This includes + * the variables: conda_store, log, stdout, stderr + * the functions: run_command, run + """ + + def __init__( + self, + conda_store=None, + stdout=None, + stderr=None, + log_level=logging.INFO, + ): + if stdout is not None and stderr is None: + stderr = stdout + + self.id = str(uuid.uuid4()) + self.stdout = stdout if stdout is not None else io.StringIO() + self.stderr = stderr if stderr is not None else io.StringIO() + self.log = logging.getLogger( + f"conda_store_server.plugins.plugin_context.{self.id}" + ) + self.log.propagate = False + self.log.addHandler(logging.StreamHandler(stream=self.stdout)) + self.log.setLevel(log_level) + self.conda_store = conda_store + + def run_command(self, command: str, redirect_stderr: bool = True, **kwargs): + """Runs command and immediately writes to logs""" + self.log.info("Running command: %s", command) + + # Unlike subprocess.run, Popen doesn't support the check argument, so + # ignore it. The code below always checks the return code + kwargs.pop("check", None) + + # https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running + with subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT if redirect_stderr else subprocess.PIPE, + bufsize=1, + universal_newlines=True, + **kwargs, + ) as proc: + for line in proc.stdout: + self.stdout.write(line) + if not redirect_stderr: + for line in proc.stderr: + self.stderr.write(line) + + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, proc.args) diff --git a/conda-store-server/conda_store_server/plugins/plugin_manager.py b/conda-store-server/conda_store_server/plugins/plugin_manager.py new file mode 100644 index 000000000..0b59b0039 --- /dev/null +++ b/conda-store-server/conda_store_server/plugins/plugin_manager.py @@ -0,0 +1,40 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import pluggy + +from conda_store_server.exception import CondaStorePluginNotFoundError +from conda_store_server.plugins import BUILTIN_PLUGINS +from conda_store_server.plugins.types import types + + +class PluginManager(pluggy.PluginManager): + """ + PluginManager extends pluggy's plugin manager in order to extend + functionality for + * retrieving CondaStore type plugins (eg. TypeLockPlugin), + * discovering and registering CondaStore plugins + """ + + def get_lock_plugins(self) -> dict[str, types.TypeLockPlugin]: + """Returns a dict of lock plugin name to class""" + plugins = [item for items in self.hook.lock_plugins() for item in items] + return {p.name.lower(): p for p in plugins} + + def get_lock_plugin(self, name: str) -> types.TypeLockPlugin: + """Returns a lock plugin by name""" + lockers = self.get_lock_plugins() + + if name not in lockers: + raise CondaStorePluginNotFoundError( + plugin=name, available_plugins=lockers.keys() + ) + + return lockers[name] + + def collect_plugins(self) -> None: + """Registers all availble plugins""" + # TODO: support loading user defined plugins (eg. https://github.com/conda/conda/blob/cf3a0fa9ce01ada7a4a0c934e17be44b94d4eb91/conda/plugins/manager.py#L131) + for plugin in BUILTIN_PLUGINS: + self.register(plugin) diff --git a/conda-store-server/conda_store_server/plugins/types/__init__.py b/conda-store-server/conda_store_server/plugins/types/__init__.py new file mode 100644 index 000000000..b559bd2ff --- /dev/null +++ b/conda-store-server/conda_store_server/plugins/types/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/conda-store-server/conda_store_server/plugins/types/lock.py b/conda-store-server/conda_store_server/plugins/types/lock.py new file mode 100644 index 000000000..89ae70cc4 --- /dev/null +++ b/conda-store-server/conda_store_server/plugins/types/lock.py @@ -0,0 +1,33 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import typing + +from conda_store_server._internal import conda_utils, schema +from conda_store_server.plugins.plugin_context import PluginContext + + +class LockPlugin: + """ + Interface for the lock plugin. This plugin is responsible for solving the + environment given some set of packages. + + * :meth: `lock_environment` + """ + + def lock_environment( + self, + context: PluginContext, + spec: schema.CondaSpecification, + platforms: typing.List[str] = [conda_utils.conda_platform()], + ) -> str: + """ + Solve the environment and generate a lockfile for a given spec on given platforms + + :param context: plugin context for execution + :param spec: the conda specification to solve + :param platforms: list of platforms (or subdirs) to solve for + :return: string contents of the lockfile + """ + raise NotImplementedError diff --git a/conda-store-server/conda_store_server/plugins/types/types.py b/conda-store-server/conda_store_server/plugins/types/types.py new file mode 100644 index 000000000..4626eccb1 --- /dev/null +++ b/conda-store-server/conda_store_server/plugins/types/types.py @@ -0,0 +1,21 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from typing import NamedTuple + +from conda_store_server.plugins.types.lock import LockPlugin + + +class TypeLockPlugin(NamedTuple): + """ + Return type to use when defining a conda store lock plugin hook. + + :param name: plugin name (e.g., ``my-nice-lock-plugin``). + :param synopsis: a brief description of the plugin + :param backend: Type that will be instantiated as the lock backend. + """ + + name: str + synopsis: str + backend: type[LockPlugin] diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 76518cea0..cc24b1c1f 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ "flower", "itsdangerous", "jinja2", + "pluggy", "pyjwt", "psycopg2-binary", "pymysql", diff --git a/conda-store-server/tests/_internal/action/test_actions.py b/conda-store-server/tests/_internal/action/test_actions.py index d6e4a594c..661b16722 100644 --- a/conda-store-server/tests/_internal/action/test_actions.py +++ b/conda-store-server/tests/_internal/action/test_actions.py @@ -11,7 +11,6 @@ from unittest import mock import pytest -import yaml import yarl from celery.result import AsyncResult from conda.base.context import context as conda_base_context @@ -23,7 +22,6 @@ from conda_store_server._internal import action, conda_utils, orm, schema, server, utils from conda_store_server._internal.action import ( generate_constructor_installer, - generate_lockfile, ) from conda_store_server.server.auth import DummyAuthentication @@ -72,114 +70,6 @@ def test_function(context): assert not context.result.exists() -@pytest.mark.parametrize( - "specification", - [ - "simple_specification", - "simple_specification_with_pip", - ], -) -@mock.patch.object(generate_lockfile, "yaml", wraps=yaml) -@mock.patch("conda_store_server._internal.action.generate_lockfile.logged_command") -@mock.patch("conda_store_server._internal.action.generate_lockfile.run_lock") -@pytest.mark.long_running_test -def test_solve_lockfile( - mock_run_lock, - mock_logged_command, - mock_yaml, - conda_store, - specification, - request, -): - """Test that the call to conda_lock.run_lock is formed correctly. - - Mock out logged_command, which is used to call `conda info`; this is an - extremely slow call, and we aren't testing any aspect of it here. - - Mock out the yaml package, it is used to load the lockfile normally - generated by conda-lock. - """ - - # Dump dummy data to the expected lockfile output location - def run_lock_side_effect(lockfile_path, **kwargs): - with open(lockfile_path, "w") as f: - yaml.dump({"foo": "bar"}, f) - - mock_run_lock.side_effect = run_lock_side_effect - - platforms = [conda_utils.conda_platform()] - specification = request.getfixturevalue(specification) - if specification.variables is None: - cuda_version = None - else: - cuda_version = specification.variables.get("CONDA_OVERRIDE_CUDA") - - context = generate_lockfile.action_solve_lockfile( - conda_command=conda_store.conda_command, - specification=specification, - platforms=platforms, - ) - - # Check that the call to `conda_lock` is correctly formed - mock_run_lock.assert_called_once() - call_args = mock_run_lock.call_args_list[0][1] - assert str(call_args["environment_files"][0]).endswith("environment.yaml") - assert call_args["platforms"] == platforms - assert str(call_args["lockfile_path"]).endswith("conda-lock.yaml") - assert call_args["conda_exe"] == conda_store.conda_command - assert call_args["with_cuda"] == cuda_version - - assert context.result["foo"] == "bar" - - -def test_solve_lockfile_valid_conda_flags(conda_store, simple_specification): - context = action.action_solve_lockfile( - conda_command=conda_store.conda_command, - specification=simple_specification, - platforms=[conda_utils.conda_platform()], - conda_flags="--strict-channel-priority", - ) - assert len(context.result["package"]) != 0 - - -# Checks that conda_flags is used by conda-lock -@pytest.mark.long_running_test -def test_solve_lockfile_invalid_conda_flags(conda_store, simple_specification): - with pytest.raises( - Exception, match=(r"Command.*--this-is-invalid.*returned non-zero exit status") - ): - action.action_solve_lockfile( - conda_command=conda_store.conda_command, - specification=simple_specification, - platforms=[conda_utils.conda_platform()], - conda_flags="--this-is-invalid", - ) - - -@pytest.mark.parametrize( - "specification", - [ - "simple_specification", - "simple_specification_with_pip", - ], -) -@pytest.mark.long_running_test -def test_solve_lockfile_multiple_platforms(conda_store, specification, request): - specification = request.getfixturevalue(specification) - context = action.action_solve_lockfile( - conda_command=conda_store.conda_command, - specification=specification, - platforms=["osx-64", "linux-64", "win-64", "osx-arm64"], - ) - assert len(context.result["package"]) != 0 - - -def test_save_lockfile(simple_lockfile_specification): - """Ensure lockfile is saved in conda-lock `output` format""" - context = action.action_save_lockfile(specification=simple_lockfile_specification) - assert context.result == simple_lockfile_specification.lockfile.dict_for_output() - - @pytest.mark.parametrize( "specification_name", [ diff --git a/conda-store-server/tests/_internal/plugins/__init__.py b/conda-store-server/tests/_internal/plugins/__init__.py new file mode 100644 index 000000000..b559bd2ff --- /dev/null +++ b/conda-store-server/tests/_internal/plugins/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/conda-store-server/tests/_internal/plugins/lock/__init__.py b/conda-store-server/tests/_internal/plugins/lock/__init__.py new file mode 100644 index 000000000..b559bd2ff --- /dev/null +++ b/conda-store-server/tests/_internal/plugins/lock/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/conda-store-server/tests/_internal/plugins/lock/test_conda_lock.py b/conda-store-server/tests/_internal/plugins/lock/test_conda_lock.py new file mode 100644 index 000000000..1a911fa1a --- /dev/null +++ b/conda-store-server/tests/_internal/plugins/lock/test_conda_lock.py @@ -0,0 +1,111 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from unittest import mock + +import pytest +import yaml +from conda_lock._vendor.poetry.utils._compat import CalledProcessError + +from conda_store_server._internal import conda_utils +from conda_store_server._internal.plugins.lock.conda_lock import conda_lock +from conda_store_server.plugins import plugin_context + + +@pytest.mark.parametrize( + "specification", + [ + "simple_specification", + "simple_specification_with_pip", + ], +) +@mock.patch("conda_store_server._internal.plugins.lock.conda_lock.conda_lock.run_lock") +@pytest.mark.long_running_test +def test_solve_lockfile( + mock_run_lock, + conda_store, + specification, + request, +): + """Test that the call to conda_lock.run_lock is formed correctly.""" + + # Dump dummy data to the expected lockfile output location + def run_lock_side_effect(lockfile_path, **kwargs): + with open(lockfile_path, "w") as f: + yaml.dump({"foo": "bar"}, f) + + mock_run_lock.side_effect = run_lock_side_effect + + platforms = [conda_utils.conda_platform()] + specification = request.getfixturevalue(specification) + if specification.variables is None: + cuda_version = None + else: + cuda_version = specification.variables.get("CONDA_OVERRIDE_CUDA") + + locker = conda_lock.CondaLock() + lock_result = locker.lock_environment( + context=plugin_context.PluginContext(conda_store), + spec=specification, + platforms=platforms, + ) + + # Check that the call to `conda_lock` is correctly formed + mock_run_lock.assert_called_once() + call_args = mock_run_lock.call_args_list[0][1] + assert str(call_args["environment_files"][0]).endswith("environment.yaml") + assert call_args["platforms"] == platforms + assert str(call_args["lockfile_path"]).endswith("conda-lock.yaml") + assert call_args["conda_exe"] == "mamba" + assert call_args["with_cuda"] == cuda_version + + assert lock_result["foo"] == "bar" + + +def test_solve_lockfile_simple(conda_store, simple_specification): + locker = conda_lock.CondaLock() + lock_result = locker.lock_environment( + context=plugin_context.PluginContext(conda_store), + spec=simple_specification, + platforms=[conda_utils.conda_platform()], + ) + assert len(lock_result["package"]) != 0 + assert "zlib" in [pkg["name"] for pkg in lock_result["package"]] + + +@pytest.mark.parametrize( + "specification", + [ + "simple_specification", + "simple_specification_with_pip", + ], +) +@pytest.mark.long_running_test +def test_solve_lockfile_multiple_platforms(conda_store, specification, request): + specification = request.getfixturevalue(specification) + locker = conda_lock.CondaLock() + lock_result = locker.lock_environment( + context=plugin_context.PluginContext(conda_store), + spec=specification, + platforms=["win-64", "osx-arm64"], + ) + assert len(lock_result["package"]) != 0 + + +def test_solve_lockfile_invalid_conda_flags(conda_store, simple_specification): + """Checks that conda_flags is used by conda-lock""" + locker = conda_lock.CondaLock() + + # Set invalid conda flags + conda_store.conda_flags = "--this-is-invalid" + + with pytest.raises( + CalledProcessError, + match=(r"Command.*--this-is-invalid.*returned non-zero exit status"), + ): + locker.lock_environment( + context=plugin_context.PluginContext(conda_store), + spec=simple_specification, + platforms=[conda_utils.conda_platform()], + ) diff --git a/conda-store-server/tests/conftest.py b/conda-store-server/tests/conftest.py index b4048ceb6..b92507aae 100644 --- a/conda-store-server/tests/conftest.py +++ b/conda-store-server/tests/conftest.py @@ -26,6 +26,8 @@ ) from conda_store_server._internal.server import app as server_app # isort:skip +from conda_store_server.plugins import hookspec +from conda_store_server.plugins.plugin_manager import PluginManager @pytest.fixture @@ -281,6 +283,13 @@ def conda_prefix(conda_store, tmp_path, request): return conda_prefix +@pytest.fixture +def plugin_manager(): + pm = PluginManager(hookspec.spec_name) + pm.add_hookspecs(hookspec.CondaStoreSpecs) + return pm + + def _seed_conda_store( db: Session, conda_store, diff --git a/conda-store-server/tests/plugins/__init__.py b/conda-store-server/tests/plugins/__init__.py new file mode 100644 index 000000000..b559bd2ff --- /dev/null +++ b/conda-store-server/tests/plugins/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/conda-store-server/tests/plugins/test_plugin_context.py b/conda-store-server/tests/plugins/test_plugin_context.py new file mode 100644 index 000000000..e3496221c --- /dev/null +++ b/conda-store-server/tests/plugins/test_plugin_context.py @@ -0,0 +1,61 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import io +import logging +import os +import subprocess +import sys + +import pytest + +from conda_store_server.plugins.plugin_context import PluginContext + + +def test_run_command_no_logs(): + out = io.StringIO() + err = io.StringIO() + context = PluginContext(stdout=out, stderr=err, log_level=logging.ERROR) + + context.run_command(["echo", "testing"]) + assert err.getvalue() == "" + assert out.getvalue() == "testing\n" + + +def test_run_command_log_info(): + out = io.StringIO() + err = io.StringIO() + context = PluginContext(stdout=out, stderr=err, log_level=logging.INFO) + + context.run_command(["echo", "testing"]) + assert err.getvalue() == "" + assert ( + out.getvalue() + == """Running command: ['echo', 'testing'] +testing +""" + ) + + +def test_run_command_errors(): + context = PluginContext(log_level=logging.ERROR) + + with pytest.raises(subprocess.CalledProcessError): + context.run_command(["conda-store-server", "-thiswillreturnanonzeroexitcode"]) + + +@pytest.mark.skipif( + sys.platform == "win32", reason="stat is not a valid command on Windows" +) +def test_run_command_kwargs(): + """Ensure that kwargs get passed to subprocess""" + out = io.StringIO() + err = io.StringIO() + context = PluginContext(stdout=out, stderr=err, log_level=logging.ERROR) + + # set the cwd to this directory and check that this file exists + dir_path = os.path.dirname(os.path.realpath(__file__)) + context.run_command(["stat", "test_plugin_context.py"], check=True, cwd=dir_path) + assert err.getvalue() == "" + assert "File: test_plugin_context.py" in out.getvalue() diff --git a/conda-store-server/tests/plugins/test_plugin_manager.py b/conda-store-server/tests/plugins/test_plugin_manager.py new file mode 100644 index 000000000..4c447ae02 --- /dev/null +++ b/conda-store-server/tests/plugins/test_plugin_manager.py @@ -0,0 +1,41 @@ +# Copyright (c) conda-store development team. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import pytest + +from conda_store_server._internal.plugins.lock.conda_lock import conda_lock +from conda_store_server.exception import CondaStorePluginNotFoundError +from conda_store_server.plugins import BUILTIN_PLUGINS +from conda_store_server.plugins.types import types + + +def test_collect_plugins(plugin_manager): + plugin_manager.collect_plugins() + for plugin in BUILTIN_PLUGINS: + assert plugin_manager.is_registered(plugin) + + +def test_get_lock_plugins(plugin_manager): + plugin_manager.collect_plugins() + lock_plugins = plugin_manager.get_lock_plugins() + + # Ensure built in lock plugins are accounted for + assert "conda-lock" in lock_plugins + + # Ensure all plugins are lock plugins + for plugin in lock_plugins.values(): + assert isinstance(plugin, types.TypeLockPlugin) + + +def get_lock_plugin(plugin_manager): + plugin_manager.collect_plugins() + lock_plugins = plugin_manager.get_lock_plugin("conda-lock") + assert lock_plugins.name == "conda-lock" + assert lock_plugins.backend == conda_lock.CondaLock + + +def get_lock_plugin_does_not_exist(plugin_manager): + plugin_manager.collect_plugins() + with pytest.raises(CondaStorePluginNotFoundError): + plugin_manager.get_lock_plugin("i really don't exist") diff --git a/conda-store-server/tests/test_app.py b/conda-store-server/tests/test_app.py index 04b617a8d..51aa6979d 100644 --- a/conda-store-server/tests/test_app.py +++ b/conda-store-server/tests/test_app.py @@ -9,6 +9,8 @@ from conda_store_server import api from conda_store_server._internal import action, conda_utils, schema +from conda_store_server._internal.plugins.lock.conda_lock import conda_lock +from conda_store_server.exception import CondaStorePluginNotFoundError @pytest.mark.long_running_test @@ -178,3 +180,18 @@ def test_conda_store_register_environment_duplicate_force_true(db, conda_store): assert first_build_id == 1 assert second_build_id == 2 + + +def test_conda_store_get_lock_plugin(conda_store): + lock_plugin_setting = "conda-lock" + conda_store.lock_backend = lock_plugin_setting + name, plugin = conda_store.lock_plugin() + assert name == lock_plugin_setting + assert isinstance(plugin, conda_lock.CondaLock) + + +def test_conda_store_get_lock_plugin_does_not_exist(conda_store): + lock_plugin_setting = "idontexist" + conda_store.lock_backend = lock_plugin_setting + with pytest.raises(CondaStorePluginNotFoundError): + conda_store.lock_plugin() diff --git a/docusaurus-docs/conda-store/references/configuration-options.md b/docusaurus-docs/conda-store/references/configuration-options.md index 1c82c1dc4..5ee9838e3 100644 --- a/docusaurus-docs/conda-store/references/configuration-options.md +++ b/docusaurus-docs/conda-store/references/configuration-options.md @@ -227,6 +227,9 @@ the default docker image `library/debian:sid-slim`. `CondaStore.post_update_environment_build_hook` is an optional configurable to allow for custom behavior that will run after an environment's current build changes. +`CondaStore.lock_backend` is the name of the default lock plugin to use +when locking a conda environment. By default, conda-store uses [conda-lock](https://github.com/conda/conda-lock). + ### `conda_store_server.storage.S3Storage` conda-store uses [minio-py](https://github.com/minio/minio-py) as a diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 3bd7d8aa9..00e5545ab 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -80,6 +80,7 @@ outputs: - itsdangerous - jinja2 - minio + - pluggy - pydantic >=2.0 - pyjwt - python >=3.10,<3.13