Refactoring of deployments and env

.github/workflows/check-test-release.yml

@@ -7,6 +7,7 @@ on:
 
 env:
   MLEM_TESTS: "true"
+  MLEM_DEBUG: "true"
 
 jobs:
   authorize:

.pylintrc

@@ -170,7 +170,8 @@ disable=print-statement,
         redefined-builtin, # TODO: https://github.com/iterative/mlem/issues/60
         no-self-use, # TODO: https://github.com/iterative/mlem/issues/60 maybe leave it
         import-outside-toplevel,
-        wrong-import-order # handeled by isort
+        wrong-import-order, # handeled by isort
+        cannot-enumerate-pytest-fixtures # TODO: https://github.com/iterative/mlem/issues/60
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

mlem/api/commands.py

@@ -414,63 +414,57 @@ def import_object(
 
 def deploy(
     deploy_meta_or_path: Union[MlemDeployment, str],
-    model: Union[MlemModel, str] = None,
+    model: Union[MlemModel, str],
     env: Union[MlemEnv, str] = None,
     project: Optional[str] = None,
+    rev: Optional[str] = None,
     fs: Optional[AbstractFileSystem] = None,
     external: bool = None,
     index: bool = None,
     env_kwargs: Dict[str, Any] = None,
     **deploy_kwargs,
 ) -> MlemDeployment:
-    deploy_path = None
+    deploy_meta: MlemDeployment
     update = False
     if isinstance(deploy_meta_or_path, str):
-        deploy_path = deploy_meta_or_path
         try:
             deploy_meta = load_meta(
-                path=deploy_path,
+                path=deploy_meta_or_path,
                 project=project,
+                rev=rev,
                 fs=fs,
                 force_type=MlemDeployment,
             )
             update = True
-        except MlemObjectNotFound:
-            deploy_meta = None
-
+        except MlemObjectNotFound as e:
+            if env is None:
+                raise MlemError(
+                    "Please provide model and env args for new deployment"
+                ) from e
+            if not deploy_meta_or_path:
+                raise MlemError("deploy_path cannot be empty") from e
+
+            env_meta = ensure_meta(MlemEnv, env, allow_typename=True)
+            if isinstance(env_meta, type):
+                env = None
+                if env_kwargs:
+                    env = env_meta(**env_kwargs)
+            deploy_type = env_meta.deploy_type
+            deploy_meta = deploy_type(
+                env=env,
+                **deploy_kwargs,
+            )
+            deploy_meta.dump(deploy_meta_or_path, fs, project, index, external)
     else:
         deploy_meta = deploy_meta_or_path
         update = True
 
-    if deploy_meta is None:
-        if model is None or env is None:
-            raise MlemError(
-                "Please provide model and env args for new deployment"
-            )
-        if not deploy_path:
-            raise MlemError("deploy_path cannot be empty")
-        model_meta = get_model_meta(model)
-        env_meta = ensure_meta(MlemEnv, env, allow_typename=True)
-        if isinstance(env_meta, type):
-            env = None
-            if env_kwargs:
-                env = env_meta(**env_kwargs)
-        deploy_type = env_meta.deploy_type
-        deploy_meta = deploy_type(
-            model_cache=model_meta,
-            model=model_meta.make_link(),
-            env=env,
-            **deploy_kwargs,
-        )
-        deploy_meta.dump(deploy_path, fs, project, index, external)
-    else:
-        if model is not None:
-            deploy_meta.replace_model(get_model_meta(model, load_value=False))
     if update:
         pass  # todo update from deploy_args and env_args
     # ensuring links are working
     deploy_meta.get_env()
-    deploy_meta.get_model()
+    model_meta = get_model_meta(model)
 
-    deploy_meta.run()
+    deploy_meta.check_unchanged()
+    deploy_meta.deploy(model_meta)
     return deploy_meta

mlem/cli/declare.py

@@ -1,11 +1,12 @@
+import itertools
 from typing import Type
 
 from typer import Argument, Typer
 from yaml import safe_dump
 
 from ..core.base import MlemABC, build_mlem_object, load_impl_ext
 from ..core.meta_io import Location
-from ..core.objects import MlemObject
+from ..core.objects import MlemDeployment, MlemObject
 from ..utils.entrypoints import list_abstractions, list_implementations
 from .main import (
     app,
@@ -16,6 +17,9 @@
     option_project,
 )
 from .utils import (
+    CliTypeField,
+    _option_from_field,
+    _options_from_model,
     abc_fields_parameters,
     lazy_class_docstring,
     wrap_build_error,
@@ -48,6 +52,38 @@ def create_declare_mlem_object(type_name, cls: Type[MlemObject]):
             )
 
 
+def add_env_params_deployment(subtype, parent_cls: Type[MlemDeployment]):
+    try:
+        impl = load_impl_ext(parent_cls.object_type, subtype)
+    except ImportError:
+        return lambda ctx: []
+
+    assert issubclass(impl, MlemDeployment)  # just to help mypy
+    env_impl = impl.env_type
+    return lambda ctx: itertools.chain(
+        abc_fields_parameters(subtype, parent_cls)(ctx),
+        _options_from_model(env_impl, ctx, path="env", force_not_set=True),
+        (
+            _option_from_field(
+                CliTypeField(
+                    path="env",
+                    required=False,
+                    allow_none=False,
+                    type_=str,
+                    help="",
+                    default=env_impl.type,
+                    is_list=False,
+                    is_mapping=False,
+                ),
+                "env",
+            ),
+        ),
+    )
+
+
+_add_fields = {"deployment": add_env_params_deployment}
+
+
 def create_declare_mlem_object_subcommand(
     parent: Typer, subtype: str, type_name: str, parent_cls
 ):
@@ -56,7 +92,9 @@ def create_declare_mlem_object_subcommand(
         section="MLEM Objects",
         parent=parent,
         dynamic_metavar="__kwargs__",
-        dynamic_options_generator=abc_fields_parameters(subtype, parent_cls),
+        dynamic_options_generator=_add_fields.get(
+            parent_cls.object_type, abc_fields_parameters
+        )(subtype, parent_cls),
         hidden=subtype.startswith("_"),
         lazy_help=lazy_class_docstring(type_name, subtype),
     )

mlem/cli/deployment.py

@@ -4,26 +4,43 @@
 from typer import Argument, Option, Typer
 
 from mlem.cli.apply import run_apply_remote
+from mlem.cli.declare import add_env_params_deployment
 from mlem.cli.main import (
     app,
     mlem_command,
     mlem_group,
-    option_conf,
+    mlem_group_callback,
     option_data_project,
     option_data_rev,
     option_external,
+    option_file_conf,
     option_index,
     option_json,
+    option_load,
     option_method,
+    option_model,
+    option_model_project,
+    option_model_rev,
     option_project,
     option_rev,
     option_target_project,
 )
-from mlem.core.base import parse_string_conf
+from mlem.cli.utils import (
+    for_each_impl,
+    lazy_class_docstring,
+    make_not_required,
+    wrap_build_error,
+)
+from mlem.core.base import build_mlem_object
 from mlem.core.data_type import DataAnalyzer
-from mlem.core.errors import DeploymentError
+from mlem.core.errors import DeploymentError, MlemObjectNotFound
 from mlem.core.metadata import load_meta
-from mlem.core.objects import DeployState, DeployStatus, MlemDeployment
+from mlem.core.objects import (
+    DeployState,
+    DeployStatus,
+    MlemDeployment,
+    MlemModel,
+)
 from mlem.ui import echo, no_echo, set_echo
 
 deployment = Typer(
@@ -33,21 +50,27 @@
 )
 app.add_typer(deployment)
 
+deploy_run = Typer(
+    name="run",
+    help="""Deploy a model to a target environment. Can use an existing deployment
+    declaration or create a new one on-the-fly.
+    """,
+    cls=mlem_group("other"),
+    subcommand_metavar="deployment",
+)
+deployment.add_typer(deploy_run)
+
 
-@mlem_command("run", parent=deployment)
-def deploy_run(
-    path: str = Argument(
-        ...,
-        help="Path to deployment meta (will be created if it does not exist)",
-    ),
-    model: Optional[str] = Option(None, "-m", "--model", help="Path to model"),
-    env: Optional[str] = Option(
-        None, "-t", "--env", help="Path to target environment"
-    ),
+@mlem_group_callback(deploy_run, required=["model", "load"])
+def deploy_run_callback(
+    load: str = option_load("deployment"),
+    model: str = make_not_required(option_model),
+    model_project: Optional[str] = option_model_project,
+    model_rev: Optional[str] = option_model_rev,
     project: Optional[str] = option_project,
+    rev: Optional[str] = option_rev,
     external: bool = option_external,
     index: bool = option_index,
-    conf: Optional[List[str]] = option_conf(),
 ):
     """Deploy a model to a target environment. Can use an existing deployment
     declaration or create a new one on-the-fly.
@@ -64,21 +87,75 @@ def deploy_run(
     """
     from mlem.api.commands import deploy
 
-    conf = conf or []
-    env_conf = [c[len("env.") :] for c in conf if c.startswith("env.")]
-    conf = [c for c in conf if not c.startswith("env.")]
     deploy(
-        path,
-        model,
-        env,
-        project,
+        load,
+        load_meta(
+            model, project=model_project, rev=model_rev, force_type=MlemModel
+        ),
+        project=project,
+        rev=rev,
         external=external,
         index=index,
-        env_kwargs=parse_string_conf(env_conf),
-        **parse_string_conf(conf or []),
     )
 
 
+@for_each_impl(MlemDeployment)
+def create_deploy_run_command(type_name):
+    @mlem_command(
+        type_name,
+        section="deployments",
+        parent=deploy_run,
+        dynamic_metavar="__kwargs__",
+        dynamic_options_generator=add_env_params_deployment(
+            type_name, MlemDeployment
+        ),
+        hidden=type_name.startswith("_"),
+        lazy_help=lazy_class_docstring(MlemDeployment.object_type, type_name),
+        no_pass_from_parent=["file_conf"],
+    )
+    def deploy_run_command(
+        path: str = Argument(
+            ..., help="Where to save the object (.mlem file)"
+        ),
+        model: str = make_not_required(option_model),
+        model_project: Optional[str] = option_model_project,
+        model_rev: Optional[str] = option_model_rev,
+        project: Optional[str] = option_project,
+        external: bool = option_external,
+        index: bool = option_index,
+        file_conf: List[str] = option_file_conf("deployment"),
+        **__kwargs__,
+    ):
+        from mlem.api.commands import deploy
+
+        try:
+            meta = load_meta(path, project=project, force_type=MlemDeployment)
+            raise DeploymentError(
+                f"Deployment meta already exists at {meta.loc}. Please use `mlem deployment run --load <path> ...`"
+            )
+        except MlemObjectNotFound:
+            with wrap_build_error(type_name, MlemDeployment):
+                meta = build_mlem_object(
+                    MlemDeployment,
+                    type_name,
+                    str_conf=None,
+                    file_conf=file_conf,
+                    **__kwargs__,
+                ).dump(path, project=project)
+        deploy(
+            meta,
+            load_meta(
+                model,
+                project=model_project,
+                rev=model_rev,
+                force_type=MlemModel,
+            ),
+            project=project,
+            external=external,
+            index=index,
+        )
+
+
 @mlem_command("remove", parent=deployment)
 def deploy_remove(
     path: str = Argument(..., help="Path to deployment meta"),
@@ -173,7 +250,7 @@ def deploy_apply(
         )
         state: DeployState = deploy_meta.get_state()
         if (
-            state == deploy_meta.state_type()
+            state == deploy_meta.state_type(declaration=deploy_meta)
             and not deploy_meta.state_type.allow_default
         ):
             raise DeploymentError(