Release 0.3.0

.github/workflows/check-test-release.yml

@@ -7,6 +7,7 @@ on:
 
 env:
   MLEM_TESTS: "true"
+  MLEM_DEBUG: "true"
 
 jobs:
   authorize:
@@ -58,7 +59,7 @@ jobs:
         # no HDF5 support installed for tables
         - os: windows-latest
           python: "3.9"
-      fail-fast: true
+      fail-fast: false
     steps:
     - uses: actions/checkout@v3
       with:
@@ -92,6 +93,9 @@ jobs:
         pip install pre-commit .[tests]
     - run: pre-commit run pylint -a -v --show-diff-on-failure
       if: matrix.python != '3.7'
+    - name: Start minikube
+      if: matrix.os == 'ubuntu-latest' && matrix.python == '3.9'
+      uses: medyagh/setup-minikube@master
     - name: Run tests
       timeout-minutes: 40
       run: pytest

.pylintrc

@@ -170,7 +170,8 @@ disable=print-statement,
         redefined-builtin, # TODO: https://github.com/iterative/mlem/issues/60
         no-self-use, # TODO: https://github.com/iterative/mlem/issues/60 maybe leave it
         import-outside-toplevel,
-        wrong-import-order # handeled by isort
+        wrong-import-order, # handeled by isort
+        cannot-enumerate-pytest-fixtures # TODO: https://github.com/iterative/mlem/issues/60
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -369,7 +370,7 @@ indent-string='    '
 max-line-length=100
 
 # Maximum number of lines in a module.
-max-module-lines=1000
+max-module-lines=2000
 
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
@@ -389,7 +390,7 @@ ignore-comments=yes
 ignore-docstrings=yes
 
 # Ignore imports when computing similarities.
-ignore-imports=no
+ignore-imports=yes
 
 # Ignore function signatures when computing similarities.
 ignore-signatures=no

mlem/api/__init__.py

@@ -11,15 +11,13 @@
     import_object,
     init,
     link,
-    ls,
     serve,
 )
 
 __all__ = [
     "save",
     "load",
     "load_meta",
-    "ls",
     "clone",
     "init",
     "link",

mlem/api/commands.py

@@ -2,7 +2,7 @@
 MLEM's Python API
 """
 import posixpath
-from typing import Any, Dict, Iterable, List, Optional, Type, Union
+from typing import Any, Dict, Optional, Union
 
 from fsspec import AbstractFileSystem
 from fsspec.implementations.local import LocalFileSystem
@@ -14,8 +14,7 @@
     get_model_meta,
     parse_import_type_modifier,
 )
-from mlem.config import CONFIG_FILE_NAME, project_config
-from mlem.constants import PREDICT_METHOD_NAME
+from mlem.constants import MLEM_CONFIG_FILE_NAME, PREDICT_METHOD_NAME
 from mlem.core.errors import (
     InvalidArgumentError,
     MlemError,
@@ -25,7 +24,7 @@
     WrongMethodError,
 )
 from mlem.core.import_objects import ImportAnalyzer, ImportHook
-from mlem.core.meta_io import MLEM_DIR, Location, UriResolver, get_fs
+from mlem.core.meta_io import Location, get_fs
 from mlem.core.metadata import load_meta, save
 from mlem.core.objects import (
     MlemBuilder,
@@ -56,8 +55,6 @@ def apply(
     method: str = None,
     output: str = None,
     target_project: str = None,
-    index: bool = None,
-    external: bool = None,
     batch_size: Optional[int] = None,
 ) -> Optional[Any]:
     """Apply provided model against provided data
@@ -70,8 +67,6 @@ def apply(
             If more than one is available, will fail.
         output (str, optional): If value is provided,
             assume it's path and save output there.
-        index (bool): Whether to index saved output in MLEM root folder.
-        external (bool): Whether to save result outside mlem dir
 
     Returns:
         If `output=None`, returns results for given data.
@@ -103,9 +98,7 @@ def apply(
         return res
     if len(res) == 1:
         res = res[0]
-    return save(
-        res, output, project=target_project, external=external, index=index
-    )
+    return save(res, output, project=target_project)
 
 
 def apply_remote(
@@ -114,7 +107,6 @@ def apply_remote(
     method: str = None,
     output: str = None,
     target_project: str = None,
-    index: bool = False,
     **client_kwargs,
 ) -> Optional[Any]:
     """Apply provided model against provided data
@@ -127,7 +119,6 @@ def apply_remote(
             If more than one is available, will fail.
         output (str, optional): If value is provided,
             assume it's path and save output there.
-        index (bool): Whether to index saved output in MLEM root folder.
 
     Returns:
         If `output=None`, returns results for given data.
@@ -151,7 +142,7 @@ def apply_remote(
         return res
     if len(res) == 1:
         res = res[0]
-    return save(res, output, project=target_project, index=index)
+    return save(res, output, project=target_project)
 
 
 def clone(
@@ -164,8 +155,6 @@ def clone(
     target_fs: Optional[str] = None,
     follow_links: bool = True,
     load_value: bool = False,
-    index: bool = None,
-    external: bool = None,
 ) -> MlemObject:
     """Clones MLEM object from `path` to `out`
         and returns Python representation for the created object
@@ -181,8 +170,6 @@ def clone(
         follow_links (bool, optional): If object we read is a MLEM link, whether to load
             the actual object link points to. Defaults to True.
         load_value (bool, optional): Load actual python object incorporated in MlemObject. Defaults to False.
-        index: whether to index object in target project
-        external: wheter to put object inside mlem dir in target project
 
     Returns:
         MlemObject: Copy of initial object saved to `out`
@@ -202,14 +189,12 @@ def clone(
         target,
         fs=target_fs,
         project=target_project,
-        index=index,
-        external=external,
     )
 
 
 def init(path: str = ".") -> None:
-    """Creates .mlem directory in `path`"""
-    path = posixpath.join(path, MLEM_DIR)
+    """Creates mlem config in `path`"""
+    path = posixpath.join(path, MLEM_CONFIG_FILE_NAME)
     fs, path = get_fs(path)
     if fs.exists(path):
         echo(
@@ -252,9 +237,8 @@ def init(path: str = ".") -> None:
                     "<https://mlem.ai/docs/user-guide/analytics>"
                 )
             )
-        fs.makedirs(path)
         # some fs dont support creating empty dirs
-        with fs.open(posixpath.join(path, CONFIG_FILE_NAME), "w"):
+        with fs.open(path, "w"):
             pass
         echo(
             EMOJI_MLEM
@@ -273,7 +257,6 @@ def link(
     rev: Optional[str] = None,
     target: Optional[str] = None,
     target_project: Optional[str] = None,
-    external: Optional[bool] = None,
     follow_links: bool = True,
     absolute: bool = False,
 ) -> MlemLink:
@@ -288,7 +271,6 @@ def link(
             treat `target` as link name and dump link in MLEM DIR
         follow_links (bool): Whether to make link to the underlying object
             if `source` is itself a link. Defaults to True.
-        external (bool): Whether to save link outside mlem dir
         absolute (bool): Whether to make link absolute or relative to mlem project
 
     Returns:
@@ -308,7 +290,6 @@ def link(
     return source.make_link(
         target,
         project=target_project,
-        external=external,
         absolute=absolute,
     )
 
@@ -359,25 +340,6 @@ def _validate_ls_project(loc: Location, project):
         mlem_project_exists(loc.project, loc.fs, raise_on_missing=True)
 
 
-def ls(  # pylint: disable=too-many-locals
-    project: str = ".",
-    rev: Optional[str] = None,
-    fs: Optional[AbstractFileSystem] = None,
-    type_filter: Union[
-        Type[MlemObject], Iterable[Type[MlemObject]], None
-    ] = None,
-    include_links: bool = True,
-    ignore_errors: bool = False,
-) -> Dict[Type[MlemObject], List[MlemObject]]:
-    loc = UriResolver.resolve(
-        "", project=project, rev=rev, fs=fs, find_project=True
-    )
-    _validate_ls_project(loc, project)
-    return project_config(project, fs).index.list(
-        loc, type_filter, include_links, ignore_errors
-    )
-
-
 def import_object(
     path: str,
     project: Optional[str] = None,
@@ -388,13 +350,11 @@ def import_object(
     target_fs: Optional[AbstractFileSystem] = None,
     type_: Optional[str] = None,
     copy_data: bool = True,
-    external: bool = None,
-    index: bool = None,
 ):
     """Try to load an object as MLEM model (or data) and return it,
     optionally saving to the specified target location
     """
-    loc = UriResolver.resolve(path, project, rev, fs)
+    loc = Location.resolve(path, project, rev, fs)
     echo(EMOJI_LOAD + f"Importing object from {loc.uri_repr}")
     if type_ is not None:
         type_, modifier = parse_import_type_modifier(type_)
@@ -408,60 +368,61 @@ def import_object(
             target,
             fs=target_fs,
             project=target_project,
-            index=index,
-            external=external,
         )
     return meta
 
 
 def deploy(
     deploy_meta_or_path: Union[MlemDeployment, str],
-    model: Union[MlemModel, str] = None,
+    model: Union[MlemModel, str],
     env: Union[MlemEnv, str] = None,
     project: Optional[str] = None,
+    rev: Optional[str] = None,
     fs: Optional[AbstractFileSystem] = None,
-    external: bool = None,
-    index: bool = None,
+    env_kwargs: Dict[str, Any] = None,
     **deploy_kwargs,
 ) -> MlemDeployment:
-    deploy_path = None
+    deploy_meta: MlemDeployment
+    update = False
     if isinstance(deploy_meta_or_path, str):
-        deploy_path = deploy_meta_or_path
         try:
             deploy_meta = load_meta(
-                path=deploy_path,
+                path=deploy_meta_or_path,
                 project=project,
+                rev=rev,
                 fs=fs,
                 force_type=MlemDeployment,
             )
-        except MlemObjectNotFound:
-            deploy_meta = None
-
+            update = True
+        except MlemObjectNotFound as e:
+            if env is None:
+                raise MlemError(
+                    "Please provide model and env args for new deployment"
+                ) from e
+            if not deploy_meta_or_path:
+                raise MlemError("deploy_path cannot be empty") from e
+
+            env_meta = ensure_meta(MlemEnv, env, allow_typename=True)
+            if isinstance(env_meta, type):
+                env = None
+                if env_kwargs:
+                    env = env_meta(**env_kwargs)
+            deploy_type = env_meta.deploy_type
+            deploy_meta = deploy_type(
+                env=env,
+                **deploy_kwargs,
+            )
+            deploy_meta.dump(deploy_meta_or_path, fs, project)
     else:
         deploy_meta = deploy_meta_or_path
-        if model is not None:
-            deploy_meta.replace_model(get_model_meta(model))
+        update = True
 
-    if deploy_meta is None:
-        if model is None or env is None:
-            raise MlemError(
-                "Please provide model and env args for new deployment"
-            )
-        if not deploy_path:
-            raise MlemError("deploy_path cannot be empty")
-        model_meta = get_model_meta(model)
-        env_meta = ensure_meta(MlemEnv, env)
-        deploy_meta = env_meta.deploy_type(
-            model=model_meta,
-            env=env_meta,
-            env_link=env_meta.make_link(),
-            model_link=model_meta.make_link(),
-            **deploy_kwargs,
-        )
-        deploy_meta.dump(deploy_path, fs, project, index, external)
+    if update:
+        pass  # todo update from deploy_args and env_args
     # ensuring links are working
     deploy_meta.get_env()
-    deploy_meta.get_model()
+    model_meta = get_model_meta(model)
 
-    deploy_meta.run()
+    deploy_meta.check_unchanged()
+    deploy_meta.deploy(model_meta)
     return deploy_meta