add requirements builder (#434)

* add requirements builder

* add builder for virtualenv

* fix linting

* fix pylint again

* fix entrypoints

* remove upgrade deps

* add ability to install in current active venv

* fix pylint

* add ability to create conda based envs

* fix import

* fix has_conda check

* fix windows issues

* relax array comparison test

* fix typo

* use conda in github actions

* suggested improvements

* fix python version determination

* make create_virtual_env consistent

* suggested improvements

* add test for unix based req

* pass sample data

* fix test

* minor improvements

* use load_impl_ext for requirements builder

* fix tests

* add test for invalid req_type

* register conda mark in pytest

* add test for current and active venv

* fix tests

* fix test

* add conda reqs as list for the builder

* remove usage of context and protected access

* fix entrypoints

* remove pylint disable

* move CondaPackageRequirement to mlem.contrib.venv

* use materialize

* fix tests

* fix docstrings

* fix tests

* fix docs based test

* suggested changes

.github/workflows/check-test-release.yml

@@ -68,6 +68,11 @@ jobs:
     - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python }}
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: ${{ matrix.python }}
+        auto-activate-base: true
+        activate-environment: ""
     - name: get pip cache dir
       id: pip-cache-dir
       run: |

mlem/contrib/requirements.py

@@ -0,0 +1,52 @@
+"""Requirements support
+Extension type: build
+
+MlemBuilder implementation for `Requirements` which includes
+installable, conda, unix, custom, file etc. based requirements.
+"""
+import logging
+from typing import ClassVar, Optional
+
+from pydantic import validator
+
+from mlem.core.base import load_impl_ext
+from mlem.core.objects import MlemBuilder, MlemModel
+from mlem.core.requirements import Requirement
+from mlem.ui import EMOJI_OK, EMOJI_PACK, echo
+from mlem.utils.entrypoints import list_implementations
+
+REQUIREMENTS = "requirements.txt"
+
+logger = logging.getLogger(__name__)
+
+
+class RequirementsBuilder(MlemBuilder):
+    """MlemBuilder implementation for building requirements"""
+
+    type: ClassVar = "requirements"
+
+    target: Optional[str] = None
+    """Target path for requirements"""
+    req_type: str = "installable"
+    """Type of requirements, example: unix"""
+
+    @validator("req_type")
+    def get_req_type(cls, req_type):  # pylint: disable=no-self-argument
+        if req_type not in list_implementations(Requirement):
+            raise ValueError(
+                f"req_type {req_type} is not valid. Allowed options are: {list_implementations(Requirement)}"
+            )
+        return req_type
+
+    def build(self, obj: MlemModel):
+        req_type_cls = load_impl_ext(Requirement.abs_name, self.req_type)
+        assert issubclass(req_type_cls, Requirement)
+        reqs = obj.requirements.of_type(req_type_cls)
+        if self.target is None:
+            reqs_representation = [r.get_repr() for r in reqs]
+            requirement_string = " ".join(reqs_representation)
+            print(requirement_string)
+        else:
+            echo(EMOJI_PACK + "Materializing requirements...")
+            req_type_cls.materialize(reqs, self.target)
+            echo(EMOJI_OK + f"Materialized to {self.target}!")

mlem/contrib/venv.py

@@ -0,0 +1,203 @@
+"""Virtual Environments support
+Extension type: build
+
+MlemBuilder implementations for `Environments` which includes
+conda based and venv based virtual environments.
+"""
+import os
+import platform
+import subprocess
+import sys
+import venv
+from abc import abstractmethod
+from typing import ClassVar, List, Optional
+
+from mlem.core.errors import MlemError
+from mlem.core.objects import MlemBuilder, MlemModel
+from mlem.core.requirements import Requirement
+from mlem.ui import EMOJI_OK, EMOJI_PACK, echo
+
+
+def get_python_exe_in_virtual_env(env_dir: str, use_conda_env: bool = False):
+    if platform.system() == "Windows":
+        if not use_conda_env:
+            return os.path.join(env_dir, "Scripts", "python.exe")
+        return os.path.join(env_dir, "python.exe")
+    return os.path.join(env_dir, "bin", "python")
+
+
+def run_in_subprocess(cmd: List[str], error_msg: str, check_output=False):
+    try:
+        if check_output:
+            return subprocess.check_output(cmd)
+        return subprocess.run(cmd, check=True)
+    except (
+        FileNotFoundError,
+        subprocess.CalledProcessError,
+        subprocess.TimeoutExpired,
+    ) as e:
+        raise MlemError(f"{error_msg}\n{e}") from e
+
+
+class CondaPackageRequirement(Requirement):
+    """Represents a conda package that needs to be installed"""
+
+    type: ClassVar[str] = "conda"
+    package_name: str
+    """Denotes name of a package such as 'numpy'"""
+    spec: Optional[str] = None
+    """Denotes selectors for a package such as '>=1.8,<2'"""
+    channel_name: str = "conda-forge"
+    """Denotes channel from which a package is to be installed"""
+
+    def get_repr(self):
+        """
+        conda installable representation of this module
+        """
+        if self.spec is not None:
+            return f"{self.channel_name}::{self.package_name}{self.spec}"
+        return f"{self.channel_name}::{self.package_name}"
+
+    @classmethod
+    def materialize(cls, reqs, target: str):
+        raise NotImplementedError
+
+
+class EnvBuilder(MlemBuilder):
+    type: ClassVar = "env"
+
+    target: Optional[str] = "venv"
+    """Name of the virtual environment"""
+
+    @abstractmethod
+    def create_virtual_env(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_installed_packages(self, env_dir: str):
+        raise NotImplementedError
+
+
+class VenvBuilder(EnvBuilder):
+    """MlemBuilder implementation for building virtual environments"""
+
+    type: ClassVar = "venv"
+
+    no_cache: bool = False
+    """Disable cache"""
+    current_env: bool = False
+    """Whether to install in the current virtual env, must be active"""
+
+    def create_virtual_env(self):
+        env_dir = os.path.abspath(self.target)
+        venv.create(env_dir, with_pip=True)
+
+    def get_installed_packages(self, env_dir):
+        env_exe = get_python_exe_in_virtual_env(env_dir)
+        return run_in_subprocess(
+            [env_exe, "-m", "pip", "freeze"],
+            error_msg="Error running pip",
+            check_output=True,
+        )
+
+    def build(self, obj: MlemModel):
+        if self.current_env:
+            if (
+                os.getenv("VIRTUAL_ENV") is None
+                or sys.prefix == sys.base_prefix
+            ):
+                raise MlemError("No virtual environment detected.")
+            echo(EMOJI_PACK + f"Detected the virtual env {sys.prefix}")
+            env_dir = sys.prefix
+        else:
+            assert self.target is not None
+            echo(EMOJI_PACK + f"Creating virtual env {self.target}...")
+            self.create_virtual_env()
+            env_dir = os.path.abspath(self.target)
+            os.environ["VIRTUAL_ENV"] = env_dir
+
+        env_exe = get_python_exe_in_virtual_env(env_dir)
+        echo(EMOJI_PACK + "Installing the required packages...")
+        # Based on recommendation given in https://pip.pypa.io/en/latest/user_guide/#using-pip-from-your-program
+        install_cmd = [env_exe, "-m", "pip", "install"]
+        if self.no_cache:
+            install_cmd.append("--no-cache-dir")
+        install_cmd.extend(obj.requirements.to_pip())
+        run_in_subprocess(install_cmd, error_msg="Error running pip")
+        if platform.system() == "Windows":
+            activate_cmd = f"`{self.target}\\Scripts\\activate`"
+        else:
+            activate_cmd = f"`source {self.target}/bin/activate`"
+        echo(
+            EMOJI_OK
+            + f"virtual environment `{self.target}` is ready, activate with {activate_cmd}"
+        )
+        return env_dir
+
+
+class CondaBuilder(EnvBuilder):
+    """MlemBuilder implementation for building conda environments"""
+
+    type: ClassVar = "conda"
+
+    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}"
+    """The python version to use"""
+    current_env: Optional[bool] = False
+    """Whether to install in the current conda env"""
+    conda_reqs: List[CondaPackageRequirement] = []
+    """List of conda package requirements"""
+
+    def create_virtual_env(self):
+        env_dir = os.path.abspath(self.target)
+        create_cmd = ["--prefix", env_dir, f"python={self.python_version}"]
+        run_in_subprocess(
+            ["conda", "create", "-y", *create_cmd],
+            error_msg="Error running conda",
+        )
+
+    def get_installed_packages(self, env_dir):
+        return run_in_subprocess(
+            ["conda", "list", "--prefix", env_dir],
+            error_msg="Error running conda",
+            check_output=True,
+        )
+
+    def build(self, obj: MlemModel):
+        pip_based_packages = obj.requirements.to_pip()
+        conda_based_packages = [r.get_repr() for r in self.conda_reqs]
+
+        if self.current_env:
+            conda_default_env = os.getenv("CONDA_DEFAULT_ENV", None)
+            if conda_default_env == "base" or conda_default_env is None:
+                raise MlemError("No conda environment detected.")
+            echo(EMOJI_PACK + f"Detected the conda env {sys.prefix}")
+            env_dir = sys.prefix
+            env_exe = sys.executable
+        else:
+            assert self.target is not None
+            self.create_virtual_env()
+            env_dir = os.path.abspath(self.target)
+            env_exe = get_python_exe_in_virtual_env(
+                env_dir, use_conda_env=True
+            )
+        if conda_based_packages:
+            run_in_subprocess(
+                [
+                    "conda",
+                    "install",
+                    "--prefix",
+                    env_dir,
+                    "-y",
+                    *conda_based_packages,
+                ],
+                error_msg="Error running conda",
+            )
+
+        # install pip packages in conda env
+        if pip_based_packages:
+            run_in_subprocess(
+                [env_exe, "-m", "pip", "install", *pip_based_packages],
+                error_msg="Error running pip",
+            )
+
+        return env_dir

mlem/core/requirements.py

@@ -55,12 +55,28 @@ class Config:
     abs_name: ClassVar[str] = "requirement"
     type: ClassVar = ...
 
+    @abstractmethod
+    def get_repr(self):
+        raise NotImplementedError
+
+    @classmethod
+    @abstractmethod
+    def materialize(cls, reqs, target: str):
+        raise NotImplementedError
+
 
 class PythonRequirement(Requirement, ABC):
     type: ClassVar = "_python"
     module: str
     """Python module name"""
 
+    def get_repr(self):
+        raise NotImplementedError
+
+    @classmethod
+    def materialize(cls, reqs, target: str):
+        raise NotImplementedError
+
 
 class InstallableRequirement(PythonRequirement):
     """
@@ -85,14 +101,21 @@ def package(self):
             self.module, self.module
         )
 
-    def to_str(self):
+    def get_repr(self):
         """
         pip installable representation of this module
         """
         if self.version is not None:
             return f"{self.package}=={self.version}"
         return self.package
 
+    @classmethod
+    def materialize(cls, reqs, target: str):
+        reqs = [r.get_repr() for r in reqs]
+        requirement_string = "\n".join(reqs)
+        with open(os.path.join(target), "w", encoding="utf8") as fp:
+            fp.write(requirement_string + "\n")
+
     @classmethod
     def from_module(
         cls, mod: ModuleType, package_name: str = None
@@ -148,6 +171,18 @@ class CustomRequirement(PythonRequirement):
     is_package: bool
     """Whether this code should be in %name%/__init__.py"""
 
+    def get_repr(self):
+        raise NotImplementedError
+
+    @classmethod
+    def materialize(cls, reqs, target: str):
+        for cr in reqs:
+            for part, src in cr.to_sources_dict().items():
+                p = os.path.join(target, part)
+                os.makedirs(os.path.dirname(p), exist_ok=True)
+                with open(p, "wb") as f:
+                    f.write(src)
+
     @staticmethod
     def from_module(mod: ModuleType) -> "CustomRequirement":
         """
@@ -273,6 +308,9 @@ class FileRequirement(CustomRequirement):
     module: str = ""
     """Ignored"""
 
+    def get_repr(self):
+        raise NotImplementedError
+
     def to_sources_dict(self):
         """
         Mapping path -> source code for this requirement
@@ -296,6 +334,13 @@ class UnixPackageRequirement(Requirement):
     package_name: str
     """Name of the package"""
 
+    def get_repr(self):
+        return self.package_name
+
+    @classmethod
+    def materialize(cls, reqs, target: str):
+        raise NotImplementedError
+
 
 T = TypeVar("T", bound=Requirement)
 
@@ -399,11 +444,17 @@ def add(self, requirement: Requirement):
             if requirement not in self.__root__:
                 self.__root__.append(requirement)
 
+    def to_unix(self) -> List[str]:
+        """
+        :return: list of unix based packages
+        """
+        return [r.get_repr() for r in self.of_type(UnixPackageRequirement)]
+
     def to_pip(self) -> List[str]:
         """
         :return: list of pip installable packages
         """
-        return [r.to_str() for r in self.installable]
+        return [r.get_repr() for r in self.installable]
 
     def __add__(self, other: "AnyRequirements"):
         other = resolve_requirements(other)
@@ -426,12 +477,7 @@ def new(cls, requirements: "AnyRequirements" = None):
         return resolve_requirements(requirements)
 
     def materialize_custom(self, path: str):
-        for cr in self.custom:
-            for part, src in cr.to_sources_dict().items():
-                p = os.path.join(path, part)
-                os.makedirs(os.path.dirname(p), exist_ok=True)
-                with open(p, "wb") as f:
-                    f.write(src)
+        CustomRequirement.materialize(self.custom, path)
 
     @contextlib.contextmanager
     def import_custom(self):