Merge pull request #43 from biosustain/42-make-pseudobatch-installati…

…on-independent-of-cmdstan

42 make pseudobatch installation independent of cmdstan

.github/workflows/main.yaml

@@ -56,6 +56,7 @@ jobs:
       - name: Install package
         run: |
           pip install .[development]
+          pip install .[error_propagation]
 
       - name: Run tests
         run: |

.gitignore

@@ -89,6 +89,7 @@ ipython_config.py
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
 # .python-version
+.venv*
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.

Dockerfile

@@ -13,7 +13,7 @@ RUN pip install --no-cache-dir "cmdstanpy==1.0.4"
 # Install cmdstan
 RUN python -m cmdstanpy.install_cmdstan --version "${CMDSTAN_VERSION}" --cores 2
 
-RUN pip install --no-cache-dir -e "." && \
+RUN pip install --no-cache-dir -e ".[error_propagation]" && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
 

README.md

@@ -25,11 +25,33 @@ Download the excel template from the [excel-pseudobatch folder](./excel-pseudoba
 The Python package holds functions which apply the pseudobatch transformation to data either in the form of `Numpy` Arrays or `Pandas` dataframe. Please visit [our documentation]() to how to use the Python package.
 
 ## How to install?
-The Pseudobatch Python package can be install through PYPI using pip.
-```
+The Pseudobatch Python package can be install through PYPI using pip. Most of the functionality can be installed simply be calling
+
+```shell
 pip install pseudobatch
 ```
 
+The error propagation functionality requires installation of cmdstanpy and CmdStan. Thus, installing the error propagation functionality takes a few steps. First install cmdstanpy in the prefered virtual environment.
+
+```shell
+pip install cmdstanpy
+```
+
+Second, use cmdstanpy to install CmdStan. To due this you need to call use the function `cmdstanpy.install_cmdstan()`. This can for example be done by opening a python session in the terminal and run the following two Python commands
+
+```python
+import cmdstanpy
+cmdstanpy.install_cmdstan()
+```
+
+Now exit the Python session and install the remaining dependencies of the error propagation module through pip.
+
+```shell
+pip install pseudobatch[error_propagation]
+```
+
+Now the error propagation module is installed and ready to use.
+
 ## How to cite
 If you use the pseudobatch transformation please cite the original article XXX.
 

build_testing.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# remove the old virtual environment if it exists
+rm -rf .venv-docker
+
+# Create a new virtual environment
+python -m venv .venv-docker
+
+# Activate the virtual environment
+source .venv-docker/bin/activate
+
+# Install the current directory
+pip install -e '.[development]'
+
+# Run the tests
+pytest
+
+# install error_propagation
+pip install '.[error_propagation]'

pseudobatch/__init__.py

@@ -1,57 +1,8 @@
-import shutil
-import warnings
-from pathlib import Path
-from .import_from_excel import process_excel_template
-# from .data_correction import pseudobatch_transform_multiple, pseudobatch_transform, pseudobatch_transform_pandas, accumulated_dilution_factor, convert_volumetric_rates_from_pseudo_to_real, pseudobatch_transform_pandas_by_group
-import cmdstanpy
-
 from pseudobatch.data_correction import (
     pseudobatch_transform,
     pseudobatch_transform_multiple,
     pseudobatch_transform_pandas,
     hypothetical_concentration,
     metabolised_amount,
 )
-from pseudobatch.error_propagation import run_error_propagation
-
-STAN_FILES_FOLDER = Path(__file__).parent / "stan"
-CMDSTAN_VERSION = "2.31.0"
-
-
-# on Windows specifically, we should point cmdstanpy to the repackaged
-# CmdStan if it exists. This lets cmdstanpy handle the TBB path for us.
-local_cmdstan = STAN_FILES_FOLDER / f"cmdstan-{CMDSTAN_VERSION}"
-if local_cmdstan.exists():
-    cmdstanpy.set_cmdstan_path(str(local_cmdstan.resolve()))
-
-
-def load_stan_model(name: str) -> cmdstanpy.CmdStanModel:
-    """
-    Try to load precompiled Stan models. If that fails,
-    compile them.
-    """
-    try:
-        model = cmdstanpy.CmdStanModel(
-            exe_file=STAN_FILES_FOLDER / f"{name}.exe",
-            stan_file=STAN_FILES_FOLDER / f"{name}.stan",
-            compile=False,
-        )
-    except ValueError:
-        warnings.warn(f"Failed to load pre-built model '{name}.exe', compiling")
-        model = cmdstanpy.CmdStanModel(
-            stan_file=STAN_FILES_FOLDER / f"{name}.stan",
-            stanc_options={"O1": True},
-        )
-        shutil.copy(
-            model.exe_file,  # type: ignore
-            STAN_FILES_FOLDER / f"{name}.exe",
-        )
-
-    return model
-
-
-ERROR_PROPAGATION = load_stan_model("error_propagation")
-
-
-# example: just print the info of the model
-print(ERROR_PROPAGATION.exe_info())
+from .import_from_excel import process_excel_template

pseudobatch/error_propagation/__init__.py

@@ -0,0 +1,68 @@
+import shutil
+import warnings
+from pathlib import Path
+from importlib.metadata import distribution
+
+# The error propagation module requires to be installed separately.
+# Cmdstanpy is here used as an indication of whether the error
+# propagation module is installed or not. If cmdstanpy is installed,
+# the error propagation module will be loaded.
+try:
+    distribution("cmdstanpy")
+    import cmdstanpy
+
+    cmdstanpy_installed = True
+except:
+    cmdstanpy_installed = False
+    raise ImportError(
+        "cmdstanpy is not installed. To use the error propagation module, "
+        "please install cmdstanpy by running 'pip install cmdstanpy'. "
+        "Then install the CmdStan binaries by running 'cmdstanpy.install_cmdstan()'. "
+        "Finally, install the remaining dependencies for the error propagation module "
+        "by running 'pip install pseudobatch[error_propagation]'."
+    )
+
+if cmdstanpy_installed:
+    from pseudobatch.error_propagation.error_propagation import (
+        run_error_propagation,
+    )
+
+    STAN_FILES_FOLDER = Path(__file__).parent / "stan"
+    CMDSTAN_VERSION = "2.31.0"
+
+    # on Windows specifically, we should point cmdstanpy to the repackaged
+    # CmdStan if it exists. This lets cmdstanpy handle the TBB path for us.
+    local_cmdstan = STAN_FILES_FOLDER / f"cmdstan-{CMDSTAN_VERSION}"
+    if local_cmdstan.exists():
+        cmdstanpy.set_cmdstan_path(str(local_cmdstan.resolve()))
+
+    def load_stan_model(name: str) -> cmdstanpy.CmdStanModel:
+        """
+        Try to load precompiled Stan models. If that fails,
+        compile them.
+        """
+        try:
+            model = cmdstanpy.CmdStanModel(
+                exe_file=STAN_FILES_FOLDER / f"{name}.exe",
+                stan_file=STAN_FILES_FOLDER / f"{name}.stan",
+                compile=False,
+            )
+        except ValueError:
+            warnings.warn(
+                f"Failed to load pre-built model '{name}.exe', compiling"
+            )
+            model = cmdstanpy.CmdStanModel(
+                stan_file=STAN_FILES_FOLDER / f"{name}.stan",
+                stanc_options={"O1": True},
+            )
+            shutil.copy(
+                model.exe_file,  # type: ignore
+                STAN_FILES_FOLDER / f"{name}.exe",
+            )
+
+        return model
+
+    ERROR_PROPAGATION = load_stan_model("error_propagation")
+
+    # example: just print the info of the model
+    print(ERROR_PROPAGATION.exe_info())

pseudobatch/error_propagation.py → pseudobatch/error_propagation/error_propagation.py

@@ -9,7 +9,7 @@
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 
-from pseudobatch import stan
+from pseudobatch.error_propagation import stan
 from pseudobatch.util import (
     get_lognormal_params_from_quantiles,
     get_normal_params_from_quantiles,
@@ -114,7 +114,7 @@ def run_error_propagation(
 
     sd_sample_volume : Sample volume measurement error.
 
-    sd_concentration_in_feed : Error for concentration in feed measurements. 
+    sd_concentration_in_feed : Error for concentration in feed measurements.
 
     prior_input : Dictionary that can be used to load a PriorInput object.
 
@@ -134,11 +134,11 @@ def run_error_propagation(
             "then prior_cfeed must not be None."
         )
         assert all(y == 0 for y in y_concentration_in_feed), msg
-        prior_cfeed = [[0. for _ in range(S)], [1. for _ in range(S)]]
+        prior_cfeed = [[0.0 for _ in range(S)], [1.0 for _ in range(S)]]
     else:
         prior_cfeed = [
-            [p.loc for p in pi.prior_cfeed], 
-            [p.scale for p in pi.prior_cfeed]
+            [p.loc for p in pi.prior_cfeed],
+            [p.scale for p in pi.prior_cfeed],
         ]
     prior_m = [Prior0dLogNormal(pct1=1e-9, pct99=1e9) for _ in range(S)]
     prior_f = Prior0dLogNormal(pct1=1e-6, pct99=1e6)
@@ -173,7 +173,7 @@ def run_error_propagation(
         "c": ["sample", "species"],
         "v": ["sample"],
         "cfeed": ["species"],
-        "pseudobatch_c": ["sample", "species"]
+        "pseudobatch_c": ["sample", "species"],
     }
     data_prior = {**data, **{"likelihood": 0}}
     data_posterior = {**data, **{"likelihood": 1}}

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools", "wheel", "cmdstanpy>=1.0.7"]
+requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.isort]

setup.cfg

@@ -13,11 +13,8 @@ packages = find:
 include_package_data = True
 requires_python = >=3.9
 install_requires =
-    arviz
-    cmdstanpy>=1.0.7
     numpy
     pandas
-    pydantic==2.4.2
 
 [options.package_data]
 * =
@@ -37,3 +34,7 @@ documentation =
     sphinx
     nbsphinx
     pydata-sphinx-theme
+error_propagation =
+    arviz
+    cmdstanpy>=1.0.7
+    pydantic==2.4.2