Allow LocalAccessPoint to load EModels from Nexus staged data

bluepyemodel/access_point/local.py

@@ -19,6 +19,7 @@
 import glob
 import json
 import logging
+from functools import cached_property
 from itertools import chain
 from pathlib import Path
 
@@ -50,6 +51,8 @@
     "myelinated": "myelin",
 }
 
+SUPPORTED_MORPHOLOGY_EXTENSIONS = (".asc", ".swc")
+
 
 class LocalAccessPoint(DataAccessPoint):
     """Access point to access configuration files and e-models when stored locally."""
@@ -124,8 +127,6 @@ def __init__(
         self.legacy_dir_structure = legacy_dir_structure
         self.with_seeds = with_seeds
 
-        self.morph_path = None
-
         if final_path is None:
             self.final_path = self.emodel_dir / "final.json"
         else:
@@ -141,6 +142,43 @@ def __init__(
         self.pipeline_settings = self.load_pipeline_settings()
         self.unfrozen_params = None
 
+    @cached_property
+    def morph_dir(self):
+        """Return the morphology directory as read from the recipes, or fallback to 'morphology'."""
+        recipes = self.get_recipes()
+        return Path(self.emodel_dir, recipes.get("morph_path", "morphology"))
+
+    @cached_property
+    def morph_path(self):
+        """Return the path to the morphology file as read from the recipes."""
+        recipes = self.get_recipes()
+
+        morph_file = None
+        if isinstance(recipes["morphology"], str):
+            morph_file = recipes["morphology"]
+        else:
+            for _, morph_file in recipes["morphology"]:
+                if morph_file.endswith(SUPPORTED_MORPHOLOGY_EXTENSIONS):
+                    break
+
+        if not morph_file or not morph_file.endswith(SUPPORTED_MORPHOLOGY_EXTENSIONS):
+            raise FileNotFoundError(f"Morphology file not defined or not supported: {morph_file}")
+
+        morph_path = self.morph_dir / morph_file
+
+        if not morph_path.is_file():
+            raise FileNotFoundError(f"Morphology file not found: {morph_path}")
+        if str(Path.cwd()) not in str(morph_path.resolve()) and self.emodel_metadata.iteration:
+            raise FileNotFoundError(
+                "When using a githash or iteration tag, the path to the morphology must be local"
+                " otherwise it cannot be archived during the creation of the githash. To solve"
+                " this issue, you can copy the morphology from "
+                f"{morph_path.resolve()} to {Path.cwd() / 'morphologies'} and update your "
+                "recipes."
+            )
+
+        return morph_path
+
     def set_emodel(self, emodel):
         """Setter for the name of the emodel, check it exists (with or without seed) in recipe."""
         _emodel = "_".join(emodel.split("_")[:2]) if self.with_seeds else emodel
@@ -154,14 +192,48 @@ def set_emodel(self, emodel):
 
     def load_pipeline_settings(self):
         """ """
-
-        settings = self.get_recipes().get("pipeline_settings", {})
-
+        recipes = self.get_recipes()
+        settings = recipes.get("pipeline_settings", {})
+        if isinstance(settings, str):
+            # read the pipeline settings from file
+            settings = self.get_json("pipeline_settings")
         if "morph_modifiers" not in settings:
-            settings["morph_modifiers"] = self.get_recipes().get("morph_modifiers", None)
-
+            settings["morph_modifiers"] = recipes.get("morph_modifiers", None)
         return EModelPipelineSettings(**settings)
 
+    def _config_to_final(self, config):
+        """Convert the configuration stored in EM_*.json to the format used for final.json."""
+        metadata = self.emodel_metadata
+        return {
+            metadata.emodel: {
+                "iteration": metadata.iteration,
+                "emodel": metadata.emodel,
+                "etype": metadata.etype,
+                "ttype": metadata.ttype,
+                "mtype": metadata.mtype,
+                "species": metadata.species,
+                "brain_region": metadata.brain_region,
+                "synapse_class": metadata.synapse_class,
+                "score": config["fitness"],  # float
+                "parameters": config["parameter"],  # list[dict]
+                "fitness": config["score"],  # list[dict]
+                "features": config["features"],  # list[dict]
+                "validation_fitness": config["scoreValidation"],  # list[dict]
+                "validated": config["passedValidation"],  # bool
+                "seed": config["seed"],  # int
+            }
+        }
+
+    def get_final_content(self, lock_file=True):
+        """Return the final content from recipes if available, or fallback to final.json"""
+        recipes = self.get_recipes()
+        if "final" in recipes:
+            if self.final_path and self.final_path.is_file():
+                logger.warning("Ignored %s, using file from recipes", self.final_path)
+            data = self.get_json("final")
+            return self._config_to_final(data)
+        return self.get_final(lock_file=lock_file)
+
     def get_final(self, lock_file=True):
         """Get emodel dictionary from final.json."""
         if self.final_path is None:
@@ -380,18 +452,13 @@ def get_available_mechanisms(self):
 
     def get_available_morphologies(self):
         """Get the list of names of available morphologies"""
-
-        names = []
-
-        morph_dir = self.emodel_dir / "morphology"
+        morph_dir = self.morph_dir
 
         if not morph_dir.is_dir():
             return None
 
-        for morph_file in glob.glob(str(morph_dir / "*.asc")) + glob.glob(str(morph_dir / "*.swc")):
-            names.append(Path(morph_file).stem)
-
-        return set(names)
+        patterns = ["*" + ext for ext in SUPPORTED_MORPHOLOGY_EXTENSIONS]
+        return {morph_file.stem for pattern in patterns for morph_file in morph_dir.glob(pattern)}
 
     def get_model_configuration(self):
         """Get the configuration of the model, including parameters, mechanisms and distributions"""
@@ -415,7 +482,7 @@ def get_model_configuration(self):
         if isinstance(parameters["mechanisms"], dict):
             configuration.init_from_legacy_dict(parameters, self.get_morphologies())
         else:
-            configuration.init_from_dict(parameters)
+            configuration.init_from_dict(parameters, self.get_morphologies())
 
         configuration.mapping_multilocation = self.get_recipes().get("multiloc_map", None)
 
@@ -555,28 +622,6 @@ def get_morphologies(self):
         """
 
         recipes = self.get_recipes()
-
-        if isinstance(recipes["morphology"], str):
-            morph_file = recipes["morphology"]
-        else:
-            morph_file = recipes["morphology"][0][1]
-
-        if self.morph_path is None:
-            self.morph_path = Path(recipes["morph_path"]) / morph_file
-            if not self.morph_path.is_absolute():
-                self.morph_path = Path(self.emodel_dir) / self.morph_path
-        else:
-            self.morph_path = Path(self.morph_path)
-
-        if str(Path.cwd()) not in str(self.morph_path.resolve()) and self.emodel_metadata.iteration:
-            raise FileNotFoundError(
-                "When using a githash or iteration tag, the path to the morphology must be local"
-                " otherwise it cannot be archived during the creation of the githash. To solve"
-                " this issue, you can copy the morphology from "
-                f"{self.morph_path.resolve()} to {Path.cwd() / 'morphologies'} and update your "
-                "recipes."
-            )
-
         morphology_definition = {
             "name": self.morph_path.stem,
             "path": str(self.morph_path),
@@ -628,7 +673,7 @@ def format_emodel_data(self, model_data):
     def get_emodel(self, lock_file=True):
         """Get dict with parameter of single emodel (including seed if any)"""
 
-        final = self.get_final(lock_file=lock_file)
+        final = self.get_final_content(lock_file=lock_file)
 
         if self.emodel_metadata.emodel in final:
             return self.format_emodel_data(final[self.emodel_metadata.emodel])
@@ -648,7 +693,7 @@ def get_emodels(self, emodels=None):
             emodels = [self.emodel_metadata.emodel]
 
         models = []
-        for mod_data in self.get_final().values():
+        for mod_data in self.get_final_content().values():
             if mod_data["emodel"] in emodels:
                 models.append(self.format_emodel_data(mod_data))
 
@@ -692,7 +737,7 @@ def has_model_configuration(self):
         return Path(recipes["params"]).is_file()
 
     def get_emodel_etype_map(self):
-        final = self.get_final()
+        final = self.get_final_content()
         return {emodel: emodel.split("_")[0] for emodel in final}
 
     def get_emodel_names(self):
@@ -702,14 +747,14 @@ def get_emodel_names(self):
             dict: keys are emodel names with seed, values are names without seed.
         """
 
-        final = self.get_final()
+        final = self.get_final_content()
 
         return {mod_name: mod.get("emodel", mod_name) for mod_name, mod in final.items()}
 
     def has_best_model(self, seed):
         """Check if the best model has been stored."""
 
-        final = self.get_final()
+        final = self.get_final_content()
 
         model_name = self.get_model_name_for_final(seed)
 
@@ -718,7 +763,7 @@ def has_best_model(self, seed):
     def is_checked_by_validation(self, seed):
         """Check if the emodel with a given seed has been checked by Validation task."""
 
-        final = self.get_final()
+        final = self.get_final_content()
 
         model_name = self.get_model_name_for_final(seed)
 
@@ -732,7 +777,7 @@ def is_validated(self):
         """Check if enough models have been validated."""
 
         n_validated = 0
-        final = self.get_final()
+        final = self.get_final_content()
 
         for _, entry in final.items():
             if (

bluepyemodel/emodel_pipeline/emodel.py

@@ -67,7 +67,7 @@ def __init__(
             passedValidation (bool or None): did the model go through validation and if yes,
                 did it pass it successfully (None: no validation, True: passed, False: didn't pass)
             seed (str): seed used during optimisation for this emodel.
-            emodel_metadata (str): metadata of the model (emodel name, etype, ttype, ...)
+            emodel_metadata (EModelMetadata): metadata of the model (emodel name, etype, ttype, ...)
         """
 
         self.emodel_metadata = emodel_metadata

bluepyemodel/model/neuron_model_configuration.py

@@ -172,10 +172,17 @@ def _format_locations(locations):
 
         return locations
 
-    def init_from_dict(self, configuration_dict, auto_mechanism=False):
+    def init_from_dict(self, configuration_dict, morphology, auto_mechanism=False):
         """Instantiate the object from its dictionary form"""
 
         if "distributions" in configuration_dict:
+            # empty the list of distributions
+            if self.distributions and configuration_dict["distributions"]:
+                # remove default uniform distribution if added at
+                # https://github.com/BlueBrain/BluePyEModel/blob/
+                # 15b8dd2824453a8bf097b2ede13dd7ecf5d07d05/bluepyemodel/access_point/local.py#L399
+                logger.warning("Removing %s pre-existing distribution(s)", len(self.distributions))
+                self.distributions = []
             for distribution in configuration_dict["distributions"]:
                 self.add_distribution(
                     distribution["name"],
@@ -206,7 +213,8 @@ def init_from_dict(self, configuration_dict, auto_mechanism=False):
                     mechanism.get("ljp_corrected", None),
                 )
 
-        self.morphology = MorphologyConfiguration(**configuration_dict["morphology"])
+        morphology_params = {**configuration_dict["morphology"], **morphology}
+        self.morphology = MorphologyConfiguration(**morphology_params)
 
     def init_from_legacy_dict(self, parameters, morphology):
         """Instantiate the object from its legacy dictionary form"""

bluepyemodel/tasks/emodel_creation/optimisation.py

@@ -931,8 +931,7 @@ def inner(self):
             if EmodelAPIConfig().api == "nexus":
                 self.access_point.check_mettypes()
             # do this instead of just func(self) because of the yield in EModelCreation
-            for x in func(self):
-                yield x
+            yield from func(self)
 
         return inner
 

setup.py

@@ -16,13 +16,7 @@
 limitations under the License.
 """
 
-import imp
-import sys
-
-from setuptools import setup, find_packages
-
-if sys.version_info < (3, 7):
-    sys.exit("Sorry, Python < 3.7 is not supported")
+from setuptools import find_packages, setup
 
 # Read the contents of the README file
 with open("README.rst", encoding="utf-8") as f:
@@ -59,6 +53,7 @@
     long_description=README,
     long_description_content_type="text/x-rst",
     license="Apache-2.0",
+    python_requires=">=3.8",
     install_requires=[
         "numpy",
         "scipy",