Merge pull request #97 from bioimage-io/change_0.3.2

Change 0.3.2

bioimageio/spec/__init__.py

@@ -1,4 +1,4 @@
-from . import nodes
-from .utils import load_and_resolve_spec, load_model_spec, load_spec, maybe_convert_manifest, maybe_convert_model
+from . import v0_1, v0_3
+from .latest import *
 
-__version__ = nodes.FormatVersion.__args__[-1]
+__version__ = FormatVersion.__args__[-1]

bioimageio/spec/__main__.py

@@ -14,19 +14,24 @@
 app = typer.Typer()  # https://typer.tiangolo.com/
 
 
+@app.command()
+def package_model(model_yaml: Path, auto_convert: bool = False):
+    pass
+
+
 @app.command()
 def verify_spec(model_yaml: Path, auto_convert: bool = False):
     try:
-        spec_data = yaml.load(model_yaml)
+        model_data = yaml.load(model_yaml)
     except Exception as e:
         pprint(e)
         code = 1
     else:
         try:
             if auto_convert:
-                spec_data = maybe_convert_model(deepcopy(spec_data))
+                model_data = maybe_convert_model(deepcopy(model_data))
 
-            verify_model_data(spec_data)
+            verify_model_data(model_data)
         except ValidationError as e:
             pprint(e.messages)
             code = 1

bioimageio/spec/cummulative.py

@@ -0,0 +1,5 @@
+import typing
+
+from . import v0_1, v0_3
+
+FormatVersion = typing.Literal[v0_1.FormatVersion, v0_3.FormatVersion]

bioimageio/spec/exceptions.py

@@ -1,17 +1,5 @@
 from marshmallow import ValidationError
 
 
-class PyBioException(Exception):
-    pass
-
-
-class PyBioRunnerException(Exception):
-    pass
-
-
-class PyBioValidationException(PyBioException, ValidationError):
-    pass
-
-
-class PyBioUnconvertibleException(PyBioValidationException):
+class UnconvertibleError(ValidationError):
     pass

bioimageio/spec/latest/__init__.py

@@ -0,0 +1,2 @@
+from bioimageio.spec.v0_3 import *  # noqa
+from .build_spec import build_spec

bioimageio/spec/utils/build_spec.py → bioimageio/spec/latest/build_spec.py

@@ -20,7 +20,7 @@ def _get_local_path(uri, root=None):
             is_local_path = True
     if not is_local_path:
         uri = spec.fields.URI().deserialize(uri)
-        uri = spec.utils.transformers._download_uri_node_to_local_path(uri).as_posix()
+        uri = spec.download_uri_to_local_path(uri).as_posix()
     return uri
 
 

bioimageio/spec/shared/__init__.py

@@ -0,0 +1 @@
+from .common import yaml, BIOIMAGEIO_CACHE_PATH

bioimageio/spec/shared/common.py

@@ -0,0 +1,17 @@
+import os
+import pathlib
+
+from ruamel.yaml import YAML
+
+yaml = YAML(typ="safe")
+
+BIOIMAGEIO_CACHE_PATH = pathlib.Path(os.getenv("BIOIMAGEIO_CACHE_PATH", pathlib.Path.home() / "bioimageio_cache"))
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]

bioimageio/spec/validate.py → bioimageio/spec/shared/field_validators.py

@@ -28,12 +28,12 @@ def __call__(self, value: typing.Any) -> typing.Any:
             if self.is_getter_method:
                 attr = attr()
         except Exception as e:
-            raise ValidationError from e
+            raise ValidationError(str(e)) from e
 
         try:
             return all(validator(attr) for validator in self.validate)
         except Exception as e:
-            raise ValidationError(f"Invalid {self.attribute}: {str(e)}") from e
+            raise ValidationError(f"Invalid {self.attribute} ({value}): {str(e)}") from e
 
 
 class Predicate(Predicate):

bioimageio/spec/fields.py → bioimageio/spec/shared/fields.py

@@ -1,3 +1,4 @@
+"""fields to be used in the versioned schemas (may return shared raw nodes on `deserialize`"""
 from __future__ import annotations
 
 import datetime
@@ -7,12 +8,12 @@
 from urllib.parse import urlparse
 from urllib.request import url2pathname
 
-import numpy
-from marshmallow import ValidationError, fields as marshmallow_fields, validate as marshmallow_validate
 import marshmallow_union
+import numpy
+from marshmallow import ValidationError, fields as marshmallow_fields
 
-from bioimageio.spec import raw_nodes
-from bioimageio.spec.exceptions import PyBioValidationException
+from bioimageio.spec.shared import field_validators
+from . import raw_nodes
 
 
 class DocumentedField:
@@ -24,12 +25,20 @@ def __init__(
         bioimageio_description: str = "",
         bioimageio_description_order: typing.Optional[int] = None,
         bioimageio_maybe_required: bool = False,  # indicates that this field may be required, depending on other fields
+        bioimageio_examples_valid: typing.Optional[
+            typing.Sequence[typing.Any]
+        ] = None,  # valid examples to render in documentation
+        bioimageio_examples_invalid: typing.Optional[
+            typing.Sequence[typing.Any]
+        ] = None,  # invalid examples to render in documentation
         **super_kwargs,
     ):
         bases = [b.__name__ for b in self.__class__.__bases__ if issubclass(b, marshmallow_fields.Field)]
         if self.__class__.__name__ not in bases:
             bases.insert(0, self.__class__.__name__)
 
+        # if bioimageio_examples_valid is not None:
+        #     valid_examples =
         self.type_name = "→".join(bases)
         self.bioimageio_description = bioimageio_description
         self.bioimageio_description_order = bioimageio_description_order
@@ -71,7 +80,7 @@ def deserialize(self, value: typing.Any, attr: str = None, data: typing.Mapping[
             try:
                 return numpy.array(value, dtype=self.dtype)
             except ValueError as e:
-                raise PyBioValidationException(str(e)) from e
+                raise ValidationError(str(e)) from e
         else:
             return value
 
@@ -166,7 +175,7 @@ def _deserialize(self, *args, **kwargs) -> str:
         axes_str = super()._deserialize(*args, **kwargs)
         valid_axes = self.metadata.get("valid_axes", "bitczyx")
         if any(a not in valid_axes for a in axes_str):
-            raise PyBioValidationException(f"Invalid axes! Valid axes consist of: {valid_axes}")
+            raise ValidationError(f"Invalid axes! Valid axes consist of: {valid_axes}")
 
         return axes_str
 
@@ -198,12 +207,12 @@ def _deserialize(self, *args, **kwargs) -> typing.Any:
             object_name = source_str[last_dot_idx + 1 :]
 
             if not module_name:
-                raise PyBioValidationException(
+                raise ValidationError(
                     f"Missing module name in importable source: {source_str}. Is it just missing a dot?"
                 )
 
             if not object_name:
-                raise PyBioValidationException(
+                raise ValidationError(
                     f"Missing object/callable name in importable source: {source_str}. Is it just missing a dot?"
                 )
 
@@ -236,7 +245,7 @@ def _serialize(self, value, attr, obj, **kwargs) -> typing.Optional[str]:
 
 class InputShape(Union):
     def __init__(self, **super_kwargs):
-        from bioimageio.spec.schema import ImplicitInputShape
+        from .schema import ImplicitInputShape
 
         super().__init__(
             fields=[
@@ -259,7 +268,7 @@ def __init__(self, keys=String, missing=dict, bioimageio_description="Key word a
 
 class OutputShape(Union):
     def __init__(self, **super_kwargs):
-        from bioimageio.spec.schema import ImplicitOutputShape
+        from .schema import ImplicitOutputShape
 
         super().__init__(
             fields=[
@@ -283,6 +292,49 @@ def _serialize(self, value, attr, obj, **kwargs) -> typing.Optional[str]:
         return None if value is None else pathlib.Path(value).as_posix()
 
 
+class RelativeLocalPath(Path):
+    def __init__(
+        self,
+        *super_args,
+        validate: typing.Optional[
+            typing.Union[
+                typing.Callable[[typing.Any], typing.Any], typing.Iterable[typing.Callable[[typing.Any], typing.Any]]
+            ]
+        ] = None,
+        **super_kwargs,
+    ):
+        if validate is None:
+            validate = []
+        elif callable(validate):
+            validate = [validate]
+        else:
+            validate = list(validate)
+
+        super().__init__(
+            *super_args,
+            validate=validate
+            + [
+                field_validators.Predicate("is_absolute", invert_output=True, error="expected relative path."),
+                field_validators.Attribute(
+                    "as_posix",
+                    [
+                        field_validators.ContainsNoneOf(
+                            ":", error="expected local, relative file path."
+                        ),  # monkey patch to fail on urls
+                        field_validators.Predicate(
+                            "count", "..", invert_output=True, error="expected relative file path within model package."
+                        ),
+                    ],
+                    is_getter_method=True,
+                ),
+                field_validators.Predicate(
+                    "is_reserved", invert_output=True, error="invalid filename as it is a reserved by the OS."
+                ),
+            ],
+            **super_kwargs,
+        )
+
+
 class ProcMode(String):
     all_modes = ("fixed", "per_dataset", "per_sample")
 
@@ -308,7 +360,7 @@ def __init__(
         else:
             validate = [validate]
 
-        validate.append(marshmallow_validate.OneOf(self.all_modes))
+        validate.append(field_validators.OneOf(self.all_modes))
         super().__init__(validate=validate, required=required, **kwargs)
 
 
@@ -323,11 +375,11 @@ def _deserialize(self, value, attr, data, **kwargs):
         uri = urlparse(value)
 
         if uri.query:
-            raise PyBioValidationException(f"Invalid URI: {value}. We do not support query: {uri.query}")
+            raise ValidationError(f"Invalid URI: {value}. We do not support query: {uri.query}")
         if uri.fragment:
-            raise PyBioValidationException(f"Invalid URI: {value}. We do not support fragment: {uri.fragment}")
+            raise ValidationError(f"Invalid URI: {value}. We do not support fragment: {uri.fragment}")
         if uri.params:
-            raise PyBioValidationException(f"Invalid URI: {value}. We do not support params: {uri.params}")
+            raise ValidationError(f"Invalid URI: {value}. We do not support params: {uri.params}")
 
         if uri.scheme == "file":
             # account for leading '/' for windows paths, e.g. '/C:/folder'
@@ -358,9 +410,9 @@ def _deserialize(self, *args, **kwargs) -> raw_nodes.URI:
         uri = urlparse(uri_str)
 
         if uri.fragment:
-            raise PyBioValidationException(f"Invalid URI: {uri_str}. We do not support fragment: {uri.fragment}")
+            raise ValidationError(f"Invalid URI: {uri_str}. We do not support fragment: {uri.fragment}")
         if uri.params:
-            raise PyBioValidationException(f"Invalid URI: {uri_str}. We do not support params: {uri.params}")
+            raise ValidationError(f"Invalid URI: {uri_str}. We do not support params: {uri.params}")
 
         if uri.scheme == "file":
             # account for leading '/' for windows paths, e.g. '/C:/folder'

bioimageio/spec/shared/nodes.py

@@ -0,0 +1,11 @@
+"""shared nodes that shared transformer act on"""
+
+from .raw_nodes import *
+
+
+@dataclass
+class ImportedSource:
+    factory: callable
+
+    def __call__(self, *args, **kwargs):
+        return self.factory(*args, **kwargs)

bioimageio/spec/shared/raw_nodes.py

@@ -0,0 +1,64 @@
+"""shared raw nodes that shared transformer act on"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List
+
+from marshmallow import missing
+
+
+@dataclass
+class Node:
+    pass
+
+
+@dataclass
+class URI(Node):
+    """URI as  scheme:[//authority]path[?query][#fragment]"""
+
+    scheme: str = missing
+    authority: str = missing
+    path: str = missing
+    query: str = missing
+    fragment: str = missing
+
+    def __str__(self):
+        """scheme:[//authority]path[?query][#fragment]"""
+        return (
+            (self.scheme + ":" if self.scheme else "")
+            + ("//" + self.authority if self.authority else "")
+            + self.path
+            + ("?" + self.query if self.query else "")
+            + ("#" + self.fragment if self.fragment else "")
+        )
+
+
+@dataclass
+class ImportablePath(Node):
+    filepath: Path = missing
+    callable_name: str = missing
+
+
+@dataclass
+class ImportableModule(Node):
+    module_name: str = missing
+    callable_name: str = missing
+
+
+@dataclass
+class ImplicitInputShape(Node):
+    min: List[float] = missing
+    step: List[float] = missing
+
+    def __len__(self):
+        return len(self.min)
+
+
+@dataclass
+class ImplicitOutputShape(Node):
+    reference_input: str = missing
+    scale: List[float] = missing
+    offset: List[int] = missing
+
+    def __len__(self):
+        return len(self.scale)