Merge pull request #153 from sbesson/write_plate_metadata

Add API for writing HCS metadata

.github/workflows/build.yml

@@ -13,10 +13,9 @@ jobs:
           - {os: windows-latest, python_Version: '3.7', toxenv: 'py37'}
           - {os: windows-latest, python_Version: '3.8', toxenv: 'py38'}
           - {os: windows-latest, python_Version: '3.9', toxenv: 'py39'}
-          # Linux still not working
-          # {os: linux-latest, python_Version: '3.7', toxenv: 'py37'}
-          # {os: linux-latest, python_Version: '3.8', toxenv: 'py38'}
-          # {os: linux-latest, python_Version: '3.9', toxenv: 'py39'}
+          - {os: ubuntu-latest, python_Version: '3.7', toxenv: 'py37'}
+          - {os: ubuntu-latest, python_Version: '3.8', toxenv: 'py38'}
+          - {os: ubuntu-latest, python_Version: '3.9', toxenv: 'py39'}
           - {os: macos-latest, python_Version: '3.7', toxenv: 'py37'}
           - {os: macos-latest, python_Version: '3.8', toxenv: 'py38'}
           # missing numcodecs wheels on 3.9. conda not yet an option. see gh-51

ome_zarr/format.py

@@ -106,7 +106,7 @@ def init_store(self, path: str, mode: str = "r") -> FSStore:
 
         kwargs = {
             "dimension_separator": "/",
-            "normalize_keys": True,
+            "normalize_keys": False,
         }
 
         mkdir = True

ome_zarr/writer.py

@@ -2,7 +2,7 @@
 
 """
 import logging
-from typing import Any, List, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
 
 import numpy as np
 import zarr
@@ -70,6 +70,53 @@ def _validate_axes(axes: List[str], fmt: Format = CurrentFormat()) -> None:
             raise ValueError("5D data must have axes ('t', 'c', 'z', 'y', 'x')")
 
 
+def _validate_well_images(images: List, fmt: Format = CurrentFormat()) -> None:
+
+    VALID_KEYS = [
+        "acquisition",
+        "path",
+    ]
+    for index, image in enumerate(images):
+        if isinstance(image, str):
+            images[index] = {"path": str(image)}
+        elif isinstance(image, dict):
+            if any(e not in VALID_KEYS for e in image.keys()):
+                LOGGER.debug("f{image} contains unspecified keys")
+            if "path" not in image:
+                raise ValueError(f"{image} must contain a path key")
+            if not isinstance(image["path"], str):
+                raise ValueError(f"{image} path must be of string type")
+            if "acquisition" in image and not isinstance(image["acquisition"], int):
+                raise ValueError(f"{image} acquisition must be of int type")
+        else:
+            raise ValueError(f"Unrecognized type for {image}")
+
+
+def _validate_plate_acquisitions(
+    acquisitions: List[Dict], fmt: Format = CurrentFormat()
+) -> None:
+
+    VALID_KEYS = [
+        "id",
+        "name",
+        "maximumfieldcount",
+        "description",
+        "starttime",
+        "endtime",
+    ]
+    if acquisitions is None:
+        return
+    for acquisition in acquisitions:
+        if not isinstance(acquisition, dict):
+            raise ValueError(f"{acquisition} must be a dictionary")
+        if any(e not in VALID_KEYS for e in acquisition.keys()):
+            LOGGER.debug("f{acquisition} contains unspecified keys")
+        if "id" not in acquisition:
+            raise ValueError(f"{acquisition} must contain an id key")
+        if not isinstance(acquisition["id"], int):
+            raise ValueError(f"{acquisition} id must be of int type")
+
+
 def write_multiscale(
     pyramid: List,
     group: zarr.Group,
@@ -147,6 +194,85 @@ def write_multiscales_metadata(
     group.attrs["multiscales"] = multiscales
 
 
+def write_plate_metadata(
+    group: zarr.Group,
+    rows: List[str],
+    columns: List[str],
+    wells: List[str],
+    fmt: Format = CurrentFormat(),
+    acquisitions: List[dict] = None,
+    field_count: int = None,
+    name: str = None,
+) -> None:
+    """
+    Write the plate metadata in the group.
+
+    Parameters
+    ----------
+    group: zarr.Group
+      the group within the zarr store to write the metadata in.
+    rows: list of str
+      The list of names for the plate rows
+    columns: list of str
+      The list of names for the plate columns
+    wells: list of str
+      The list of paths for the well groups
+    fmt: Format
+      The format of the ome_zarr data which should be used.
+      Defaults to the most current.
+    name: str
+      The plate name
+    field_count: int
+      The maximum number of fields per view across wells
+    acquisitions: list of dict
+      A list of the various plate acquisitions
+    """
+
+    plate: Dict[str, Union[str, int, List[Dict]]] = {
+        "columns": [{"name": str(c)} for c in columns],
+        "rows": [{"name": str(r)} for r in rows],
+        "wells": [{"path": str(wp)} for wp in wells],
+        "version": fmt.version,
+    }
+    if name is not None:
+        plate["name"] = name
+    if field_count is not None:
+        plate["field_count"] = field_count
+    if acquisitions is not None:
+        _validate_plate_acquisitions(acquisitions)
+        plate["acquisitions"] = acquisitions
+    group.attrs["plate"] = plate
+
+
+def write_well_metadata(
+    group: zarr.Group,
+    images: Union[List[str], List[dict]],
+    fmt: Format = CurrentFormat(),
+) -> None:
+    """
+    Write the well metadata in the group.
+
+    Parameters
+    ----------
+    group: zarr.Group
+      the group within the zarr store to write the metadata in.
+    image_paths: list of str
+      The list of paths for the well images
+    image_acquisitions: list of int
+      The list of acquisitions for the well images
+    fmt: Format
+      The format of the ome_zarr data which should be used.
+      Defaults to the most current.
+    """
+
+    _validate_well_images(images)
+    well = {
+        "images": images,
+        "version": fmt.version,
+    }
+    group.attrs["well"] = well
+
+
 def write_image(
     image: np.ndarray,
     group: zarr.Group,

tests/test_node.py

@@ -1,8 +1,12 @@
 import pytest
+import zarr
+from numpy import zeros
 
 from ome_zarr.data import create_zarr
+from ome_zarr.format import FormatV01, FormatV02, FormatV03
 from ome_zarr.io import parse_url
-from ome_zarr.reader import Node
+from ome_zarr.reader import Label, Labels, Multiscales, Node, Plate, Well
+from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata
 
 
 class TestNode:
@@ -15,15 +19,125 @@ def test_image(self):
         node = Node(parse_url(str(self.path)), list())
         assert node.data
         assert node.metadata
+        assert len(node.specs) == 2
+        assert isinstance(node.specs[0], Multiscales)
 
     def test_labels(self):
         filename = str(self.path.join("labels"))
         node = Node(parse_url(filename), list())
         assert not node.data
         assert not node.metadata
+        assert len(node.specs) == 1
+        assert isinstance(node.specs[0], Labels)
 
     def test_label(self):
         filename = str(self.path.join("labels", "coins"))
         node = Node(parse_url(filename), list())
         assert node.data
         assert node.metadata
+        assert len(node.specs) == 2
+        assert isinstance(node.specs[0], Label)
+        assert isinstance(node.specs[1], Multiscales)
+
+
+class TestHCSNode:
+    @pytest.fixture(autouse=True)
+    def initdir(self, tmpdir):
+        self.path = tmpdir.mkdir("data")
+        self.store = parse_url(str(self.path), mode="w").store
+        self.root = zarr.group(store=self.store)
+
+    def test_minimal_plate(self):
+        write_plate_metadata(self.root, ["A"], ["1"], ["A/1"])
+        row_group = self.root.require_group("A")
+        well = row_group.require_group("1")
+        write_well_metadata(well, ["0"])
+        image = well.require_group("0")
+        write_image(zeros((1, 1, 1, 256, 256)), image)
+
+        node = Node(parse_url(str(self.path)), list())
+        assert node.data
+        assert node.metadata
+        assert len(node.specs) == 1
+        assert isinstance(node.specs[0], Plate)
+        assert node.specs[0].row_names == ["A"]
+        assert node.specs[0].col_names == ["1"]
+        assert node.specs[0].well_paths == ["A/1"]
+        assert node.specs[0].row_count == 1
+        assert node.specs[0].column_count == 1
+
+        node = Node(parse_url(str(self.path / "A" / "1")), list())
+        assert node.data
+        assert node.metadata
+        assert len(node.specs) == 1
+        assert isinstance(node.specs[0], Well)
+
+    @pytest.mark.parametrize("fmt", (FormatV01(), FormatV02(), FormatV03()))
+    def test_multiwells_plate(self, fmt):
+        row_names = ["A", "B", "C"]
+        col_names = ["1", "2", "3", "4"]
+        well_paths = ["A/1", "A/2", "A/4", "B/2", "B/3", "C/1", "C/3", "C/4"]
+        write_plate_metadata(self.root, row_names, col_names, well_paths, fmt=fmt)
+        for wp in well_paths:
+            row, col = wp.split("/")
+            row_group = self.root.require_group(row)
+            well = row_group.require_group(col)
+            write_well_metadata(well, ["0", "1", "2"], fmt=fmt)
+            for field in range(3):
+                image = well.require_group(str(field))
+                write_image(zeros((1, 1, 1, 256, 256)), image)
+
+        node = Node(parse_url(str(self.path)), list())
+        assert node.data
+        assert node.metadata
+        assert len(node.specs) == 1
+        assert isinstance(node.specs[0], Plate)
+        assert node.specs[0].row_names == row_names
+        assert node.specs[0].col_names == col_names
+        assert node.specs[0].well_paths == well_paths
+        assert node.specs[0].row_count == 3
+        assert node.specs[0].column_count == 4
+
+        for wp in well_paths:
+            node = Node(parse_url(str(self.path / wp)), list())
+            assert node.data
+            assert node.metadata
+            assert len(node.specs) == 1
+            assert isinstance(node.specs[0], Well)
+
+        empty_wells = ["A/3", "B/1", "B/4", "C/2"]
+        for wp in empty_wells:
+            assert parse_url(str(self.path / wp)) is None
+
+    @pytest.mark.xfail(reason="https://github.com/ome/ome-zarr-py/issues/145")
+    @pytest.mark.parametrize(
+        "axes, dims",
+        (
+            (["y", "x"], (256, 256)),
+            (["t", "y", "x"], (1, 256, 256)),
+            (["z", "y", "x"], (1, 256, 256)),
+            (["c", "y", "x"], (1, 256, 256)),
+            (["c", "z", "y", "x"], (1, 1, 256, 256)),
+            (["t", "z", "y", "x"], (1, 1, 256, 256)),
+            (["t", "c", "y", "x"], (1, 1, 256, 256)),
+        ),
+    )
+    def test_plate_2D5D(self, axes, dims):
+        write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], fmt=FormatV03())
+        row_group = self.root.require_group("A")
+        well = row_group.require_group("1")
+        write_well_metadata(well, ["0"], fmt=FormatV03())
+        image = well.require_group("0")
+        write_image(zeros(dims), image, fmt=FormatV03(), axes=axes)
+
+        node = Node(parse_url(str(self.path)), list())
+        assert node.data
+        assert node.metadata
+        assert len(node.specs) == 1
+        assert isinstance(node.specs[0], Plate)
+
+        node = Node(parse_url(str(self.path / "A" / "1")), list())
+        assert node.data
+        assert node.metadata
+        assert len(node.specs) == 1
+        assert isinstance(node.specs[0], Well)

tests/test_reader.py

@@ -1,8 +1,11 @@
 import pytest
+import zarr
+from numpy import zeros
 
 from ome_zarr.data import create_zarr
 from ome_zarr.io import parse_url
-from ome_zarr.reader import Node, Reader
+from ome_zarr.reader import Node, Plate, PlateLabels, Reader
+from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata
 
 
 class TestReader:
@@ -28,3 +31,49 @@ def test_label(self):
         filename = str(self.path.join("labels", "coins"))
         reader = Reader(parse_url(filename))
         assert len(list(reader())) == 3
+
+
+class TestHCSReader:
+    @pytest.fixture(autouse=True)
+    def initdir(self, tmpdir):
+        self.path = tmpdir.mkdir("data")
+        self.store = parse_url(str(self.path), mode="w").store
+        self.root = zarr.group(store=self.store)
+
+    def test_minimal_plate(self):
+        write_plate_metadata(self.root, ["A"], ["1"], ["A/1"])
+        row_group = self.root.require_group("A")
+        well = row_group.require_group("1")
+        write_well_metadata(well, ["0"])
+        image = well.require_group("0")
+        write_image(zeros((1, 1, 1, 256, 256)), image)
+
+        reader = Reader(parse_url(str(self.path)))
+        nodes = list(reader())
+        assert len(nodes) == 2
+        assert len(nodes[0].specs) == 1
+        assert isinstance(nodes[0].specs[0], Plate)
+        assert len(nodes[1].specs) == 1
+        assert isinstance(nodes[1].specs[0], PlateLabels)
+
+    def test_multiwells_plate(self):
+        row_names = ["A", "B", "C"]
+        col_names = ["1", "2", "3", "4"]
+        well_paths = ["A/1", "A/2", "A/4", "B/2", "B/3", "C/1", "C/3", "C/4"]
+        write_plate_metadata(self.root, row_names, col_names, well_paths)
+        for wp in well_paths:
+            row, col = wp.split("/")
+            row_group = self.root.require_group(row)
+            well = row_group.require_group(col)
+            write_well_metadata(well, ["0", "1", "2"])
+            for field in range(3):
+                image = well.require_group(str(field))
+                write_image(zeros((1, 1, 1, 256, 256)), image)
+
+        reader = Reader(parse_url(str(self.path)))
+        nodes = list(reader())
+        assert len(nodes) == 2
+        assert len(nodes[0].specs) == 1
+        assert isinstance(nodes[0].specs[0], Plate)
+        assert len(nodes[1].specs) == 1
+        assert isinstance(nodes[1].specs[0], PlateLabels)