From f16f15451ff445bfe89570050f664ac37607ed2f Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 5 Jan 2022 11:26:10 +0000 Subject: [PATCH 1/7] Add new API for writing plate and well metadata Define two new methods write_plate_metadata and write_well_metadata including internal validation methods --- ome_zarr/writer.py | 128 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/ome_zarr/writer.py b/ome_zarr/writer.py index 0e94a467..d5c09f13 100644 --- a/ome_zarr/writer.py +++ b/ome_zarr/writer.py @@ -2,7 +2,7 @@ """ import logging -from typing import Any, List, Tuple, Union +from typing import Any, Dict, List, Tuple, Union import numpy as np import zarr @@ -70,6 +70,53 @@ def _validate_axes(axes: List[str], fmt: Format = CurrentFormat()) -> None: raise ValueError("5D data must have axes ('t', 'c', 'z', 'y', 'x')") +def _validate_well_images(images: List, fmt: Format = CurrentFormat()) -> None: + + VALID_KEYS = [ + "acquisition", + "path", + ] + for index, image in enumerate(images): + if isinstance(image, str): + images[index] = {"path": str(image)} + elif isinstance(image, dict): + if not all(e in VALID_KEYS for e in image.keys()): + raise ValueError(f"{image} contains invalid keys") + if "path" not in image: + raise ValueError(f"{image} must contain a path key") + if not isinstance(image["path"], str): + raise ValueError(f"{image} path must be of string type") + if "acquisition" in image and not isinstance(image["acquisition"], int): + raise ValueError(f"{image} acquisition must be of int type") + else: + raise ValueError(f"Unrecognized type for {image}") + + +def _validate_plate_acquisitions( + acquisitions: List[Dict], fmt: Format = CurrentFormat() +) -> None: + + VALID_KEYS = [ + "id", + "name", + "maximumfieldcount", + "description", + "starttime", + "endtime", + ] + if acquisitions is None: + return + for acquisition in acquisitions: + if not isinstance(acquisition, dict): + raise ValueError(f"{acquisition} must be a dictionary") + if not all(e in VALID_KEYS for e in acquisition.keys()): + raise ValueError(f"{acquisition} contains invalid keys") + if "id" not in acquisition: + raise ValueError(f"{acquisition} must contain an id key") + if not isinstance(acquisition["id"], int): + raise ValueError(f"{acquisition} id must be of int type") + + def write_multiscale( pyramid: List, group: zarr.Group, @@ -147,6 +194,85 @@ def write_multiscales_metadata( group.attrs["multiscales"] = multiscales +def write_plate_metadata( + group: zarr.Group, + rows: List[str], + columns: List[str], + wells: List[str], + fmt: Format = CurrentFormat(), + acquisitions: List[dict] = None, + field_count: int = None, + name: str = None, +) -> None: + """ + Write the plate metadata in the group. + + Parameters + ---------- + group: zarr.Group + the group within the zarr store to write the metadata in. + rows: list of str + The list of names for the plate rows + columns: list of str + The list of names for the plate columns + wells: list of str + The list of paths for the well groups + fmt: Format + The format of the ome_zarr data which should be used. + Defaults to the most current. + name: str + The plate name + field_count: int + The maximum number of fields per view across wells + acquisitions: list of dict + A list of the various plate acquisitions + """ + + plate = { + "columns": [{"name": str(c)} for c in columns], + "rows": [{"name": str(r)} for r in rows], + "wells": [{"path": str(wp)} for wp in wells], + "version": fmt.version, + } + if name is not None: + plate["name"] = name + if field_count is not None: + plate["field_count"] = field_count + if acquisitions is not None: + _validate_plate_acquisitions(acquisitions) + plate["acquisitions"] = acquisitions + group.attrs["plate"] = plate + + +def write_well_metadata( + group: zarr.Group, + images: Union[List[str], List[dict]], + fmt: Format = CurrentFormat(), +) -> None: + """ + Write the well metadata in the group. + + Parameters + ---------- + group: zarr.Group + the group within the zarr store to write the metadata in. + image_paths: list of str + The list of paths for the well images + image_acquisitions: list of int + The list of acquisitions for the well images + fmt: Format + The format of the ome_zarr data which should be used. + Defaults to the most current. + """ + + _validate_well_images(images) + well = { + "images": images, + "version": fmt.version, + } + group.attrs["well"] = well + + def write_image( image: np.ndarray, group: zarr.Group, From fab102ab287ec88202d33f108fc470b61f9807d1 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 5 Jan 2022 11:27:29 +0000 Subject: [PATCH 2/7] Update node, reader and writer unit tests to cover plate/well specification --- tests/test_node.py | 116 +++++++++++++++++++++- tests/test_reader.py | 51 +++++++++- tests/test_writer.py | 222 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 386 insertions(+), 3 deletions(-) diff --git a/tests/test_node.py b/tests/test_node.py index dff986d8..aede5b99 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -1,8 +1,12 @@ import pytest +import zarr +from numpy import zeros from ome_zarr.data import create_zarr +from ome_zarr.format import FormatV01, FormatV02, FormatV03 from ome_zarr.io import parse_url -from ome_zarr.reader import Node +from ome_zarr.reader import Label, Labels, Multiscales, Node, Plate, Well +from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata class TestNode: @@ -15,15 +19,125 @@ def test_image(self): node = Node(parse_url(str(self.path)), list()) assert node.data assert node.metadata + assert len(node.specs) == 2 + assert isinstance(node.specs[0], Multiscales) def test_labels(self): filename = str(self.path.join("labels")) node = Node(parse_url(filename), list()) assert not node.data assert not node.metadata + assert len(node.specs) == 1 + assert isinstance(node.specs[0], Labels) def test_label(self): filename = str(self.path.join("labels", "coins")) node = Node(parse_url(filename), list()) assert node.data assert node.metadata + assert len(node.specs) == 2 + assert isinstance(node.specs[0], Label) + assert isinstance(node.specs[1], Multiscales) + + +class TestHCSNode: + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + self.store = parse_url(str(self.path), mode="w").store + self.root = zarr.group(store=self.store) + + def test_minimal_plate(self): + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"]) + row_group = self.root.require_group("A") + well = row_group.require_group("1") + write_well_metadata(well, ["0"]) + image = well.require_group("0") + write_image(zeros((1, 1, 1, 256, 256)), image) + + node = Node(parse_url(str(self.path)), list()) + assert node.data + assert node.metadata + assert len(node.specs) == 1 + assert isinstance(node.specs[0], Plate) + assert node.specs[0].row_names == ["A"] + assert node.specs[0].col_names == ["1"] + assert node.specs[0].well_paths == ["A/1"] + assert node.specs[0].row_count == 1 + assert node.specs[0].column_count == 1 + + node = Node(parse_url(str(self.path / "A" / "1")), list()) + assert node.data + assert node.metadata + assert len(node.specs) == 1 + assert isinstance(node.specs[0], Well) + + @pytest.mark.parametrize("fmt", (FormatV01(), FormatV02(), FormatV03())) + def test_multiwells_plate(self, fmt): + row_names = ["A", "B", "C"] + col_names = ["1", "2", "3", "4"] + well_paths = ["A/1", "A/2", "A/4", "B/2", "B/3", "C/1", "C/3", "C/4"] + write_plate_metadata(self.root, row_names, col_names, well_paths, fmt=fmt) + for wp in well_paths: + row, col = wp.split("/") + row_group = self.root.require_group(row) + well = row_group.require_group(col) + write_well_metadata(well, ["0", "1", "2"], fmt=fmt) + for field in range(3): + image = well.require_group(str(field)) + write_image(zeros((1, 1, 1, 256, 256)), image) + + node = Node(parse_url(str(self.path)), list()) + assert node.data + assert node.metadata + assert len(node.specs) == 1 + assert isinstance(node.specs[0], Plate) + assert node.specs[0].row_names == row_names + assert node.specs[0].col_names == col_names + assert node.specs[0].well_paths == well_paths + assert node.specs[0].row_count == 3 + assert node.specs[0].column_count == 4 + + for wp in well_paths: + node = Node(parse_url(str(self.path / wp)), list()) + assert node.data + assert node.metadata + assert len(node.specs) == 1 + assert isinstance(node.specs[0], Well) + + empty_wells = ["A/3", "B/1", "B/4", "C/2"] + for wp in empty_wells: + assert parse_url(str(self.path / wp)) is None + + @pytest.mark.xfail(reason="https://github.com/ome/ome-zarr-py/issues/145") + @pytest.mark.parametrize( + "axes, dims", + ( + (["y", "x"], (256, 256)), + (["t", "y", "x"], (1, 256, 256)), + (["z", "y", "x"], (1, 256, 256)), + (["c", "y", "x"], (1, 256, 256)), + (["c", "z", "y", "x"], (1, 1, 256, 256)), + (["t", "z", "y", "x"], (1, 1, 256, 256)), + (["t", "c", "y", "x"], (1, 1, 256, 256)), + ), + ) + def test_plate_2D5D(self, axes, dims): + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], fmt=FormatV03()) + row_group = self.root.require_group("A") + well = row_group.require_group("1") + write_well_metadata(well, ["0"], fmt=FormatV03()) + image = well.require_group("0") + write_image(zeros(dims), image, fmt=FormatV03(), axes=axes) + + node = Node(parse_url(str(self.path)), list()) + assert node.data + assert node.metadata + assert len(node.specs) == 1 + assert isinstance(node.specs[0], Plate) + + node = Node(parse_url(str(self.path / "A" / "1")), list()) + assert node.data + assert node.metadata + assert len(node.specs) == 1 + assert isinstance(node.specs[0], Well) diff --git a/tests/test_reader.py b/tests/test_reader.py index f1bc20a1..f82eb59e 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,8 +1,11 @@ import pytest +import zarr +from numpy import zeros from ome_zarr.data import create_zarr from ome_zarr.io import parse_url -from ome_zarr.reader import Node, Reader +from ome_zarr.reader import Node, Plate, PlateLabels, Reader +from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata class TestReader: @@ -28,3 +31,49 @@ def test_label(self): filename = str(self.path.join("labels", "coins")) reader = Reader(parse_url(filename)) assert len(list(reader())) == 3 + + +class TestHCSReader: + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + self.store = parse_url(str(self.path), mode="w").store + self.root = zarr.group(store=self.store) + + def test_minimal_plate(self): + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"]) + row_group = self.root.require_group("A") + well = row_group.require_group("1") + write_well_metadata(well, ["0"]) + image = well.require_group("0") + write_image(zeros((1, 1, 1, 256, 256)), image) + + reader = Reader(parse_url(str(self.path))) + nodes = list(reader()) + assert len(nodes) == 2 + assert len(nodes[0].specs) == 1 + assert isinstance(nodes[0].specs[0], Plate) + assert len(nodes[1].specs) == 1 + assert isinstance(nodes[1].specs[0], PlateLabels) + + def test_multiwells_plate(self): + row_names = ["A", "B", "C"] + col_names = ["1", "2", "3", "4"] + well_paths = ["A/1", "A/2", "A/4", "B/2", "B/3", "C/1", "C/3", "C/4"] + write_plate_metadata(self.root, row_names, col_names, well_paths) + for wp in well_paths: + row, col = wp.split("/") + row_group = self.root.require_group(row) + well = row_group.require_group(col) + write_well_metadata(well, ["0", "1", "2"]) + for field in range(3): + image = well.require_group(str(field)) + write_image(zeros((1, 1, 1, 256, 256)), image) + + reader = Reader(parse_url(str(self.path))) + nodes = list(reader()) + assert len(nodes) == 2 + assert len(nodes[0].specs) == 1 + assert isinstance(nodes[0].specs[0], Plate) + assert len(nodes[1].specs) == 1 + assert isinstance(nodes[1].specs[0], PlateLabels) diff --git a/tests/test_writer.py b/tests/test_writer.py index 5110b3d4..1ef7b05d 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -4,7 +4,7 @@ import pytest import zarr -from ome_zarr.format import FormatV01, FormatV02, FormatV03 +from ome_zarr.format import CurrentFormat, FormatV01, FormatV02, FormatV03 from ome_zarr.io import parse_url from ome_zarr.reader import Multiscales, Reader from ome_zarr.scale import Scaler @@ -12,6 +12,8 @@ _validate_axes_names, write_image, write_multiscales_metadata, + write_plate_metadata, + write_well_metadata, ) @@ -200,3 +202,221 @@ def test_axes_ignored(self, fmt): def test_invalid_0_3_axes(self, axes): with pytest.raises(ValueError): write_multiscales_metadata(self.root, ["0"], fmt=FormatV03(), axes=axes) + + +class TestPlateMetadata: + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = pathlib.Path(tmpdir.mkdir("data")) + self.store = parse_url(self.path, mode="w").store + self.root = zarr.group(store=self.store) + + def test_minimal_plate(self): + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"]) + assert "plate" in self.root.attrs + assert self.root.attrs["plate"]["columns"] == [{"name": "1"}] + assert self.root.attrs["plate"]["rows"] == [{"name": "A"}] + assert self.root.attrs["plate"]["version"] == CurrentFormat().version + assert self.root.attrs["plate"]["wells"] == [{"path": "A/1"}] + assert "name" not in self.root.attrs["plate"] + assert "field_count" not in self.root.attrs["plate"] + assert "acquisitions" not in self.root.attrs["plate"] + + def test_12wells_plate(self): + rows = ["A", "B", "C", "D"] + cols = ["1", "2", "3"] + wells = [ + "A/1", + "A/2", + "A/3", + "B/1", + "B/2", + "B/3", + "C/1", + "C/2", + "C/3", + "D/1", + "D/2", + "D/3", + ] + write_plate_metadata(self.root, rows, cols, wells) + assert "plate" in self.root.attrs + assert self.root.attrs["plate"]["columns"] == [ + {"name": "1"}, + {"name": "2"}, + {"name": "3"}, + ] + assert self.root.attrs["plate"]["rows"] == [ + {"name": "A"}, + {"name": "B"}, + {"name": "C"}, + {"name": "D"}, + ] + assert self.root.attrs["plate"]["version"] == CurrentFormat().version + assert self.root.attrs["plate"]["wells"] == [ + {"path": "A/1"}, + {"path": "A/2"}, + {"path": "A/3"}, + {"path": "B/1"}, + {"path": "B/2"}, + {"path": "B/3"}, + {"path": "C/1"}, + {"path": "C/2"}, + {"path": "C/3"}, + {"path": "D/1"}, + {"path": "D/2"}, + {"path": "D/3"}, + ] + assert "name" not in self.root.attrs["plate"] + assert "field_count" not in self.root.attrs["plate"] + assert "acquisitions" not in self.root.attrs["plate"] + + @pytest.mark.parametrize("fmt", (FormatV01(), FormatV02(), FormatV03())) + def test_plate_version(self, fmt): + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], fmt=fmt) + assert "plate" in self.root.attrs + assert self.root.attrs["plate"]["columns"] == [{"name": "1"}] + assert self.root.attrs["plate"]["rows"] == [{"name": "A"}] + assert self.root.attrs["plate"]["version"] == fmt.version + assert self.root.attrs["plate"]["wells"] == [{"path": "A/1"}] + assert "name" not in self.root.attrs["plate"] + assert "field_count" not in self.root.attrs["plate"] + assert "acquisitions" not in self.root.attrs["plate"] + + def test_plate_name(self): + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], name="test") + assert "plate" in self.root.attrs + assert self.root.attrs["plate"]["columns"] == [{"name": "1"}] + assert self.root.attrs["plate"]["name"] == "test" + assert self.root.attrs["plate"]["rows"] == [{"name": "A"}] + assert self.root.attrs["plate"]["version"] == CurrentFormat().version + assert self.root.attrs["plate"]["wells"] == [{"path": "A/1"}] + assert "field_count" not in self.root.attrs["plate"] + assert "acquisitions" not in self.root.attrs["plate"] + + def test_field_count(self): + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], field_count=10) + assert "plate" in self.root.attrs + assert self.root.attrs["plate"]["columns"] == [{"name": "1"}] + assert self.root.attrs["plate"]["field_count"] == 10 + assert self.root.attrs["plate"]["rows"] == [{"name": "A"}] + assert self.root.attrs["plate"]["version"] == CurrentFormat().version + assert self.root.attrs["plate"]["wells"] == [{"path": "A/1"}] + assert "name" not in self.root.attrs["plate"] + assert "acquisitions" not in self.root.attrs["plate"] + + def test_acquisitions_minimal(self): + a = [{"id": 1}, {"id": 2}, {"id": 3}] + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], acquisitions=a) + assert "plate" in self.root.attrs + assert self.root.attrs["plate"]["acquisitions"] == a + assert self.root.attrs["plate"]["columns"] == [{"name": "1"}] + assert self.root.attrs["plate"]["rows"] == [{"name": "A"}] + assert self.root.attrs["plate"]["version"] == CurrentFormat().version + assert self.root.attrs["plate"]["wells"] == [{"path": "A/1"}] + assert "name" not in self.root.attrs["plate"] + assert "field_count" not in self.root.attrs["plate"] + + def test_acquisitions_maximal(self): + a = [ + { + "id": 1, + "name": "acquisition_1", + "description": " first acquisition", + "maximumfieldcount": 2, + "starttime": 1343749391000, + "endtime": 1343749392000, + } + ] + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], acquisitions=a) + assert "plate" in self.root.attrs + assert self.root.attrs["plate"]["acquisitions"] == a + assert self.root.attrs["plate"]["columns"] == [{"name": "1"}] + assert self.root.attrs["plate"]["rows"] == [{"name": "A"}] + assert self.root.attrs["plate"]["version"] == CurrentFormat().version + assert self.root.attrs["plate"]["wells"] == [{"path": "A/1"}] + assert "name" not in self.root.attrs["plate"] + assert "field_count" not in self.root.attrs["plate"] + + @pytest.mark.parametrize( + "acquisitions", + ( + [0, 1], + [{"name": "0"}, {"name": "1"}], + [{"id": 0, "invalid_key": "0"}], + [{"id": "0"}, {"id": "1"}], + ), + ) + def test_invalid_acquisitions(self, acquisitions): + with pytest.raises(ValueError): + write_plate_metadata( + self.root, ["A"], ["1"], ["A/1"], acquisitions=acquisitions + ) + + +class TestWellMetadata: + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = pathlib.Path(tmpdir.mkdir("data")) + self.store = parse_url(self.path, mode="w").store + self.root = zarr.group(store=self.store) + + @pytest.mark.parametrize("images", (["0"], [{"path": "0"}])) + def test_minimal_well(self, images): + write_well_metadata(self.root, images) + assert "well" in self.root.attrs + assert self.root.attrs["well"]["images"] == [{"path": "0"}] + assert self.root.attrs["well"]["version"] == CurrentFormat().version + + @pytest.mark.parametrize( + "images", + ( + ["0", "1", "2"], + [ + {"path": "0"}, + {"path": "1"}, + {"path": "2"}, + ], + ), + ) + def test_multiple_images(self, images): + write_well_metadata(self.root, images) + assert "well" in self.root.attrs + assert self.root.attrs["well"]["images"] == [ + {"path": "0"}, + {"path": "1"}, + {"path": "2"}, + ] + assert self.root.attrs["well"]["version"] == CurrentFormat().version + + @pytest.mark.parametrize("fmt", (FormatV01(), FormatV02(), FormatV03())) + def test_version(self, fmt): + write_well_metadata(self.root, ["0"], fmt=fmt) + assert "well" in self.root.attrs + assert self.root.attrs["well"]["images"] == [{"path": "0"}] + assert self.root.attrs["well"]["version"] == fmt.version + + def test_multiple_acquisitions(self): + images = [ + {"path": "0", "acquisition": 1}, + {"path": "1", "acquisition": 2}, + {"path": "2", "acquisition": 3}, + ] + write_well_metadata(self.root, images) + assert "well" in self.root.attrs + assert self.root.attrs["well"]["images"] == images + assert self.root.attrs["well"]["version"] == CurrentFormat().version + + @pytest.mark.parametrize( + "images", + ( + [{"acquisition": 0}, {"acquisition": 1}], + [{"path": "0", "acquisition": "0"}, {"path": "1", "acquisition": "1"}], + [{"path": 0}, {"path": 1}], + [{"path": "0", "name": "0"}, {"path": "1", "name": "1"}], + [0, 1], + ), + ) + def test_invalid_images(self, images): + with pytest.raises(ValueError): + write_well_metadata(self.root, images) From 7105153076b7e74cef5daf2bcfea26985bd65fb4 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 5 Jan 2022 11:55:53 +0000 Subject: [PATCH 3/7] Specify types for plate dictionary using type annotation --- ome_zarr/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/writer.py b/ome_zarr/writer.py index d5c09f13..a8f0b79b 100644 --- a/ome_zarr/writer.py +++ b/ome_zarr/writer.py @@ -228,7 +228,7 @@ def write_plate_metadata( A list of the various plate acquisitions """ - plate = { + plate: Dict[str, Union[str, int, List[Dict]]] = { "columns": [{"name": str(c)} for c in columns], "rows": [{"name": str(r)} for r in rows], "wells": [{"path": str(wp)} for wp in wells], From f468a0a213a1a084d7ec314e72b9b97398d46573 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 5 Jan 2022 14:37:17 +0000 Subject: [PATCH 4/7] ome_zarr.format: do not normalize_keys For writing HCS datasets where group names are case-sensitive (e.g. A), this option alters the name and creates a mismatch with the plate metadata. --- ome_zarr/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/format.py b/ome_zarr/format.py index d8c3a77b..2a6ab021 100644 --- a/ome_zarr/format.py +++ b/ome_zarr/format.py @@ -106,7 +106,7 @@ def init_store(self, path: str, mode: str = "r") -> FSStore: kwargs = { "dimension_separator": "/", - "normalize_keys": True, + "normalize_keys": False, } mkdir = True From d1311aa883c345781f369a117dfcd1bde32d5130 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 5 Jan 2022 14:39:15 +0000 Subject: [PATCH 5/7] Re-enable Ubuntu tests using various Python versions --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11bfe93e..c0852cdd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,10 +13,9 @@ jobs: - {os: windows-latest, python_Version: '3.7', toxenv: 'py37'} - {os: windows-latest, python_Version: '3.8', toxenv: 'py38'} - {os: windows-latest, python_Version: '3.9', toxenv: 'py39'} - # Linux still not working - # {os: linux-latest, python_Version: '3.7', toxenv: 'py37'} - # {os: linux-latest, python_Version: '3.8', toxenv: 'py38'} - # {os: linux-latest, python_Version: '3.9', toxenv: 'py39'} + - {os: ubuntu-latest, python_Version: '3.7', toxenv: 'py37'} + - {os: ubuntu-latest, python_Version: '3.8', toxenv: 'py38'} + - {os: ubuntu-latest, python_Version: '3.9', toxenv: 'py39'} - {os: macos-latest, python_Version: '3.7', toxenv: 'py37'} - {os: macos-latest, python_Version: '3.8', toxenv: 'py38'} # missing numcodecs wheels on 3.9. conda not yet an option. see gh-51 From 12b341d712a4fdaed29e5094a6f27a1cdf922e09 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Sun, 9 Jan 2022 11:47:46 +0000 Subject: [PATCH 6/7] Be lenient about unspecified keys in ome_zarr.writer --- ome_zarr/writer.py | 8 ++++---- tests/test_writer.py | 21 +++++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ome_zarr/writer.py b/ome_zarr/writer.py index a8f0b79b..28c6d1b7 100644 --- a/ome_zarr/writer.py +++ b/ome_zarr/writer.py @@ -80,8 +80,8 @@ def _validate_well_images(images: List, fmt: Format = CurrentFormat()) -> None: if isinstance(image, str): images[index] = {"path": str(image)} elif isinstance(image, dict): - if not all(e in VALID_KEYS for e in image.keys()): - raise ValueError(f"{image} contains invalid keys") + if any(e not in VALID_KEYS for e in image.keys()): + LOGGER.debug("f{image} contains unspecified keys") if "path" not in image: raise ValueError(f"{image} must contain a path key") if not isinstance(image["path"], str): @@ -109,8 +109,8 @@ def _validate_plate_acquisitions( for acquisition in acquisitions: if not isinstance(acquisition, dict): raise ValueError(f"{acquisition} must be a dictionary") - if not all(e in VALID_KEYS for e in acquisition.keys()): - raise ValueError(f"{acquisition} contains invalid keys") + if not any(e not in VALID_KEYS for e in acquisition.keys()): + LOGGER.debug("f{acquisition} contains unspecified keys") if "id" not in acquisition: raise ValueError(f"{acquisition} must contain an id key") if not isinstance(acquisition["id"], int): diff --git a/tests/test_writer.py b/tests/test_writer.py index 1ef7b05d..2f0897ef 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -347,11 +347,10 @@ def test_acquisitions_maximal(self): [{"id": "0"}, {"id": "1"}], ), ) - def test_invalid_acquisitions(self, acquisitions): - with pytest.raises(ValueError): - write_plate_metadata( - self.root, ["A"], ["1"], ["A/1"], acquisitions=acquisitions - ) + def test_unspecified_acquisition_keys(self, acquisitions): + a = [{"id": 0, "invalid_key": "0"}] + write_plate_metadata(self.root, ["A"], ["1"], ["A/1"], acquisitions=a) + assert "plate" in self.root.attrs class TestWellMetadata: @@ -413,10 +412,20 @@ def test_multiple_acquisitions(self): [{"acquisition": 0}, {"acquisition": 1}], [{"path": "0", "acquisition": "0"}, {"path": "1", "acquisition": "1"}], [{"path": 0}, {"path": 1}], - [{"path": "0", "name": "0"}, {"path": "1", "name": "1"}], [0, 1], ), ) def test_invalid_images(self, images): with pytest.raises(ValueError): write_well_metadata(self.root, images) + + def test_unspecified_images_keys(self): + images = [ + {"path": "0", "acquisition": 1, "unspecified_key": "alpha"}, + {"path": "1", "acquisition": 2, "unspecified_key": "beta"}, + {"path": "2", "acquisition": 3, "unspecified_key": "gamma"}, + ] + write_well_metadata(self.root, images) + assert "well" in self.root.attrs + assert self.root.attrs["well"]["images"] == images + assert self.root.attrs["well"]["version"] == CurrentFormat().version From ff197a0d4b278d29e6aa5ff27a7372cef1904fc1 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Mon, 10 Jan 2022 09:33:23 +0000 Subject: [PATCH 7/7] Remove unexpected not --- ome_zarr/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/writer.py b/ome_zarr/writer.py index 28c6d1b7..9c36603a 100644 --- a/ome_zarr/writer.py +++ b/ome_zarr/writer.py @@ -109,7 +109,7 @@ def _validate_plate_acquisitions( for acquisition in acquisitions: if not isinstance(acquisition, dict): raise ValueError(f"{acquisition} must be a dictionary") - if not any(e not in VALID_KEYS for e in acquisition.keys()): + if any(e not in VALID_KEYS for e in acquisition.keys()): LOGGER.debug("f{acquisition} contains unspecified keys") if "id" not in acquisition: raise ValueError(f"{acquisition} must contain an id key")