From 10df6d43ab06cc63ee7c9a3e65583943fae7e9dd Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Fri, 11 Jun 2021 23:26:26 +0200 Subject: [PATCH 01/22] Update reader.py to v0.3 --- ome_zarr/reader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 08e3335c..c6e7ee06 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -262,10 +262,15 @@ def matches(zarr: ZarrLocation) -> bool: def __init__(self, node: Node) -> None: super().__init__(node) + axes_values = {"t", "c", "z", "y", "x"} try: multiscales = self.lookup("multiscales", []) version = multiscales[0].get("version", "0.1") datasets = multiscales[0]["datasets"] + # axes field was introduced in 0.3, before all data was 5d + axes = multiscales[0].get("axes", ["t", "c", "z", "y", "x"]) + if len(set(axes) - axes_values) > 0: + raise RuntimeError(f"Invalid axes names: {set(axes) - axes_values}") datasets = [d["path"] for d in datasets] self.datasets: List[str] = datasets LOGGER.info("datasets %s", datasets) @@ -274,14 +279,13 @@ def __init__(self, node: Node) -> None: return # EARLY EXIT for resolution in self.datasets: - # data.shape is (t, c, z, y, x) by convention data: da.core.Array = self.array(resolution, version) chunk_sizes = [ str(c[0]) + (" (+ %s)" % c[-1] if c[-1] != c[0] else "") for c in data.chunks ] LOGGER.info("resolution: %s", resolution) - LOGGER.info(" - shape (t, c, z, y, x) = %s", data.shape) + LOGGER.info(" - shape %s = %s", axes, data.shape) LOGGER.info(" - chunks = %s", chunk_sizes) LOGGER.info(" - dtype = %s", data.dtype) node.data.append(data) From 36e37cfd9b5dac9d92a1f9b6c72808b9d4909c0f Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Mon, 14 Jun 2021 13:59:07 +0200 Subject: [PATCH 02/22] Remove opencv dependency and use skimage.resize for nearest-neighbor interpolation --- environment.yml | 2 -- ome_zarr/scale.py | 15 ++++++++------- setup.py | 1 - 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/environment.yml b/environment.yml index 52a34855..2dbb7d64 100644 --- a/environment.yml +++ b/environment.yml @@ -8,9 +8,7 @@ dependencies: - ipython - mypy - omero-py - - opencv - pip - - py-opencv - pytest - requests - s3fs diff --git a/ome_zarr/scale.py b/ome_zarr/scale.py index dc92374d..c9194cef 100644 --- a/ome_zarr/scale.py +++ b/ome_zarr/scale.py @@ -9,11 +9,10 @@ from dataclasses import dataclass from typing import Callable, Iterator, List -import cv2 import numpy as np import zarr from scipy.ndimage import zoom -from skimage.transform import downscale_local_mean, pyramid_gaussian, pyramid_laplacian +from skimage.transform import downscale_local_mean, pyramid_gaussian, pyramid_laplacian, resize from .io import parse_url @@ -123,7 +122,7 @@ def __create_group( def nearest(self, base: np.ndarray) -> List[np.ndarray]: """ - Downsample using :func:`cv2.resize`. + Downsample using :func:`skimage.transform.resize`. The :const:`cvs2.INTER_NEAREST` interpolation method is used. """ @@ -131,11 +130,13 @@ def nearest(self, base: np.ndarray) -> List[np.ndarray]: def __nearest(self, plane: np.ndarray, sizeY: int, sizeX: int) -> np.ndarray: """Apply the 2-dimensional transformation.""" - return cv2.resize( + return resize( plane, - dsize=(sizeY // self.downscale, sizeX // self.downscale), - interpolation=cv2.INTER_NEAREST, - ) + output_shape=(sizeY // self.downscale, sizeX // self.downscale), + order=0, + preserve_range=True, + anti_aliasing=False + ).astype(plane.dtype) def gaussian(self, base: np.ndarray) -> List[np.ndarray]: """Downsample using :func:`skimage.transform.pyramid_gaussian`.""" diff --git a/setup.py b/setup.py index fac91c28..991862cf 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,6 @@ def read(fname): install_requires += (["scikit-image"],) install_requires += (["toolz"],) install_requires += (["vispy"],) -install_requires += (["opencv-contrib-python-headless"],) setup( From cfe03e386490276a2c59c1e72c3824d46380d260 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 12:00:49 +0000 Subject: [PATCH 03/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .isort.cfg | 2 +- ome_zarr/scale.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 324d69b4..a43a1678 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] -known_third_party = cv2,dask,numpy,pytest,scipy,setuptools,skimage,vispy,zarr +known_third_party = dask,numpy,pytest,scipy,setuptools,skimage,vispy,zarr multi_line_output=6 include_trailing_comma=False force_grid_wrap=0 diff --git a/ome_zarr/scale.py b/ome_zarr/scale.py index c9194cef..4509a2a3 100644 --- a/ome_zarr/scale.py +++ b/ome_zarr/scale.py @@ -12,7 +12,12 @@ import numpy as np import zarr from scipy.ndimage import zoom -from skimage.transform import downscale_local_mean, pyramid_gaussian, pyramid_laplacian, resize +from skimage.transform import ( + downscale_local_mean, + pyramid_gaussian, + pyramid_laplacian, + resize, +) from .io import parse_url @@ -135,7 +140,7 @@ def __nearest(self, plane: np.ndarray, sizeY: int, sizeX: int) -> np.ndarray: output_shape=(sizeY // self.downscale, sizeX // self.downscale), order=0, preserve_range=True, - anti_aliasing=False + anti_aliasing=False, ).astype(plane.dtype) def gaussian(self, base: np.ndarray) -> List[np.ndarray]: From f28b05128618bde7b25f26fe6c9a799119b2ae65 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Tue, 15 Jun 2021 23:09:21 +0200 Subject: [PATCH 04/22] Add FormatV03 --- ome_zarr/format.py | 14 +++++++++++++- ome_zarr/reader.py | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ome_zarr/format.py b/ome_zarr/format.py index 1771d0dc..4f552ea2 100644 --- a/ome_zarr/format.py +++ b/ome_zarr/format.py @@ -13,6 +13,7 @@ def format_implementations() -> Iterator["Format"]: """ Return an instance of each format implementation, newest to oldest. """ + yield FormatV03() yield FormatV02() yield FormatV01() @@ -124,4 +125,15 @@ def init_store(self, path: str, mode: str = "r") -> FSStore: return store -CurrentFormat = FormatV02 +class FormatV03(FormatV02): # inherits from V02 to avoid code duplication + """ + Changelog: variable number of dimensions (up to 5), + introduce axes field in multiscales (June 2021) + """ + + @property + def version(self) -> str: + return "0.3" + + +CurrentFormat = FormatV03 diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 21b0ce19..9a4e2054 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -265,7 +265,7 @@ def __init__(self, node: Node) -> None: axes_values = {"t", "c", "z", "y", "x"} try: multiscales = self.lookup("multiscales", []) - version = multiscales[0].get("version", "0.1") + version = multiscales[0].get("version", "0.1") # should this be matched with Format.version? datasets = multiscales[0]["datasets"] # axes field was introduced in 0.3, before all data was 5d axes = multiscales[0].get("axes", ["t", "c", "z", "y", "x"]) @@ -336,7 +336,7 @@ def __init__(self, node: Node) -> None: color = ch.get("color", None) if color is not None: - rgb = [(int(color[i : i + 2], 16) / 255) for i in range(0, 6, 2)] + rgb = [(int(color[i: i + 2], 16) / 255) for i in range(0, 6, 2)] # TODO: make this value an enumeration if model == "greyscale": rgb = [1, 1, 1] From 49017d7c8c6d2a19f0181d7ff67b8c045ee8ed80 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 15 Jun 2021 21:09:55 +0000 Subject: [PATCH 05/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ome_zarr/reader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 9a4e2054..015ad324 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -265,7 +265,9 @@ def __init__(self, node: Node) -> None: axes_values = {"t", "c", "z", "y", "x"} try: multiscales = self.lookup("multiscales", []) - version = multiscales[0].get("version", "0.1") # should this be matched with Format.version? + version = multiscales[0].get( + "version", "0.1" + ) # should this be matched with Format.version? datasets = multiscales[0]["datasets"] # axes field was introduced in 0.3, before all data was 5d axes = multiscales[0].get("axes", ["t", "c", "z", "y", "x"]) @@ -336,7 +338,7 @@ def __init__(self, node: Node) -> None: color = ch.get("color", None) if color is not None: - rgb = [(int(color[i: i + 2], 16) / 255) for i in range(0, 6, 2)] + rgb = [(int(color[i : i + 2], 16) / 255) for i in range(0, 6, 2)] # TODO: make this value an enumeration if model == "greyscale": rgb = [1, 1, 1] From 5740ba15c73629e80fcc990ba867af51bee60dd7 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Wed, 7 Jul 2021 22:01:03 +0200 Subject: [PATCH 06/22] Fix tests --- ome_zarr/reader.py | 2 +- tests/test_ome_zarr.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 9a4e2054..61b5c90e 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -268,7 +268,7 @@ def __init__(self, node: Node) -> None: version = multiscales[0].get("version", "0.1") # should this be matched with Format.version? datasets = multiscales[0]["datasets"] # axes field was introduced in 0.3, before all data was 5d - axes = multiscales[0].get("axes", ["t", "c", "z", "y", "x"]) + axes = tuple(multiscales[0].get("axes", ["t", "c", "z", "y", "x"])) if len(set(axes) - axes_values) > 0: raise RuntimeError(f"Invalid axes names: {set(axes) - axes_values}") datasets = [d["path"] for d in datasets] diff --git a/tests/test_ome_zarr.py b/tests/test_ome_zarr.py index dacf0189..ed34d34b 100644 --- a/tests/test_ome_zarr.py +++ b/tests/test_ome_zarr.py @@ -9,7 +9,7 @@ def log_strings(idx, t, c, z, y, x, ct, cc, cz, cy, cx, dtype): yield f"resolution: {idx}" - yield f" - shape (t, c, z, y, x) = ({t}, {c}, {z}, {y}, {x})" + yield f" - shape ('t', 'c', 'z', 'y', 'x') = ({t}, {c}, {z}, {y}, {x})" yield f" - chunks = ['{ct}', '{cc}', '{cz}', '{cx}', '{cy}']" yield f" - dtype = {dtype}" From e07194d18468be2ba7ff659d1616ae30a26b0da9 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Tue, 10 Aug 2021 13:27:02 +0200 Subject: [PATCH 07/22] Add test for scaler --- tests/test_scaler.py | 52 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/test_scaler.py diff --git a/tests/test_scaler.py b/tests/test_scaler.py new file mode 100644 index 00000000..37a42111 --- /dev/null +++ b/tests/test_scaler.py @@ -0,0 +1,52 @@ +import numpy as np +import pytest +from ome_zarr.scale import Scaler + + +class TestScaler: + @pytest.fixture(params=((1, 2, 1, 256, 256),)) + def shape(self, request): + return request.param + + def create_data(self, shape, dtype=np.uint8, mean_val=10): + rng = np.random.default_rng(0) + return rng.poisson(mean_val, size=shape).astype(dtype) + + def check_downscaled(self, downscaled, shape, scale_factor=2): + expected_shape = shape + for data in downscaled: + assert data.shape == expected_shape + expected_shape = expected_shape[:-2] + tuple(sh // scale_factor for sh in expected_shape[-2:]) + + def test_nearest(self, shape): + data = self.create_data(shape) + scaler = Scaler() + downscaled = scaler.nearest(data) + self.check_downscaled(downscaled, shape) + + # this fails because of wrong channel dimension; mark expected failure? + def test_gaussian(self, shape): + data = self.create_data(shape) + scaler = Scaler() + downscaled = scaler.gaussian(data) + self.check_downscaled(downscaled, shape) + + # this fails because of wrong channel dimension; mark expected failure? + def test_laplacian(self, shape): + data = self.create_data(shape) + scaler = Scaler() + downscaled = scaler.laplacian(data) + self.check_downscaled(downscaled, shape) + + def test_local_mean(self, shape): + data = self.create_data(shape) + scaler = Scaler() + downscaled = scaler.local_mean(data) + self.check_downscaled(downscaled, shape) + + @pytest.mark.skip(reason="This test does not terminate") + def test_zoom(self, shape): + data = self.create_data(shape) + scaler = Scaler() + downscaled = scaler.zoom(data) + self.check_downscaled(downscaled, shape) From e971facc506b5c19fc793ec2b6de821b77d33be1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 11:27:21 +0000 Subject: [PATCH 08/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_scaler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_scaler.py b/tests/test_scaler.py index 37a42111..b7a8f3d5 100644 --- a/tests/test_scaler.py +++ b/tests/test_scaler.py @@ -1,5 +1,6 @@ import numpy as np import pytest + from ome_zarr.scale import Scaler @@ -16,7 +17,9 @@ def check_downscaled(self, downscaled, shape, scale_factor=2): expected_shape = shape for data in downscaled: assert data.shape == expected_shape - expected_shape = expected_shape[:-2] + tuple(sh // scale_factor for sh in expected_shape[-2:]) + expected_shape = expected_shape[:-2] + tuple( + sh // scale_factor for sh in expected_shape[-2:] + ) def test_nearest(self, shape): data = self.create_data(shape) From b986523ec359a239df79041d9384b2992cddb75b Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Tue, 10 Aug 2021 13:45:04 +0200 Subject: [PATCH 09/22] Update test writer --- tests/test_writer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_writer.py b/tests/test_writer.py index 3ac821d4..ba1ab554 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -4,6 +4,7 @@ import pytest import zarr +from ome_zarr.format import FormatV01, FormatV02, FormatV03 from ome_zarr.io import parse_url from ome_zarr.reader import Multiscales, Reader from ome_zarr.scale import Scaler @@ -33,10 +34,14 @@ def scaler(self, request): else: return None - def test_writer(self, shape, scaler): + @pytest.fixture(params=[FormatV01, FormatV02, FormatV03], ids=["v0.1", "v0.2", "v0.3"]) + def format_version(self, request): + return request.param + + def test_writer(self, shape, scaler, format_version): data = self.create_data(shape) - write_image(image=data, group=self.group, chunks=(128, 128), scaler=scaler) + write_image(image=data, group=self.group, chunks=(128, 128), scaler=scaler, fmt=format_version()) # Verify reader = Reader(parse_url(f"{self.path}/test")) @@ -44,3 +49,4 @@ def test_writer(self, shape, scaler): assert Multiscales.matches(node.zarr) assert node.data[0].shape == shape assert node.data[0].chunks == ((1,), (2,), (1,), (128, 128), (128, 128)) + assert np.allclose(data, node.data[0][...].compute()) From 221c0e456f0b5691e5f1ed9fb58e07f275c62b58 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 11:45:44 +0000 Subject: [PATCH 10/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_writer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_writer.py b/tests/test_writer.py index ba1ab554..28679f20 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -34,14 +34,22 @@ def scaler(self, request): else: return None - @pytest.fixture(params=[FormatV01, FormatV02, FormatV03], ids=["v0.1", "v0.2", "v0.3"]) + @pytest.fixture( + params=[FormatV01, FormatV02, FormatV03], ids=["v0.1", "v0.2", "v0.3"] + ) def format_version(self, request): return request.param def test_writer(self, shape, scaler, format_version): data = self.create_data(shape) - write_image(image=data, group=self.group, chunks=(128, 128), scaler=scaler, fmt=format_version()) + write_image( + image=data, + group=self.group, + chunks=(128, 128), + scaler=scaler, + fmt=format_version(), + ) # Verify reader = Reader(parse_url(f"{self.path}/test")) From ebdeab25edc277a518bc86cdaa9625cd5c7c22ab Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Tue, 10 Aug 2021 14:02:25 +0200 Subject: [PATCH 11/22] Add test for ome_zarr info cli info with s3 address --- ome_zarr/io.py | 2 +- ome_zarr/reader.py | 2 +- tests/test_cli.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ome_zarr/io.py b/ome_zarr/io.py index 4d93d211..41456f70 100644 --- a/ome_zarr/io.py +++ b/ome_zarr/io.py @@ -48,7 +48,7 @@ def __init__( self.__init_metadata() detected = detect_format(self.__metadata) if detected != fmt: - LOGGER.warning(f"version mistmatch: detected:{detected}, requested:{fmt}") + LOGGER.warning(f"version mismatch: detected:{detected}, requested:{fmt}") self.__fmt = detected self.__store == detected.init_store(self.__path, mode) self.__init_metadata() diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 0d4019dd..fea949b8 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -338,7 +338,7 @@ def __init__(self, node: Node) -> None: color = ch.get("color", None) if color is not None: - rgb = [(int(color[i : i + 2], 16) / 255) for i in range(0, 6, 2)] + rgb = [(int(color[i: i + 2], 16) / 255) for i in range(0, 6, 2)] # TODO: make this value an enumeration if model == "greyscale": rgb = [1, 1, 1] diff --git a/tests/test_cli.py b/tests/test_cli.py index 0c61461b..e7d8bc41 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,6 +14,15 @@ class TestCli: def initdir(self, tmpdir): self.path = (tmpdir / "data").mkdir() + # TODO extend with test data for 0.2 + @pytest.fixture(params=["0.1", "0.3"], ids=["v0.1", "v0.3"]) + def s3_address(self, request): + urls = { + "0.1": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/6001240.zarr", + "0.3": "https://minio-dev.openmicroscopy.org/idr/v0.3/idr0077-valuchova-flowerlightsheet/9836842.zarr" + } + return urls[request.param] + def test_coins_info(self): filename = str(self.path) + "-1" main(["create", "--method=coins", filename]) @@ -32,6 +41,9 @@ def test_astronaut_download(self, tmpdir): main(["download", filename, f"--output={out}"]) main(["info", f"{out}/{basename}"]) + def test_s3_info(self, s3_address): + main(["info", s3_address]) + def test_strip_prefix_relative(self): top = Path(".") / "d" mid = Path(".") / "d" / "e" From ebc0ed9d7f4cf3f6ba6f895e07a83e9e4e26ffd7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 12:04:47 +0000 Subject: [PATCH 12/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ome_zarr/reader.py | 2 +- tests/test_cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index fea949b8..0d4019dd 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -338,7 +338,7 @@ def __init__(self, node: Node) -> None: color = ch.get("color", None) if color is not None: - rgb = [(int(color[i: i + 2], 16) / 255) for i in range(0, 6, 2)] + rgb = [(int(color[i : i + 2], 16) / 255) for i in range(0, 6, 2)] # TODO: make this value an enumeration if model == "greyscale": rgb = [1, 1, 1] diff --git a/tests/test_cli.py b/tests/test_cli.py index e7d8bc41..0e1d49d4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,7 +19,7 @@ def initdir(self, tmpdir): def s3_address(self, request): urls = { "0.1": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/6001240.zarr", - "0.3": "https://minio-dev.openmicroscopy.org/idr/v0.3/idr0077-valuchova-flowerlightsheet/9836842.zarr" + "0.3": "https://minio-dev.openmicroscopy.org/idr/v0.3/idr0077-valuchova-flowerlightsheet/9836842.zarr", } return urls[request.param] From a6ea341cdc27d7afbc82b7024004630e1461e072 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Tue, 10 Aug 2021 18:37:12 +0200 Subject: [PATCH 13/22] Fix black --- tests/test_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0e1d49d4..726db309 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,7 +19,10 @@ def initdir(self, tmpdir): def s3_address(self, request): urls = { "0.1": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/6001240.zarr", - "0.3": "https://minio-dev.openmicroscopy.org/idr/v0.3/idr0077-valuchova-flowerlightsheet/9836842.zarr", + "0.3": ( + "https://minio-dev.openmicroscopy.org/idr/v0.3/" + "idr0077-valuchova-flowerlightsheet/9836842.zarr" + ), } return urls[request.param] From 0ba85e79d2b28d2185c8bc7300639cf7838c4918 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Wed, 11 Aug 2021 12:45:53 +0200 Subject: [PATCH 14/22] Update ome_zarr/reader.py Co-authored-by: Will Moore --- ome_zarr/reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 0d4019dd..73c54ecb 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -273,6 +273,7 @@ def __init__(self, node: Node) -> None: axes = tuple(multiscales[0].get("axes", ["t", "c", "z", "y", "x"])) if len(set(axes) - axes_values) > 0: raise RuntimeError(f"Invalid axes names: {set(axes) - axes_values}") + node.metadata["axes"] = axes datasets = [d["path"] for d in datasets] self.datasets: List[str] = datasets LOGGER.info("datasets %s", datasets) From 329c8525bd7381e4b97015ad9dc2c9debe961ff1 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Wed, 11 Aug 2021 15:03:23 +0200 Subject: [PATCH 15/22] Fix typo in ZarrLocation --- ome_zarr/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/io.py b/ome_zarr/io.py index 41456f70..14ac56fb 100644 --- a/ome_zarr/io.py +++ b/ome_zarr/io.py @@ -50,7 +50,7 @@ def __init__( if detected != fmt: LOGGER.warning(f"version mismatch: detected:{detected}, requested:{fmt}") self.__fmt = detected - self.__store == detected.init_store(self.__path, mode) + self.__store = detected.init_store(self.__path, mode) self.__init_metadata() def __init_metadata(self) -> None: From 5dfd9739e2007aab91e30bb9203b9632b051a565 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Wed, 11 Aug 2021 16:06:20 +0200 Subject: [PATCH 16/22] Mark expected failures with xfail --- ome_zarr/format.py | 2 +- tests/test_scaler.py | 6 ++++-- tests/test_upgrade.py | 7 ++++++- tests/test_writer.py | 15 ++++++++++----- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ome_zarr/format.py b/ome_zarr/format.py index 4f552ea2..d8c3a77b 100644 --- a/ome_zarr/format.py +++ b/ome_zarr/format.py @@ -79,7 +79,7 @@ def matches(self, metadata: dict) -> bool: return version == self.version def init_store(self, path: str, mode: str = "r") -> FSStore: - store = FSStore(path, mode=mode) + store = FSStore(path, mode=mode, dimension_separator=".") LOGGER.debug(f"Created legacy flat FSStore({path}, {mode})") return store diff --git a/tests/test_scaler.py b/tests/test_scaler.py index b7a8f3d5..aaf280ab 100644 --- a/tests/test_scaler.py +++ b/tests/test_scaler.py @@ -27,14 +27,16 @@ def test_nearest(self, shape): downscaled = scaler.nearest(data) self.check_downscaled(downscaled, shape) - # this fails because of wrong channel dimension; mark expected failure? + # this fails because of wrong channel dimension; need to fix in follow-up PR + @pytest.mark.xfail def test_gaussian(self, shape): data = self.create_data(shape) scaler = Scaler() downscaled = scaler.gaussian(data) self.check_downscaled(downscaled, shape) - # this fails because of wrong channel dimension; mark expected failure? + # this fails because of wrong channel dimension; need to fix in follow-up PR + @pytest.mark.xfail def test_laplacian(self, shape): data = self.create_data(shape) scaler = Scaler() diff --git a/tests/test_upgrade.py b/tests/test_upgrade.py index e7787e13..dbd49307 100644 --- a/tests/test_upgrade.py +++ b/tests/test_upgrade.py @@ -7,6 +7,7 @@ from ome_zarr.format import CurrentFormat from ome_zarr.format import FormatV01 as V01 from ome_zarr.format import FormatV02 as V02 +from ome_zarr.format import FormatV03 as V03 from ome_zarr.io import parse_url from ome_zarr.reader import Multiscales, Reader from ome_zarr.writer import write_image @@ -45,8 +46,12 @@ def test_pre_created(self, request, path, version): @pytest.mark.parametrize( "version", ( - pytest.param(V01(), id="V01"), + pytest.param( + V01(), + id="V01", + marks=pytest.mark.xfail(reason="issues with dimension_separator")), pytest.param(V02(), id="V02"), + pytest.param(V03(), id="V03"), ), ) def test_newly_created(self, version): diff --git a/tests/test_writer.py b/tests/test_writer.py index 28679f20..fdfbae00 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -34,12 +34,17 @@ def scaler(self, request): else: return None - @pytest.fixture( - params=[FormatV01, FormatV02, FormatV03], ids=["v0.1", "v0.2", "v0.3"] + @pytest.mark.parametrize( + "format_version", + ( + pytest.param( + FormatV01, + id="V01", + marks=pytest.mark.xfail(reason="issues with dimension_separator")), + pytest.param(FormatV02, id="V02"), + pytest.param(FormatV03, id="V03"), + ), ) - def format_version(self, request): - return request.param - def test_writer(self, shape, scaler, format_version): data = self.create_data(shape) From d9f994dce1bd4e81d0592b231a17a04bff674601 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 14:06:40 +0000 Subject: [PATCH 17/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_upgrade.py | 3 ++- tests/test_writer.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_upgrade.py b/tests/test_upgrade.py index dbd49307..591312b5 100644 --- a/tests/test_upgrade.py +++ b/tests/test_upgrade.py @@ -49,7 +49,8 @@ def test_pre_created(self, request, path, version): pytest.param( V01(), id="V01", - marks=pytest.mark.xfail(reason="issues with dimension_separator")), + marks=pytest.mark.xfail(reason="issues with dimension_separator"), + ), pytest.param(V02(), id="V02"), pytest.param(V03(), id="V03"), ), diff --git a/tests/test_writer.py b/tests/test_writer.py index fdfbae00..b65a5e25 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -40,7 +40,8 @@ def scaler(self, request): pytest.param( FormatV01, id="V01", - marks=pytest.mark.xfail(reason="issues with dimension_separator")), + marks=pytest.mark.xfail(reason="issues with dimension_separator"), + ), pytest.param(FormatV02, id="V02"), pytest.param(FormatV03, id="V03"), ), From 84d8a54f68156ca7dc0d93af22e64083de8ead34 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 12 Aug 2021 12:27:47 +0200 Subject: [PATCH 18/22] Cherry-pick isory fix: 561df25dc77e758864bab94470213c8cc8df8166 --- .isort.cfg | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index ca5ad42c..b3d36cbe 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,8 @@ [settings] known_third_party = dask,numpy,pytest,scipy,setuptools,skimage,zarr -multi_line_output=6 -include_trailing_comma=False -force_grid_wrap=0 -use_parentheses=True -line_length=120 +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +use_parentheses = True +ensure_newline_before_comments = True +line_length = 88 From 79e53c002d185479d2fb52801140bfd4e072e078 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 12 Aug 2021 12:36:41 +0200 Subject: [PATCH 19/22] Add v0.2 test data --- tests/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 726db309..d7a9bfa6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,11 +14,11 @@ class TestCli: def initdir(self, tmpdir): self.path = (tmpdir / "data").mkdir() - # TODO extend with test data for 0.2 - @pytest.fixture(params=["0.1", "0.3"], ids=["v0.1", "v0.3"]) + @pytest.fixture(params=["0.1", "0.2", "0.3"], ids=["v0.1", "v0.2", "v0.3"]) def s3_address(self, request): urls = { "0.1": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/6001240.zarr", + "0.2": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.2/6001240.zarr", "0.3": ( "https://minio-dev.openmicroscopy.org/idr/v0.3/" "idr0077-valuchova-flowerlightsheet/9836842.zarr" From cf1d2432cf1643d4c196616bef3fa91d767e1153 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 25 Aug 2021 09:59:24 +0200 Subject: [PATCH 20/22] Remove xfail after dim separator fix (Close #104) With the 2.9 series of zarr-python, both dimension separators should now be functional. This will let us remove the spec restriction in future versions. --- tests/test_upgrade.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_upgrade.py b/tests/test_upgrade.py index 591312b5..7243e12f 100644 --- a/tests/test_upgrade.py +++ b/tests/test_upgrade.py @@ -46,11 +46,7 @@ def test_pre_created(self, request, path, version): @pytest.mark.parametrize( "version", ( - pytest.param( - V01(), - id="V01", - marks=pytest.mark.xfail(reason="issues with dimension_separator"), - ), + pytest.param(V01(), id="V01"), pytest.param(V02(), id="V02"), pytest.param(V03(), id="V03"), ), From 741e952730bd88e4e9f1e62e2fec532bedb00cd2 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 25 Aug 2021 10:04:56 +0200 Subject: [PATCH 21/22] Skip fsspec 2021.7.0 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7b3fdb07..eff62a7b 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,7 @@ def read(fname): install_requires += (["numpy"],) install_requires += (["dask"],) install_requires += (["zarr>=2.8.1"],) -install_requires += (["fsspec>=0.9.0"],) -install_requires += (["s3fs"],) +install_requires += (["fsspec[s3fs]!=2021.07.0"],) install_requires += (["aiohttp"],) install_requires += (["requests"],) install_requires += (["scikit-image"],) From 93d096474a445f4cdb7aee194520f7ddee77a972 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 25 Aug 2021 15:27:22 +0200 Subject: [PATCH 22/22] Use official upload of v0.3 data --- tests/test_cli.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index d7a9bfa6..47dc207b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,10 +19,7 @@ def s3_address(self, request): urls = { "0.1": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/6001240.zarr", "0.2": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.2/6001240.zarr", - "0.3": ( - "https://minio-dev.openmicroscopy.org/idr/v0.3/" - "idr0077-valuchova-flowerlightsheet/9836842.zarr" - ), + "0.3": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.3/9836842.zarr", } return urls[request.param]