Merge pull request #89 from constantinpape/patch-1

Update reader.py to v0.3

.isort.cfg

@@ -1,7 +1,8 @@
 [settings]
-known_third_party = cv2,dask,numpy,pytest,scipy,setuptools,skimage,zarr
-multi_line_output=6
-include_trailing_comma=False
-force_grid_wrap=0
-use_parentheses=True
-line_length=120
+known_third_party = dask,numpy,pytest,scipy,setuptools,skimage,zarr
+multi_line_output = 3
+include_trailing_comma = True
+force_grid_wrap = 0
+use_parentheses = True
+ensure_newline_before_comments = True
+line_length = 88

environment.yml

@@ -8,9 +8,7 @@ dependencies:
   - ipython
   - mypy
   - omero-py
-  - opencv
   - pip
-  - py-opencv
   - pytest
   - requests
   - s3fs

ome_zarr/format.py

@@ -13,6 +13,7 @@ def format_implementations() -> Iterator["Format"]:
     """
     Return an instance of each format implementation, newest to oldest.
     """
+    yield FormatV03()
     yield FormatV02()
     yield FormatV01()
 
@@ -78,7 +79,7 @@ def matches(self, metadata: dict) -> bool:
         return version == self.version
 
     def init_store(self, path: str, mode: str = "r") -> FSStore:
-        store = FSStore(path, mode=mode)
+        store = FSStore(path, mode=mode, dimension_separator=".")
         LOGGER.debug(f"Created legacy flat FSStore({path}, {mode})")
         return store
 
@@ -124,4 +125,15 @@ def init_store(self, path: str, mode: str = "r") -> FSStore:
         return store
 
 
-CurrentFormat = FormatV02
+class FormatV03(FormatV02):  # inherits from V02 to avoid code duplication
+    """
+    Changelog: variable number of dimensions (up to 5),
+    introduce axes field in multiscales (June 2021)
+    """
+
+    @property
+    def version(self) -> str:
+        return "0.3"
+
+
+CurrentFormat = FormatV03

ome_zarr/io.py

@@ -48,9 +48,9 @@ def __init__(
         self.__init_metadata()
         detected = detect_format(self.__metadata)
         if detected != fmt:
-            LOGGER.warning(f"version mistmatch: detected:{detected}, requested:{fmt}")
+            LOGGER.warning(f"version mismatch: detected:{detected}, requested:{fmt}")
             self.__fmt = detected
-            self.__store == detected.init_store(self.__path, mode)
+            self.__store = detected.init_store(self.__path, mode)
             self.__init_metadata()
 
     def __init_metadata(self) -> None:

ome_zarr/reader.py

@@ -261,10 +261,18 @@ def matches(zarr: ZarrLocation) -> bool:
     def __init__(self, node: Node) -> None:
         super().__init__(node)
 
+        axes_values = {"t", "c", "z", "y", "x"}
         try:
             multiscales = self.lookup("multiscales", [])
-            version = multiscales[0].get("version", "0.1")
+            version = multiscales[0].get(
+                "version", "0.1"
+            )  # should this be matched with Format.version?
             datasets = multiscales[0]["datasets"]
+            # axes field was introduced in 0.3, before all data was 5d
+            axes = tuple(multiscales[0].get("axes", ["t", "c", "z", "y", "x"]))
+            if len(set(axes) - axes_values) > 0:
+                raise RuntimeError(f"Invalid axes names: {set(axes) - axes_values}")
+            node.metadata["axes"] = axes
             datasets = [d["path"] for d in datasets]
             self.datasets: List[str] = datasets
             LOGGER.info("datasets %s", datasets)
@@ -273,14 +281,13 @@ def __init__(self, node: Node) -> None:
             return  # EARLY EXIT
 
         for resolution in self.datasets:
-            # data.shape is (t, c, z, y, x) by convention
             data: da.core.Array = self.array(resolution, version)
             chunk_sizes = [
                 str(c[0]) + (" (+ %s)" % c[-1] if c[-1] != c[0] else "")
                 for c in data.chunks
             ]
             LOGGER.info("resolution: %s", resolution)
-            LOGGER.info(" - shape (t, c, z, y, x) = %s", data.shape)
+            LOGGER.info(" - shape %s = %s", axes, data.shape)
             LOGGER.info(" - chunks =  %s", chunk_sizes)
             LOGGER.info(" - dtype = %s", data.dtype)
             node.data.append(data)

ome_zarr/scale.py

@@ -9,11 +9,15 @@
 from dataclasses import dataclass
 from typing import Callable, Iterator, List
 
-import cv2
 import numpy as np
 import zarr
 from scipy.ndimage import zoom
-from skimage.transform import downscale_local_mean, pyramid_gaussian, pyramid_laplacian
+from skimage.transform import (
+    downscale_local_mean,
+    pyramid_gaussian,
+    pyramid_laplacian,
+    resize,
+)
 
 from .io import parse_url
 
@@ -123,19 +127,21 @@ def __create_group(
 
     def nearest(self, base: np.ndarray) -> List[np.ndarray]:
         """
-        Downsample using :func:`cv2.resize`.
+        Downsample using :func:`skimage.transform.resize`.
 
         The :const:`cvs2.INTER_NEAREST` interpolation method is used.
         """
         return self._by_plane(base, self.__nearest)
 
     def __nearest(self, plane: np.ndarray, sizeY: int, sizeX: int) -> np.ndarray:
         """Apply the 2-dimensional transformation."""
-        return cv2.resize(
+        return resize(
             plane,
-            dsize=(sizeY // self.downscale, sizeX // self.downscale),
-            interpolation=cv2.INTER_NEAREST,
-        )
+            output_shape=(sizeY // self.downscale, sizeX // self.downscale),
+            order=0,
+            preserve_range=True,
+            anti_aliasing=False,
+        ).astype(plane.dtype)
 
     def gaussian(self, base: np.ndarray) -> List[np.ndarray]:
         """Downsample using :func:`skimage.transform.pyramid_gaussian`."""

setup.py

@@ -18,13 +18,11 @@ def read(fname):
 install_requires += (["numpy"],)
 install_requires += (["dask"],)
 install_requires += (["zarr>=2.8.1"],)
-install_requires += (["fsspec>=0.9.0"],)
-install_requires += (["s3fs"],)
+install_requires += (["fsspec[s3fs]!=2021.07.0"],)
 install_requires += (["aiohttp"],)
 install_requires += (["requests"],)
 install_requires += (["scikit-image"],)
 install_requires += (["toolz"],)
-install_requires += (["opencv-contrib-python-headless"],)
 
 
 setup(

tests/test_cli.py

@@ -14,6 +14,15 @@ class TestCli:
     def initdir(self, tmpdir):
         self.path = (tmpdir / "data").mkdir()
 
+    @pytest.fixture(params=["0.1", "0.2", "0.3"], ids=["v0.1", "v0.2", "v0.3"])
+    def s3_address(self, request):
+        urls = {
+            "0.1": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.1/6001240.zarr",
+            "0.2": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.2/6001240.zarr",
+            "0.3": "https://s3.embassy.ebi.ac.uk/idr/zarr/v0.3/9836842.zarr",
+        }
+        return urls[request.param]
+
     def test_coins_info(self):
         filename = str(self.path) + "-1"
         main(["create", "--method=coins", filename])
@@ -32,6 +41,9 @@ def test_astronaut_download(self, tmpdir):
         main(["download", filename, f"--output={out}"])
         main(["info", f"{out}/{basename}"])
 
+    def test_s3_info(self, s3_address):
+        main(["info", s3_address])
+
     def test_strip_prefix_relative(self):
         top = Path(".") / "d"
         mid = Path(".") / "d" / "e"

tests/test_ome_zarr.py

@@ -9,7 +9,7 @@
 
 def log_strings(idx, t, c, z, y, x, ct, cc, cz, cy, cx, dtype):
     yield f"resolution: {idx}"
-    yield f" - shape (t, c, z, y, x) = ({t}, {c}, {z}, {y}, {x})"
+    yield f" - shape ('t', 'c', 'z', 'y', 'x') = ({t}, {c}, {z}, {y}, {x})"
     yield f" - chunks =  ['{ct}', '{cc}', '{cz}', '{cx}', '{cy}']"
     yield f" - dtype = {dtype}"
 

tests/test_scaler.py

@@ -0,0 +1,57 @@
+import numpy as np
+import pytest
+
+from ome_zarr.scale import Scaler
+
+
+class TestScaler:
+    @pytest.fixture(params=((1, 2, 1, 256, 256),))
+    def shape(self, request):
+        return request.param
+
+    def create_data(self, shape, dtype=np.uint8, mean_val=10):
+        rng = np.random.default_rng(0)
+        return rng.poisson(mean_val, size=shape).astype(dtype)
+
+    def check_downscaled(self, downscaled, shape, scale_factor=2):
+        expected_shape = shape
+        for data in downscaled:
+            assert data.shape == expected_shape
+            expected_shape = expected_shape[:-2] + tuple(
+                sh // scale_factor for sh in expected_shape[-2:]
+            )
+
+    def test_nearest(self, shape):
+        data = self.create_data(shape)
+        scaler = Scaler()
+        downscaled = scaler.nearest(data)
+        self.check_downscaled(downscaled, shape)
+
+    # this fails because of wrong channel dimension; need to fix in follow-up PR
+    @pytest.mark.xfail
+    def test_gaussian(self, shape):
+        data = self.create_data(shape)
+        scaler = Scaler()
+        downscaled = scaler.gaussian(data)
+        self.check_downscaled(downscaled, shape)
+
+    # this fails because of wrong channel dimension; need to fix in follow-up PR
+    @pytest.mark.xfail
+    def test_laplacian(self, shape):
+        data = self.create_data(shape)
+        scaler = Scaler()
+        downscaled = scaler.laplacian(data)
+        self.check_downscaled(downscaled, shape)
+
+    def test_local_mean(self, shape):
+        data = self.create_data(shape)
+        scaler = Scaler()
+        downscaled = scaler.local_mean(data)
+        self.check_downscaled(downscaled, shape)
+
+    @pytest.mark.skip(reason="This test does not terminate")
+    def test_zoom(self, shape):
+        data = self.create_data(shape)
+        scaler = Scaler()
+        downscaled = scaler.zoom(data)
+        self.check_downscaled(downscaled, shape)

tests/test_upgrade.py

@@ -7,6 +7,7 @@
 from ome_zarr.format import CurrentFormat
 from ome_zarr.format import FormatV01 as V01
 from ome_zarr.format import FormatV02 as V02
+from ome_zarr.format import FormatV03 as V03
 from ome_zarr.io import parse_url
 from ome_zarr.reader import Multiscales, Reader
 from ome_zarr.writer import write_image
@@ -47,6 +48,7 @@ def test_pre_created(self, request, path, version):
         (
             pytest.param(V01(), id="V01"),
             pytest.param(V02(), id="V02"),
+            pytest.param(V03(), id="V03"),
         ),
     )
     def test_newly_created(self, version):

tests/test_writer.py

@@ -4,6 +4,7 @@
 import pytest
 import zarr
 
+from ome_zarr.format import FormatV01, FormatV02, FormatV03
 from ome_zarr.io import parse_url
 from ome_zarr.reader import Multiscales, Reader
 from ome_zarr.scale import Scaler
@@ -33,14 +34,33 @@ def scaler(self, request):
         else:
             return None
 
-    def test_writer(self, shape, scaler):
+    @pytest.mark.parametrize(
+        "format_version",
+        (
+            pytest.param(
+                FormatV01,
+                id="V01",
+                marks=pytest.mark.xfail(reason="issues with dimension_separator"),
+            ),
+            pytest.param(FormatV02, id="V02"),
+            pytest.param(FormatV03, id="V03"),
+        ),
+    )
+    def test_writer(self, shape, scaler, format_version):
 
         data = self.create_data(shape)
-        write_image(image=data, group=self.group, chunks=(128, 128), scaler=scaler)
+        write_image(
+            image=data,
+            group=self.group,
+            chunks=(128, 128),
+            scaler=scaler,
+            fmt=format_version(),
+        )
 
         # Verify
         reader = Reader(parse_url(f"{self.path}/test"))
         node = list(reader())[0]
         assert Multiscales.matches(node.zarr)
         assert node.data[0].shape == shape
         assert node.data[0].chunks == ((1,), (2,), (1,), (128, 128), (128, 128))
+        assert np.allclose(data, node.data[0][...].compute())