Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use nested chunk store for labels #82

Merged
merged 5 commits into from
Oct 12, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def get_long_description() -> str:
author="The Open Microscopy Team",
author_email="",
python_requires=">=3",
install_requires=["omero-py>=5.6.0", "ome-zarr"],
install_requires=["omero-py>=5.6.0", "ome-zarr>=0.2.0"],
long_description=long_description,
keywords=["OMERO.CLI", "plugin"],
url="https://github.com/ome/omero-cli-zarr/",
Expand Down
24 changes: 19 additions & 5 deletions src/omero_zarr/masks.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
from omero.model import MaskI, PolygonI
from omero.rtypes import unwrap
from skimage.draw import polygon as sk_polygon
from zarr.convenience import open as zarr_open
from zarr.hierarchy import open_group

from .util import print_status
from .util import open_store, print_status

# Mapping of dimension names to axes in the Zarr
DIMENSION_ORDER: Dict[str, int] = {
Expand Down Expand Up @@ -278,7 +278,8 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None:
assert input_pyramid.load(Multiscales), "No multiscales metadata found"
input_pyramid_levels = len(input_pyramid.data)

root = zarr_open(filename)
store = open_store(filename)
root = open_group(store)

if current_path in root.group_keys():
out_labels = getattr(root, current_path)
Expand All @@ -302,17 +303,30 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None:
ignored_dimensions,
check_overlaps=True,
)
# For v0.3 ngff we want to reduce the number of dimensions to
# match the dims of the Image.
dims_to_squeeze = []
axes = []
for dim, size in enumerate(self.image_shape):
if size == 1:
dims_to_squeeze.append(dim)
else:
axes.append("tczyx"[dim])
labels = np.squeeze(labels, axis=tuple(dims_to_squeeze))

scaler = Scaler(max_layer=input_pyramid_levels)
label_pyramid = scaler.nearest(labels)
pyramid_grp = out_labels.require_group(name)

write_multiscale(label_pyramid, pyramid_grp) # TODO: dtype, chunks, overwite
write_multiscale(
label_pyramid, pyramid_grp, axes=axes
) # TODO: dtype, chunks, overwite

# Specify and store metadata
image_label_colors: List[JSONDict] = []
label_properties: List[JSONDict] = []
image_label = {
"version": "0.2",
"version": "0.3",
"colors": image_label_colors,
"properties": label_properties,
"source": {"image": source_image_link},
Expand Down
20 changes: 3 additions & 17 deletions src/omero_zarr/raw_pixels.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,10 @@
from omero.rtypes import unwrap
from skimage.transform import resize
from zarr.hierarchy import Array, Group, open_group
from zarr.storage import FSStore

from . import __version__
from . import ngff_version as VERSION
from .util import print_status


def _open_store(name: str) -> FSStore:
"""
Create an FSStore instance that supports nested storage of chunks.
"""
return FSStore(
name,
auto_mkdir=True,
key_separator="/",
normalize_keys=False,
mode="w",
)
from .util import open_store, print_status


def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -> None:
Expand All @@ -36,7 +22,7 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -

name = os.path.join(target_dir, "%s.zarr" % image.id)
print(f"Exporting to {name} ({VERSION})")
store = _open_store(name)
store = open_store(name)
root = open_group(store)
n_levels, axes = add_image(image, root, cache_dir=cache_dir)
add_multiscales_metadata(root, axes, n_levels)
Expand Down Expand Up @@ -226,7 +212,7 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace)
target_dir = args.output
cache_dir = target_dir if args.cache_numpy else None
name = os.path.join(target_dir, "%s.zarr" % plate.id)
store = _open_store(name)
store = open_store(name)
print(f"Exporting to {name} ({VERSION})")
root = open_group(store)

Expand Down
15 changes: 15 additions & 0 deletions src/omero_zarr/util.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import time

from zarr.storage import FSStore


def print_status(t0: int, t: int, count: int, total: int) -> None:
"""Prints percent done and ETA.
Expand All @@ -18,3 +20,16 @@ def print_status(t0: int, t: int, count: int, total: int) -> None:
eta = "NA"
status = f"{percent_done:.2f}% done, ETA: {eta}"
print(status, end="\r", flush=True)


def open_store(name: str) -> FSStore:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Generally ok with this refactoring, but if it's going public we need to keep ome/ome-zarr-py#110 and similar in mind. i.e. we may not want to control this in the future.

"""
Create an FSStore instance that supports nested storage of chunks.
"""
return FSStore(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this logic could be delegated to ome_zarr.format down the line? This would also make the format version used by this plugin more explicit.

Immediate caveat looking at https://github.com/ome/ome-zarr-py/blob/62b49da4b7200384dac8dec8d9fce48bd727a967/ome_zarr/format.py#L101-L125 is that normalize_keys would need to configurable (unless it could be set to True here?).

Copy link
Member Author

@will-moore will-moore Sep 17, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was set to False because otherwise path with A/1 was normalised to a/1: #59 (comment)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That may have been because upstream was doing something wonky. Again, I think that's all settled:

zarr-developers/zarr-python#755

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I want this repo to depend on that fix zarr>=2.8.3 I'd either need to update that in ome-zarr-py (currently "zarr>=2.8.1") or add zarr as a dependency to this repo?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Happy for it to happen in either as long as we eventually unify. fwiw, zarr is now at 2.10.*

name,
auto_mkdir=True,
key_separator="/",
normalize_keys=False,
mode="w",
)