From 7b0a239b9f17cbd416063863c82ca52ade00a02e Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 20 Oct 2020 14:37:10 +0200 Subject: [PATCH 01/21] HCS: add Plate spec --- ome_zarr/reader.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 2cd93a7b..1261eae5 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -2,6 +2,7 @@ import logging from abc import ABC +from string import ascii_uppercase from typing import Any, Dict, Iterator, List, Optional, Type, Union, cast import dask.array as da @@ -48,6 +49,8 @@ def __init__( self.specs.append(Multiscales(self)) if OMERO.matches(zarr): self.specs.append(OMERO(self)) + if Plate.matches(zarr): + self.specs.append(Plate(self)) @property def visible(self) -> bool: @@ -336,6 +339,34 @@ def __init__(self, node: Node) -> None: LOGGER.error(f"failed to parse metadata: {e}") +class Plate(Spec): + @staticmethod + def matches(zarr: BaseZarrLocation) -> bool: + return bool("plate" in zarr.root_attrs) + + def __init__(self, node: Node) -> None: + super().__init__(node) + # TODO: start checking metadata version + self.plate_data = self.lookup("plate", {}) + self.rows = self.plate_data.get("rows", 0) + self.cols = self.plate_data.get("columns", 0) + + # FIXME: shouldn't hard code + self.acquisitions = ["PlateAcquisition Name 0"] + self.fields = ["Field_1"] + self.row_labels = ascii_uppercase[0 : self.rows] + self.col_labels = range(1, self.cols + 1) + + for acq in self.acquisitions: + for row in self.row_labels: + for col in self.col_labels: + for field in self.fields: + path = f"{acq}/{row}/{col}/{field}" + child_zarr = self.zarr.create(path) + if child_zarr.exists(): + node.add(child_zarr, visibility=True) + + class Reader: """Parses the given Zarr instance into a collection of Nodes properly ordered depending on context. From c9f4901ff42519910c6e87ebd25d715bedad6547 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 21 Oct 2020 09:22:51 +0200 Subject: [PATCH 02/21] HCS: only first field is visible --- ome_zarr/reader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 1261eae5..32df81e6 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -353,10 +353,11 @@ def __init__(self, node: Node) -> None: # FIXME: shouldn't hard code self.acquisitions = ["PlateAcquisition Name 0"] - self.fields = ["Field_1"] + self.fields = ["Field_1", "Field_2", "Field_3", "Field_4"] self.row_labels = ascii_uppercase[0 : self.rows] self.col_labels = range(1, self.cols + 1) + visibility = True for acq in self.acquisitions: for row in self.row_labels: for col in self.col_labels: @@ -364,7 +365,8 @@ def __init__(self, node: Node) -> None: path = f"{acq}/{row}/{col}/{field}" child_zarr = self.zarr.create(path) if child_zarr.exists(): - node.add(child_zarr, visibility=True) + node.add(child_zarr, visibility=visibility) + visibility = False class Reader: From f97f5a862091c95139b1b70d7f1c0d2d6712c45e Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 21 Oct 2020 21:28:50 +0100 Subject: [PATCH 03/21] Stitch Plate Wells into pyramid --- ome_zarr/reader.py | 117 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 105 insertions(+), 12 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 32df81e6..121f1348 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -5,6 +5,7 @@ from string import ascii_uppercase from typing import Any, Dict, Iterator, List, Optional, Type, Union, cast +from dask import delayed import dask.array as da from vispy.color import Colormap @@ -352,21 +353,113 @@ def __init__(self, node: Node) -> None: self.cols = self.plate_data.get("columns", 0) # FIXME: shouldn't hard code - self.acquisitions = ["PlateAcquisition Name 0"] - self.fields = ["Field_1", "Field_2", "Field_3", "Field_4"] + self.acquisitions = ["0"] + self.fields = ["Field_1",] self.row_labels = ascii_uppercase[0 : self.rows] self.col_labels = range(1, self.cols + 1) - visibility = True - for acq in self.acquisitions: - for row in self.row_labels: - for col in self.col_labels: - for field in self.fields: - path = f"{acq}/{row}/{col}/{field}" - child_zarr = self.zarr.create(path) - if child_zarr.exists(): - node.add(child_zarr, visibility=visibility) - visibility = False + # visibility = True + # for acq in self.acquisitions: + # for row in self.row_labels: + # for col in self.col_labels: + # for field in self.fields: + # path = f"{acq}/{row}/{col}/{field}" + # child_zarr = self.zarr.create(path) + # if child_zarr.exists(): + # node.add(child_zarr, visibility=visibility) + # visibility = False + + self.get_pyramid_lazy(node) + + + def get_pyramid_lazy(self, node): + """ + Return a pyramid of dask data, where the highest resolution is the + stitched full-resolution images. + """ + + run = '0' + rows = self.rows + cols = self.cols + row_labels = self.row_labels + + # Get the first image... + path = f"{run}/A/1/Field_1" + image_zarr = self.zarr.create(path) + image_node = Node(image_zarr, node) + + numpy_type = image_node.data[0].dtype + img_shape = image_node.data[0].shape + + size_t = img_shape[0] + size_c = img_shape[1] + size_z = img_shape[2] + + size_x = img_shape[3] + size_y = img_shape[4] + # Assume this matches the sizes of the downsampled images in each field + # Probably better to look it up - Assumed to be same for every image + tile_sizes = [] + for l in range(5): + tile_sizes.append((size_x, size_y)) + size_x = size_x // 2 + size_y = size_y // 2 + + def get_tile(tile_name): + """ tile_name is 'level,z,c,t,row,col' """ + level, z, c, t, row, col = [int(n) for n in tile_name.split(",")] + path = f'{run}/{row_labels[row]}/{col + 1}/Field_1/{level}' + print('get_tile', path) + + data = self.zarr.load(path) + tile = data[z, c, t] + print('data.shape', data.shape, tile.shape) + return tile + + lazy_reader = delayed(get_tile) + + def get_lazy_plane(level, z, c, t): + lazy_rows = [] + # For level 0, return whole image for each tile + for row in range(rows): + lazy_row = [] + for col in range(cols): + tile_name = "%s,%s,%s,%s,%s,%s" % (level, z, c, t, row, col) + print('tile_name', tile_name) + tile_size = tile_sizes[level] + lazy_tile = da.from_delayed(lazy_reader(tile_name), shape=tile_size, dtype=numpy_type) + lazy_row.append(lazy_tile) + lazy_row = da.concatenate(lazy_row, axis=1) + print('lazy_row.shape', lazy_row.shape) + lazy_rows.append(lazy_row) + return da.concatenate(lazy_rows, axis=0) + + pyramid = [] + + # This should create a pyramid of levels, but causes seg faults! + # for level in range(5): + for level in [0]: + print('level', level) + t_stacks = [] + for t in range(size_t): + c_stacks = [] + for c in range(size_c): + z_stack = [] + for z in range(size_z): + lazy_plane = get_lazy_plane(level, z, c, t) + print('lazy_plane', lazy_plane.shape) + z_stack.append(lazy_plane) + c_stacks.append(da.stack(z_stack)) + t_stacks.append(da.stack(c_stacks)) + pyramid.append(da.stack(t_stacks)) + print("pyramid shape...") + for l in pyramid: + print(l.shape) + + # Set the node.data to be pyramid view of the plate + node.data = pyramid + # Use the first image's metadata for viewing the whole Plate + node.metadata = image_node.metadata class Reader: From 9aa51716a168f3e9a42880a68707d5e0a0298d61 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 22 Oct 2020 10:25:57 +0200 Subject: [PATCH 04/21] Cleanup and fix mypy --- ome_zarr/reader.py | 68 +++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 121f1348..a90b8e1c 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -5,8 +5,9 @@ from string import ascii_uppercase from typing import Any, Dict, Iterator, List, Optional, Type, Union, cast -from dask import delayed import dask.array as da +import numpy as np +from dask import delayed from vispy.color import Colormap from .io import BaseZarrLocation @@ -353,32 +354,20 @@ def __init__(self, node: Node) -> None: self.cols = self.plate_data.get("columns", 0) # FIXME: shouldn't hard code - self.acquisitions = ["0"] - self.fields = ["Field_1",] + self.acquisitions = ["PlateAcquisition Name 0", "0"] + self.fields = ["Field_1", "Field_2", "Field_3", "Field_4"] self.row_labels = ascii_uppercase[0 : self.rows] self.col_labels = range(1, self.cols + 1) - # visibility = True - # for acq in self.acquisitions: - # for row in self.row_labels: - # for col in self.col_labels: - # for field in self.fields: - # path = f"{acq}/{row}/{col}/{field}" - # child_zarr = self.zarr.create(path) - # if child_zarr.exists(): - # node.add(child_zarr, visibility=visibility) - # visibility = False - self.get_pyramid_lazy(node) - - def get_pyramid_lazy(self, node): + def get_pyramid_lazy(self, node: Node) -> None: """ Return a pyramid of dask data, where the highest resolution is the stitched full-resolution images. """ - run = '0' + run = "0" rows = self.rows cols = self.cols row_labels = self.row_labels @@ -400,38 +389,39 @@ def get_pyramid_lazy(self, node): # Assume this matches the sizes of the downsampled images in each field # Probably better to look it up - Assumed to be same for every image tile_sizes = [] - for l in range(5): + for _level in range(5): tile_sizes.append((size_x, size_y)) size_x = size_x // 2 size_y = size_y // 2 - def get_tile(tile_name): + def get_tile(tile_name: str) -> np.ndarray: """ tile_name is 'level,z,c,t,row,col' """ level, z, c, t, row, col = [int(n) for n in tile_name.split(",")] - path = f'{run}/{row_labels[row]}/{col + 1}/Field_1/{level}' - print('get_tile', path) + path = f"{run}/{row_labels[row]}/{col + 1}/Field_1/{level}" + print("get_tile", path) data = self.zarr.load(path) tile = data[z, c, t] - print('data.shape', data.shape, tile.shape) + print("data.shape", data.shape, tile.shape) return tile lazy_reader = delayed(get_tile) - def get_lazy_plane(level, z, c, t): + def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: lazy_rows = [] # For level 0, return whole image for each tile for row in range(rows): - lazy_row = [] + lazy_row: List[da.Array] = [] for col in range(cols): - tile_name = "%s,%s,%s,%s,%s,%s" % (level, z, c, t, row, col) - print('tile_name', tile_name) + tile_name = f"{level},{z},{c},{t},{row},{col}" + print("tile_name", tile_name) tile_size = tile_sizes[level] - lazy_tile = da.from_delayed(lazy_reader(tile_name), shape=tile_size, dtype=numpy_type) + lazy_tile = da.from_delayed( + lazy_reader(tile_name), shape=tile_size, dtype=numpy_type + ) lazy_row.append(lazy_tile) - lazy_row = da.concatenate(lazy_row, axis=1) - print('lazy_row.shape', lazy_row.shape) - lazy_rows.append(lazy_row) + lazy_rows.append(da.concatenate(lazy_row, axis=1)) + print("lazy_row.shape", lazy_rows[-1].shape) return da.concatenate(lazy_rows, axis=0) pyramid = [] @@ -439,7 +429,7 @@ def get_lazy_plane(level, z, c, t): # This should create a pyramid of levels, but causes seg faults! # for level in range(5): for level in [0]: - print('level', level) + print("level", level) t_stacks = [] for t in range(size_t): c_stacks = [] @@ -447,19 +437,29 @@ def get_lazy_plane(level, z, c, t): z_stack = [] for z in range(size_z): lazy_plane = get_lazy_plane(level, z, c, t) - print('lazy_plane', lazy_plane.shape) + print("lazy_plane", lazy_plane.shape) z_stack.append(lazy_plane) c_stacks.append(da.stack(z_stack)) t_stacks.append(da.stack(c_stacks)) pyramid.append(da.stack(t_stacks)) print("pyramid shape...") - for l in pyramid: - print(l.shape) + for stack in pyramid: + print(stack.shape) # Set the node.data to be pyramid view of the plate node.data = pyramid # Use the first image's metadata for viewing the whole Plate node.metadata = image_node.metadata + visibility = True + for acq in self.acquisitions: + for row in self.row_labels: + for col in self.col_labels: + for field in self.fields: + path = f"{acq}/{row}/{col}/{field}" + child_zarr = self.zarr.create(path) + if child_zarr.exists(): + node.add(child_zarr, visibility=visibility) + visibility = False class Reader: From b8e130dd259711be93fb499d02c7fe0a93a9036b Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 22 Oct 2020 15:51:12 +0100 Subject: [PATCH 05/21] Calculate optimal resolution level so stitched plate < 3000 pixels --- ome_zarr/reader.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index a90b8e1c..b4e4a902 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -349,6 +349,7 @@ def matches(zarr: BaseZarrLocation) -> bool: def __init__(self, node: Node) -> None: super().__init__(node) # TODO: start checking metadata version +<<<<<<< HEAD self.plate_data = self.lookup("plate", {}) self.rows = self.plate_data.get("rows", 0) self.cols = self.plate_data.get("columns", 0) @@ -358,6 +359,8 @@ def __init__(self, node: Node) -> None: self.fields = ["Field_1", "Field_2", "Field_3", "Field_4"] self.row_labels = ascii_uppercase[0 : self.rows] self.col_labels = range(1, self.cols + 1) +======= +>>>>>>> f316d76... Calculate optimal resolution level so stitched plate < 3000 pixels self.get_pyramid_lazy(node) @@ -366,14 +369,29 @@ def get_pyramid_lazy(self, node: Node) -> None: Return a pyramid of dask data, where the highest resolution is the stitched full-resolution images. """ + self.plate_data = self.lookup("plate", {}) + print('self.plate_data', self.plate_data) + self.rows = self.plate_data.get("rows", 0) + self.cols = self.plate_data.get("columns", 0) +<<<<<<< HEAD run = "0" +======= + # FIXME: shouldn't hard code + self.acquisitions = ["0"] + self.fields = ["Field_1",] + self.row_labels = ascii_uppercase[0 : self.rows] + self.col_labels = range(1, self.cols + 1) + + # TODO: support more Acquisitions - just 1st for now + run = self.acquisitions[0] +>>>>>>> f316d76... Calculate optimal resolution level so stitched plate < 3000 pixels rows = self.rows cols = self.cols row_labels = self.row_labels # Get the first image... - path = f"{run}/A/1/Field_1" + path = f"{run}/{row_labels[0]}/{self.col_labels[0]}/{self.fields[0]}" image_zarr = self.zarr.create(path) image_node = Node(image_zarr, node) @@ -386,6 +404,19 @@ def get_pyramid_lazy(self, node: Node) -> None: size_x = img_shape[3] size_y = img_shape[4] + + # FIXME - if only returning a single stiched plate (not a pyramid) + # need to decide optimal size. E.g. longest side < 3000 + TARGET_SIZE = 3000 + plate_width = cols * size_x + plate_height = cols * size_y + longest_side = max(plate_width, plate_height) + target_level = 0 + while longest_side > TARGET_SIZE: + longest_side = longest_side // 2 + target_level += 1 + + print('target_level', target_level) # Assume this matches the sizes of the downsampled images in each field # Probably better to look it up - Assumed to be same for every image tile_sizes = [] @@ -428,8 +459,9 @@ def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: # This should create a pyramid of levels, but causes seg faults! # for level in range(5): - for level in [0]: - print("level", level) + + for level in [target_level]: + print('level', level) t_stacks = [] for t in range(size_t): c_stacks = [] From 5cda8030cdece63cd3f8c4210e49436c7dbff976 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 22 Oct 2020 21:26:10 +0100 Subject: [PATCH 06/21] Add 'plates' metadata to layers in napari --- ome_zarr/reader.py | 37 +++++++++---------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index b4e4a902..80118c78 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -348,20 +348,6 @@ def matches(zarr: BaseZarrLocation) -> bool: def __init__(self, node: Node) -> None: super().__init__(node) - # TODO: start checking metadata version -<<<<<<< HEAD - self.plate_data = self.lookup("plate", {}) - self.rows = self.plate_data.get("rows", 0) - self.cols = self.plate_data.get("columns", 0) - - # FIXME: shouldn't hard code - self.acquisitions = ["PlateAcquisition Name 0", "0"] - self.fields = ["Field_1", "Field_2", "Field_3", "Field_4"] - self.row_labels = ascii_uppercase[0 : self.rows] - self.col_labels = range(1, self.cols + 1) -======= ->>>>>>> f316d76... Calculate optimal resolution level so stitched plate < 3000 pixels - self.get_pyramid_lazy(node) def get_pyramid_lazy(self, node: Node) -> None: @@ -374,9 +360,6 @@ def get_pyramid_lazy(self, node: Node) -> None: self.rows = self.plate_data.get("rows", 0) self.cols = self.plate_data.get("columns", 0) -<<<<<<< HEAD - run = "0" -======= # FIXME: shouldn't hard code self.acquisitions = ["0"] self.fields = ["Field_1",] @@ -385,7 +368,6 @@ def get_pyramid_lazy(self, node: Node) -> None: # TODO: support more Acquisitions - just 1st for now run = self.acquisitions[0] ->>>>>>> f316d76... Calculate optimal resolution level so stitched plate < 3000 pixels rows = self.rows cols = self.cols row_labels = self.row_labels @@ -482,16 +464,15 @@ def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: node.data = pyramid # Use the first image's metadata for viewing the whole Plate node.metadata = image_node.metadata - visibility = True - for acq in self.acquisitions: - for row in self.row_labels: - for col in self.col_labels: - for field in self.fields: - path = f"{acq}/{row}/{col}/{field}" - child_zarr = self.zarr.create(path) - if child_zarr.exists(): - node.add(child_zarr, visibility=visibility) - visibility = False + + node.metadata.update({ + "metadata": { + "plate": { + "rows": rows, + "columns": cols, + } + } + }) class Reader: From 31d9da6c5d295566b43d87ab3b8ba88ef409b777 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 22 Oct 2020 22:08:21 +0100 Subject: [PATCH 07/21] black fix --- ome_zarr/reader.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 80118c78..1ede8e7e 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -356,13 +356,15 @@ def get_pyramid_lazy(self, node: Node) -> None: stitched full-resolution images. """ self.plate_data = self.lookup("plate", {}) - print('self.plate_data', self.plate_data) + print("self.plate_data", self.plate_data) self.rows = self.plate_data.get("rows", 0) self.cols = self.plate_data.get("columns", 0) # FIXME: shouldn't hard code self.acquisitions = ["0"] - self.fields = ["Field_1",] + self.fields = [ + "Field_1", + ] self.row_labels = ascii_uppercase[0 : self.rows] self.col_labels = range(1, self.cols + 1) @@ -398,7 +400,7 @@ def get_pyramid_lazy(self, node: Node) -> None: longest_side = longest_side // 2 target_level += 1 - print('target_level', target_level) + print("target_level", target_level) # Assume this matches the sizes of the downsampled images in each field # Probably better to look it up - Assumed to be same for every image tile_sizes = [] @@ -443,7 +445,7 @@ def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: # for level in range(5): for level in [target_level]: - print('level', level) + print("level", level) t_stacks = [] for t in range(size_t): c_stacks = [] @@ -465,14 +467,7 @@ def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: # Use the first image's metadata for viewing the whole Plate node.metadata = image_node.metadata - node.metadata.update({ - "metadata": { - "plate": { - "rows": rows, - "columns": cols, - } - } - }) + node.metadata.update({"metadata": {"plate": {"rows": rows, "columns": cols,}}}) class Reader: From c0c308fb5939f28aa27b9a8905b778a8a7f0b804 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 23 Oct 2020 07:22:21 +0100 Subject: [PATCH 08/21] Add well outlines plates in napari --- ome_zarr/napari.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index ec172b5f..ecea79f1 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -7,6 +7,7 @@ import logging import warnings from typing import Any, Callable, Dict, Iterator, List, Optional +from string import ascii_uppercase from .data import CHANNEL_DIMENSION from .io import parse_url @@ -77,6 +78,54 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: LOGGER.debug(f"Transformed: {rv}") results.append(rv) + # the 'metadata' dict takes any extra info, not supported + # by napari. If we have 'plate' info, add a shapes layer + if "metadata" in metadata: + if "plate" in metadata["metadata"]: + plate_info = metadata["metadata"]["plate"] + plate_width = shape[-1] + plate_height = shape[-2] + rows = plate_info['rows'] + columns = plate_info['columns'] + well_width = plate_width / columns + well_height = plate_height / rows + labels = [] + outlines = [] + for row in range(rows): + for column in range(columns): + x1 = int(column * well_width) + x2 = int((column + 1) * well_width) + y1 = int(row * well_height) + y2 = int((row + 1) * well_height) + # Since napari will only place labels 'outside' a bounding box + # we have a line along top of Well, with label below + outlines.append([[y1, x1], [y1, x2]]) + label = f'{ascii_uppercase[row]}{column + 1}' + labels.append(label) + # Well bounding box, with no label + outlines.append([ + [ y1, x1], + [ y2, x1], + [ y2, x2], + [ y1, x2], + ]) + labels.append('') + text_parameters = { + 'text': '{well}', + 'size': 12, + 'color': 'white', + 'anchor': 'lower_left', + 'translation': [5, 5], + } + rv: LayerData = (outlines,{ + 'edge_color': 'white', + 'face_color': 'transparent', + 'name': 'Well Labels', + 'text': text_parameters, + 'properties': {'well' : labels} + }, "shapes") + results.append(rv) + return results return f From b88240e34893c87c928f8e623b801097a79ae662 Mon Sep 17 00:00:00 2001 From: William Moore Date: Sun, 25 Oct 2020 06:40:26 +0000 Subject: [PATCH 09/21] Pick pyramid level based on first image pyramid --- ome_zarr/reader.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 1ede8e7e..a0577aa0 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -382,6 +382,8 @@ def get_pyramid_lazy(self, node: Node) -> None: numpy_type = image_node.data[0].dtype img_shape = image_node.data[0].shape + img_pyramid_shapes = [d.shape for d in image_node.data] + print('img_pyramid_shapes', img_pyramid_shapes) size_t = img_shape[0] size_c = img_shape[1] size_z = img_shape[2] @@ -391,14 +393,18 @@ def get_pyramid_lazy(self, node: Node) -> None: # FIXME - if only returning a single stiched plate (not a pyramid) # need to decide optimal size. E.g. longest side < 3000 - TARGET_SIZE = 3000 + TARGET_SIZE = 1500 plate_width = cols * size_x plate_height = cols * size_y longest_side = max(plate_width, plate_height) target_level = 0 - while longest_side > TARGET_SIZE: - longest_side = longest_side // 2 - target_level += 1 + for level, shape in enumerate(img_pyramid_shapes): + plate_width = cols * shape[-1] + plate_height = rows * shape[-2] + longest_side = max(plate_width, plate_height) + target_level = level + if longest_side <= TARGET_SIZE: + break print("target_level", target_level) # Assume this matches the sizes of the downsampled images in each field @@ -413,10 +419,10 @@ def get_tile(tile_name: str) -> np.ndarray: """ tile_name is 'level,z,c,t,row,col' """ level, z, c, t, row, col = [int(n) for n in tile_name.split(",")] path = f"{run}/{row_labels[row]}/{col + 1}/Field_1/{level}" - print("get_tile", path) + print(f"LOADING tile... {path} for t:{t} c:{c} z:{z}") data = self.zarr.load(path) - tile = data[z, c, t] + tile = data[t, c, z] print("data.shape", data.shape, tile.shape) return tile @@ -429,7 +435,7 @@ def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: lazy_row: List[da.Array] = [] for col in range(cols): tile_name = f"{level},{z},{c},{t},{row},{col}" - print("tile_name", tile_name) + print(f"creating lazy_reader. level:{level} Z:{z} c:{c} t:{t} row:{row} col:{col}") tile_size = tile_sizes[level] lazy_tile = da.from_delayed( lazy_reader(tile_name), shape=tile_size, dtype=numpy_type @@ -443,9 +449,7 @@ def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: # This should create a pyramid of levels, but causes seg faults! # for level in range(5): - for level in [target_level]: - print("level", level) t_stacks = [] for t in range(size_t): c_stacks = [] From 1f51e55045fbed1b695120725185a09c168e8978 Mon Sep 17 00:00:00 2001 From: William Moore Date: Sun, 25 Oct 2020 23:11:03 +0000 Subject: [PATCH 10/21] Read plateAcquisitions from plate_data --- ome_zarr/reader.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index a0577aa0..ee21a30f 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -359,9 +359,8 @@ def get_pyramid_lazy(self, node: Node) -> None: print("self.plate_data", self.plate_data) self.rows = self.plate_data.get("rows", 0) self.cols = self.plate_data.get("columns", 0) + self.acquisitions = self.plate_data.get('plateAcquisitions', [{"path": "0"}]) - # FIXME: shouldn't hard code - self.acquisitions = ["0"] self.fields = [ "Field_1", ] @@ -369,7 +368,7 @@ def get_pyramid_lazy(self, node: Node) -> None: self.col_labels = range(1, self.cols + 1) # TODO: support more Acquisitions - just 1st for now - run = self.acquisitions[0] + run = self.acquisitions[0]['path'] rows = self.rows cols = self.cols row_labels = self.row_labels From 5195e7f8311eef67ae3a7a0134ea89e77c3a1746 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 26 Oct 2020 06:50:04 +0000 Subject: [PATCH 11/21] lazy get_tile() returns 5D data, not 2D plane This means that you need much fewer lazy tile readers. Hopefully dask doesn't read more data than it needs --- ome_zarr/reader.py | 51 +++++++++++++++------------------------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index ee21a30f..af55b63a 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -382,13 +382,14 @@ def get_pyramid_lazy(self, node: Node) -> None: img_shape = image_node.data[0].shape img_pyramid_shapes = [d.shape for d in image_node.data] + print('img_pyramid_shapes', img_pyramid_shapes) size_t = img_shape[0] size_c = img_shape[1] size_z = img_shape[2] - size_x = img_shape[3] - size_y = img_shape[4] + size_y = img_shape[3] + size_x = img_shape[4] # FIXME - if only returning a single stiched plate (not a pyramid) # need to decide optimal size. E.g. longest side < 3000 @@ -406,64 +407,44 @@ def get_pyramid_lazy(self, node: Node) -> None: break print("target_level", target_level) - # Assume this matches the sizes of the downsampled images in each field - # Probably better to look it up - Assumed to be same for every image - tile_sizes = [] - for _level in range(5): - tile_sizes.append((size_x, size_y)) - size_x = size_x // 2 - size_y = size_y // 2 def get_tile(tile_name: str) -> np.ndarray: """ tile_name is 'level,z,c,t,row,col' """ - level, z, c, t, row, col = [int(n) for n in tile_name.split(",")] + level, row, col = [int(n) for n in tile_name.split(",")] path = f"{run}/{row_labels[row]}/{col + 1}/Field_1/{level}" - print(f"LOADING tile... {path} for t:{t} c:{c} z:{z}") + print(f"LOADING tile... {path}") data = self.zarr.load(path) - tile = data[t, c, z] - print("data.shape", data.shape, tile.shape) - return tile + print("...data.shape", data.shape) + return data lazy_reader = delayed(get_tile) - def get_lazy_plane(level: int, z: int, c: int, t: int) -> da.Array: + def get_lazy_plate(level: int) -> da.Array: lazy_rows = [] # For level 0, return whole image for each tile for row in range(rows): lazy_row: List[da.Array] = [] for col in range(cols): - tile_name = f"{level},{z},{c},{t},{row},{col}" - print(f"creating lazy_reader. level:{level} Z:{z} c:{c} t:{t} row:{row} col:{col}") - tile_size = tile_sizes[level] + tile_name = f"{level},{row},{col}" + print(f"creating lazy_reader. level:{level} row:{row} col:{col}") + tile_size = img_pyramid_shapes[level] lazy_tile = da.from_delayed( lazy_reader(tile_name), shape=tile_size, dtype=numpy_type ) lazy_row.append(lazy_tile) - lazy_rows.append(da.concatenate(lazy_row, axis=1)) + lazy_rows.append(da.concatenate(lazy_row, axis=4)) print("lazy_row.shape", lazy_rows[-1].shape) - return da.concatenate(lazy_rows, axis=0) + return da.concatenate(lazy_rows, axis=3) pyramid = [] # This should create a pyramid of levels, but causes seg faults! # for level in range(5): for level in [target_level]: - t_stacks = [] - for t in range(size_t): - c_stacks = [] - for c in range(size_c): - z_stack = [] - for z in range(size_z): - lazy_plane = get_lazy_plane(level, z, c, t) - print("lazy_plane", lazy_plane.shape) - z_stack.append(lazy_plane) - c_stacks.append(da.stack(z_stack)) - t_stacks.append(da.stack(c_stacks)) - pyramid.append(da.stack(t_stacks)) - print("pyramid shape...") - for stack in pyramid: - print(stack.shape) + lazy_plate = get_lazy_plate(level) + print("lazy_plate", lazy_plate.shape) + pyramid.append(lazy_plate) # Set the node.data to be pyramid view of the plate node.data = pyramid From 1b406e7037db4e7900323c9e82d8e810987baafc Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 27 Oct 2020 13:21:49 +0000 Subject: [PATCH 12/21] Black fixes --- ome_zarr/reader.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index af55b63a..9056fdd4 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -359,7 +359,7 @@ def get_pyramid_lazy(self, node: Node) -> None: print("self.plate_data", self.plate_data) self.rows = self.plate_data.get("rows", 0) self.cols = self.plate_data.get("columns", 0) - self.acquisitions = self.plate_data.get('plateAcquisitions', [{"path": "0"}]) + self.acquisitions = self.plate_data.get("plateAcquisitions", [{"path": "0"}]) self.fields = [ "Field_1", @@ -368,7 +368,7 @@ def get_pyramid_lazy(self, node: Node) -> None: self.col_labels = range(1, self.cols + 1) # TODO: support more Acquisitions - just 1st for now - run = self.acquisitions[0]['path'] + run = self.acquisitions[0]["path"] rows = self.rows cols = self.cols row_labels = self.row_labels @@ -383,10 +383,7 @@ def get_pyramid_lazy(self, node: Node) -> None: img_pyramid_shapes = [d.shape for d in image_node.data] - print('img_pyramid_shapes', img_pyramid_shapes) - size_t = img_shape[0] - size_c = img_shape[1] - size_z = img_shape[2] + print("img_pyramid_shapes", img_pyramid_shapes) size_y = img_shape[3] size_x = img_shape[4] From c27d0bcbf50735b7f1953f2e46e89ea37e9d4886 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 27 Oct 2020 13:24:43 +0000 Subject: [PATCH 13/21] Black fixes napari.py --- ome_zarr/napari.py | 47 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index ecea79f1..52686076 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -85,8 +85,8 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: plate_info = metadata["metadata"]["plate"] plate_width = shape[-1] plate_height = shape[-2] - rows = plate_info['rows'] - columns = plate_info['columns'] + rows = plate_info["rows"] + columns = plate_info["columns"] well_width = plate_width / columns well_height = plate_height / rows labels = [] @@ -99,31 +99,32 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: y2 = int((row + 1) * well_height) # Since napari will only place labels 'outside' a bounding box # we have a line along top of Well, with label below - outlines.append([[y1, x1], [y1, x2]]) - label = f'{ascii_uppercase[row]}{column + 1}' + outlines.append([[y1, x1], [y1, x2]]) + label = f"{ascii_uppercase[row]}{column + 1}" labels.append(label) # Well bounding box, with no label - outlines.append([ - [ y1, x1], - [ y2, x1], - [ y2, x2], - [ y1, x2], - ]) - labels.append('') + outlines.append( + [[y1, x1], [y2, x1], [y2, x2], [y1, x2],] + ) + labels.append("") text_parameters = { - 'text': '{well}', - 'size': 12, - 'color': 'white', - 'anchor': 'lower_left', - 'translation': [5, 5], + "text": "{well}", + "size": 12, + "color": "white", + "anchor": "lower_left", + "translation": [5, 5], } - rv: LayerData = (outlines,{ - 'edge_color': 'white', - 'face_color': 'transparent', - 'name': 'Well Labels', - 'text': text_parameters, - 'properties': {'well' : labels} - }, "shapes") + rv: LayerData = ( + outlines, + { + "edge_color": "white", + "face_color": "transparent", + "name": "Well Labels", + "text": text_parameters, + "properties": {"well": labels}, + }, + "shapes", + ) results.append(rv) return results From 0c7ddb7bbc554ba21a4ffdeb3fd8e657c3af7312 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 27 Oct 2020 13:30:36 +0000 Subject: [PATCH 14/21] More Black fixes --- ome_zarr/napari.py | 7 ++++--- ome_zarr/reader.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 52686076..a00b8693 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -97,14 +97,15 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: x2 = int((column + 1) * well_width) y1 = int(row * well_height) y2 = int((row + 1) * well_height) - # Since napari will only place labels 'outside' a bounding box - # we have a line along top of Well, with label below + # Since napari will only place labels 'outside' a + # bounding box we have a line + # along top of Well, with label below outlines.append([[y1, x1], [y1, x2]]) label = f"{ascii_uppercase[row]}{column + 1}" labels.append(label) # Well bounding box, with no label outlines.append( - [[y1, x1], [y2, x1], [y2, x2], [y1, x2],] + [[y1, x1], [y2, x1], [y2, x2], [y1, x2]] ) labels.append("") text_parameters = { diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 9056fdd4..923b393a 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -448,7 +448,7 @@ def get_lazy_plate(level: int) -> da.Array: # Use the first image's metadata for viewing the whole Plate node.metadata = image_node.metadata - node.metadata.update({"metadata": {"plate": {"rows": rows, "columns": cols,}}}) + node.metadata.update({"metadata": {"plate": {"rows": rows, "columns": cols}}}) class Reader: From f9651699ee1ee2c2dc6e56005ed3c9347ce5a017 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 27 Oct 2020 13:41:33 +0000 Subject: [PATCH 15/21] Fix mypy --- ome_zarr/napari.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index a00b8693..9d7d8380 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -6,8 +6,8 @@ import logging import warnings -from typing import Any, Callable, Dict, Iterator, List, Optional from string import ascii_uppercase +from typing import Any, Callable, Dict, Iterator, List, Optional from .data import CHANNEL_DIMENSION from .io import parse_url @@ -115,7 +115,7 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: "anchor": "lower_left", "translation": [5, 5], } - rv: LayerData = ( + shapes: LayerData = ( outlines, { "edge_color": "white", @@ -126,7 +126,7 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: }, "shapes", ) - results.append(rv) + results.append(shapes) return results From ecc285953f54b38593a44c901190422af69eb7b4 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 27 Oct 2020 13:46:52 +0000 Subject: [PATCH 16/21] Add comment --- ome_zarr/reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 923b393a..1c10221d 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -448,6 +448,7 @@ def get_lazy_plate(level: int) -> da.Array: # Use the first image's metadata for viewing the whole Plate node.metadata = image_node.metadata + # "metadata" dict gets added to each 'plate' layer in napari node.metadata.update({"metadata": {"plate": {"rows": rows, "columns": cols}}}) From a78f37cd5beb45dd37aa1e94fd001fcd45fd1ae8 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 29 Oct 2020 10:57:29 +0000 Subject: [PATCH 17/21] Add all plate metadata to layer.metadata --- ome_zarr/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 1c10221d..e5b52e1e 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -449,7 +449,7 @@ def get_lazy_plate(level: int) -> da.Array: node.metadata = image_node.metadata # "metadata" dict gets added to each 'plate' layer in napari - node.metadata.update({"metadata": {"plate": {"rows": rows, "columns": cols}}}) + node.metadata.update({"metadata": {"plate": self.plate_data}}) class Reader: From d73c3e78f77f58a994fa90b989b78403c32bf76e Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 5 Nov 2020 14:23:04 +0000 Subject: [PATCH 18/21] Update to use spec from omero-cli-zarr PR 42. Handle sparse wells --- ome_zarr/napari.py | 9 ++++---- ome_zarr/reader.py | 53 +++++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/ome_zarr/napari.py b/ome_zarr/napari.py index 9d7d8380..1322a252 100644 --- a/ome_zarr/napari.py +++ b/ome_zarr/napari.py @@ -6,7 +6,6 @@ import logging import warnings -from string import ascii_uppercase from typing import Any, Callable, Dict, Iterator, List, Optional from .data import CHANNEL_DIMENSION @@ -85,8 +84,8 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: plate_info = metadata["metadata"]["plate"] plate_width = shape[-1] plate_height = shape[-2] - rows = plate_info["rows"] - columns = plate_info["columns"] + rows = len(plate_info["rows"]) + columns = len(plate_info["columns"]) well_width = plate_width / columns well_height = plate_height / rows labels = [] @@ -101,7 +100,9 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: # bounding box we have a line # along top of Well, with label below outlines.append([[y1, x1], [y1, x2]]) - label = f"{ascii_uppercase[row]}{column + 1}" + row_name = plate_info["rows"][row]["name"] + col_name = plate_info["columns"][column]["name"] + label = f"{row_name}{col_name}" labels.append(label) # Well bounding box, with no label outlines.append( diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index e5b52e1e..a40ba601 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -2,7 +2,6 @@ import logging from abc import ABC -from string import ascii_uppercase from typing import Any, Dict, Iterator, List, Optional, Type, Union, cast import dask.array as da @@ -356,25 +355,23 @@ def get_pyramid_lazy(self, node: Node) -> None: stitched full-resolution images. """ self.plate_data = self.lookup("plate", {}) - print("self.plate_data", self.plate_data) - self.rows = self.plate_data.get("rows", 0) - self.cols = self.plate_data.get("columns", 0) - self.acquisitions = self.plate_data.get("plateAcquisitions", [{"path": "0"}]) + print("plate_data", self.plate_data) + self.rows = self.plate_data.get("rows") + self.columns = self.plate_data.get("columns") + self.acquisitions = self.plate_data.get("acquisitions", [{"path": "0"}]) + first_field = "0" + row_names = [row["name"] for row in self.rows] + col_names = [col["name"] for col in self.columns] - self.fields = [ - "Field_1", - ] - self.row_labels = ascii_uppercase[0 : self.rows] - self.col_labels = range(1, self.cols + 1) + well_paths = [well["path"] for well in self.plate_data.get("wells")] + well_paths.sort() # TODO: support more Acquisitions - just 1st for now - run = self.acquisitions[0]["path"] - rows = self.rows - cols = self.cols - row_labels = self.row_labels - + run = self.acquisitions[0]["path"].strip("/") + row_count = len(self.rows) + column_count = len(self.columns) # Get the first image... - path = f"{run}/{row_labels[0]}/{self.col_labels[0]}/{self.fields[0]}" + path = f"{well_paths[0]}/{first_field}" image_zarr = self.zarr.create(path) image_node = Node(image_zarr, node) @@ -389,15 +386,15 @@ def get_pyramid_lazy(self, node: Node) -> None: size_x = img_shape[4] # FIXME - if only returning a single stiched plate (not a pyramid) - # need to decide optimal size. E.g. longest side < 3000 + # need to decide optimal size. E.g. longest side < 1500 TARGET_SIZE = 1500 - plate_width = cols * size_x - plate_height = cols * size_y + plate_width = column_count * size_x + plate_height = row_count * size_y longest_side = max(plate_width, plate_height) target_level = 0 for level, shape in enumerate(img_pyramid_shapes): - plate_width = cols * shape[-1] - plate_height = rows * shape[-2] + plate_width = column_count * shape[-1] + plate_height = row_count * shape[-2] longest_side = max(plate_width, plate_height) target_level = level if longest_side <= TARGET_SIZE: @@ -408,11 +405,15 @@ def get_pyramid_lazy(self, node: Node) -> None: def get_tile(tile_name: str) -> np.ndarray: """ tile_name is 'level,z,c,t,row,col' """ level, row, col = [int(n) for n in tile_name.split(",")] - path = f"{run}/{row_labels[row]}/{col + 1}/Field_1/{level}" + path = f"{run}/{row_names[row]}/{col_names[col]}/{first_field}/{level}" print(f"LOADING tile... {path}") - data = self.zarr.load(path) - print("...data.shape", data.shape) + try: + data = self.zarr.load(path) + print("...data.shape", data.shape) + except ValueError: + print(f"Failed to load {path}") + data = np.zeros(img_pyramid_shapes[level], dtype=numpy_type) return data lazy_reader = delayed(get_tile) @@ -420,9 +421,9 @@ def get_tile(tile_name: str) -> np.ndarray: def get_lazy_plate(level: int) -> da.Array: lazy_rows = [] # For level 0, return whole image for each tile - for row in range(rows): + for row in range(row_count): lazy_row: List[da.Array] = [] - for col in range(cols): + for col in range(column_count): tile_name = f"{level},{row},{col}" print(f"creating lazy_reader. level:{level} row:{row} col:{col}") tile_size = img_pyramid_shapes[level] From 3e52d43be4d9758ab27fff64915fc0c0e35760d3 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 5 Nov 2020 15:14:08 +0000 Subject: [PATCH 19/21] Support loading of HCS Well, show fields in grid --- ome_zarr/reader.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index a40ba601..bb02b9fb 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -1,6 +1,7 @@ """Reading logic for ome-zarr.""" import logging +import math from abc import ABC from typing import Any, Dict, Iterator, List, Optional, Type, Union, cast @@ -52,6 +53,8 @@ def __init__( self.specs.append(OMERO(self)) if Plate.matches(zarr): self.specs.append(Plate(self)) + if Well.matches(zarr): + self.specs.append(Well(self)) @property def visible(self) -> bool: @@ -340,6 +343,68 @@ def __init__(self, node: Node) -> None: LOGGER.error(f"failed to parse metadata: {e}") +class Well(Spec): + @staticmethod + def matches(zarr: BaseZarrLocation) -> bool: + return bool("well" in zarr.root_attrs) + + def __init__(self, node: Node) -> None: + super().__init__(node) + self.well_data = self.lookup("well", {}) + print("well_data", self.well_data) + + image_paths = [image["path"] for image in self.well_data.get("images")] + field_count = len(image_paths) + column_count = math.ceil(math.sqrt(field_count)) + row_count = math.ceil(field_count / column_count) + print("column_count", column_count, "row_count", row_count) + + # Use first Field for rendering settings, shape etc. + image_zarr = self.zarr.create(image_paths[0]) + image_node = Node(image_zarr, node) + print("image_node", image_node.metadata) + level = 0 # load full resolution image + numpy_type = image_node.data[level].dtype + img_shape = image_node.data[level].shape + + # stitch full-resolution images into a grid + def get_field(tile_name: str) -> np.ndarray: + """ tile_name is 'row,col' """ + row, col = [int(n) for n in tile_name.split(",")] + field_index = (column_count * row) + col + path = f"{field_index}/{level}" + print(f"LOADING tile... {path}") + try: + data = self.zarr.load(path) + print("...data.shape", data.shape) + except ValueError: + print(f"Failed to load {path}") + data = np.zeros(img_shape, dtype=numpy_type) + return data + + lazy_reader = delayed(get_field) + + def get_lazy_well() -> da.Array: + lazy_rows = [] + # For level 0, return whole image for each tile + for row in range(row_count): + lazy_row: List[da.Array] = [] + for col in range(column_count): + tile_name = f"{row},{col}" + print(f"creating lazy_reader. row:{row} col:{col}") + lazy_tile = da.from_delayed( + lazy_reader(tile_name), shape=img_shape, dtype=numpy_type + ) + lazy_row.append(lazy_tile) + lazy_rows.append(da.concatenate(lazy_row, axis=4)) + print("lazy_row.shape", lazy_rows[-1].shape) + return da.concatenate(lazy_rows, axis=3) + + node.data = [get_lazy_well()] + print("node.data.shape", node.data[0].shape) + node.metadata = image_node.metadata + + class Plate(Spec): @staticmethod def matches(zarr: BaseZarrLocation) -> bool: From 5efffc72d7b0deb5a840a863f7440d14af718533 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 12 Nov 2020 13:05:37 +0000 Subject: [PATCH 20/21] Support hierarchy with no acquisition path --- ome_zarr/reader.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index bb02b9fb..d482a578 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -423,7 +423,7 @@ def get_pyramid_lazy(self, node: Node) -> None: print("plate_data", self.plate_data) self.rows = self.plate_data.get("rows") self.columns = self.plate_data.get("columns") - self.acquisitions = self.plate_data.get("acquisitions", [{"path": "0"}]) + self.acquisitions = self.plate_data.get("acquisitions") first_field = "0" row_names = [row["name"] for row in self.rows] col_names = [col["name"] for col in self.columns] @@ -431,8 +431,14 @@ def get_pyramid_lazy(self, node: Node) -> None: well_paths = [well["path"] for well in self.plate_data.get("wells")] well_paths.sort() - # TODO: support more Acquisitions - just 1st for now - run = self.acquisitions[0]["path"].strip("/") + run = "" + # TEMP - support acquisition path in plate/acq/row/col hierarchy + # remove when we don't want to support dev versions of ome-zarr plate data + if len(self.acquisitions) > 0: + run = self.acquisitions[0].get("path", "") + if len(run) > 0 and not run.endswith("/"): + run = run + "/" + row_count = len(self.rows) column_count = len(self.columns) # Get the first image... @@ -470,7 +476,7 @@ def get_pyramid_lazy(self, node: Node) -> None: def get_tile(tile_name: str) -> np.ndarray: """ tile_name is 'level,z,c,t,row,col' """ level, row, col = [int(n) for n in tile_name.split(",")] - path = f"{run}/{row_names[row]}/{col_names[col]}/{first_field}/{level}" + path = f"{run}{row_names[row]}/{col_names[col]}/{first_field}/{level}" print(f"LOADING tile... {path}") try: From 450483efe16f10c29f8ef3bfc9d390766136e6a7 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 16 Nov 2020 14:58:56 +0000 Subject: [PATCH 21/21] Remove print or use LOGGER instead --- ome_zarr/reader.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index d482a578..fd6e8b04 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -351,18 +351,16 @@ def matches(zarr: BaseZarrLocation) -> bool: def __init__(self, node: Node) -> None: super().__init__(node) self.well_data = self.lookup("well", {}) - print("well_data", self.well_data) + LOGGER.info("well_data: %s", self.well_data) image_paths = [image["path"] for image in self.well_data.get("images")] field_count = len(image_paths) column_count = math.ceil(math.sqrt(field_count)) row_count = math.ceil(field_count / column_count) - print("column_count", column_count, "row_count", row_count) # Use first Field for rendering settings, shape etc. image_zarr = self.zarr.create(image_paths[0]) image_node = Node(image_zarr, node) - print("image_node", image_node.metadata) level = 0 # load full resolution image numpy_type = image_node.data[level].dtype img_shape = image_node.data[level].shape @@ -373,12 +371,11 @@ def get_field(tile_name: str) -> np.ndarray: row, col = [int(n) for n in tile_name.split(",")] field_index = (column_count * row) + col path = f"{field_index}/{level}" - print(f"LOADING tile... {path}") + LOGGER.debug(f"LOADING tile... {path}") try: data = self.zarr.load(path) - print("...data.shape", data.shape) except ValueError: - print(f"Failed to load {path}") + LOGGER.error(f"Failed to load {path}") data = np.zeros(img_shape, dtype=numpy_type) return data @@ -391,17 +388,15 @@ def get_lazy_well() -> da.Array: lazy_row: List[da.Array] = [] for col in range(column_count): tile_name = f"{row},{col}" - print(f"creating lazy_reader. row:{row} col:{col}") + LOGGER.debug(f"creating lazy_reader. row:{row} col:{col}") lazy_tile = da.from_delayed( lazy_reader(tile_name), shape=img_shape, dtype=numpy_type ) lazy_row.append(lazy_tile) lazy_rows.append(da.concatenate(lazy_row, axis=4)) - print("lazy_row.shape", lazy_rows[-1].shape) return da.concatenate(lazy_rows, axis=3) node.data = [get_lazy_well()] - print("node.data.shape", node.data[0].shape) node.metadata = image_node.metadata @@ -420,7 +415,7 @@ def get_pyramid_lazy(self, node: Node) -> None: stitched full-resolution images. """ self.plate_data = self.lookup("plate", {}) - print("plate_data", self.plate_data) + LOGGER.info("plate_data", self.plate_data) self.rows = self.plate_data.get("rows") self.columns = self.plate_data.get("columns") self.acquisitions = self.plate_data.get("acquisitions") @@ -451,7 +446,7 @@ def get_pyramid_lazy(self, node: Node) -> None: img_pyramid_shapes = [d.shape for d in image_node.data] - print("img_pyramid_shapes", img_pyramid_shapes) + LOGGER.debug("img_pyramid_shapes", img_pyramid_shapes) size_y = img_shape[3] size_x = img_shape[4] @@ -471,19 +466,18 @@ def get_pyramid_lazy(self, node: Node) -> None: if longest_side <= TARGET_SIZE: break - print("target_level", target_level) + LOGGER.debug("target_level", target_level) def get_tile(tile_name: str) -> np.ndarray: """ tile_name is 'level,z,c,t,row,col' """ level, row, col = [int(n) for n in tile_name.split(",")] path = f"{run}{row_names[row]}/{col_names[col]}/{first_field}/{level}" - print(f"LOADING tile... {path}") + LOGGER.debug(f"LOADING tile... {path}") try: data = self.zarr.load(path) - print("...data.shape", data.shape) except ValueError: - print(f"Failed to load {path}") + LOGGER.error(f"Failed to load {path}") data = np.zeros(img_pyramid_shapes[level], dtype=numpy_type) return data @@ -496,14 +490,15 @@ def get_lazy_plate(level: int) -> da.Array: lazy_row: List[da.Array] = [] for col in range(column_count): tile_name = f"{level},{row},{col}" - print(f"creating lazy_reader. level:{level} row:{row} col:{col}") + LOGGER.debug( + f"creating lazy_reader. level:{level} row:{row} col:{col}" + ) tile_size = img_pyramid_shapes[level] lazy_tile = da.from_delayed( lazy_reader(tile_name), shape=tile_size, dtype=numpy_type ) lazy_row.append(lazy_tile) lazy_rows.append(da.concatenate(lazy_row, axis=4)) - print("lazy_row.shape", lazy_rows[-1].shape) return da.concatenate(lazy_rows, axis=3) pyramid = [] @@ -512,7 +507,7 @@ def get_lazy_plate(level: int) -> da.Array: # for level in range(5): for level in [target_level]: lazy_plate = get_lazy_plate(level) - print("lazy_plate", lazy_plate.shape) + LOGGER.debug("lazy_plate", lazy_plate.shape) pyramid.append(lazy_plate) # Set the node.data to be pyramid view of the plate