From c4b1c43d69d751b71418e7a642bc8a68278911d4 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Tue, 17 Aug 2021 22:50:23 +0100 Subject: [PATCH 1/5] Migrate bioformats2raw logic to _bf_export - pass --series argument using image.series - pass --no-root-group and --no-ome-meta - rename exported image folder as image_id.zarr --- src/omero_zarr/cli.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index d5cae42..ed2f636 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -237,19 +237,8 @@ def polygons(self, args: argparse.Namespace) -> None: def export(self, args: argparse.Namespace) -> None: if isinstance(args.object, ImageI): image = self._lookup(self.gateway, "Image", args.object.id) - inplace = image.getInplaceImport() - if args.bf: - if self.client is None: - raise Exception("This cannot happen") # mypy is confused - prx, desc = self.client.getManagedRepository(description=True) - repo_path = Path(desc._path._val) / Path(desc._name._val) - if inplace: - for p in image.getImportedImageFilePaths()["client_paths"]: - self._bf_export(Path("/") / Path(p), args) - else: - for p in image.getImportedImageFilePaths()["server_paths"]: - self._bf_export(repo_path / p, args) + self._bf_export(image, args) else: image_to_zarr(image, args) elif isinstance(args.object, PlateI): @@ -266,7 +255,16 @@ def _lookup( self.ctx.die(110, f"No such {otype}: {oid}") return obj - def _bf_export(self, abs_path: Path, args: argparse.Namespace) -> None: + def _bf_export(self, image: BlitzObjectWrapper, args: argparse.Namespace) -> None: + if image.getInplaceImport(): + p = image.getImportedImageFilePaths()["client_paths"][0] + abs_path = Path("/") / Path(p) + else: + if self.client is None: + raise Exception("This cannot happen") # mypy is confused + prx, desc = self.client.getManagedRepository(description=True) + p = image.getImportedImageFilePaths()["server_paths"][0] + abs_path = Path(desc._path._val) / Path(desc._name._val) / Path(p) target = (Path(args.output) or Path.cwd()) / Path(abs_path).name cmd: List[str] = [ @@ -283,6 +281,9 @@ def _bf_export(self, abs_path: Path, args: argparse.Namespace) -> None: cmd.append(f"--resolutions={args.resolutions}") if args.max_workers: cmd.append(f"--max_workers={args.max_workers}") + cmd.append(f"--series={image.series}") + cmd.append("--no-root-group") + cmd.append("--no-ome-meta-export") self.ctx.dbg(" ".join(cmd)) process = subprocess.Popen( @@ -294,7 +295,11 @@ def _bf_export(self, abs_path: Path, args: argparse.Namespace) -> None: if stderr: self.ctx.err(stderr.decode("utf-8")) if process.returncode == 0: - self.ctx.out(f"Image exported to {target.resolve()}") + image_source = target / "0" + image_dest = (Path(args.output) or Path.cwd()) / f"{image.id}.zarr" + image_source.rename(image_dest) + target.rmdir() + self.ctx.out(f"Image exported to {image_dest.resolve()}") try: From 27620bdeb31a01ad7baf14131479786c4d371d92 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 18 Aug 2021 14:10:25 +0100 Subject: [PATCH 2/5] Add checks for directory existence --- src/omero_zarr/cli.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index ed2f636..3a859f5 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -266,6 +266,12 @@ def _bf_export(self, image: BlitzObjectWrapper, args: argparse.Namespace) -> Non p = image.getImportedImageFilePaths()["server_paths"][0] abs_path = Path(desc._path._val) / Path(desc._name._val) / Path(p) target = (Path(args.output) or Path.cwd()) / Path(abs_path).name + image_target = (Path(args.output) or Path.cwd()) / f"{image.id}.zarr" + + if target.exists(): + self.ctx.die(111, f"{target.resolve()} already exists") + if image_target.exists(): + self.ctx.die(111, f"{image_target.resolve()} already exists") cmd: List[str] = [ "bioformats2raw", @@ -296,10 +302,9 @@ def _bf_export(self, image: BlitzObjectWrapper, args: argparse.Namespace) -> Non self.ctx.err(stderr.decode("utf-8")) if process.returncode == 0: image_source = target / "0" - image_dest = (Path(args.output) or Path.cwd()) / f"{image.id}.zarr" - image_source.rename(image_dest) + image_source.rename(image_target) target.rmdir() - self.ctx.out(f"Image exported to {image_dest.resolve()}") + self.ctx.out(f"Image exported to {image_target.resolve()}") try: From 91bf1a1c0029dc9b7d35dd73a2d4e6ec120c53b1 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 18 Aug 2021 14:20:05 +0100 Subject: [PATCH 3/5] Split add_group_metadata into multiscales/omero specifications --- src/omero_zarr/raw_pixels.py | 38 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index b69e260..3b85e0c 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -39,7 +39,8 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) - store = _open_store(name) root = open_group(store) n_levels, axes = add_image(image, root, cache_dir=cache_dir) - add_group_metadata(root, image, axes, n_levels) + add_multiscales_metadata(root, axes, n_levels) + add_omero_metadata(root, image) add_toplevel_metadata(root) print("Finished.") @@ -268,7 +269,8 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) col_group = row_group.require_group(col) field_group = col_group.require_group(field_name) n_levels, axes = add_image(img, field_group, cache_dir=cache_dir) - add_group_metadata(field_group, img, axes, n_levels) + add_multiscales_metadata(field_group, axes, n_levels) + add_omero_metadata(field_group, img) # Update Well metadata after each image col_group.attrs["well"] = {"images": fields, "version": VERSION} max_fields = max(max_fields, field + 1) @@ -283,26 +285,12 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) print("Finished.") -def add_group_metadata( +def add_multiscales_metadata( zarr_root: Group, - image: Optional[omero.gateway.ImageWrapper], axes: List[str], resolutions: int = 1, ) -> None: - if image: - image_data = { - "id": 1, - "channels": [channelMarshal(c) for c in image.getChannels()], - "rdefs": { - "model": (image.isGreyscaleRenderingModel() and "greyscale" or "color"), - "defaultZ": image._re.getDefaultZ(), - "defaultT": image._re.getDefaultT(), - }, - "version": VERSION, - } - zarr_root.attrs["omero"] = image_data - image._closeRE() multiscales = [ { "version": "0.3", @@ -313,6 +301,22 @@ def add_group_metadata( zarr_root.attrs["multiscales"] = multiscales +def add_omero_metadata(zarr_root: Group, image: omero.gateway.ImageWrapper) -> None: + + image_data = { + "id": 1, + "channels": [channelMarshal(c) for c in image.getChannels()], + "rdefs": { + "model": (image.isGreyscaleRenderingModel() and "greyscale" or "color"), + "defaultZ": image._re.getDefaultZ(), + "defaultT": image._re.getDefaultT(), + }, + "version": VERSION, + } + zarr_root.attrs["omero"] = image_data + image._closeRE() + + def add_toplevel_metadata(zarr_root: Group) -> None: zarr_root.attrs["_creator"] = {"name": "omero-zarr", "version": __version__} From 0835e4f141fafff55417f98196031d20ace6d58c Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 18 Aug 2021 14:47:29 +0100 Subject: [PATCH 4/5] Populate OMERO and top-level metadata in bf2raw Zarr image Pass -profile black to isort pre-commit to solve incompatibility when committing --- .pre-commit-config.yaml | 1 + src/omero_zarr/cli.py | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e2e676..3d7303d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,7 @@ repos: rev: 5.9.3 hooks: - id: isort + args: ["--profile", "black"] - repo: https://github.com/psf/black rev: 21.7b0 diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 3a859f5..59c6e38 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -8,9 +8,16 @@ from omero.cli import CLI, BaseControl, Parser, ProxyStringType from omero.gateway import BlitzGateway, BlitzObjectWrapper from omero.model import ImageI, PlateI +from zarr.hierarchy import open_group +from zarr.storage import FSStore from .masks import MASK_DTYPE_SIZE, image_shapes_to_zarr, plate_shapes_to_zarr -from .raw_pixels import image_to_zarr, plate_to_zarr +from .raw_pixels import ( + add_omero_metadata, + add_toplevel_metadata, + image_to_zarr, + plate_to_zarr, +) HELP = """Export data in zarr format. @@ -306,6 +313,17 @@ def _bf_export(self, image: BlitzObjectWrapper, args: argparse.Namespace) -> Non target.rmdir() self.ctx.out(f"Image exported to {image_target.resolve()}") + # Add OMERO metadata + store = FSStore( + str(image_target.resolve()), + auto_mkdir=False, + normalize_keys=False, + mode="w", + ) + root = open_group(store) + add_omero_metadata(root, image) + add_toplevel_metadata(root) + try: register("zarr", ZarrControl, HELP) # type: ignore From 7ae24db5e8d31ffdd443c611b975dc6a7ef663c9 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Thu, 19 Aug 2021 22:43:06 +0100 Subject: [PATCH 5/5] Improve --bf arguments description --- src/omero_zarr/cli.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 59c6e38..ec4716c 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -196,19 +196,29 @@ def _configure(self, parser: Parser) -> None: export.add_argument( "--bf", action="store_true", - help="Use bioformats2raw to export the image.", + help="Use bioformats2raw to export the image. Requires" + " bioformats2raw 0.3.0 or higher.", ) export.add_argument( - "--tile_width", default=None, help="For use with bioformats2raw" + "--tile_width", + default=None, + help="Maximum tile width to read (only for use with bioformats2raw)", ) export.add_argument( - "--tile_height", default=None, help="For use with bioformats2raw" + "--tile_height", + default=None, + help="Maximum tile height to read (only for use with bioformats2raw)", ) export.add_argument( - "--resolutions", default=None, help="For use with bioformats2raw" + "--resolutions", + default=None, + help="Number of pyramid resolutions to generate" + " (only for use with bioformats2raw)", ) export.add_argument( - "--max_workers", default=None, help="For use with bioformats2raw" + "--max_workers", + default=None, + help="Maximum number of workers (only for use with bioformats2raw)", ) export.add_argument( "object",