From f26475e320914186a43ea8427b58567719307e03 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Mon, 15 Apr 2024 17:03:23 -0400 Subject: [PATCH 1/9] support s3 --- .../experiments/datasplits/datasplit_generator.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dacapo/experiments/datasplits/datasplit_generator.py b/dacapo/experiments/datasplits/datasplit_generator.py index fac37ed7e..5424fd991 100644 --- a/dacapo/experiments/datasplits/datasplit_generator.py +++ b/dacapo/experiments/datasplits/datasplit_generator.py @@ -1,11 +1,12 @@ from dacapo.experiments.tasks import TaskConfig -from pathlib import Path +from upath import UPath as Path from typing import List from enum import Enum, EnumMeta from funlib.geometry import Coordinate from typing import Union, Optional import zarr +from zarr.n5 import N5FSStore from dacapo.experiments.datasplits.datasets.arrays import ( ZarrArrayConfig, ZarrArray, @@ -21,7 +22,7 @@ logger = logging.getLogger(__name__) -def is_zarr_group(file_name: str, dataset: str): +def is_zarr_group(file_name: Path, dataset: str): """ Check if the dataset is a Zarr group. If the dataset is a Zarr group, it will return True, otherwise False. @@ -40,7 +41,10 @@ def is_zarr_group(file_name: str, dataset: str): Notes: This function is used to check if the dataset is a Zarr group. """ - zarr_file = zarr.open(str(file_name)) + if file_name.suffix == ".n5": + zarr_file = zarr.open(N5FSStore(str(file_name)), mode="r") + else: + zarr_file = zarr.open(str(file_name), mode="r") return isinstance(zarr_file[dataset], zarr.hierarchy.Group) @@ -762,7 +766,7 @@ def __generate_semantic_seg_dataset_crop(self, dataset: DatasetSpec): # f"Processing raw_container:{raw_container} raw_dataset:{raw_dataset} gt_path:{gt_path} gt_dataset:{gt_dataset}" # ) - if is_zarr_group(str(raw_container), raw_dataset): + if is_zarr_group(raw_container, raw_dataset): raw_config = get_right_resolution_array_config( raw_container, raw_dataset, self.input_resolution, "raw" ) @@ -789,7 +793,7 @@ def __generate_semantic_seg_dataset_crop(self, dataset: DatasetSpec): raise FileNotFoundError( f"GT path {gt_path/current_class_dataset} does not exist." ) - if is_zarr_group(str(gt_path), current_class_dataset): + if is_zarr_group(gt_path, current_class_dataset): gt_config = get_right_resolution_array_config( gt_path, current_class_dataset, self.output_resolution, "gt" ) From c0b71a3930aa465d68fdf924c8182835a49e1cc8 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Mon, 15 Apr 2024 17:15:45 -0400 Subject: [PATCH 2/9] support s3 --- .../experiments/datasplits/datasets/arrays/zarr_array.py | 8 +++++++- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py index 2399b5b17..de9623cc3 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py +++ b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py @@ -9,6 +9,7 @@ import lazy_property import numpy as np import zarr +from zarr.n5 import N5FSStore from collections import OrderedDict import logging @@ -358,7 +359,12 @@ def data(self) -> Any: Notes: This method is used to return the data of the array. """ - zarr_container = zarr.open(str(self.file_name)) + file_name = str(self.file_name) + # Zarr library does not detect the store for N5 datasets + if file_name.endswith(".n5"): + zarr_container = zarr.open(N5FSStore(str(file_name)), mode="r") + else: + zarr_container = zarr.open(str(file_name), mode="r") return zarr_container[self.dataset] def __getitem__(self, roi: Roi) -> np.ndarray: diff --git a/pyproject.toml b/pyproject.toml index 0ef3a39c2..f9acee29c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "universal-pathlib>=0.2.2,<1.0.0", "numpy>=1.22.4", "pyyaml", "zarr", @@ -56,6 +55,7 @@ dependencies = [ "click", "pyyaml", "scipy", + "universal_pathlib", ] # extras From 75f568864e18c0f6b285a74f7a0298c73aca1a1d Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Mon, 15 Apr 2024 17:18:25 -0400 Subject: [PATCH 3/9] Delete pyproject.toml --- pyproject.toml | 217 ------------------------------------------------- 1 file changed, 217 deletions(-) delete mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index f9acee29c..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,217 +0,0 @@ -# https://peps.python.org/pep-0517/ -[build-system] -requires = ["hatchling", "hatch-vcs"] -build-backend = "hatchling.build" - -# https://peps.python.org/pep-0621/ -[project] -name = "dacapo-ml" -description = "Framework for deployment of volumetric machine learning models, and easy composition of training jobs." -readme = "README.md" -requires-python = ">=3.10" -# license = { text = "BSD 3-Clause License" } -authors = [ - { email = "pattonw@janelia.hhmi.org", name = "Will Patton" }, - { email = "rhoadesj@hhmi.org", name = "Jeff Rhoades" }, - { email = "zouinkhim@hhmi.org", name = "Marwan Zouinkhi" }, - { email = "funkej@janelia.hhmi.org", name = "Jan Funke" }, -] -classifiers = [ - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Typing :: Typed", -] -dynamic = ["version"] -dependencies = [ - "numpy>=1.22.4", - "pyyaml", - "zarr", - "cattrs", - "pymongo", - "tqdm", - "simpleitk", - "lazy-property", - "neuroglancer", - "torch", - "fibsem_tools", - "attrs", - "bokeh", - "numpy-indexed>=0.3.7", - "daisy", - "funlib.math>=0.1", - "funlib.geometry>=0.2", - "mwatershed>=0.1", - "cellmap-models", - "funlib.persistence>=0.3.0", - "gunpowder>=1.3", - "lsds", - "xarray", - "cattrs", - "numpy-indexed", - "click", - "pyyaml", - "scipy", - "universal_pathlib", - ] - -# extras -# https://peps.python.org/pep-0621/#dependencies-optional-dependencies -[project.optional-dependencies] -test = ["pytest==7.4.4", "pytest-cov", "pytest-lazy-fixture"] -dev = [ - "black", - "mypy", - "pdbpp", - "rich", - "ruff", - "pre-commit", -] -docs = [ - "sphinx-autodoc-typehints", - "sphinx-autoapi", - "sphinx-click", - "sphinx-rtd-theme", - "myst-parser", -] -examples = [ - "ipython", - "ipykernel", - "jupyter", -] -pretrained = [ - "pyqt5", - "empanada-napari", - "cellmap-models", - ] -all = ["dacapo-ml[test,dev,docs,examples,pretrained]"] - -[project.urls] -homepage = "https://github.io/janelia-cellmap/dacapo" -repository = "https://github.com/janelia-cellmap/dacapo" - -# https://hatch.pypa.io/latest/config/metadata/ -[tool.hatch.version] -source = "vcs" - -# https://hatch.pypa.io/latest/config/build/#file-selection -# [tool.hatch.build.targets.sdist] -# include = ["/src", "/tests"] -[tool.hatch.metadata] -allow-direct-references = true - -[tool.hatch.build.targets.wheel] -packages = ["dacapo"] - -[project.scripts] -dacapo = "dacapo.cli:cli" - -# https://github.com/charliermarsh/ruff -[tool.ruff] -line-length = 88 -target-version = "py310" -src = ["dacapo"] - -[tool.ruff.lint] -# https://beta.ruff.rs/docs/rules/ -# We may want to enable some of these options later -select = [ - "E", # style errors -# "W", # style warnings - "F", # flakes -# "D", # pydocstyle -# "I", # isort -# "UP", # pyupgrade -# "C4", # flake8-comprehensions -# "B", # flake8-bugbear -# "A001", # flake8-builtins -# "RUF", # ruff-specific rules -] -extend-ignore = ["E501"] - -[tool.ruff.lint.per-file-ignores] -"tests/*.py" = ["D", "S"] -"__init__.py" = ["F401"] - -# https://docs.pytest.org/en/6.2.x/customize.html -[tool.pytest.ini_options] -minversion = "6.0" -testpaths = ["tests"] -filterwarnings = [ - "error", - "ignore::DeprecationWarning", - ] - -# https://mypy.readthedocs.io/en/stable/config_file.html -[tool.mypy] -files = "dacapo/**/" -strict = false -disallow_any_generics = false -disallow_subclassing_any = false -show_error_codes = true -pretty = true -exclude = [ - "scratch/*", - "examples/*", -] - - -# # module specific overrides -[[tool.mypy.overrides]] -module = [ - "cellmap_models.*", - "funlib.*", - "toml.*", - "gunpowder.*", - "scipy.*", - "augment.*", - "tifffile.*", - "daisy.*", - "lazy_property.*", - "skimage.*", - "fibsem_tools.*", - "neuroglancer.*", - "tqdm.*", - "zarr.*", - "pymongo.*", - "bson.*", - "affogato.*", - "SimpleITK.*", - "bokeh.*", - "lsd.*", - "yaml.*", - "pytest_lazyfixture.*", - "neuclease.dvid.*", - "mwatershed.*", - "numpy_indexed.*", - "empanada_napari.*", - "napari.*", - "empanada.*", - "IPython.*", -] -ignore_missing_imports = true - -# https://coverage.readthedocs.io/en/6.4/config.html -[tool.coverage.report] -exclude_lines = [ - "pragma: no cover", - "if TYPE_CHECKING:", - "@overload", - "except ImportError", - "\\.\\.\\.", - "raise NotImplementedError()", -] -[tool.coverage.run] -source = ["dacapo"] - -# https://github.com/mgedmin/check-manifest#configuration -[tool.check-manifest] -ignore = [ - ".github_changelog_generator", - ".pre-commit-config.yaml", - ".ruff_cache/**/*", - "tests/**/*", -] From 3aae94e3cc2075c3cda172ab48d89e82c1a7c1ec Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Mon, 15 Apr 2024 17:19:42 -0400 Subject: [PATCH 4/9] recover pyproject --- pyproject.toml | 217 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..0ef3a39c2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,217 @@ +# https://peps.python.org/pep-0517/ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +# https://peps.python.org/pep-0621/ +[project] +name = "dacapo-ml" +description = "Framework for deployment of volumetric machine learning models, and easy composition of training jobs." +readme = "README.md" +requires-python = ">=3.10" +# license = { text = "BSD 3-Clause License" } +authors = [ + { email = "pattonw@janelia.hhmi.org", name = "Will Patton" }, + { email = "rhoadesj@hhmi.org", name = "Jeff Rhoades" }, + { email = "zouinkhim@hhmi.org", name = "Marwan Zouinkhi" }, + { email = "funkej@janelia.hhmi.org", name = "Jan Funke" }, +] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Typing :: Typed", +] +dynamic = ["version"] +dependencies = [ + "universal-pathlib>=0.2.2,<1.0.0", + "numpy>=1.22.4", + "pyyaml", + "zarr", + "cattrs", + "pymongo", + "tqdm", + "simpleitk", + "lazy-property", + "neuroglancer", + "torch", + "fibsem_tools", + "attrs", + "bokeh", + "numpy-indexed>=0.3.7", + "daisy", + "funlib.math>=0.1", + "funlib.geometry>=0.2", + "mwatershed>=0.1", + "cellmap-models", + "funlib.persistence>=0.3.0", + "gunpowder>=1.3", + "lsds", + "xarray", + "cattrs", + "numpy-indexed", + "click", + "pyyaml", + "scipy", + ] + +# extras +# https://peps.python.org/pep-0621/#dependencies-optional-dependencies +[project.optional-dependencies] +test = ["pytest==7.4.4", "pytest-cov", "pytest-lazy-fixture"] +dev = [ + "black", + "mypy", + "pdbpp", + "rich", + "ruff", + "pre-commit", +] +docs = [ + "sphinx-autodoc-typehints", + "sphinx-autoapi", + "sphinx-click", + "sphinx-rtd-theme", + "myst-parser", +] +examples = [ + "ipython", + "ipykernel", + "jupyter", +] +pretrained = [ + "pyqt5", + "empanada-napari", + "cellmap-models", + ] +all = ["dacapo-ml[test,dev,docs,examples,pretrained]"] + +[project.urls] +homepage = "https://github.io/janelia-cellmap/dacapo" +repository = "https://github.com/janelia-cellmap/dacapo" + +# https://hatch.pypa.io/latest/config/metadata/ +[tool.hatch.version] +source = "vcs" + +# https://hatch.pypa.io/latest/config/build/#file-selection +# [tool.hatch.build.targets.sdist] +# include = ["/src", "/tests"] +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["dacapo"] + +[project.scripts] +dacapo = "dacapo.cli:cli" + +# https://github.com/charliermarsh/ruff +[tool.ruff] +line-length = 88 +target-version = "py310" +src = ["dacapo"] + +[tool.ruff.lint] +# https://beta.ruff.rs/docs/rules/ +# We may want to enable some of these options later +select = [ + "E", # style errors +# "W", # style warnings + "F", # flakes +# "D", # pydocstyle +# "I", # isort +# "UP", # pyupgrade +# "C4", # flake8-comprehensions +# "B", # flake8-bugbear +# "A001", # flake8-builtins +# "RUF", # ruff-specific rules +] +extend-ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +"tests/*.py" = ["D", "S"] +"__init__.py" = ["F401"] + +# https://docs.pytest.org/en/6.2.x/customize.html +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = ["tests"] +filterwarnings = [ + "error", + "ignore::DeprecationWarning", + ] + +# https://mypy.readthedocs.io/en/stable/config_file.html +[tool.mypy] +files = "dacapo/**/" +strict = false +disallow_any_generics = false +disallow_subclassing_any = false +show_error_codes = true +pretty = true +exclude = [ + "scratch/*", + "examples/*", +] + + +# # module specific overrides +[[tool.mypy.overrides]] +module = [ + "cellmap_models.*", + "funlib.*", + "toml.*", + "gunpowder.*", + "scipy.*", + "augment.*", + "tifffile.*", + "daisy.*", + "lazy_property.*", + "skimage.*", + "fibsem_tools.*", + "neuroglancer.*", + "tqdm.*", + "zarr.*", + "pymongo.*", + "bson.*", + "affogato.*", + "SimpleITK.*", + "bokeh.*", + "lsd.*", + "yaml.*", + "pytest_lazyfixture.*", + "neuclease.dvid.*", + "mwatershed.*", + "numpy_indexed.*", + "empanada_napari.*", + "napari.*", + "empanada.*", + "IPython.*", +] +ignore_missing_imports = true + +# https://coverage.readthedocs.io/en/6.4/config.html +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "@overload", + "except ImportError", + "\\.\\.\\.", + "raise NotImplementedError()", +] +[tool.coverage.run] +source = ["dacapo"] + +# https://github.com/mgedmin/check-manifest#configuration +[tool.check-manifest] +ignore = [ + ".github_changelog_generator", + ".pre-commit-config.yaml", + ".ruff_cache/**/*", + "tests/**/*", +] From 412789c0aa62900cdae9db8bced97c6cce78d687 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Mon, 15 Apr 2024 17:51:13 -0400 Subject: [PATCH 5/9] fix mode error --- .../experiments/datasplits/datasets/arrays/zarr_array.py | 8 +++++--- dacapo/experiments/datasplits/datasplit_generator.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py index de9623cc3..6e2b092b8 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py +++ b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py @@ -85,7 +85,7 @@ class ZarrArray(Array): This class is used to create a zarr array. """ - def __init__(self, array_config): + def __init__(self, array_config, mode="w"): """ Initializes the array type 'raw' and name for the DummyDataset instance. @@ -105,9 +105,11 @@ def __init__(self, array_config): self.file_name = array_config.file_name self.dataset = array_config.dataset + self._mode = mode self._attributes = self.data.attrs self._axes = array_config._axes self.snap_to_grid = array_config.snap_to_grid + def __str__(self): """ @@ -362,9 +364,9 @@ def data(self) -> Any: file_name = str(self.file_name) # Zarr library does not detect the store for N5 datasets if file_name.endswith(".n5"): - zarr_container = zarr.open(N5FSStore(str(file_name)), mode="r") + zarr_container = zarr.open(N5FSStore(str(file_name)), mode=self._mode) else: - zarr_container = zarr.open(str(file_name), mode="r") + zarr_container = zarr.open(str(file_name), mode=self._mode) return zarr_container[self.dataset] def __getitem__(self, roi: Roi) -> np.ndarray: diff --git a/dacapo/experiments/datasplits/datasplit_generator.py b/dacapo/experiments/datasplits/datasplit_generator.py index 5424fd991..e4b97a205 100644 --- a/dacapo/experiments/datasplits/datasplit_generator.py +++ b/dacapo/experiments/datasplits/datasplit_generator.py @@ -71,7 +71,7 @@ def resize_if_needed( Notes: This function is used to resize the array if needed. """ - zarr_array = ZarrArray(array_config) + zarr_array = ZarrArray(array_config,mode="r") raw_voxel_size = zarr_array.voxel_size raw_upsample = raw_voxel_size / target_resolution @@ -126,7 +126,7 @@ def get_right_resolution_array_config( dataset=str(current_dataset_path), snap_to_grid=target_resolution, ) - zarr_array = ZarrArray(zarr_config) + zarr_array = ZarrArray(zarr_config,mode="r") while ( all([z < t for (z, t) in zip(zarr_array.voxel_size, target_resolution)]) and Path(container, Path(dataset, f"s{level+1}")).exists() @@ -139,7 +139,7 @@ def get_right_resolution_array_config( snap_to_grid=target_resolution, ) - zarr_array = ZarrArray(zarr_config) + zarr_array = ZarrArray(zarr_config,mode = "r") return resize_if_needed(zarr_config, target_resolution, extra_str) From 74335c41126fd55c61038184838a33088f3dfd85 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Mon, 15 Apr 2024 21:49:52 -0400 Subject: [PATCH 6/9] Update zarr_array.py --- dacapo/experiments/datasplits/datasets/arrays/zarr_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py index 6e2b092b8..b3e72b555 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py +++ b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py @@ -85,7 +85,7 @@ class ZarrArray(Array): This class is used to create a zarr array. """ - def __init__(self, array_config, mode="w"): + def __init__(self, array_config, mode="a"): """ Initializes the array type 'raw' and name for the DummyDataset instance. From a413ac36ff0d7ccc6e8dbda9c33a489669a8838c Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Tue, 16 Apr 2024 09:43:25 -0400 Subject: [PATCH 7/9] zarr array mode as part of the config --- .../datasplits/datasets/arrays/zarr_array.py | 5 ++--- .../datasplits/datasets/arrays/zarr_array_config.py | 3 +++ dacapo/experiments/datasplits/datasplit_generator.py | 10 +++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py index b3e72b555..a44a1d6a6 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py +++ b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py @@ -85,7 +85,7 @@ class ZarrArray(Array): This class is used to create a zarr array. """ - def __init__(self, array_config, mode="a"): + def __init__(self, array_config): """ Initializes the array type 'raw' and name for the DummyDataset instance. @@ -104,8 +104,7 @@ def __init__(self, array_config, mode="a"): self.name = array_config.name self.file_name = array_config.file_name self.dataset = array_config.dataset - - self._mode = mode + self._mode = array_config.mode self._attributes = self.data.attrs self._axes = array_config._axes self.snap_to_grid = array_config.snap_to_grid diff --git a/dacapo/experiments/datasplits/datasets/arrays/zarr_array_config.py b/dacapo/experiments/datasplits/datasets/arrays/zarr_array_config.py index af6f9dd20..b67717647 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/zarr_array_config.py +++ b/dacapo/experiments/datasplits/datasets/arrays/zarr_array_config.py @@ -55,6 +55,9 @@ class ZarrArrayConfig(ArrayConfig): _axes: Optional[List[str]] = attr.ib( default=None, metadata={"help_text": "The axes of your data!"} ) + mode: Optional[str] = attr.ib( + default="a", metadata={"help_text": "The access mode!"} + ) def verify(self) -> Tuple[bool, str]: """ diff --git a/dacapo/experiments/datasplits/datasplit_generator.py b/dacapo/experiments/datasplits/datasplit_generator.py index e4b97a205..54df330b6 100644 --- a/dacapo/experiments/datasplits/datasplit_generator.py +++ b/dacapo/experiments/datasplits/datasplit_generator.py @@ -71,7 +71,7 @@ def resize_if_needed( Notes: This function is used to resize the array if needed. """ - zarr_array = ZarrArray(array_config,mode="r") + zarr_array = ZarrArray(array_config) raw_voxel_size = zarr_array.voxel_size raw_upsample = raw_voxel_size / target_resolution @@ -125,8 +125,9 @@ def get_right_resolution_array_config( file_name=container, dataset=str(current_dataset_path), snap_to_grid=target_resolution, + mode = "r" ) - zarr_array = ZarrArray(zarr_config,mode="r") + zarr_array = ZarrArray(zarr_config) while ( all([z < t for (z, t) in zip(zarr_array.voxel_size, target_resolution)]) and Path(container, Path(dataset, f"s{level+1}")).exists() @@ -137,9 +138,10 @@ def get_right_resolution_array_config( file_name=container, dataset=str(Path(dataset, f"s{level}")), snap_to_grid=target_resolution, + mode = "r" ) - zarr_array = ZarrArray(zarr_config,mode = "r") + zarr_array = ZarrArray(zarr_config) return resize_if_needed(zarr_config, target_resolution, extra_str) @@ -776,6 +778,7 @@ def __generate_semantic_seg_dataset_crop(self, dataset: DatasetSpec): name=f"raw_{raw_container.stem}_uint8", file_name=raw_container, dataset=raw_dataset, + mode="r", ), self.input_resolution, "raw", @@ -803,6 +806,7 @@ def __generate_semantic_seg_dataset_crop(self, dataset: DatasetSpec): name=f"gt_{gt_path.stem}_{current_class_dataset}_uint8", file_name=gt_path, dataset=current_class_dataset, + mode="r", ), self.output_resolution, "gt", From 20b967decf9b267e6de28c8c473a0acb37d58f33 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Tue, 16 Apr 2024 09:53:51 -0400 Subject: [PATCH 8/9] remove _ --- dacapo/experiments/datasplits/datasets/arrays/zarr_array.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py index a44a1d6a6..4812911c7 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py +++ b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py @@ -104,7 +104,7 @@ def __init__(self, array_config): self.name = array_config.name self.file_name = array_config.file_name self.dataset = array_config.dataset - self._mode = array_config.mode + self.mode = array_config.mode self._attributes = self.data.attrs self._axes = array_config._axes self.snap_to_grid = array_config.snap_to_grid @@ -363,9 +363,9 @@ def data(self) -> Any: file_name = str(self.file_name) # Zarr library does not detect the store for N5 datasets if file_name.endswith(".n5"): - zarr_container = zarr.open(N5FSStore(str(file_name)), mode=self._mode) + zarr_container = zarr.open(N5FSStore(str(file_name)), mode=self.mode) else: - zarr_container = zarr.open(str(file_name), mode=self._mode) + zarr_container = zarr.open(str(file_name), mode=self.mode) return zarr_container[self.dataset] def __getitem__(self, roi: Roi) -> np.ndarray: From e3afa41429f45c55e5110b65603e5e19b0974b53 Mon Sep 17 00:00:00 2001 From: Marwan Zouinkhi Date: Tue, 16 Apr 2024 10:03:33 -0400 Subject: [PATCH 9/9] mode property --- .../datasplits/datasets/arrays/zarr_array.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py index 4812911c7..c976f7d96 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py +++ b/dacapo/experiments/datasplits/datasets/arrays/zarr_array.py @@ -104,7 +104,7 @@ def __init__(self, array_config): self.name = array_config.name self.file_name = array_config.file_name self.dataset = array_config.dataset - self.mode = array_config.mode + self._mode = array_config.mode self._attributes = self.data.attrs self._axes = array_config._axes self.snap_to_grid = array_config.snap_to_grid @@ -144,6 +144,14 @@ def __repr__(self): """ return f"ZarrArray({self.file_name}, {self.dataset})" + + @property + def mode(self): + if not hasattr(self, "_mode"): + self._mode = "a" + if self._mode not in ["r", "w", "a"]: + raise ValueError(f"Mode {self._mode} not in ['r', 'w', 'a']") + return self._mode @property def attrs(self): @@ -413,6 +421,7 @@ def create_from_array_identifier( num_channels, voxel_size, dtype, + mode = "a", write_size=None, name=None, overwrite=False,