From 37506b3b49d3005452b532a7d5cc4ca6a6c58108 Mon Sep 17 00:00:00 2001 From: Seth Hinz Date: Wed, 9 Oct 2024 10:08:30 -0700 Subject: [PATCH] On-demand folder creation --- softcopy/zarr_copier.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/softcopy/zarr_copier.py b/softcopy/zarr_copier.py index 1aa8403..b366271 100644 --- a/softcopy/zarr_copier.py +++ b/softcopy/zarr_copier.py @@ -112,9 +112,11 @@ def __init__(self, source: Path, destination: Path, n_copy_procs: int = 1, log: def start(self): self._log.debug("Creating zarr folder structure in destination.") - ZarrCopier.create_zarr_folder_structure( - self._destination, self._zarr_format, self._files_nd, self._dimension_separator, self._log - ) + # ZarrCopier.create_zarr_folder_structure( + # self._destination, self._zarr_format, self._files_nd, self._dimension_separator, self._log + # ) + ZarrCopier.create_mandatory_folders(self._destination, self._zarr_format, self._log) + self._log.warning("Using WIP on-demand folder creation!") self._log.debug("Starting copy processes.") for _ in range(self._n_copy_procs): @@ -309,6 +311,17 @@ def create_zarr_folder_structure( log.critical(f"Unsupported zarr version {zarr_format}") exit(1) + @staticmethod + def create_mandatory_folders(zarr_archive_location, zarr_format, log: Logger = LOG): + """ + This is used for an on-demand folder creation mode that is under development in softcopy. Not stable, will + be reorganized + """ + if zarr_format == 2: + zarr_archive_location.mkdir(parents=True, exist_ok=True) + else: + (zarr_archive_location / "c").mkdir(parents=True, exist_ok=True) + def _copy_worker( queue: Queue, @@ -327,7 +340,10 @@ def _copy_worker( srcfile = data.get_path(files_nd, source, dimension_separator, zarr_format) destfile = data.get_path(files_nd, destination, dimension_separator, zarr_format) print(f"Copying {srcfile} to {destfile}") - # destfile = PackedName3.get_path(data, files_nd, destination) + + # TODO: this is just for on-demand folder creation. it slows things down, so we should + # make it optional in the targets.yaml + destfile.parent.mkdir(parents=True, exist_ok=True) copyfile(srcfile, destfile) # Increment the copy count only if this is a data chunk