Append References

CHANGELOG.md

@@ -1,5 +1,9 @@
 # HDMF-ZARR Changelog
 
+## 0.9.0 (Upcoming)
+### Enhancements
+* Added support for appending a dataset of references. @mavaylon1 [#203]([https://github.com/hdmf-dev/hdmf-zarr/pull/172](https://github.com/hdmf-dev/hdmf-zarr/pull/203))
+
 ## 0.8.0 (June 4, 2024)
 ### Bug Fixes
 * Fixed bug when opening a file in with `mode=r+`. The file will open without using the consolidated metadata. @mavaylon1 [#182](https://github.com/hdmf-dev/hdmf-zarr/issues/182)

pyproject.toml

@@ -29,7 +29,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Medical Science Apps."
 ]
 dependencies = [
-    'hdmf>=3.9.0',
+    'hdmf>=3.14.2',
     'zarr>=2.11.0, <3.0', # pin below 3.0 until HDMF-zarr supports zarr 3.0
     'numpy>=1.24, <2.0', # pin below 2.0 until HDMF supports numpy 2.0
     'numcodecs>=0.9.1',
@@ -109,7 +109,7 @@ force-exclude = '''
 '''
 
 [tool.ruff]
-select = ["E", "F", "T100", "T201", "T203"]
+lint.select = ["E", "F", "T100", "T201", "T203"]
 exclude = [
   ".git",
   ".tox",
@@ -128,5 +128,5 @@ line-length = 120
 "src/*/__init__.py" = ["F401"]
 "test_gallery.py" = ["T201"]
 
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
 max-complexity = 17

requirements-min.txt

@@ -1,4 +1,4 @@
-hdmf==3.12.0
+hdmf==3.14.2
 zarr==2.11.0
 numcodecs==0.9.1
 pynwb==2.5.0

src/hdmf_zarr/backend.py

@@ -757,7 +757,9 @@ def __get_ref(self, ref_object, export_source=None):
             builder = ref_object.builder
         else:
             builder = self.manager.build(ref_object)
+
         path = self.__get_path(builder)
+
         # TODO Add to get region for region references.
         #      Also add  {'name': 'region', 'type': (slice, list, tuple),
         #      'doc': 'the region reference indexing object',  'default': None},
@@ -1279,7 +1281,7 @@ def __list_fill__(self, parent, name, data, options=None):  # noqa: C901
         dset.attrs['zarr_dtype'] = type_str
 
         # Write the data to file
-        if dtype == object:
+        if dtype == object: # noqa: E721
             for c in np.ndindex(data_shape):
                 o = data
                 for i in c:
@@ -1313,7 +1315,7 @@ def __scalar_fill__(self, parent, name, data, options=None):
             except Exception as exc:
                 msg = 'cannot add %s to %s - could not determine type' % (name, parent.name)
                 raise Exception(msg) from exc
-        if dtype == object:
+        if dtype == object: # noqa: E721
             io_settings['object_codec'] = self.__codec_cls()
 
         dset = parent.require_dataset(name, shape=(1, ), dtype=dtype, **io_settings)

src/hdmf_zarr/zarr_utils.py

@@ -10,6 +10,7 @@
 from zarr import Array as ZarrArray
 
 from hdmf.build import DatasetBuilder
+from hdmf.data_utils import append_data
 from hdmf.array import Array
 from hdmf.query import HDMFDataset, ReferenceResolver, ContainerResolver, BuilderResolver
 from hdmf.utils import docval, popargs, get_docval
@@ -73,6 +74,14 @@ def __iter__(self):
     def __next__(self):
         return self._get_ref(super().__next__())
 
+    def append(self, arg):
+        # Create Builder
+        builder = self.io.manager.build(arg)
+
+        # Create ZarrReference
+        ref = self.io._ZarrIO__get_ref(builder)
+        append_data(self.dataset, ref)
+
 
 class BuilderResolverMixin(BuilderResolver):  # refactor to backend/utils.py
     """

tests/unit/test_zarrio.py

@@ -16,9 +16,12 @@
 from zarr.storage import (DirectoryStore,
                           TempStore,
                           NestedDirectoryStore)
+from tests.unit.utils import (Baz, BazData, BazBucket, get_baz_buildmanager)
 import zarr
 from hdmf_zarr.backend import ZarrIO
 import os
+import shutil
+import warnings
 
 
 CUR_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -182,3 +185,47 @@ def test_force_open_without_consolidated_fails(self):
             except ValueError as e:
                 self.fail("ZarrIO.__open_file_consolidated raised an unexpected ValueError: {}".format(e))
 
+
+class TestDatasetofReferences(ZarrStoreTestCase):
+    def setUp(self):
+        self.store_path = "test_io.zarr"
+        self.store = DirectoryStore(self.store_path)
+
+    def tearDown(self):
+        """
+        Remove all files and folders defined by self.store_path
+        """
+        paths = self.store_path if isinstance(self.store_path, list) else [self.store_path, ]
+        for path in paths:
+            if os.path.exists(path):
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+                elif os.path.isfile(path):
+                    os.remove(path)
+                else:
+                    warnings.warn("Could not remove: %s" % path)
+
+    def test_append_references(self):
+        # Setup a file container with references
+        num_bazs = 10
+        bazs = []  # set up dataset of references
+        for i in range(num_bazs):
+            bazs.append(Baz(name='baz%d' % i))
+        baz_data = BazData(name='baz_data', data=bazs)
+        container = BazBucket(bazs=bazs, baz_data=baz_data)
+        manager = get_baz_buildmanager()
+        # write to file
+        with ZarrIO(self.store, manager=manager, mode='w') as writer:
+            writer.write(container=container)
+        # read from file and validate references
+        with ZarrIO(self.store, manager=manager, mode='r+') as append_io:
+            read_container = append_io.read()
+            DoR = read_container.baz_data.data
+            DoR.append(DoR[0])
+
+            expected =  {'source': '.', 'path': '/bazs/baz0',
+                         'object_id': DoR[0].object_id,
+                         'source_object_id': container.object_id}
+
+            self.assertEqual(len(read_container.baz_data.data), 11)
+            self.assertDictEqual(read_container.baz_data.data.dataset[10], expected)