Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix handling of read h5py string datasets #1189

Merged
merged 3 commits into from
Sep 13, 2024
Merged

Fix handling of read h5py string datasets #1189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a1db9f8
Fix handling of read h5py string datasets
rly Sep 5, 2024
2935d64
Update changelog
rly Sep 5, 2024
9ecdf4f
Run tests differently for h5py < 3
rly Sep 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# HDMF Changelog

## HDMF 3.14.5 (September 6, 2024)

### Bug fixes
- Fixed bug in writing of string arrays to an HDF5 file that were read from an HDF5 file that was introduced in 3.14.4. @rly @stephprince
[#1189](https://github.com/hdmf-dev/hdmf/pull/1189)

## HDMF 3.14.4 (September 4, 2024)

### Enhancements
Expand Down
5 changes: 4 additions & 1 deletion src/hdmf/build/objectmapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,10 @@ def __get_data_type(cls, spec):
def __convert_string(self, value, spec):
"""Convert string types to the specified dtype."""
def __apply_string_type(value, string_type):
if isinstance(value, (list, tuple, np.ndarray, DataIO)):
# NOTE: if a user passes a h5py.Dataset that is not wrapped with a hdmf.utils.StrDataset,
# then this conversion may not be correct. Users should unpack their string h5py.Datasets
# into a numpy array (or wrap them in StrDataset) before passing them to a container object.
if hasattr(value, '__iter__') and not isinstance(value, (str, bytes)):
return [__apply_string_type(item, string_type) for item in value]
else:
return string_type(value)
Expand Down
2 changes: 1 addition & 1 deletion src/hdmf/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1140,7 +1140,7 @@ def update(self, other):

@docval_macro('array_data')
class StrDataset(h5py.Dataset):
"""Wrapper to decode strings on reading the dataset"""
"""Wrapper to decode strings on reading the dataset. Use only for h5py 3+."""
def __init__(self, dset, encoding, errors='strict'):
self.dset = dset
if encoding is None:
Expand Down
131 changes: 130 additions & 1 deletion tests/unit/build_tests/test_io_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from hdmf.utils import docval, getargs
from hdmf.utils import StrDataset, docval, getargs
from hdmf import Container, Data
from hdmf.backends.hdf5 import H5DataIO
from hdmf.build import (GroupBuilder, DatasetBuilder, ObjectMapper, BuildManager, TypeMap, LinkBuilder,
Expand All @@ -7,12 +7,15 @@
from hdmf.spec import (GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog, RefSpec,
LinkSpec)
from hdmf.testing import TestCase
import h5py
from abc import ABCMeta, abstractmethod
import unittest
import numpy as np

from tests.unit.helpers.utils import CORE_NAMESPACE, create_test_type_map

H5PY_3 = h5py.__version__.startswith('3')


class Bar(Container):

Expand Down Expand Up @@ -460,6 +463,132 @@ def test_build_3d_ndarray(self):
np.testing.assert_array_equal(builder.get('data').data, str_array_3d)
np.testing.assert_array_equal(builder.get('attr_array'), str_array_3d)

@unittest.skipIf(not H5PY_3, "Use StrDataset only for h5py 3+")
def test_build_1d_h5py_3_dataset(self):
bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec(
doc='an example dataset',
dtype='text',
name='data',
shape=(None, ),
attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')],
)
],
attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text',
shape=(None, ))],
)
type_map = self.customSetUp(bar_spec)
type_map.register_map(Bar, BarMapper)
# create in-memory hdf5 file that is discarded after closing
with h5py.File("test.h5", "w", driver="core", backing_store=False) as f:
str_array_1d = np.array(
['aa', 'bb', 'cc', 'dd'],
dtype=h5py.special_dtype(vlen=str)
)
# wrap the dataset in a StrDataset to mimic how HDF5IO would read this dataset with h5py 3+
dataset = StrDataset(f.create_dataset('data', data=str_array_1d), None)
bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset)
builder = type_map.build(bar_inst)
np.testing.assert_array_equal(builder.get('data').data, dataset[:])
np.testing.assert_array_equal(builder.get('attr_array'), dataset[:])

@unittest.skipIf(not H5PY_3, "Use StrDataset only for h5py 3+")
def test_build_3d_h5py_3_dataset(self):
bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec(
doc='an example dataset',
dtype='text',
name='data',
shape=(None, None, None),
attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')],
)
],
attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text',
shape=(None, None, None))],
)
type_map = self.customSetUp(bar_spec)
type_map.register_map(Bar, BarMapper)
# create in-memory hdf5 file that is discarded after closing
with h5py.File("test.h5", "w", driver="core", backing_store=False) as f:
str_array_3d = np.array(
[[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]],
dtype=h5py.special_dtype(vlen=str)
)
# wrap the dataset in a StrDataset to mimic how HDF5IO would read this dataset with h5py 3+
dataset = StrDataset(f.create_dataset('data', data=str_array_3d), None)
bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset)
builder = type_map.build(bar_inst)
np.testing.assert_array_equal(builder.get('data').data, dataset[:])
np.testing.assert_array_equal(builder.get('attr_array'), dataset[:])

@unittest.skipIf(H5PY_3, "Create dataset differently for h5py < 3")
def test_build_1d_h5py_2_dataset(self):
bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec(
doc='an example dataset',
dtype='text',
name='data',
shape=(None, ),
attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')],
)
],
attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text',
shape=(None, ))],
)
type_map = self.customSetUp(bar_spec)
type_map.register_map(Bar, BarMapper)
# create in-memory hdf5 file that is discarded after closing
with h5py.File("test.h5", "w", driver="core", backing_store=False) as f:
str_array_1d = np.array(
['aa', 'bb', 'cc', 'dd'],
dtype=h5py.special_dtype(vlen=str)
)
dataset = f.create_dataset('data', data=str_array_1d)
bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset)
builder = type_map.build(bar_inst)
np.testing.assert_array_equal(builder.get('data').data, dataset[:])
np.testing.assert_array_equal(builder.get('attr_array'), dataset[:])

@unittest.skipIf(H5PY_3, "Create dataset differently for h5py < 3")
def test_build_3d_h5py_2_dataset(self):
bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec(
doc='an example dataset',
dtype='text',
name='data',
shape=(None, None, None),
attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')],
)
],
attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text',
shape=(None, None, None))],
)
type_map = self.customSetUp(bar_spec)
type_map.register_map(Bar, BarMapper)
# create in-memory hdf5 file that is discarded after closing
with h5py.File("test.h5", "w", driver="core", backing_store=False) as f:
str_array_3d = np.array(
[[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]],
dtype=h5py.special_dtype(vlen=str)
)
dataset = f.create_dataset('data', data=str_array_3d)
bar_inst = Bar('my_bar', dataset, 'value1', 10, attr_array=dataset)
builder = type_map.build(bar_inst)
np.testing.assert_array_equal(builder.get('data').data, dataset[:])
np.testing.assert_array_equal(builder.get('attr_array'), dataset[:])

def test_build_dataio(self):
bar_spec = GroupSpec('A test group specification with a data type',
data_type_def='Bar',
Expand Down
Loading