From 0345a46972e169a44e21eec3b4adbe92007fdcc7 Mon Sep 17 00:00:00 2001 From: bendichter Date: Fri, 28 Jul 2023 17:27:05 -0400 Subject: [PATCH 01/15] default load_namespaces=True --- docs/gallery/advanced_io/streaming.py | 4 ++-- docs/gallery/general/extensions.py | 2 +- src/pynwb/__init__.py | 9 +++++---- tests/back_compat/test_read.py | 2 +- tests/read_dandi/test_read_dandi.py | 2 +- tests/unit/test_file.py | 4 ++-- tests/validation/test_validate.py | 8 +++----- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/gallery/advanced_io/streaming.py b/docs/gallery/advanced_io/streaming.py index b3800584a..0294255fd 100644 --- a/docs/gallery/advanced_io/streaming.py +++ b/docs/gallery/advanced_io/streaming.py @@ -69,7 +69,7 @@ # next, open the file with fs.open(s3_url, "rb") as f: with h5py.File(f) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: + with pynwb.NWBHDF5IO(file=file) as io: nwbfile = io.read() print(nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:]) @@ -111,7 +111,7 @@ from pynwb import NWBHDF5IO - with NWBHDF5IO(s3_url, mode='r', load_namespaces=True, driver='ros3') as io: + with NWBHDF5IO(s3_url, mode='r', driver='ros3') as io: nwbfile = io.read() print(nwbfile) print(nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:]) diff --git a/docs/gallery/general/extensions.py b/docs/gallery/general/extensions.py index fa4f4cbb7..5140c531b 100644 --- a/docs/gallery/general/extensions.py +++ b/docs/gallery/general/extensions.py @@ -254,7 +254,7 @@ def __init__(self, **kwargs): # explicitly specify this. This behavior is enabled by the *load_namespaces* # argument to the :py:class:`~pynwb.NWBHDF5IO` constructor. -with NWBHDF5IO("cache_spec_example.nwb", mode="r", load_namespaces=True) as io: +with NWBHDF5IO("cache_spec_example.nwb", mode="r") as io: nwbfile = io.read() #################### diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 181079970..8c89e466f 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -207,7 +207,7 @@ class NWBHDF5IO(_HDF5IO): 'default': 'r'}, {'name': 'load_namespaces', 'type': bool, 'doc': 'whether or not to load cached namespaces from given path - not applicable in write mode', - 'default': False}, + 'default': True}, {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None}, {'name': 'extensions', 'type': (str, TypeMap, list), 'doc': 'a path to a namespace, a TypeMap, or a list consisting paths to namespaces and TypeMaps', @@ -223,14 +223,15 @@ def __init__(self, **kwargs): popargs('path', 'mode', 'manager', 'extensions', 'load_namespaces', 'file', 'comm', 'driver', 'external_resources_path', kwargs) # Define the BuildManager to use + if mode in 'wx': + # namespaces are not loaded when creating an NWBHDF5IO object in write mode + load_namespaces = False + if load_namespaces: if manager is not None: warn("loading namespaces from file - ignoring 'manager'") if extensions is not None: warn("loading namespaces from file - ignoring 'extensions' argument") - # namespaces are not loaded when creating an NWBHDF5IO object in write mode - if 'w' in mode or mode == 'x': - raise ValueError("cannot load namespaces from file when writing to it") tm = get_type_map() super().load_namespaces(tm, path, file=file_obj, driver=driver) diff --git a/tests/back_compat/test_read.py b/tests/back_compat/test_read.py index 919ae6bde..be02c7205 100644 --- a/tests/back_compat/test_read.py +++ b/tests/back_compat/test_read.py @@ -43,7 +43,7 @@ def test_read(self): with self.subTest(file=f.name): with warnings.catch_warnings(record=True) as warnings_on_read: warnings.simplefilter("always") - with NWBHDF5IO(str(f), 'r', load_namespaces=True) as io: + with NWBHDF5IO(str(f), 'r') as io: errors = validate(io) io.read() for w in warnings_on_read: diff --git a/tests/read_dandi/test_read_dandi.py b/tests/read_dandi/test_read_dandi.py index 84e9f3f62..7c7115799 100644 --- a/tests/read_dandi/test_read_dandi.py +++ b/tests/read_dandi/test_read_dandi.py @@ -41,7 +41,7 @@ def test_read_first_nwb_asset(self): s3_url = first_asset.get_content_url(follow_redirects=1, strip_query=True) try: - with NWBHDF5IO(path=s3_url, load_namespaces=True, driver="ros3") as io: + with NWBHDF5IO(path=s3_url, driver="ros3") as io: io.read() except Exception as e: print(traceback.format_exc()) diff --git a/tests/unit/test_file.py b/tests/unit/test_file.py index bb5c9c1e1..de1d06178 100644 --- a/tests/unit/test_file.py +++ b/tests/unit/test_file.py @@ -542,7 +542,7 @@ def test_simple(self): lab='Chang Lab') with NWBHDF5IO(self.path, 'w') as io: io.write(nwbfile) - with NWBHDF5IO(self.path, 'r', load_namespaces=True) as reader: + with NWBHDF5IO(self.path, 'r') as reader: nwbfile = reader.read() @@ -564,7 +564,7 @@ def test_simple(self): io.write(nwbfile, cache_spec=False) with self.assertWarnsWith(UserWarning, "No cached namespaces found in %s" % self.path): - with NWBHDF5IO(self.path, 'r', load_namespaces=True) as reader: + with NWBHDF5IO(self.path, 'r') as reader: nwbfile = reader.read() diff --git a/tests/validation/test_validate.py b/tests/validation/test_validate.py index 813f8d4e3..72418ac00 100644 --- a/tests/validation/test_validate.py +++ b/tests/validation/test_validate.py @@ -222,13 +222,13 @@ def test_validate_io_cached(self): def test_validate_io_cached_extension(self): """Test that validating a file with cached spec against its cached namespaces succeeds.""" - with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r', load_namespaces=True) as io: + with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r') as io: errors = validate(io) self.assertEqual(errors, []) def test_validate_io_cached_extension_pass_ns(self): """Test that validating a file with cached extension spec against the extension namespace succeeds.""" - with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r', load_namespaces=True) as io: + with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r') as io: errors = validate(io, 'ndx-testextension') self.assertEqual(errors, []) @@ -237,9 +237,7 @@ def test_validate_io_cached_core_with_io(self): For back-compatability, test that validating a file with cached extension spec against the core namespace succeeds when using the `io` + `namespace` keywords. """ - with NWBHDF5IO( - path='tests/back_compat/2.1.0_nwbfile_with_extension.nwb', mode='r', load_namespaces=True - ) as io: + with NWBHDF5IO(path='tests/back_compat/2.1.0_nwbfile_with_extension.nwb', mode='r') as io: results = validate(io=io, namespace="core") self.assertEqual(results, []) From 792451d0b5c503fbf394a4e64e499ccbef16ec7f Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Fri, 28 Jul 2023 17:29:54 -0400 Subject: [PATCH 02/15] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7dbad481..e286a0dd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Enhancements and minor changes - Add `TimeSeries.get_timestamps`. @bendichter [#1741](https://github.com/NeurodataWithoutBorders/pynwb/pull/1741) +- For `NWBHDF5IO()`, change the default of arg `load_namespaces` from `False` to `True`. @bendichter [#1748](https://github.com/NeurodataWithoutBorders/pynwb/pull/1748) ## PyNWB 2.4.1 (July 26, 2023) - Stop running validation tests as part of integration tests. They cause issues in CI and can be run separately. @rly [#1740](https://github.com/NeurodataWithoutBorders/pynwb/pull/1740) From 382f0fbf215fd589129007b948d380b26c583625 Mon Sep 17 00:00:00 2001 From: bendichter Date: Fri, 28 Jul 2023 17:56:38 -0400 Subject: [PATCH 03/15] change logic around to ignore load_namespaces when it cannot be used --- docs/gallery/advanced_io/linking_data.py | 81 +++++++++++------------- src/pynwb/__init__.py | 8 +-- 2 files changed, 39 insertions(+), 50 deletions(-) diff --git a/docs/gallery/advanced_io/linking_data.py b/docs/gallery/advanced_io/linking_data.py index 082aa3c51..168708888 100644 --- a/docs/gallery/advanced_io/linking_data.py +++ b/docs/gallery/advanced_io/linking_data.py @@ -6,57 +6,50 @@ PyNWB supports linking between files using external links. -""" +Example Use Case: Integrating data from multiple files +--------------------------------------------------------- -#################### -# Example Use Case: Integrating data from multiple files -# --------------------------------------------------------- -# -# NBWContainer classes (e.g., :py:class:`~pynwb.base.TimeSeries`) support the integration of data stored in external -# HDF5 files with NWB data files via external links. To make things more concrete, let's look at the following use -# case. We want to simultaneously record multiple data streams during data acquisition. Using the concept of external -# links allows us to save each data stream to an external HDF5 files during data acquisition and to -# afterwards link the data into a single NWB:N file. In this case, each recording becomes represented by a -# separate file-system object that can be set as read-only once the experiment is done. In the following -# we are using :py:meth:`~pynwb.base.TimeSeries` as an example, but the same approach works for other -# NWBContainers as well. -# +NBWContainer classes (e.g., :py:class:`~pynwb.base.TimeSeries`) support the integration of data stored in external +HDF5 files with NWB data files via external links. To make things more concrete, let's look at the following use +case. We want to simultaneously record multiple data streams during data acquisition. Using the concept of external +links allows us to save each data stream to an external HDF5 files during data acquisition and to +afterwards link the data into a single NWB:N file. In this case, each recording becomes represented by a +separate file-system object that can be set as read-only once the experiment is done. In the following +we are using :py:meth:`~pynwb.base.TimeSeries` as an example, but the same approach works for other +NWBContainers as well. -#################### -# .. tip:: -# -# The same strategies we use here for creating External Links also apply to Soft Links. -# The main difference between soft and external links is that soft links point to other -# objects within the same file while external links point to objects in external files. -# +.. tip:: -#################### -# .. tip:: -# -# In the case of :py:meth:`~pynwb.base.TimeSeries`, the uncorrected timestamps generated by the acquisition -# system can be stored (or linked) in the *sync* group. In the NWB:N format, hardware-recorded time data -# must then be corrected to a common time base (e.g., timestamps from all hardware sources aligned) before -# it can be included in the *timestamps* of the *TimeSeries*. This means, in the case -# of :py:meth:`~pynwb.base.TimeSeries` we need to be careful that we are not including data with incompatible -# timestamps in the same file when using external links. -# + The same strategies we use here for creating External Links also apply to Soft Links. + The main difference between soft and external links is that soft links point to other + objects within the same file while external links point to objects in external files. -#################### -# .. warning:: -# -# External links can become stale/break. Since external links are pointing to data in other files -# external links may become invalid any time files are modified on the file system, e.g., renamed, -# moved or access permissions are changed. -# + .. tip:: -#################### -# Creating test data -# --------------------------- -# -# In the following we are creating two :py:meth:`~pynwb.base.TimeSeries` each written to a separate file. -# We then show how we can integrate these files into a single NWBFile. + In the case of :py:meth:`~pynwb.base.TimeSeries`, the uncorrected timestamps generated by the acquisition + system can be stored (or linked) in the *sync* group. In the NWB:N format, hardware-recorded time data + must then be corrected to a common time base (e.g., timestamps from all hardware sources aligned) before + it can be included in the *timestamps* of the *TimeSeries*. This means, in the case + of :py:meth:`~pynwb.base.TimeSeries` we need to be careful that we are not including data with incompatible + timestamps in the same file when using external links. + + +.. warning:: + + External links can become stale/break. Since external links are pointing to data in other files + external links may become invalid any time files are modified on the file system, e.g., renamed, + moved or access permissions are changed. + + +Creating test data +--------------------------- + +In the following we are creating two :py:meth:`~pynwb.base.TimeSeries` each written to a separate file. +We then show how we can integrate these files into a single NWBFile. +""" # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_linking_data.png' + from datetime import datetime from uuid import uuid4 diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 8c89e466f..6b5f1388c 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -222,16 +222,12 @@ def __init__(self, **kwargs): path, mode, manager, extensions, load_namespaces, file_obj, comm, driver, external_resources_path =\ popargs('path', 'mode', 'manager', 'extensions', 'load_namespaces', 'file', 'comm', 'driver', 'external_resources_path', kwargs) + # Define the BuildManager to use - if mode in 'wx': - # namespaces are not loaded when creating an NWBHDF5IO object in write mode + if mode in 'wx' or manager is not None or extensions is not None: load_namespaces = False if load_namespaces: - if manager is not None: - warn("loading namespaces from file - ignoring 'manager'") - if extensions is not None: - warn("loading namespaces from file - ignoring 'extensions' argument") tm = get_type_map() super().load_namespaces(tm, path, file=file_obj, driver=driver) From 26faa0cdc0ad6cbf43e9f8361b9bcbc773310534 Mon Sep 17 00:00:00 2001 From: bendichter Date: Sat, 29 Jul 2023 09:16:52 -0400 Subject: [PATCH 04/15] flake8 --- src/pynwb/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 6b5f1388c..7f9545ee2 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -4,7 +4,6 @@ import os.path from pathlib import Path from copy import deepcopy -from warnings import warn import h5py from hdmf.spec import NamespaceCatalog From 2c6a415832b9a765bd84198a45bc1dec14e451d6 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Sat, 29 Jul 2023 06:31:29 -0700 Subject: [PATCH 05/15] Update __init__.py --- src/pynwb/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 7f9545ee2..c071df5fb 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -205,7 +205,8 @@ class NWBHDF5IO(_HDF5IO): 'doc': 'the mode to open the HDF5 file with, one of ("w", "r", "r+", "a", "w-", "x")', 'default': 'r'}, {'name': 'load_namespaces', 'type': bool, - 'doc': 'whether or not to load cached namespaces from given path - not applicable in write mode', + 'doc': ('whether or not to load cached namespaces from given path - not applicable in write mode ' + 'or when `manager` is not None or when `extensions` is not None'), 'default': True}, {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None}, {'name': 'extensions', 'type': (str, TypeMap, list), From 9d5ab3381094ea63a68e85ae32ba7e9e5da489ff Mon Sep 17 00:00:00 2001 From: bendichter Date: Mon, 31 Jul 2023 13:07:04 -0400 Subject: [PATCH 06/15] fix back-compatibility test --- tests/back_compat/test_import_structure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/back_compat/test_import_structure.py b/tests/back_compat/test_import_structure.py index e5f931f5d..79d4f6ad0 100644 --- a/tests/back_compat/test_import_structure.py +++ b/tests/back_compat/test_import_structure.py @@ -82,7 +82,6 @@ def test_outer_import_structure(self): "spec", "testing", "validate", - "warn", ] for member in expected_structure: self.assertIn(member=member, container=current_structure) From 6c941a0dbef9b616dd7cec12e9318c56823aad7d Mon Sep 17 00:00:00 2001 From: bendichter Date: Mon, 31 Jul 2023 13:26:08 -0400 Subject: [PATCH 07/15] flake8 --- src/pynwb/misc.py | 2 +- tests/integration/hdf5/test_ecephys.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pynwb/misc.py b/src/pynwb/misc.py index 098fce1de..4d977b4f2 100644 --- a/src/pynwb/misc.py +++ b/src/pynwb/misc.py @@ -75,7 +75,7 @@ def __init__(self, **kwargs): {'name': 'features', 'type': (list, np.ndarray), 'doc': 'the feature values for this time point'}) def add_features(self, **kwargs): time, features = getargs('time', 'features', kwargs) - if type(self.timestamps) == list and type(self.data) is list: + if isinstance(self.timestamps, list) and isinstance(self.data, list): self.timestamps.append(time) self.data.append(features) else: diff --git a/tests/integration/hdf5/test_ecephys.py b/tests/integration/hdf5/test_ecephys.py index 9d810270c..d36b8d871 100644 --- a/tests/integration/hdf5/test_ecephys.py +++ b/tests/integration/hdf5/test_ecephys.py @@ -106,8 +106,9 @@ def setUpTwoElectricalSeries(self): data2 = list(zip(reversed(range(10)), reversed(range(10, 20)))) timestamps = list(map(lambda x: x/10., range(10))) es1 = ElectricalSeries(name='test_eS1', data=data1, electrodes=region1, timestamps=timestamps) - es2 = ElectricalSeries(name='test_eS2', data=data2, electrodes=region2, channel_conversion=[4., .4], - timestamps=timestamps) + es2 = ElectricalSeries( + name='test_eS2', data=data2, electrodes=region2, channel_conversion=[4., .4], timestamps=timestamps + ) return es1, es2 def addContainer(self, nwbfile): From f8e35d360c5488110ed6a9555a0b0c94a22d0795 Mon Sep 17 00:00:00 2001 From: bendichter Date: Mon, 31 Jul 2023 16:08:44 -0400 Subject: [PATCH 08/15] flake8 --- tests/integration/hdf5/test_ecephys.py | 14 ++++++++++++-- tests/unit/test_ecephys.py | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/integration/hdf5/test_ecephys.py b/tests/integration/hdf5/test_ecephys.py index d36b8d871..dec040830 100644 --- a/tests/integration/hdf5/test_ecephys.py +++ b/tests/integration/hdf5/test_ecephys.py @@ -1,7 +1,17 @@ from hdmf.common import DynamicTableRegion -from pynwb.ecephys import ElectrodeGroup, ElectricalSeries, FilteredEphys, LFP, Clustering, ClusterWaveforms,\ - SpikeEventSeries, EventWaveform, EventDetection, FeatureExtraction +from pynwb.ecephys import ( + ElectrodeGroup, + ElectricalSeries, + FilteredEphys, + LFP, + Clustering, + ClusterWaveforms, + SpikeEventSeries, + EventWaveform, + EventDetection, + FeatureExtraction, +) from pynwb.device import Device from pynwb.file import ElectrodeTable as get_electrode_table from pynwb.testing import NWBH5IOMixin, AcquisitionH5IOMixin, TestCase diff --git a/tests/unit/test_ecephys.py b/tests/unit/test_ecephys.py index 6cdfcd59e..26320394b 100644 --- a/tests/unit/test_ecephys.py +++ b/tests/unit/test_ecephys.py @@ -2,8 +2,18 @@ import numpy as np -from pynwb.ecephys import ElectricalSeries, SpikeEventSeries, EventDetection, Clustering, EventWaveform,\ - ClusterWaveforms, LFP, FilteredEphys, FeatureExtraction, ElectrodeGroup +from pynwb.ecephys import ( + ElectricalSeries, + SpikeEventSeries, + EventDetection, + Clustering, + EventWaveform, + ClusterWaveforms, + LFP, + FilteredEphys, + FeatureExtraction, + ElectrodeGroup, +) from pynwb.device import Device from pynwb.file import ElectrodeTable from pynwb.testing import TestCase From df2a32ed80531b19c56b05df417723432c1f6ca7 Mon Sep 17 00:00:00 2001 From: bendichter Date: Mon, 31 Jul 2023 20:32:41 -0400 Subject: [PATCH 09/15] remove warning from test --- tests/unit/test_file.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_file.py b/tests/unit/test_file.py index de1d06178..15ac4b0a4 100644 --- a/tests/unit/test_file.py +++ b/tests/unit/test_file.py @@ -563,9 +563,8 @@ def test_simple(self): with NWBHDF5IO(self.path, 'w') as io: io.write(nwbfile, cache_spec=False) - with self.assertWarnsWith(UserWarning, "No cached namespaces found in %s" % self.path): - with NWBHDF5IO(self.path, 'r') as reader: - nwbfile = reader.read() + with NWBHDF5IO(self.path, 'r') as reader: + nwbfile = reader.read() class TestTimestampsRefDefault(TestCase): From 8087153b9208c54c498dd5c6cf86ffa3834de194 Mon Sep 17 00:00:00 2001 From: bendichter Date: Thu, 21 Sep 2023 09:22:27 -0400 Subject: [PATCH 10/15] rmv useless test --- tests/unit/test_file.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/unit/test_file.py b/tests/unit/test_file.py index 15ac4b0a4..d4918b0eb 100644 --- a/tests/unit/test_file.py +++ b/tests/unit/test_file.py @@ -554,18 +554,6 @@ def setUp(self): def tearDown(self): remove_test_file(self.path) - def test_simple(self): - nwbfile = NWBFile(' ', ' ', - datetime.now(tzlocal()), - file_create_date=datetime.now(tzlocal()), - institution='University of California, San Francisco', - lab='Chang Lab') - with NWBHDF5IO(self.path, 'w') as io: - io.write(nwbfile, cache_spec=False) - - with NWBHDF5IO(self.path, 'r') as reader: - nwbfile = reader.read() - class TestTimestampsRefDefault(TestCase): def setUp(self): From 8b86a20f436164197bc72aa7bbd7b50fa5175859 Mon Sep 17 00:00:00 2001 From: rly Date: Wed, 4 Oct 2023 00:25:55 -0700 Subject: [PATCH 11/15] Update tests, validate script for default load ns --- src/pynwb/__init__.py | 1 - src/pynwb/validate.py | 1 + tests/back_compat/test_read.py | 26 +++++++++----- tests/validation/test_validate.py | 58 +++++++++++++------------------ 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 0a20f2140..a341e8fb0 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -227,7 +227,6 @@ def __init__(self, **kwargs): load_namespaces = False if load_namespaces: - tm = get_type_map() super().load_namespaces(tm, path, file=file_obj, driver=driver) manager = BuildManager(tm) diff --git a/src/pynwb/validate.py b/src/pynwb/validate.py index 23b3aee6f..62aa41426 100644 --- a/src/pynwb/validate.py +++ b/src/pynwb/validate.py @@ -156,6 +156,7 @@ def validate(**kwargs): file=sys.stderr, ) else: + io_kwargs.update(load_namespaces=False) namespaces_to_validate = [CORE_NAMESPACE] if namespace is not None: diff --git a/tests/back_compat/test_read.py b/tests/back_compat/test_read.py index be02c7205..792d26e7a 100644 --- a/tests/back_compat/test_read.py +++ b/tests/back_compat/test_read.py @@ -31,6 +31,16 @@ class TestReadOldVersions(TestCase): "- expected an array of shape '[None]', got non-array data 'one publication'")], } + def get_io(self, path): + """Get an NWBHDF5IO object for the given path.""" + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"Ignoring cached namespace .*", + category=UserWarning, + ) + return NWBHDF5IO(str(path), 'r') + def test_read(self): """Test reading and validating all NWB files in the same folder as this file. @@ -43,7 +53,7 @@ def test_read(self): with self.subTest(file=f.name): with warnings.catch_warnings(record=True) as warnings_on_read: warnings.simplefilter("always") - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: errors = validate(io) io.read() for w in warnings_on_read: @@ -69,28 +79,28 @@ def test_read(self): def test_read_timeseries_no_data(self): """Test that a TimeSeries written without data is read with data set to the default value.""" f = Path(__file__).parent / '1.5.1_timeseries_no_data.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() np.testing.assert_array_equal(read_nwbfile.acquisition['test_timeseries'].data, TimeSeries.DEFAULT_DATA) def test_read_timeseries_no_unit(self): """Test that an ImageSeries written without unit is read with unit set to the default value.""" f = Path(__file__).parent / '1.5.1_timeseries_no_unit.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertEqual(read_nwbfile.acquisition['test_timeseries'].unit, TimeSeries.DEFAULT_UNIT) def test_read_imageseries_no_data(self): """Test that an ImageSeries written without data is read with data set to the default value.""" f = Path(__file__).parent / '1.5.1_imageseries_no_data.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() np.testing.assert_array_equal(read_nwbfile.acquisition['test_imageseries'].data, ImageSeries.DEFAULT_DATA) def test_read_imageseries_no_unit(self): """Test that an ImageSeries written without unit is read with unit set to the default value.""" f = Path(__file__).parent / '1.5.1_imageseries_no_unit.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertEqual(read_nwbfile.acquisition['test_imageseries'].unit, ImageSeries.DEFAULT_UNIT) @@ -100,7 +110,7 @@ def test_read_imageseries_non_external_format(self): f = Path(__file__).parent / fbase expected_warning = self.expected_warnings[fbase][0] with self.assertWarnsWith(UserWarning, expected_warning): - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertEqual(read_nwbfile.acquisition['test_imageseries'].format, "tiff") @@ -110,13 +120,13 @@ def test_read_imageseries_nonmatch_starting_frame(self): f = Path(__file__).parent / fbase expected_warning = self.expected_warnings[fbase][0] with self.assertWarnsWith(UserWarning, expected_warning): - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() np.testing.assert_array_equal(read_nwbfile.acquisition['test_imageseries'].starting_frame, [1, 2, 3]) def test_read_subject_no_age__reference(self): """Test that reading a Subject without an age__reference set with NWB schema 2.5.0 sets the value to None""" f = Path(__file__).parent / '2.2.0_subject_no_age__reference.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertIsNone(read_nwbfile.subject.age__reference) diff --git a/tests/validation/test_validate.py b/tests/validation/test_validate.py index 637f3d82a..6aa2ee25e 100644 --- a/tests/validation/test_validate.py +++ b/tests/validation/test_validate.py @@ -199,64 +199,54 @@ class TestValidateFunction(TestCase): # 1.0.3_nwbfile.nwb has cached "core" specification # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specificaitions + def get_io(self, path): + """Get an NWBHDF5IO object for the given path, ignoring the warning about ignoring cached namespaces.""" + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"Ignoring cached namespace .*", + category=UserWarning, + ) + return NWBHDF5IO(str(path), 'r') + def test_validate_io_no_cache(self): """Test that validating a file with no cached spec against the core namespace succeeds.""" - with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.0.2_nwbfile.nwb') as io: errors = validate(io) self.assertEqual(errors, []) def test_validate_io_no_cache_bad_ns(self): """Test that validating a file with no cached spec against a specified, unknown namespace fails.""" - with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.0.2_nwbfile.nwb') as io: with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): validate(io, 'notfound') def test_validate_io_cached(self): """Test that validating a file with cached spec against its cached namespace succeeds.""" - with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.1.2_nwbfile.nwb') as io: errors = validate(io) self.assertEqual(errors, []) def test_validate_io_cached_extension(self): """Test that validating a file with cached spec against its cached namespaces succeeds.""" - with warnings.catch_warnings(record=True): - warnings.filterwarnings( - "ignore", - message=r"Ignoring cached namespace .*", - category=UserWarning, - ) - with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r') as io: - errors = validate(io) - self.assertEqual(errors, []) + with self.get_io('tests/back_compat/2.1.0_nwbfile_with_extension.nwb') as io: + errors = validate(io) + self.assertEqual(errors, []) def test_validate_io_cached_extension_pass_ns(self): """Test that validating a file with cached extension spec against the extension namespace succeeds.""" - with warnings.catch_warnings(record=True): - warnings.filterwarnings( - "ignore", - message=r"Ignoring cached namespace .*", - category=UserWarning, - ) - with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r') as io: - errors = validate(io, 'ndx-testextension') - self.assertEqual(errors, []) + with self.get_io('tests/back_compat/2.1.0_nwbfile_with_extension.nwb') as io: + errors = validate(io, 'ndx-testextension') + self.assertEqual(errors, []) def test_validate_io_cached_core_with_io(self): """ For back-compatability, test that validating a file with cached extension spec against the core namespace succeeds when using the `io` + `namespace` keywords. """ - with warnings.catch_warnings(record=True): - warnings.filterwarnings( - "ignore", - message=r"Ignoring cached namespace .*", - category=UserWarning, - ) - with NWBHDF5IO( - path='tests/back_compat/2.1.0_nwbfile_with_extension.nwb', mode='r' - ) as io: - results = validate(io=io, namespace="core") - self.assertEqual(results, []) + with self.get_io(path='tests/back_compat/2.1.0_nwbfile_with_extension.nwb') as io: + results = validate(io=io, namespace="core") + self.assertEqual(results, []) def test_validate_file_cached_extension(self): """ @@ -310,13 +300,13 @@ def test_validate_file_cached_no_cache_bad_ns(self): def test_validate_io_cached_bad_ns(self): """Test that validating a file with cached spec against a specified, unknown namespace fails.""" - with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.1.2_nwbfile.nwb') as io: with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): validate(io, 'notfound') def test_validate_io_cached_hdmf_common(self): """Test that validating a file with cached spec against the hdmf-common namespace fails.""" - with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.1.2_nwbfile.nwb') as io: # TODO this error should not be different from the error when using the validate script above msg = "builder must have data type defined with attribute 'data_type'" with self.assertRaisesWith(ValueError, msg): From ac71956610dd44839b90fba21fcbc6ff061c5619 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 4 Oct 2023 01:30:47 -0700 Subject: [PATCH 12/15] Update CHANGELOG.md --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb221674e..e64483434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,15 @@ # PyNWB Changelog +## PyNWB 2.6.0 (Upcoming) + +### Enhancements and minor changes +- For `NWBHDF5IO()`, change the default of arg `load_namespaces` from `False` to `True`. @bendichter [#1748](https://github.com/NeurodataWithoutBorders/pynwb/pull/1748) + ## PyNWB 2.5.0 (August 18, 2023) ### Enhancements and minor changes - Add `TimeSeries.get_timestamps()`. @bendichter [#1741](https://github.com/NeurodataWithoutBorders/pynwb/pull/1741) - Add `TimeSeries.get_data_in_units()`. @bendichter [#1745](https://github.com/NeurodataWithoutBorders/pynwb/pull/1745) -- For `NWBHDF5IO()`, change the default of arg `load_namespaces` from `False` to `True`. @bendichter [#1748](https://github.com/NeurodataWithoutBorders/pynwb/pull/1748) - Updated `ExternalResources` name change to `HERD`, along with HDMF 3.9.0 being the new minimum. @mavaylon1 [#1754](https://github.com/NeurodataWithoutBorders/pynwb/pull/1754) ### Documentation and tutorial enhancements From 2384a69a52c789e864c9dce785d05e536f0b6081 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 4 Oct 2023 12:05:07 -0700 Subject: [PATCH 13/15] Update docs/gallery/advanced_io/linking_data.py Co-authored-by: Ben Dichter --- docs/gallery/advanced_io/linking_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gallery/advanced_io/linking_data.py b/docs/gallery/advanced_io/linking_data.py index 168708888..4f85786cf 100644 --- a/docs/gallery/advanced_io/linking_data.py +++ b/docs/gallery/advanced_io/linking_data.py @@ -27,7 +27,7 @@ .. tip:: In the case of :py:meth:`~pynwb.base.TimeSeries`, the uncorrected timestamps generated by the acquisition - system can be stored (or linked) in the *sync* group. In the NWB:N format, hardware-recorded time data + system can be stored (or linked) in the *sync* group. In the NWB format, hardware-recorded time data must then be corrected to a common time base (e.g., timestamps from all hardware sources aligned) before it can be included in the *timestamps* of the *TimeSeries*. This means, in the case of :py:meth:`~pynwb.base.TimeSeries` we need to be careful that we are not including data with incompatible From a0daffa9a6822698d79d6eefa254223733b844ca Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 4 Oct 2023 12:05:13 -0700 Subject: [PATCH 14/15] Update docs/gallery/advanced_io/linking_data.py Co-authored-by: Ben Dichter --- docs/gallery/advanced_io/linking_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gallery/advanced_io/linking_data.py b/docs/gallery/advanced_io/linking_data.py index 4f85786cf..82824f6cd 100644 --- a/docs/gallery/advanced_io/linking_data.py +++ b/docs/gallery/advanced_io/linking_data.py @@ -13,7 +13,7 @@ HDF5 files with NWB data files via external links. To make things more concrete, let's look at the following use case. We want to simultaneously record multiple data streams during data acquisition. Using the concept of external links allows us to save each data stream to an external HDF5 files during data acquisition and to -afterwards link the data into a single NWB:N file. In this case, each recording becomes represented by a +afterwards link the data into a single NWB file. In this case, each recording becomes represented by a separate file-system object that can be set as read-only once the experiment is done. In the following we are using :py:meth:`~pynwb.base.TimeSeries` as an example, but the same approach works for other NWBContainers as well. From 930eef69da457f49efb55ccdf5379852f72bf608 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 4 Oct 2023 12:23:06 -0700 Subject: [PATCH 15/15] Update test_file.py --- tests/unit/test_file.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_file.py b/tests/unit/test_file.py index 15ac4b0a4..756009ff3 100644 --- a/tests/unit/test_file.py +++ b/tests/unit/test_file.py @@ -527,6 +527,7 @@ def test_subject_age_duration(self): class TestCacheSpec(TestCase): + """Test whether the file can be written and read when caching the spec.""" def setUp(self): self.path = 'unittest_cached_spec.nwb' @@ -535,7 +536,7 @@ def tearDown(self): remove_test_file(self.path) def test_simple(self): - nwbfile = NWBFile(' ', ' ', + nwbfile = NWBFile('sess_desc', 'identifier', datetime.now(tzlocal()), file_create_date=datetime.now(tzlocal()), institution='University of California, San Francisco', @@ -544,9 +545,11 @@ def test_simple(self): io.write(nwbfile) with NWBHDF5IO(self.path, 'r') as reader: nwbfile = reader.read() + assert nwbfile.session_description == "sess_desc" class TestNoCacheSpec(TestCase): + """Test whether the file can be written and read when not caching the spec.""" def setUp(self): self.path = 'unittest_cached_spec.nwb' @@ -555,7 +558,7 @@ def tearDown(self): remove_test_file(self.path) def test_simple(self): - nwbfile = NWBFile(' ', ' ', + nwbfile = NWBFile('sess_desc', 'identifier', datetime.now(tzlocal()), file_create_date=datetime.now(tzlocal()), institution='University of California, San Francisco', @@ -565,6 +568,7 @@ def test_simple(self): with NWBHDF5IO(self.path, 'r') as reader: nwbfile = reader.read() + assert nwbfile.session_description == "sess_desc" class TestTimestampsRefDefault(TestCase):