Merge branch 'dev' into upgrade-validator

CHANGELOG.md

@@ -1,12 +1,15 @@
 # PyNWB Changelog
 
-## PyNWB 2.8.3 (Upcoming)
+## PyNWB 2.8.3 (November 19, 2024)
 
 ### Enhancements and minor changes
 * Added `NWBHDF5IO.read_nwb` convenience method to simplify reading an NWB file. @h-mayorquin [#1979](https://github.com/NeurodataWithoutBorders/pynwb/pull/1979)
+* Removed unused references to region references and builders in preparation for changes in HDMF 4.0. @rly [#1991](https://github.com/NeurodataWithoutBorders/pynwb/pull/1991)
 
 ### Documentation and tutorial enhancements
 - Added documentation example for `SpikeEventSeries`. @stephprince [#1983](https://github.com/NeurodataWithoutBorders/pynwb/pull/1983)
+- Added documentation example for `AnnotationSeries`. @stephprince [#1989](https://github.com/NeurodataWithoutBorders/pynwb/pull/1989)
+- Added documentation example for `DecompositionSeries`. @stephprince [#1981](https://github.com/NeurodataWithoutBorders/pynwb/pull/1981)
 
 ### Performance
 - Cache global type map to speed import 3X. @sneakers-the-rat [#1931](https://github.com/NeurodataWithoutBorders/pynwb/pull/1931)

docs/gallery/domain/ecephys.py

@@ -31,7 +31,9 @@
 from dateutil.tz import tzlocal
 
 from pynwb import NWBHDF5IO, NWBFile
+
 from pynwb.ecephys import LFP, ElectricalSeries, SpikeEventSeries
+from pynwb.misc import DecompositionSeries
 
 #######################
 # Creating and Writing NWB files
@@ -241,6 +243,68 @@
 )
 ecephys_module.add(lfp)
 
+#######################
+# If the derived data is filtered but not downsampled, you can store the data in an
+# :py:class:`~pynwb.ecephys.ElectricalSeries` object in a :py:class:`~pynwb.ecephys.FilteredEphys` object
+# instead of a :py:class:`~pynwb.ecephys.LFP` object.
+
+from pynwb.ecephys import FilteredEphys
+
+filtered_data = np.random.randn(50, 12)
+filtered_electrical_series = ElectricalSeries(
+    name="FilteredElectricalSeries",
+    description="Filtered data",
+    data=filtered_data,
+    electrodes=all_table_region,
+    starting_time=0.0,
+    rate=200.0,
+)
+
+filtered_ephys = FilteredEphys(electrical_series=filtered_electrical_series)
+ecephys_module.add(filtered_ephys)
+
+################################
+# In some cases, you may want to further process the LFP data and decompose the signal into different frequency bands 
+# to use for other downstream analyses. You can store the processed data from these spectral analyses using a
+# :py:class:`~pynwb.misc.DecompositionSeries` object. This object allows you to include metadata about the frequency
+# bands and metric used (e.g., power, phase, amplitude), as well as link the decomposed data to the original 
+# :py:class:`~pynwb.base.TimeSeries` signal the data was derived from.
+
+#######################
+# .. note:: When adding data to :py:class:`~pynwb.misc.DecompositionSeries`, the ``data`` argument is assumed to be 
+#           3D where the first dimension is time, the second dimension is channels, and the third dimension is bands.
+
+
+bands = dict(theta=(4.0, 12.0), 
+             beta=(12.0, 30.0), 
+             gamma=(30.0, 80.0))  # in Hz
+phase_data = np.random.randn(50, 12, len(bands))  # 50 samples, 12 channels, 3 frequency bands  
+
+decomp_series = DecompositionSeries(
+    name="theta",
+    description="phase of bandpass filtered LFP data",
+    data=phase_data,
+    metric='phase',
+    rate=200.0,
+    source_channels=all_table_region,
+    source_timeseries=lfp_electrical_series,
+)
+
+for band_name, band_limits in bands.items():
+    decomp_series.add_band(
+        band_name=band_name,
+        band_limits=band_limits,
+        band_mean=np.nan,
+        band_stdev=np.nan,
+    )
+
+ecephys_module.add(decomp_series)
+
+#######################
+# The frequency band information can also be viewed as a pandas DataFrame.
+
+decomp_series.bands.to_dataframe()
+
 ####################
 # .. _units_electrode:
 #
@@ -269,6 +333,10 @@
 
 #######################
 # The :py:class:`~pynwb.misc.Units` table can also be converted to a pandas :py:class:`~pandas.DataFrame`.
+#
+# The :py:class:`~pynwb.misc.Units` table can contain simply the spike times of sorted units, or you can also include
+# individual and mean waveform information in some of the optional, predefined :py:class:`~pynwb.misc.Units` table
+# columns: ``waveform_mean``, ``waveform_sd``, or ``waveforms``.
 
 nwbfile.units.to_dataframe()
 
@@ -287,44 +355,50 @@
     description="shank0",
 )
 
-
-spike_events = SpikeEventSeries(name='SpikeEvents_Shank0',
-                                description="events detected with 100uV threshold",
-                                data=spike_snippets,
-                                timestamps=np.arange(20),
-                                electrodes=shank0)
+spike_events = SpikeEventSeries(
+    name='SpikeEvents_Shank0',
+    description="events detected with 100uV threshold",
+    data=spike_snippets,
+    timestamps=np.arange(20),
+    electrodes=shank0,
+)
 nwbfile.add_acquisition(spike_events)
 
-#######################
-# Designating electrophysiology data
-# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-#
-# As mentioned above, :py:class:`~pynwb.ecephys.ElectricalSeries` objects
-# are meant for storing specific types of extracellular recordings. In addition to this
-# :py:class:`~pynwb.base.TimeSeries` class, NWB provides some :ref:`modules_overview`
-# for designating the type of data you are storing. We will briefly discuss them here, and refer the reader to
-# :py:mod:`API documentation <pynwb.ecephys>` and :ref:`basics` for more details on
-# using these objects.
-#
-# For storing unsorted spiking data, there are two options. Which one you choose depends on what data you 
-# have available. If you need to store the complete, continuous raw voltage traces, you should store the traces with
-# :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition <basic_timeseries>` data, and use
-# the :py:class:`~pynwb.ecephys.EventDetection` class for identifying the spike events in your raw traces.
+############################################
+# If you need to store the complete, continuous raw voltage traces, along with unsorted spike times, you should store
+# the traces with :py:class:`~pynwb.ecephys.ElectricalSeries` objects as :ref:`acquisition <basic_timeseries>` data,
+# and use the :py:class:`~pynwb.ecephys.EventDetection` class to identify the spike events in your raw traces.
+
+from pynwb.ecephys import EventDetection
+
+event_detection = EventDetection(
+    name="threshold_events",
+    detection_method="thresholding, 1.5 * std",
+    source_electricalseries=raw_electrical_series,
+    source_idx=[1000, 2000, 3000],
+    times=[.033, .066, .099],
+)
+
+ecephys_module.add(event_detection)
+
+######################################
 # If you do not want to store the raw voltage traces and only the waveform 'snippets' surrounding spike events,
 # you should use :py:class:`~pynwb.ecephys.SpikeEventSeries` objects.
 #
-# The results of spike sorting (or clustering) should be stored in the top-level :py:class:`~pynwb.misc.Units` table.
-# The :py:class:`~pynwb.misc.Units` table can contain simply the spike times of sorted units, or you can also include 
-# individual and mean waveform information in some of the optional, predefined :py:class:`~pynwb.misc.Units` table 
-# columns: ``waveform_mean``, ``waveform_sd``, or ``waveforms``.
-#
-# For local field potential data, there are two options. Again, which one you choose depends on what data you
-# have available. With both options, you should store your traces with :py:class:`~pynwb.ecephys.ElectricalSeries`
-# objects. If you are storing unfiltered local field potential data, you should store
-# the :py:class:`~pynwb.ecephys.ElectricalSeries` objects in :py:class:`~pynwb.ecephys.LFP` data interface object(s).
-# If you have filtered LFP data, you should store the :py:class:`~pynwb.ecephys.ElectricalSeries` objects  in
-# :py:class:`~pynwb.ecephys.FilteredEphys` data interface object(s).
+# NWB also provides a way to store features of spikes, such as principal components, using the
+# :py:class:`~pynwb.ecephys.FeatureExtraction` class.
+
+from pynwb.ecephys import FeatureExtraction
+
+feature_extraction = FeatureExtraction(
+    name="PCA_features",
+    electrodes=all_table_region,
+    description=["PC1", "PC2", "PC3", "PC4"],
+    times=[.033, .066, .099],
+    features=np.random.rand(3, 12, 4),  # time, channel, feature
+)
 
+ecephys_module.add(feature_extraction)
 
 ####################
 # .. _ecephys_writing:

docs/gallery/general/plot_file.py

@@ -131,6 +131,7 @@
 from pynwb import NWBHDF5IO, NWBFile, TimeSeries
 from pynwb.behavior import Position, SpatialSeries
 from pynwb.file import Subject
+from pynwb.misc import AnnotationSeries
 
 ####################
 # .. _basics_nwbfile:
@@ -285,6 +286,29 @@
 # or using the method :py:meth:`.NWBFile.get_acquisition`:
 nwbfile.get_acquisition("test_timeseries")
 
+####################
+# Other Types of Time Series
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^
+#
+# As mentioned previously, there are many subtypes of :py:class:`~pynwb.base.TimeSeries` that are used to store
+# different kinds of data. One example is :py:class:`~pynwb.misc.AnnotationSeries`, a subclass of 
+# :py:class:`~pynwb.base.TimeSeries` that stores text-based records about the experiment. Similarly to our
+# :py:class:`~pynwb.base.TimeSeries` example above, we can create an :py:class:`~pynwb.misc.AnnotationSeries` 
+# object with text information about a stimulus and add it to the stimulus group in 
+# the :py:class:`~pynwb.file.NWBFile`. 
+
+annotations = AnnotationSeries(
+    name='airpuffs',
+    data=['Left Airpuff', 'Right Airpuff', 'Right Airpuff'],
+    description='Airpuff events delivered to the animal',
+    timestamps=[1.0, 3.0, 8.0],
+)
+
+nwbfile.add_stimulus(annotations)
+
+####################
+# This approach of creating a :py:class:`~pynwb.base.TimeSeries` object and adding it to the appropriate 
+# :py:class:`~pynwb.file.NWBFile` group can be used for all subtypes of :py:class:`~pynwb.base.TimeSeries` data.
 
 ####################
 # .. _basic_spatialseries:

docs/source/conf.py

@@ -168,6 +168,10 @@ def __call__(self, filename):
 nitpick_ignore = [('py:class', 'Intracomm'),
                   ('py:class', 'BaseStorageSpec')]
 
+linkcheck_ignore = [
+    r'https://training.incf.org/*'  # temporary ignore until SSL certificate issue is resolved
+]
+
 suppress_warnings = ["config.cache"]
 
 # Add any paths that contain templates here, relative to this directory.

docs/source/overview_software_architecture.rst

@@ -69,7 +69,7 @@ Builder
    * :py:class:`~hdmf.build.builders.GroupBuilder` - represents a collection of objects
    * :py:class:`~hdmf.build.builders.DatasetBuilder` - represents data
    * :py:class:`~hdmf.build.builders.LinkBuilder` - represents soft-links
-   * :py:class:`~hdmf.build.builders.RegionBuilder` - represents a slice into data (Subclass of :py:class:`~hdmf.build.builders.DatasetBuilder`)
+   * :py:class:`~hdmf.build.builders.ReferenceBuilder` - represents a reference to another group or dataset
 
 * **Main Module:** :py:mod:`hdmf.build.builders`
 

requirements-opt.txt

@@ -5,4 +5,4 @@ oaklib==0.5.32; python_version >= "3.9"
 # for streaming tests
 fsspec==2024.10.0
 requests==2.32.3
-aiohttp==3.10.10
+aiohttp==3.10.11

src/pynwb/io/core.py

@@ -1,4 +1,4 @@
-from hdmf.build import ObjectMapper, RegionBuilder
+from hdmf.build import ObjectMapper
 from hdmf.common import VectorData
 from hdmf.utils import getargs, docval
 from hdmf.spec import AttributeSpec
@@ -40,19 +40,6 @@ def __init__(self, spec):
         self.map_spec('description', spec.get_attribute('notes'))
 
 
-class NWBTableRegionMap(NWBDataMap):
-
-    @ObjectMapper.constructor_arg('table')
-    def carg_table(self, builder, manager):
-        return manager.construct(builder.data.builder)
-
-    @ObjectMapper.constructor_arg('region')
-    def carg_region(self, builder, manager):
-        if not isinstance(builder.data, RegionBuilder):
-            raise ValueError("'builder' must be a RegionBuilder")
-        return builder.data.region
-
-
 @register_map(VectorData)
 class VectorDataMap(ObjectMapper):