From a70bb403f9c4cc0e7667721a979a709a4e1bf23e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 5 Nov 2024 16:00:16 -0600 Subject: [PATCH 1/5] add always_write_timestamps to imaging --- pyproject.toml | 26 +++++++------- .../ophys/baseimagingextractorinterface.py | 2 ++ .../tools/roiextractors/roiextractors.py | 34 +++++++++++++++---- .../tools/testing/mock_interfaces.py | 2 ++ tests/test_ophys/test_ophys_interfaces.py | 14 ++++++-- 5 files changed, 55 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a83380467..53f85c2d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -270,50 +270,50 @@ icephys = [ ## Ophys brukertiff = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", "tifffile>=2023.3.21", ] caiman = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] cnmfe = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] extract = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] hdf5 = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] micromanagertiff = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", "tifffile>=2023.3.21", ] miniscope = [ "natsort>=8.3.1", "ndx-miniscope>=0.5.1", - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] sbx = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] scanimage = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", "scanimage-tiff-reader>=1.4.1", ] sima = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] suite2p = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", ] tdt_fp = [ "ndx-fiber-photometry", - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", "tdt", ] tiff = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", "tiffile>=2018.10.18", ] ophys = [ diff --git a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py index 5125af3cc..2489af80a 100644 --- a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py @@ -146,6 +146,7 @@ def add_to_nwbfile( parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", stub_test: bool = False, stub_frames: int = 100, + always_write_timestamps: bool = False, ): from ...tools.roiextractors import add_imaging_to_nwbfile @@ -162,4 +163,5 @@ def add_to_nwbfile( photon_series_type=photon_series_type, photon_series_index=photon_series_index, parent_container=parent_container, + always_write_timestamps=always_write_timestamps, ) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index f28631c77..27d3b5f9c 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -445,6 +445,7 @@ def add_photon_series_to_nwbfile( parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, + always_write_timestamps: bool = False, ) -> NWBFile: """ Auxiliary static method for nwbextractor. @@ -472,6 +473,11 @@ def add_photon_series_to_nwbfile( iterator_type: str, default: 'v2' The type of iterator to use when adding the photon series to the NWB file. iterator_options: dict, optional + always_write_timestamps : bool, default: False + Set to True to always write timestamps. + By default (False), the function checks if the timestamps are uniformly sampled, and if so, stores the data + using a regular sampling rate instead of explicit timestamps. If set to True, timestamps will be written + explicitly, regardless of whether the sampling rate is uniform. Returns ------- @@ -530,16 +536,23 @@ def add_photon_series_to_nwbfile( photon_series_kwargs.update(dimension=imaging.get_image_size()) # Add timestamps or rate - if imaging.has_time_vector(): + if always_write_timestamps: timestamps = imaging.frame_to_time(np.arange(imaging.get_num_frames())) - estimated_rate = calculate_regular_series_rate(series=timestamps) + photon_series_kwargs.update(timestamps=timestamps) + else: + imaging_has_timestamps = imaging.has_time_vector() + if imaging_has_timestamps: + timestamps = imaging.frame_to_time(np.arange(imaging.get_num_frames())) + estimated_rate = calculate_regular_series_rate(series=timestamps) + starting_time = timestamps[0] + else: + estimated_rate = float(imaging.get_sampling_frequency()) + starting_time = 0.0 + if estimated_rate: - photon_series_kwargs.update(starting_time=timestamps[0], rate=estimated_rate) + photon_series_kwargs.update(rate=estimated_rate, starting_time=starting_time) else: - photon_series_kwargs.update(timestamps=timestamps, rate=None) - else: - rate = float(imaging.get_sampling_frequency()) - photon_series_kwargs.update(rate=rate) + photon_series_kwargs.update(timestamps=timestamps) # Add the photon series to the nwbfile (either as OnePhotonSeries or TwoPhotonSeries) photon_series = dict( @@ -682,6 +695,7 @@ def add_imaging_to_nwbfile( iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", + always_write_timestamps: bool = False, ) -> NWBFile: """ Add imaging data from an ImagingExtractor object to an NWBFile. @@ -705,6 +719,11 @@ def add_imaging_to_nwbfile( parent_container : {"acquisition", "processing/ophys"}, optional Specifies the parent container to which the photon series should be added, either as part of "acquisition" or under the "processing/ophys" module, by default "acquisition". + always_write_timestamps : bool, default: False + Set to True to always write timestamps. + By default (False), the function checks if the timestamps are uniformly sampled, and if so, stores the data + using a regular sampling rate instead of explicit timestamps. If set to True, timestamps will be written + explicitly, regardless of whether the sampling rate is uniform. Returns ------- @@ -722,6 +741,7 @@ def add_imaging_to_nwbfile( iterator_type=iterator_type, iterator_options=iterator_options, parent_container=parent_container, + always_write_timestamps=always_write_timestamps, ) return nwbfile diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 04dc57250..47332f265 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -260,6 +260,7 @@ def __init__( sampling_frequency=sampling_frequency, dtype=dtype, verbose=verbose, + seed=seed, ) self.verbose = verbose @@ -334,6 +335,7 @@ def __init__( has_deconvolved_signal=has_deconvolved_signal, has_neuropil_signal=has_neuropil_signal, verbose=verbose, + seed=seed, ) def get_metadata(self) -> dict: diff --git a/tests/test_ophys/test_ophys_interfaces.py b/tests/test_ophys/test_ophys_interfaces.py index 4381faf8b..e7e29edff 100644 --- a/tests/test_ophys/test_ophys_interfaces.py +++ b/tests/test_ophys/test_ophys_interfaces.py @@ -1,3 +1,5 @@ +import numpy as np + from neuroconv.tools.testing.data_interface_mixins import ( ImagingExtractorInterfaceTestMixin, SegmentationExtractorInterfaceTestMixin, @@ -12,9 +14,15 @@ class TestMockImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = MockImagingInterface interface_kwargs = dict() - # TODO: fix this by setting a seed on the dummy imaging extractor - def test_all_conversion_checks(self): - pass + def test_always_write_timestamps(self, setup_interface): + # By default the MockImagingInterface has a uniform sampling rate + + nwbfile = self.interface.create_nwbfile(always_write_timestamps=True) + two_photon_series = nwbfile.acquisition["TwoPhotonSeries"] + imaging = self.interface.imaging_extractor + expected_timestamps = imaging.frame_to_time(np.arange(imaging.get_num_frames())) + + np.testing.assert_array_equal(two_photon_series.timestamps[:], expected_timestamps) class TestMockSegmentationInterface(SegmentationExtractorInterfaceTestMixin): From 9d8131aec479292d7a573f6483b06c165d347069 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 5 Nov 2024 16:14:20 -0600 Subject: [PATCH 2/5] changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa679434a..19735bc31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bug Fixes ## Features +* Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) ## Improvements @@ -45,7 +46,7 @@ * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) * Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) * Added a `MockSortingInterface` for testing purposes. [PR #1065](https://github.com/catalystneuro/neuroconv/pull/1065) -* BaseRecordingInterfaces have a new conversion options `always_write_timestamps` that ca be used to force writing timestamps even if neuroconv heuristic indicates regular sampling rate [PR #1091](https://github.com/catalystneuro/neuroconv/pull/1091) +* BaseRecordingInterfaces have a new conversion options `always_write_timestamps` that can be used to force writing timestamps even if neuroconv heuristic indicates regular sampling rate [PR #1091](https://github.com/catalystneuro/neuroconv/pull/1091) ## Improvements From cd411c58ad7462ac13d0e1b62892a74fcc8cb344 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 5 Nov 2024 16:16:08 -0600 Subject: [PATCH 3/5] cancel new test meanwhile --- tests/test_ophys/test_ophys_interfaces.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_ophys/test_ophys_interfaces.py b/tests/test_ophys/test_ophys_interfaces.py index e7e29edff..3ea329e5f 100644 --- a/tests/test_ophys/test_ophys_interfaces.py +++ b/tests/test_ophys/test_ophys_interfaces.py @@ -24,6 +24,10 @@ def test_always_write_timestamps(self, setup_interface): np.testing.assert_array_equal(two_photon_series.timestamps[:], expected_timestamps) + # Remove this after roiextractors 0.5.10 is released + def test_all_conversion_checks(self): + pass + class TestMockSegmentationInterface(SegmentationExtractorInterfaceTestMixin): From 62ac15f43970899a84c1e292ee682d4a01f0effa Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 6 Nov 2024 13:14:47 -0600 Subject: [PATCH 4/5] update roiextractor dependency to just released and drop compression options --- pyproject.toml | 10 +++++----- .../behavior/video/videodatainterface.py | 13 ------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 53f85c2d7..2022e6597 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -270,20 +270,20 @@ icephys = [ ## Ophys brukertiff = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", "tifffile>=2023.3.21", ] caiman = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] cnmfe = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] extract = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] hdf5 = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] micromanagertiff = [ "roiextractors>=0.5.9", diff --git a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py index a544f9c27..aaa875f3e 100644 --- a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py @@ -269,8 +269,6 @@ def add_to_nwbfile( chunk_data: bool = True, module_name: Optional[str] = None, module_description: Optional[str] = None, - compression: Optional[str] = "gzip", - compression_options: Optional[int] = None, ): """ Convert the video data files to :py:class:`~pynwb.image.ImageSeries` and write them in the @@ -431,17 +429,6 @@ def add_to_nwbfile( pbar.update(1) iterable = video - # TODO: remove completely after 03/1/2024 - if compression is not None or compression_options is not None: - warnings.warn( - message=( - "Specifying compression methods and their options for this interface has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) - image_series_kwargs.update(data=iterable) if timing_type == "starting_time and rate": From cb57c061247ad24ca908a0a818c72af34433f622 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 6 Nov 2024 14:14:03 -0600 Subject: [PATCH 5/5] pyproject toml full update --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2022e6597..5efd432f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -286,30 +286,30 @@ hdf5 = [ "roiextractors>=0.5.10", ] micromanagertiff = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", "tifffile>=2023.3.21", ] miniscope = [ "natsort>=8.3.1", "ndx-miniscope>=0.5.1", - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] sbx = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] scanimage = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", "scanimage-tiff-reader>=1.4.1", ] sima = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] suite2p = [ - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", ] tdt_fp = [ "ndx-fiber-photometry", - "roiextractors>=0.5.9", + "roiextractors>=0.5.10", "tdt", ] tiff = [