[Pydantic II] Change all paths to pydantic types

.github/workflows/testing.yml

@@ -34,6 +34,7 @@ jobs:
           python -m pip install -U pip  # Official recommended way
           git config --global user.email "[email protected]"
           git config --global user.name "CI Almighty"
+
       - name: Install NeuroConv with minimal requirements
         run: pip install .
       - name: Test initial import of all non-lazy dependencies
@@ -42,13 +43,15 @@ jobs:
         uses: ./.github/actions/install-wine
         with:
           os: ${{ runner.os }}
+
       - name: Install NeuroConv with testing requirements
         run: pip install .[test]
       - name: Run import tests
         run: |
           pytest tests/imports.py::TestImportStructure::test_top_level
           pytest tests/imports.py::TestImportStructure::test_tools
           pytest tests/imports.py::TestImportStructure::test_datainterfaces
+
       - name: Run minimal tests
         run: pytest tests/test_minimal -vv -rsx -n auto --dist loadscope
 

CHANGELOG.md

@@ -2,7 +2,9 @@
 
 ### Deprecations
 * Deprecated  `WaveformExtractor` usage. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821)
-* Deprecated use of `compression` and `compression_options` in `VideoInterface` [PR #1005](https://github.com/catalystneuro/neuroconv/pull/10)
+* Deprecated use of `compression` and `compression_options` in `VideoInterface` [PR #1005](https://github.com/catalystneuro/neuroconv/pull/1005)
+* `get_schema_from_method_signature` has been deprecated; please use `get_json_schema_from_method_signature` instead. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016)
+* `neuroconv.utils.FilePathType` and `neuroconv.utils.FolderPathType` have been deprecated; please use `pydantic.FilePath` and `pydantic.DirectoryPath` instead. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017)
 
 ### Features
 * Added MedPCInterface for operant behavioral output files. [PR #883](https://github.com/catalystneuro/neuroconv/pull/883)
@@ -14,20 +16,21 @@
 * Data interfaces `run_conversion` method now performs metadata validation before running the conversion. [PR #949](https://github.com/catalystneuro/neuroconv/pull/949)
 * Introduced `null_values_for_properties` to `add_units_table` to give user control over null values behavior [PR #989](https://github.com/catalystneuro/neuroconv/pull/989)
 
-
 ### Bug fixes
 * Fixed the default naming of multiple electrical series in the `SpikeGLXConverterPipe`. [PR #957](https://github.com/catalystneuro/neuroconv/pull/957)
 * Write new properties to the electrode table use the global identifier channel_name, group [PR #984](https://github.com/catalystneuro/neuroconv/pull/984)
 * Removed a bug where int64 was casted lossy to float [PR #989](https://github.com/catalystneuro/neuroconv/pull/989)
 
-
 ### Improvements
 * The `OpenEphysBinaryRecordingInterface` now uses `lxml` for extracting the session start time from the settings.xml file and does not depend on `pyopenephys` anymore. [PR #971](https://github.com/catalystneuro/neuroconv/pull/971)
 * Swap the majority of package setup and build steps to `pyproject.toml` instead of `setup.py`. [PR #955](https://github.com/catalystneuro/neuroconv/pull/955)
 * The `DeeplabcutInterface` now skips inferring timestamps from movie when timestamps are specified, running faster. [PR #967](https://github.com/catalystneuro/neuroconv/pull/967)
 * Improve metadata writing for SpikeGLX data interface. Added contact ids, shank ids and, remove references to shanks for neuropixels 1.0. Also deprecated the previous neuroconv exclusive property "electrode_shank_number` [PR #986](https://github.com/catalystneuro/neuroconv/pull/986)
 * Add tqdm with warning to DeepLabCut interface [PR #1006](https://github.com/catalystneuro/neuroconv/pull/1006)
 * `BaseRecordingInterface` now calls default metadata when metadata is not passing mimicking `run_conversion` behavior. [PR #1012](https://github.com/catalystneuro/neuroconv/pull/1012)
+* Added `get_json_schema_from_method_signature` which constructs Pydantic models automatically from the signature of any function with typical annotation types used throughout NeuroConv. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016)
+* Replaced all interface annotations with Pydantic types. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017)
+
 
 
 ## v0.5.0 (July 17, 2024)

src/neuroconv/basedatainterface.py

@@ -6,6 +6,7 @@
 from typing import Literal, Optional, Tuple, Union
 
 from jsonschema.validators import validate
+from pydantic import FilePath
 from pynwb import NWBFile
 
 from .tools.nwb_helpers import (
@@ -19,7 +20,7 @@
 from .tools.nwb_helpers._metadata_and_file_helpers import _resolve_backend
 from .utils import (
     NWBMetaDataEncoder,
-    get_schema_from_method_signature,
+    get_json_schema_from_method_signature,
     load_dict_from_file,
 )
 from .utils.dict import DeepDict
@@ -36,16 +37,12 @@ class BaseDataInterface(ABC):
     @classmethod
     def get_source_schema(cls) -> dict:
         """Infer the JSON schema for the source_data from the method signature (annotation typing)."""
-        return get_schema_from_method_signature(cls, exclude=["source_data"])
+        return get_json_schema_from_method_signature(cls, exclude=["source_data"])
 
     def __init__(self, verbose: bool = False, **source_data):
         self.verbose = verbose
         self.source_data = source_data
 
-    def get_conversion_options_schema(self) -> dict:
-        """Infer the JSON schema for the conversion options from the method signature (annotation typing)."""
-        return get_schema_from_method_signature(self.add_to_nwbfile, exclude=["nwbfile", "metadata"])
-
     def get_metadata_schema(self) -> dict:
         """Retrieve JSON schema for metadata."""
         metadata_schema = load_dict_from_file(Path(__file__).parent / "schemas" / "base_metadata_schema.json")
@@ -79,6 +76,10 @@ def validate_metadata(self, metadata: dict, append_mode: bool = False) -> None:
 
         validate(instance=decoded_metadata, schema=metdata_schema)
 
+    def get_conversion_options_schema(self) -> dict:
+        """Infer the JSON schema for the conversion options from the method signature (annotation typing)."""
+        return get_json_schema_from_method_signature(self.add_to_nwbfile, exclude=["nwbfile", "metadata"])
+
     def create_nwbfile(self, metadata: Optional[dict] = None, **conversion_options) -> NWBFile:
         """
         Create and return an in-memory pynwb.NWBFile object with this interface's data added to it.
@@ -121,7 +122,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, **conversion_options) -> None:
 
     def run_conversion(
         self,
-        nwbfile_path: str,
+        nwbfile_path: FilePath,
         nwbfile: Optional[NWBFile] = None,
         metadata: Optional[dict] = None,
         overwrite: bool = False,

src/neuroconv/datainterfaces/behavior/audio/audiointerface.py

@@ -4,6 +4,7 @@
 
 import numpy as np
 import scipy
+from pydantic import FilePath
 from pynwb import NWBFile
 
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
@@ -27,7 +28,7 @@ class AudioInterface(BaseTemporalAlignmentInterface):
     associated_suffixes = (".wav",)
     info = "Interface for writing audio recordings to an NWB file."
 
-    def __init__(self, file_paths: list, verbose: bool = False):
+    def __init__(self, file_paths: List[FilePath], verbose: bool = False):
         """
         Data interface for writing acoustic recordings to an NWB file.
 

src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py

@@ -10,7 +10,7 @@
 from pynwb import NWBFile
 from ruamel.yaml import YAML
 
-from ....utils import FilePathType
+from ....utils import FilePath
 
 
 def _read_config(config_file_path):
@@ -303,9 +303,9 @@ def _write_pes_to_nwbfile(
 
 def add_subject_to_nwbfile(
     nwbfile: NWBFile,
-    h5file: FilePathType,
+    h5file: FilePath,
     individual_name: str,
-    config_file: FilePathType,
+    config_file: FilePath,
     timestamps: Optional[Union[List, np.ndarray]] = None,
     pose_estimation_container_kwargs: Optional[dict] = None,
 ) -> NWBFile:

src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py

@@ -2,10 +2,10 @@
 from typing import List, Optional, Union
 
 import numpy as np
+from pydantic import FilePath
 from pynwb.file import NWBFile
 
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
-from ....utils import FilePathType
 
 
 class DeepLabCutInterface(BaseTemporalAlignmentInterface):
@@ -27,8 +27,8 @@ def get_source_schema(cls) -> dict:
 
     def __init__(
         self,
-        file_path: FilePathType,
-        config_file_path: FilePathType,
+        file_path: FilePath,
+        config_file_path: FilePath,
         subject_name: str = "ind1",
         verbose: bool = True,
     ):
@@ -37,9 +37,9 @@ def __init__(
 
         Parameters
         ----------
-        file_path : FilePathType
+        file_path : FilePath
             path to the h5 file output by dlc.
-        config_file_path : FilePathType
+        config_file_path : FilePath
             path to .yml config file
         subject_name : str, default: "ind1"
             the name of the subject for which the :py:class:`~pynwb.file.NWBFile` is to be created.

src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py

@@ -6,12 +6,13 @@
 from typing import Optional, Union
 
 import numpy as np
+from pydantic import FilePath
 from pynwb.behavior import Position, SpatialSeries
 from pynwb.file import NWBFile
 
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
 from ....tools import get_module
-from ....utils import FilePathType, calculate_regular_series_rate
+from ....utils import calculate_regular_series_rate
 
 
 class FicTracDataInterface(BaseTemporalAlignmentInterface):
@@ -155,22 +156,22 @@ def get_source_schema(cls) -> dict:
 
     def __init__(
         self,
-        file_path: FilePathType,
+        file_path: FilePath,
         radius: Optional[float] = None,
-        configuration_file_path: Optional[FilePathType] = None,
+        configuration_file_path: Optional[FilePath] = None,
         verbose: bool = True,
     ):
         """
         Interface for writing FicTrac files to nwb.
 
         Parameters
         ----------
-        file_path : a string or a path
+        file_path : FilePath
             Path to the .dat file (the output of fictrac)
         radius : float, optional
             The radius of the ball in meters. If provided the radius is stored as a conversion factor
             and the units are set to meters. If not provided the units are set to radians.
-        configuration_file_path : a string or a path, optional
+        configuration_file_path : FilePath, optional
             Path to the .txt file with the configuration metadata. Usually called config.txt
         verbose : bool, default: True
             controls verbosity. ``True`` by default.
@@ -358,8 +359,8 @@ def set_aligned_starting_time(self, aligned_starting_time):
 
 
 def extract_session_start_time(
-    file_path: FilePathType,
-    configuration_file_path: Optional[FilePathType] = None,
+    file_path: FilePath,
+    configuration_file_path: Optional[FilePath] = None,
 ) -> Union[datetime, None]:
     """
     Extract the session start time from a FicTrac data file or its configuration file.
@@ -378,9 +379,9 @@ def extract_session_start_time(
 
     Parameters
     ----------
-    file_path : FilePathType
+    file_path : FilePath
         Path to the FicTrac data file.
-    configuration_file_path : Optional[FilePathType]
+    configuration_file_path : FilePath, optional
         Path to the FicTrac configuration file. If omitted, the function defaults to searching for
         "fictrac_config.txt" in the same directory as the data file.
 
@@ -413,13 +414,13 @@ def extract_session_start_time(
     return None
 
 
-def parse_fictrac_config(file_path: FilePathType) -> dict:
+def parse_fictrac_config(file_path: FilePath) -> dict:
     """
     Parse a FicTrac configuration file and return a dictionary of its parameters.
 
     Parameters
     ----------
-    file_path : str, Path
+    file_path : FilePath
         Path to the configuration file in txt format.
 
     Returns

src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py

@@ -1,14 +1,14 @@
 from copy import deepcopy
 from typing import Optional
 
+from pydantic import FilePath
 from pynwb import NWBFile
 
 from neuroconv import NWBConverter
 from neuroconv.datainterfaces import LightningPoseDataInterface, VideoInterface
 from neuroconv.tools.nwb_helpers import make_or_load_nwbfile
 from neuroconv.utils import (
     DeepDict,
-    FilePathType,
     dict_deep_update,
     get_schema_from_method_signature,
 )
@@ -28,9 +28,9 @@ def get_source_schema(cls):
 
     def __init__(
         self,
-        file_path: FilePathType,
-        original_video_file_path: FilePathType,
-        labeled_video_file_path: Optional[FilePathType] = None,
+        file_path: FilePath,
+        original_video_file_path: FilePath,
+        labeled_video_file_path: Optional[FilePath] = None,
         image_series_original_video_name: Optional[str] = None,
         image_series_labeled_video_name: Optional[str] = None,
         verbose: bool = True,
@@ -41,11 +41,11 @@ def __init__(
 
         Parameters
         ----------
-        file_path : a string or a path
+        file_path : FilePath
             Path to the .csv file that contains the predictions from Lightning Pose.
-        original_video_file_path : a string or a path
+        original_video_file_path : FilePath
             Path to the original video file (.mp4).
-        labeled_video_file_path : a string or a path, optional
+        labeled_video_file_path : FilePath, optional
             Path to the labeled video file (.mp4).
         image_series_original_video_name: string, optional
             The name of the ImageSeries to add for the original video.
@@ -169,7 +169,7 @@ def add_to_nwbfile(
 
     def run_conversion(
         self,
-        nwbfile_path: Optional[str] = None,
+        nwbfile_path: Optional[FilePath] = None,
         nwbfile: Optional[NWBFile] = None,
         metadata: Optional[dict] = None,
         overwrite: bool = False,

src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py

@@ -5,13 +5,13 @@
 from typing import Optional, Tuple
 
 import numpy as np
+from pydantic import FilePath
 from pynwb import NWBFile
 
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
 from ....tools import get_module
 from ....utils import (
     DeepDict,
-    FilePathType,
     calculate_regular_series_rate,
     get_base_schema,
 )
@@ -60,19 +60,19 @@ def get_metadata_schema(self) -> dict:
 
     def __init__(
         self,
-        file_path: FilePathType,
-        original_video_file_path: FilePathType,
-        labeled_video_file_path: Optional[FilePathType] = None,
+        file_path: FilePath,
+        original_video_file_path: FilePath,
+        labeled_video_file_path: Optional[FilePath] = None,
         verbose: bool = True,
     ):
         """
         Interface for writing pose estimation data from the Lightning Pose algorithm.
 
         Parameters
         ----------
-        file_path : a string or a path
+        file_path : FilePath
             Path to the .csv file that contains the predictions from Lightning Pose.
-        original_video_file_path : a string or a path
+        original_video_file_path : FilePath
             Path to the original video file (.mp4).
         labeled_video_file_path : a string or a path, optional
             Path to the labeled video file (.mp4).

src/neuroconv/datainterfaces/behavior/medpc/medpc_helpers.py

@@ -1,9 +1,9 @@
 import numpy as np
 
-from neuroconv.utils import FilePathType
+from neuroconv.utils import FilePath
 
 
-def get_medpc_variables(file_path: FilePathType, variable_names: list) -> dict:
+def get_medpc_variables(file_path: FilePath, variable_names: list) -> dict:
     """
     Get the values of the given single-line variables from a MedPC file for all sessions in that file.
 
@@ -85,7 +85,7 @@ def _get_session_lines(lines: list, session_conditions: dict, start_variable: st
 
 
 def read_medpc_file(
-    file_path: FilePathType,
+    file_path: FilePath,
     medpc_name_to_info_dict: dict,
     session_conditions: dict,
     start_variable: str,