From beb48d97185c28af2fe1b35285b827af00b03b2b Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:17:24 -0400 Subject: [PATCH 001/118] [Pydantic II] Change all interfaces to pydantic types (#1017) Authored-by: CodyCBakerPhD --- CHANGELOG.md | 2 ++ src/neuroconv/basedatainterface.py | 11 ++++---- .../behavior/audio/audiointerface.py | 3 ++- .../behavior/deeplabcut/_dlc_utils.py | 7 +++-- .../deeplabcut/deeplabcutdatainterface.py | 10 +++---- .../behavior/fictrac/fictracdatainterface.py | 23 ++++++++-------- .../lightningpose/lightningposeconverter.py | 16 ++++++------ .../lightningposedatainterface.py | 12 ++++----- .../behavior/medpc/medpc_helpers.py | 7 +++-- .../behavior/medpc/medpcdatainterface.py | 7 ++--- .../miniscope/miniscopedatainterface.py | 7 ++--- .../neuralynx/neuralynx_nvt_interface.py | 7 ++--- .../behavior/neuralynx/nvt_utils.py | 6 ++--- .../behavior/sleap/sleap_utils.py | 5 ++-- .../behavior/sleap/sleapdatainterface.py | 8 +++--- .../behavior/video/video_utils.py | 10 +++---- .../behavior/video/videodatainterface.py | 5 ++-- .../ecephys/axona/axona_utils.py | 23 ++++++++-------- .../ecephys/axona/axonadatainterface.py | 11 ++++---- .../ecephys/biocam/biocamdatainterface.py | 5 ++-- .../blackrock/blackrockdatainterface.py | 10 ++++--- .../cellexplorer/cellexplorerdatainterface.py | 14 +++++----- .../ecephys/edf/edfdatainterface.py | 5 ++-- .../ecephys/intan/intandatainterface.py | 5 ++-- .../ecephys/kilosort/kilosortdatainterface.py | 5 ++-- .../ecephys/maxwell/maxonedatainterface.py | 9 ++++--- .../ecephys/mcsraw/mcsrawdatainterface.py | 5 ++-- .../ecephys/mearec/mearecdatainterface.py | 5 ++-- .../neuroscope/neuroscopedatainterface.py | 14 +++++----- .../ecephys/openephys/_openephys_utils.py | 5 ++-- .../openephys/openephysbinarydatainterface.py | 8 +++--- .../openephys/openephysdatainterface.py | 5 ++-- .../openephys/openephyslegacydatainterface.py | 7 ++--- .../openephyssortingdatainterface.py | 6 +++-- .../ecephys/phy/phydatainterface.py | 5 ++-- .../ecephys/plexon/plexondatainterface.py | 9 ++++--- .../ecephys/spike2/spike2datainterface.py | 8 +++--- .../spikegadgets/spikegadgetsdatainterface.py | 8 +++--- .../ecephys/spikeglx/spikeglx_utils.py | 4 +-- .../ecephys/spikeglx/spikeglxconverter.py | 8 +++--- .../ecephys/spikeglx/spikeglxdatainterface.py | 5 ++-- .../ecephys/spikeglx/spikeglxnidqinterface.py | 5 ++-- .../ecephys/tdt/tdtdatainterface.py | 5 ++-- .../icephys/abf/abfdatainterface.py | 10 ++++--- .../ophys/brukertiff/brukertiffconverter.py | 15 ++++++----- .../brukertiff/brukertiffdatainterface.py | 14 +++++----- .../ophys/caiman/caimandatainterface.py | 7 ++--- .../ophys/cnmfe/cnmfedatainterface.py | 5 ++-- .../ophys/extract/extractdatainterface.py | 5 ++-- .../ophys/hdf5/hdf5datainterface.py | 8 +++--- .../micromanagertiffdatainterface.py | 4 +-- .../ophys/miniscope/miniscopeconverter.py | 5 ++-- .../miniscopeimagingdatainterface.py | 7 ++--- .../ophys/sbx/sbxdatainterface.py | 5 ++-- .../scanimage/scanimageimaginginterfaces.py | 26 +++++++++---------- .../ophys/sima/simadatainterface.py | 7 ++--- .../ophys/suite2p/suite2pdatainterface.py | 11 ++++---- .../tdt_fp/tdtfiberphotometrydatainterface.py | 7 ++--- .../ophys/tiff/tiffdatainterface.py | 5 ++-- .../text/csv/csvtimeintervalsinterface.py | 4 +-- .../text/excel/exceltimeintervalsinterface.py | 6 ++--- .../text/timeintervalsinterface.py | 6 ++--- src/neuroconv/nwbconverter.py | 3 ++- src/neuroconv/tools/data_transfers/_dandi.py | 7 +++-- src/neuroconv/tools/neo/neo.py | 4 +-- .../tools/roiextractors/roiextractors.py | 6 ++--- .../tools/spikeinterface/spikeinterface.py | 22 ++++++++-------- .../tools/testing/mock_ttl_signals.py | 5 ++-- .../_yaml_conversion_specification.py | 9 ++++--- src/neuroconv/utils/__init__.py | 5 ---- src/neuroconv/utils/dict.py | 5 ++-- src/neuroconv/utils/types.py | 7 +---- tests/test_behavior/test_audio_interface.py | 4 +-- 73 files changed, 308 insertions(+), 271 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6952f7e71..4e0f198d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * Changed the spikeinterface.tool functions (e.g. `add_recording`, `add_sorting`) to have `_to_nwbfile` as suffix [PR #1015](https://github.com/catalystneuro/neuroconv/pull/1015) * Deprecated use of `compression` and `compression_options` in `VideoInterface` [PR #1005](https://github.com/catalystneuro/neuroconv/pull/1005) * `get_schema_from_method_signature` has been deprecated; please use `get_json_schema_from_method_signature` instead. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016) +* `neuroconv.utils.FilePathType` and `neuroconv.utils.FolderPathType` have been deprecated; please use `pydantic.FilePath` and `pydantic.DirectoryPath` instead. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017) ### Features * Added MedPCInterface for operant behavioral output files. [PR #883](https://github.com/catalystneuro/neuroconv/pull/883) @@ -30,6 +31,7 @@ * Add tqdm with warning to DeepLabCut interface [PR #1006](https://github.com/catalystneuro/neuroconv/pull/1006) * `BaseRecordingInterface` now calls default metadata when metadata is not passing mimicking `run_conversion` behavior. [PR #1012](https://github.com/catalystneuro/neuroconv/pull/1012) * Added `get_json_schema_from_method_signature` which constructs Pydantic models automatically from the signature of any function with typical annotation types used throughout NeuroConv. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016) +* Replaced all interface annotations with Pydantic types. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017) diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 9de06e827..0c9b9d813 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -6,6 +6,7 @@ from typing import Literal, Optional, Tuple, Union from jsonschema.validators import validate +from pydantic import FilePath from pynwb import NWBFile from .tools.nwb_helpers import ( @@ -42,10 +43,6 @@ def __init__(self, verbose: bool = False, **source_data): self.verbose = verbose self.source_data = source_data - def get_conversion_options_schema(self) -> dict: - """Infer the JSON schema for the conversion options from the method signature (annotation typing).""" - return get_json_schema_from_method_signature(self.add_to_nwbfile, exclude=["nwbfile", "metadata"]) - def get_metadata_schema(self) -> dict: """Retrieve JSON schema for metadata.""" metadata_schema = load_dict_from_file(Path(__file__).parent / "schemas" / "base_metadata_schema.json") @@ -79,6 +76,10 @@ def validate_metadata(self, metadata: dict, append_mode: bool = False) -> None: validate(instance=decoded_metadata, schema=metdata_schema) + def get_conversion_options_schema(self) -> dict: + """Infer the JSON schema for the conversion options from the method signature (annotation typing).""" + return get_json_schema_from_method_signature(self.add_to_nwbfile, exclude=["nwbfile", "metadata"]) + def create_nwbfile(self, metadata: Optional[dict] = None, **conversion_options) -> NWBFile: """ Create and return an in-memory pynwb.NWBFile object with this interface's data added to it. @@ -121,7 +122,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, **conversion_options) -> None: def run_conversion( self, - nwbfile_path: str, + nwbfile_path: FilePath, nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, diff --git a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py index 6054a65b9..532ad5b42 100644 --- a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py +++ b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py @@ -4,6 +4,7 @@ import numpy as np import scipy +from pydantic import FilePath from pynwb import NWBFile from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -27,7 +28,7 @@ class AudioInterface(BaseTemporalAlignmentInterface): associated_suffixes = (".wav",) info = "Interface for writing audio recordings to an NWB file." - def __init__(self, file_paths: list, verbose: bool = False): + def __init__(self, file_paths: List[FilePath], verbose: bool = False): """ Data interface for writing acoustic recordings to an NWB file. diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index ddea0751d..39245c307 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -7,11 +7,10 @@ import numpy as np import pandas as pd import yaml +from pydantic import FilePath from pynwb import NWBFile from ruamel.yaml import YAML -from ....utils import FilePathType - def _read_config(config_file_path): """ @@ -303,9 +302,9 @@ def _write_pes_to_nwbfile( def add_subject_to_nwbfile( nwbfile: NWBFile, - h5file: FilePathType, + h5file: FilePath, individual_name: str, - config_file: FilePathType, + config_file: FilePath, timestamps: Optional[Union[List, np.ndarray]] = None, pose_estimation_container_kwargs: Optional[dict] = None, ) -> NWBFile: diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index a0bfa3fb4..f35f3854c 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -2,10 +2,10 @@ from typing import List, Optional, Union import numpy as np +from pydantic import FilePath from pynwb.file import NWBFile from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface -from ....utils import FilePathType class DeepLabCutInterface(BaseTemporalAlignmentInterface): @@ -27,8 +27,8 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, - config_file_path: FilePathType, + file_path: FilePath, + config_file_path: FilePath, subject_name: str = "ind1", verbose: bool = True, ): @@ -37,9 +37,9 @@ def __init__( Parameters ---------- - file_path : FilePathType + file_path : FilePath path to the h5 file output by dlc. - config_file_path : FilePathType + config_file_path : FilePath path to .yml config file subject_name : str, default: "ind1" the name of the subject for which the :py:class:`~pynwb.file.NWBFile` is to be created. diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index da52b10b7..13136b691 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -6,12 +6,13 @@ from typing import Optional, Union import numpy as np +from pydantic import FilePath from pynwb.behavior import Position, SpatialSeries from pynwb.file import NWBFile from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface from ....tools import get_module -from ....utils import FilePathType, calculate_regular_series_rate +from ....utils import calculate_regular_series_rate class FicTracDataInterface(BaseTemporalAlignmentInterface): @@ -155,9 +156,9 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, radius: Optional[float] = None, - configuration_file_path: Optional[FilePathType] = None, + configuration_file_path: Optional[FilePath] = None, verbose: bool = True, ): """ @@ -165,12 +166,12 @@ def __init__( Parameters ---------- - file_path : a string or a path + file_path : FilePath Path to the .dat file (the output of fictrac) radius : float, optional The radius of the ball in meters. If provided the radius is stored as a conversion factor and the units are set to meters. If not provided the units are set to radians. - configuration_file_path : a string or a path, optional + configuration_file_path : FilePath, optional Path to the .txt file with the configuration metadata. Usually called config.txt verbose : bool, default: True controls verbosity. ``True`` by default. @@ -358,8 +359,8 @@ def set_aligned_starting_time(self, aligned_starting_time): def extract_session_start_time( - file_path: FilePathType, - configuration_file_path: Optional[FilePathType] = None, + file_path: FilePath, + configuration_file_path: Optional[FilePath] = None, ) -> Union[datetime, None]: """ Extract the session start time from a FicTrac data file or its configuration file. @@ -378,9 +379,9 @@ def extract_session_start_time( Parameters ---------- - file_path : FilePathType + file_path : FilePath Path to the FicTrac data file. - configuration_file_path : Optional[FilePathType] + configuration_file_path : FilePath, optional Path to the FicTrac configuration file. If omitted, the function defaults to searching for "fictrac_config.txt" in the same directory as the data file. @@ -413,13 +414,13 @@ def extract_session_start_time( return None -def parse_fictrac_config(file_path: FilePathType) -> dict: +def parse_fictrac_config(file_path: FilePath) -> dict: """ Parse a FicTrac configuration file and return a dictionary of its parameters. Parameters ---------- - file_path : str, Path + file_path : FilePath Path to the configuration file in txt format. Returns diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py index 3907134a6..20c080a6e 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py @@ -1,6 +1,7 @@ from copy import deepcopy from typing import List, Optional +from pydantic import FilePath from pynwb import NWBFile from neuroconv import NWBConverter @@ -8,7 +9,6 @@ from neuroconv.tools.nwb_helpers import make_or_load_nwbfile from neuroconv.utils import ( DeepDict, - FilePathType, dict_deep_update, get_schema_from_method_signature, ) @@ -28,9 +28,9 @@ def get_source_schema(cls): def __init__( self, - file_path: FilePathType, - original_video_file_path: FilePathType, - labeled_video_file_path: Optional[FilePathType] = None, + file_path: FilePath, + original_video_file_path: FilePath, + labeled_video_file_path: Optional[FilePath] = None, image_series_original_video_name: Optional[str] = None, image_series_labeled_video_name: Optional[str] = None, verbose: bool = True, @@ -41,11 +41,11 @@ def __init__( Parameters ---------- - file_path : a string or a path + file_path : FilePath Path to the .csv file that contains the predictions from Lightning Pose. - original_video_file_path : a string or a path + original_video_file_path : FilePath Path to the original video file (.mp4). - labeled_video_file_path : a string or a path, optional + labeled_video_file_path : FilePath, optional Path to the labeled video file (.mp4). image_series_original_video_name: string, optional The name of the ImageSeries to add for the original video. @@ -160,7 +160,7 @@ def add_to_nwbfile( def run_conversion( self, - nwbfile_path: Optional[str] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py index 28c93db9c..b9761b2c6 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py @@ -5,13 +5,13 @@ from typing import Optional, Tuple import numpy as np +from pydantic import FilePath from pynwb import NWBFile from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface from ....tools import get_module from ....utils import ( DeepDict, - FilePathType, calculate_regular_series_rate, get_base_schema, ) @@ -60,9 +60,9 @@ def get_metadata_schema(self) -> dict: def __init__( self, - file_path: FilePathType, - original_video_file_path: FilePathType, - labeled_video_file_path: Optional[FilePathType] = None, + file_path: FilePath, + original_video_file_path: FilePath, + labeled_video_file_path: Optional[FilePath] = None, verbose: bool = True, ): """ @@ -70,9 +70,9 @@ def __init__( Parameters ---------- - file_path : a string or a path + file_path : FilePath Path to the .csv file that contains the predictions from Lightning Pose. - original_video_file_path : a string or a path + original_video_file_path : FilePath Path to the original video file (.mp4). labeled_video_file_path : a string or a path, optional Path to the labeled video file (.mp4). diff --git a/src/neuroconv/datainterfaces/behavior/medpc/medpc_helpers.py b/src/neuroconv/datainterfaces/behavior/medpc/medpc_helpers.py index 3610c4a49..8e8b7289b 100644 --- a/src/neuroconv/datainterfaces/behavior/medpc/medpc_helpers.py +++ b/src/neuroconv/datainterfaces/behavior/medpc/medpc_helpers.py @@ -1,9 +1,8 @@ import numpy as np +from pydantic import FilePath -from neuroconv.utils import FilePathType - -def get_medpc_variables(file_path: FilePathType, variable_names: list) -> dict: +def get_medpc_variables(file_path: FilePath, variable_names: list) -> dict: """ Get the values of the given single-line variables from a MedPC file for all sessions in that file. @@ -85,7 +84,7 @@ def _get_session_lines(lines: list, session_conditions: dict, start_variable: st def read_medpc_file( - file_path: FilePathType, + file_path: FilePath, medpc_name_to_info_dict: dict, session_conditions: dict, start_variable: str, diff --git a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py index 267f006f5..0c109f559 100644 --- a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py @@ -1,12 +1,13 @@ from typing import Optional import numpy as np +from pydantic import FilePath from pynwb.behavior import BehavioralEpochs, IntervalSeries from pynwb.file import NWBFile from neuroconv.basetemporalalignmentinterface import BaseTemporalAlignmentInterface from neuroconv.tools import get_package, nwb_helpers -from neuroconv.utils import DeepDict, FilePathType +from neuroconv.utils import DeepDict from .medpc_helpers import read_medpc_file @@ -42,7 +43,7 @@ class MedPCInterface(BaseTemporalAlignmentInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, session_conditions: dict, start_variable: str, metadata_medpc_name_to_info_dict: dict, @@ -54,7 +55,7 @@ def __init__( Parameters ---------- - file_path : FilePathType + file_path : FilePath Path to the MedPC file. session_conditions : dict The conditions that define the session. The keys are the names of the single-line variables (ex. 'Start Date') diff --git a/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py b/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py index e55fec432..6a46e84fe 100644 --- a/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py @@ -1,10 +1,11 @@ from pathlib import Path +from pydantic import DirectoryPath from pynwb import NWBFile from .... import BaseDataInterface from ....tools import get_package -from ....utils import DeepDict, FolderPathType +from ....utils import DeepDict class MiniscopeBehaviorInterface(BaseDataInterface): @@ -23,13 +24,13 @@ def get_source_schema(cls) -> dict: ] = "The main Miniscope folder. The movie files are expected to be in sub folders within the main folder." return source_schema - def __init__(self, folder_path: FolderPathType): + def __init__(self, folder_path: DirectoryPath): """ Initialize reading recordings from the Miniscope behavioral camera. Parameters ---------- - folder_path : FolderPathType + folder_path : DirectoryPath The path that points to the main Miniscope folder. The movie files are expected to be in sub folders within the main folder. """ diff --git a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py index 414b2d0d5..51e04821d 100644 --- a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py +++ b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py @@ -2,12 +2,13 @@ from typing import Optional import numpy as np +from pydantic import FilePath from pynwb import NWBFile from pynwb.behavior import CompassDirection, Position, SpatialSeries from .nvt_utils import read_data, read_header from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface -from ....utils import DeepDict, FilePathType, NWBMetaDataEncoder, get_base_schema +from ....utils import DeepDict, NWBMetaDataEncoder, get_base_schema from ....utils.path import infer_path @@ -19,13 +20,13 @@ class NeuralynxNvtInterface(BaseTemporalAlignmentInterface): associated_suffixes = (".nvt",) info = "Interface for writing Neuralynx position tracking .nvt files to NWB." - def __init__(self, file_path: FilePathType, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = True): """ Interface for writing Neuralynx .nvt files to nwb. Parameters ---------- - file_path : FilePathType + file_path : FilePath Path to the .nvt file verbose : bool, default: True controls verbosity. diff --git a/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py b/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py index 278433b5c..99a22e577 100644 --- a/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py +++ b/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py @@ -8,9 +8,7 @@ from typing import Dict, List, Union import numpy as np - -# Constants for header size and record format -from neuroconv.utils import FilePathType +from pydantic import FilePath HEADER_SIZE = 16 * 1024 @@ -120,7 +118,7 @@ def read_data(filename: str) -> Dict[str, np.ndarray]: return {name: records[name].squeeze() for name, *_ in RECORD_FORMAT} -def truncate_file(source_filename: FilePathType, dest_filename: str, n_records: int = 10) -> None: # pragma: no cover +def truncate_file(source_filename: FilePath, dest_filename: str, n_records: int = 10) -> None: # pragma: no cover """ Creates a new .nvt file with the same header and truncated data. Useful for creating test files. diff --git a/src/neuroconv/datainterfaces/behavior/sleap/sleap_utils.py b/src/neuroconv/datainterfaces/behavior/sleap/sleap_utils.py index 16d998053..b0dae3a3a 100644 --- a/src/neuroconv/datainterfaces/behavior/sleap/sleap_utils.py +++ b/src/neuroconv/datainterfaces/behavior/sleap/sleap_utils.py @@ -1,8 +1,9 @@ +from pydantic import FilePath + from ....tools import get_package -from ....utils import FilePathType -def extract_timestamps(video_file_path: FilePathType) -> list: +def extract_timestamps(video_file_path: FilePath) -> list: """Extract the timestamps using pyav Parameters diff --git a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py index a689d970e..a74b90a67 100644 --- a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py @@ -2,12 +2,12 @@ from typing import Optional import numpy as np +from pydantic import FilePath from pynwb.file import NWBFile from .sleap_utils import extract_timestamps from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface from ....tools import get_package -from ....utils import FilePathType class SLEAPInterface(BaseTemporalAlignmentInterface): @@ -29,8 +29,8 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, - video_file_path: Optional[FilePathType] = None, + file_path: FilePath, + video_file_path: Optional[FilePath] = None, verbose: bool = True, frames_per_second: Optional[float] = None, ): @@ -39,7 +39,7 @@ def __init__( Parameters ---------- - file_path : FilePathType + file_path : FilePath Path to the .slp file (the output of sleap) verbose : bool, default: True controls verbosity. ``True`` by default. diff --git a/src/neuroconv/datainterfaces/behavior/video/video_utils.py b/src/neuroconv/datainterfaces/behavior/video/video_utils.py index 5000c468b..78c66472a 100644 --- a/src/neuroconv/datainterfaces/behavior/video/video_utils.py +++ b/src/neuroconv/datainterfaces/behavior/video/video_utils.py @@ -3,15 +3,13 @@ import numpy as np from hdmf.data_utils import GenericDataChunkIterator +from pydantic import FilePath from tqdm import tqdm from ....tools import get_package -from ....utils import FilePathType -def get_video_timestamps( - file_path: FilePathType, max_frames: Optional[int] = None, display_progress: bool = True -) -> list: +def get_video_timestamps(file_path: FilePath, max_frames: Optional[int] = None, display_progress: bool = True) -> list: """Extract the timestamps of the video located in file_path Parameters @@ -36,7 +34,7 @@ def get_video_timestamps( class VideoCaptureContext: """Retrieving video metadata and frames using a context manager.""" - def __init__(self, file_path: FilePathType): + def __init__(self, file_path: FilePath): cv2 = get_package(package_name="cv2", installation_instructions="pip install opencv-python-headless") self.vc = cv2.VideoCapture(filename=file_path) @@ -178,7 +176,7 @@ class VideoDataChunkIterator(GenericDataChunkIterator): def __init__( self, - video_file: FilePathType, + video_file: FilePath, buffer_gb: float = None, chunk_shape: tuple = None, stub_test: bool = False, diff --git a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py index 56880b820..ca651e597 100644 --- a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py @@ -6,6 +6,7 @@ import numpy as np import psutil from hdmf.data_utils import DataChunkIterator +from pydantic import FilePath from pynwb import NWBFile from pynwb.image import ImageSeries from tqdm import tqdm @@ -29,7 +30,7 @@ class VideoInterface(BaseDataInterface): def __init__( self, - file_paths: list, + file_paths: List[FilePath], verbose: bool = False, *, metadata_key_name: str = "Videos", @@ -39,7 +40,7 @@ def __init__( Parameters ---------- - file_paths : list of FilePathTypes + file_paths : list of FilePaths Many video storage formats segment a sequence of videos over the course of the experiment. Pass the file paths for this videos as a list in sorted, consecutive order. metadata_key_name : str, optional diff --git a/src/neuroconv/datainterfaces/ecephys/axona/axona_utils.py b/src/neuroconv/datainterfaces/ecephys/axona/axona_utils.py index ae095c19d..f7db5924d 100644 --- a/src/neuroconv/datainterfaces/ecephys/axona/axona_utils.py +++ b/src/neuroconv/datainterfaces/ecephys/axona/axona_utils.py @@ -4,12 +4,11 @@ import dateutil import numpy as np +from pydantic import FilePath from pynwb.behavior import Position, SpatialSeries -from ....utils import FilePathType - -def get_eeg_sampling_frequency(file_path: FilePathType) -> float: +def get_eeg_sampling_frequency(file_path: FilePath) -> float: """ Read sampling frequency from .eegX or .egfX file header. @@ -30,7 +29,7 @@ def get_eeg_sampling_frequency(file_path: FilePathType) -> float: # Helper functions for AxonaLFPDataInterface -def read_eeg_file_lfp_data(file_path: FilePathType) -> np.memmap: +def read_eeg_file_lfp_data(file_path: FilePath) -> np.memmap: """ Read LFP data from Axona `.eegX` or `.egfX` file. @@ -64,7 +63,7 @@ def read_eeg_file_lfp_data(file_path: FilePathType) -> np.memmap: return eeg_data -def get_all_file_paths(file_path: FilePathType) -> list: +def get_all_file_paths(file_path: FilePath) -> list: """ Read LFP file_paths of `.eeg` or `.egf` files in file_path's directory. E.g. if file_path='/my/directory/my_file.eeg', all .eeg channels will be @@ -89,7 +88,7 @@ def get_all_file_paths(file_path: FilePathType) -> list: return path_list -def read_all_eeg_file_lfp_data(file_path: FilePathType) -> np.ndarray: +def read_all_eeg_file_lfp_data(file_path: FilePath) -> np.ndarray: """ Read LFP data from all Axona `.eeg` or `.egf` files in file_path's directory. E.g. if file_path='/my/directory/my_file.eeg', all .eeg channels will be conactenated @@ -122,7 +121,7 @@ def read_all_eeg_file_lfp_data(file_path: FilePathType) -> np.ndarray: # Helper functions for AxonaPositionDataInterface -def parse_generic_header(file_path: FilePathType, params: Union[list, set]) -> dict: +def parse_generic_header(file_path: FilePath, params: Union[list, set]) -> dict: """ Given a binary file with phrases and line breaks, enters the first word of a phrase as dictionary key and the following @@ -159,7 +158,7 @@ def parse_generic_header(file_path: FilePathType, params: Union[list, set]) -> d return header -def read_axona_iso_datetime(set_file: FilePathType) -> str: +def read_axona_iso_datetime(set_file: FilePath) -> str: """ Creates datetime object (y, m, d, h, m, s) from .set file header and converts it to ISO 8601 format @@ -173,7 +172,7 @@ def read_axona_iso_datetime(set_file: FilePathType) -> str: return dateutil.parser.parse(date_string + " " + time_string).isoformat() -def get_header_bstring(file: FilePathType) -> bytes: +def get_header_bstring(file: FilePath) -> bytes: """ Scan file for the occurrence of 'data_start' and return the header as byte string @@ -198,7 +197,7 @@ def get_header_bstring(file: FilePathType) -> bytes: return header -def read_bin_file_position_data(bin_file_path: FilePathType) -> np.ndarray: +def read_bin_file_position_data(bin_file_path: FilePath) -> np.ndarray: """ Read position data from Axona `.bin` file (if present). @@ -289,7 +288,7 @@ def read_bin_file_position_data(bin_file_path: FilePathType) -> np.ndarray: return pos_data -def read_pos_file_position_data(pos_file_path: FilePathType) -> np.ndarray: +def read_pos_file_position_data(pos_file_path: FilePath) -> np.ndarray: """ Read position data from Axona `.pos` file. @@ -359,7 +358,7 @@ def read_pos_file_position_data(pos_file_path: FilePathType) -> np.ndarray: return pos_data -def get_position_object(file_path: FilePathType) -> Position: +def get_position_object(file_path: FilePath) -> Position: """ Read position data from .bin or .pos file and convert to pynwb.behavior.SpatialSeries objects. If possible it should always diff --git a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py index bd3fe06fa..69612c85c 100644 --- a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py @@ -1,5 +1,6 @@ """Collection of Axona interfaces.""" +from pydantic import FilePath from pynwb import NWBFile from .axona_utils import ( @@ -11,7 +12,7 @@ from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....basedatainterface import BaseDataInterface from ....tools.nwb_helpers import get_module -from ....utils import FilePathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class AxonaRecordingInterface(BaseRecordingExtractorInterface): @@ -29,12 +30,12 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to .bin file." return source_schema - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Parameters ---------- - file_path: FilePathType + file_path: FilePath Path to .bin file. verbose: bool, optional, default: True es_key: str, default: "ElectricalSeries" @@ -125,7 +126,7 @@ def get_source_schema(cls) -> dict: type="object", ) - def __init__(self, file_path: FilePathType, noise_std: float = 3.5): + def __init__(self, file_path: FilePath, noise_std: float = 3.5): super().__init__(filename=file_path, noise_std=noise_std) self.source_data = dict(file_path=file_path, noise_std=noise_std) @@ -147,7 +148,7 @@ def get_source_schema(cls) -> dict: additionalProperties=False, ) - def __init__(self, file_path: FilePathType): + def __init__(self, file_path: FilePath): data = read_all_eeg_file_lfp_data(file_path).T sampling_frequency = get_eeg_sampling_frequency(file_path) super().__init__(traces_list=[data], sampling_frequency=sampling_frequency) diff --git a/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py b/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py index 8df5a1fc9..f12f3a93d 100644 --- a/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py @@ -1,5 +1,6 @@ +from pydantic import FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils.types import FilePathType class BiocamRecordingInterface(BaseRecordingExtractorInterface): @@ -19,7 +20,7 @@ def get_source_schema(cls) -> dict: schema["properties"]["file_path"]["description"] = "Path to the .bwr file." return schema - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for Biocam. diff --git a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py index 460ada6ce..19e34fdcd 100644 --- a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py @@ -1,10 +1,12 @@ from pathlib import Path from typing import Optional +from pydantic import FilePath + from .header_tools import parse_nev_basic_header, parse_nsx_basic_header from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ..basesortingextractorinterface import BaseSortingExtractorInterface -from ....utils import FilePathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class BlackrockRecordingInterface(BaseRecordingExtractorInterface): @@ -25,8 +27,8 @@ def get_source_schema(cls): def __init__( self, - file_path: FilePathType, - nsx_override: Optional[FilePathType] = None, + file_path: FilePath, + nsx_override: Optional[FilePath] = None, verbose: bool = True, es_key: str = "ElectricalSeries", ): @@ -81,7 +83,7 @@ def get_source_schema(cls) -> dict: metadata_schema["properties"]["file_path"].update(description="Path to Blackrock .nev file.") return metadata_schema - def __init__(self, file_path: FilePathType, sampling_frequency: float = None, verbose: bool = True): + def __init__(self, file_path: FilePath, sampling_frequency: float = None, verbose: bool = True): """ Parameters ---------- diff --git a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py index e9e7a703f..731ce981b 100644 --- a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py @@ -3,15 +3,15 @@ import numpy as np import scipy +from pydantic import DirectoryPath, FilePath from pynwb import NWBFile from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ..basesortingextractorinterface import BaseSortingExtractorInterface from ....tools import get_package -from ....utils import FilePathType, FolderPathType -def add_channel_metadata_to_recoder(recording_extractor, folder_path: FolderPathType): +def add_channel_metadata_to_recoder(recording_extractor, folder_path: DirectoryPath): """ Main function to add channel metadata to a recording extractor from a CellExplorer session. The metadata is added as channel properties to the recording extractor. @@ -73,7 +73,7 @@ def add_channel_metadata_to_recoder(recording_extractor, folder_path: FolderPath def add_channel_metadata_to_recorder_from_session_file( recording_extractor, - folder_path: FolderPathType, + folder_path: DirectoryPath, ): """ Extracts channel metadata from the CellExplorer's `session.mat` file and adds @@ -177,7 +177,7 @@ def add_channel_metadata_to_recorder_from_session_file( def add_channel_metadata_to_recorder_from_channel_map_file( recording_extractor, - folder_path: FolderPathType, + folder_path: DirectoryPath, ): """ Extracts channel metadata from the `chanMap.mat` file used by Kilosort and adds @@ -294,7 +294,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["folder_path"]["description"] = "Folder containing the .session.mat file" return source_schema - def __init__(self, folder_path: FolderPathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Parameters @@ -374,7 +374,7 @@ class CellExplorerLFPInterface(CellExplorerRecordingInterface): sampling_frequency_key = "srLfp" binary_file_extension = "lfp" - def __init__(self, folder_path: FolderPathType, verbose: bool = True, es_key: str = "ElectricalSeriesLFP"): + def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str = "ElectricalSeriesLFP"): super().__init__(folder_path, verbose, es_key) def add_to_nwbfile( @@ -411,7 +411,7 @@ class CellExplorerSortingInterface(BaseSortingExtractorInterface): associated_suffixes = (".mat", ".sessionInfo", ".spikes", ".cellinfo") info = "Interface for CellExplorer sorting data." - def __init__(self, file_path: FilePathType, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = True): """ Initialize read of Cell Explorer file. diff --git a/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py b/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py index 13659b450..119e9f8d2 100644 --- a/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py @@ -1,6 +1,7 @@ +from pydantic import FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....tools import get_package -from ....utils.types import FilePathType class EDFRecordingInterface(BaseRecordingExtractorInterface): @@ -22,7 +23,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .edf file." return source_schema - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for EDF. Currently, only continuous EDF+ files (EDF+C) and original EDF files (EDF) are supported diff --git a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py index ce3504055..d28214598 100644 --- a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py @@ -2,11 +2,12 @@ from typing import Optional from packaging.version import Version +from pydantic import FilePath from pynwb.ecephys import ElectricalSeries from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....tools import get_package_version -from ....utils import FilePathType, get_schema_from_hdmf_class +from ....utils import get_schema_from_hdmf_class class IntanRecordingInterface(BaseRecordingExtractorInterface): @@ -29,7 +30,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, stream_id: Optional[str] = None, verbose: bool = True, es_key: str = "ElectricalSeries", diff --git a/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py b/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py index 37f812142..fc6765823 100644 --- a/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py @@ -1,5 +1,6 @@ +from pydantic import DirectoryPath + from ..basesortingextractorinterface import BaseSortingExtractorInterface -from ....utils import FolderPathType class KiloSortSortingInterface(BaseSortingExtractorInterface): @@ -19,7 +20,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, keep_good_only: bool = False, verbose: bool = True, ): diff --git a/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py b/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py index 73f951270..11902f81b 100644 --- a/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py @@ -3,8 +3,9 @@ from platform import system from typing import Optional +from pydantic import DirectoryPath, FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils.types import FilePathType, FolderPathType class MaxOneRecordingInterface(BaseRecordingExtractorInterface): # pragma: no cover @@ -22,7 +23,7 @@ class MaxOneRecordingInterface(BaseRecordingExtractorInterface): # pragma: no c @staticmethod def auto_install_maxwell_hdf5_compression_plugin( - hdf5_plugin_path: Optional[FolderPathType] = None, download_plugin: bool = True + hdf5_plugin_path: Optional[DirectoryPath] = None, download_plugin: bool = True ) -> None: """ If you do not yet have the Maxwell compression plugin installed, this function will automatically install it. @@ -43,8 +44,8 @@ def auto_install_maxwell_hdf5_compression_plugin( def __init__( self, - file_path: FilePathType, - hdf5_plugin_path: Optional[FolderPathType] = None, + file_path: FilePath, + hdf5_plugin_path: Optional[DirectoryPath] = None, download_plugin: bool = True, verbose: bool = True, es_key: str = "ElectricalSeries", diff --git a/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py b/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py index 230b1d044..ff8e82139 100644 --- a/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py @@ -1,5 +1,6 @@ +from pydantic import FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils.types import FilePathType class MCSRawRecordingInterface(BaseRecordingExtractorInterface): @@ -19,7 +20,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .raw file." return source_schema - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for MCSRaw. diff --git a/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py b/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py index f69c312fd..9d3797e0f 100644 --- a/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py @@ -1,8 +1,9 @@ import json +from pydantic import FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....utils.json_schema import NWBMetaDataEncoder -from ....utils.types import FilePathType class MEArecRecordingInterface(BaseRecordingExtractorInterface): @@ -22,7 +23,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the MEArec .h5 file." return source_schema - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for MEArec. diff --git a/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py index cfb09e009..d3f9fdfb4 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py @@ -2,6 +2,7 @@ from typing import Optional import numpy as np +from pydantic import DirectoryPath, FilePath from .neuroscope_utils import ( get_channel_groups, @@ -13,7 +14,6 @@ from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ..basesortingextractorinterface import BaseSortingExtractorInterface from ....tools import get_package -from ....utils import FilePathType, FolderPathType def filter_non_neural_channels(recording_extractor, xml_file_path: str): @@ -125,9 +125,9 @@ def get_ecephys_metadata(xml_file_path: str) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, gain: Optional[float] = None, - xml_file_path: Optional[FilePathType] = None, + xml_file_path: Optional[FilePath] = None, verbose: bool = True, es_key: str = "ElectricalSeries", ): @@ -202,9 +202,9 @@ def get_source_schema(self) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, gain: Optional[float] = None, - xml_file_path: Optional[FilePathType] = None, + xml_file_path: Optional[FilePath] = None, ): """ Load and prepare lfp data and corresponding metadata from the Neuroscope format (.eeg or .lfp files). @@ -267,10 +267,10 @@ def get_source_schema(self) -> dict: def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, keep_mua_units: bool = True, exclude_shanks: Optional[list] = None, - xml_file_path: Optional[FilePathType] = None, + xml_file_path: Optional[FilePath] = None, verbose: bool = True, ): """ diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/_openephys_utils.py b/src/neuroconv/datainterfaces/ecephys/openephys/_openephys_utils.py index 2dd972aea..2a8128c6e 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/_openephys_utils.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/_openephys_utils.py @@ -4,8 +4,7 @@ from warnings import warn from lxml import etree - -from ....utils import FolderPathType +from pydantic import DirectoryPath def _get_session_start_time(element: etree.Element) -> Union[datetime, None]: @@ -28,7 +27,7 @@ def _get_session_start_time(element: etree.Element) -> Union[datetime, None]: return session_start_time -def _read_settings_xml(folder_path: FolderPathType) -> etree.Element: +def _read_settings_xml(folder_path: DirectoryPath) -> etree.Element: """ Read the settings.xml file from an OpenEphys binary recording folder. Returns the root element of the XML tree. diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py index 88a700dd9..ccbd3cbbd 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py @@ -1,7 +1,9 @@ from typing import List, Optional +from pydantic import DirectoryPath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils import FolderPathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class OpenEphysBinaryRecordingInterface(BaseRecordingExtractorInterface): @@ -18,7 +20,7 @@ class OpenEphysBinaryRecordingInterface(BaseRecordingExtractorInterface): ExtractorName = "OpenEphysBinaryRecordingExtractor" @classmethod - def get_stream_names(cls, folder_path: FolderPathType) -> List[str]: + def get_stream_names(cls, folder_path: DirectoryPath) -> List[str]: from spikeinterface.extractors import OpenEphysBinaryRecordingExtractor stream_names, _ = OpenEphysBinaryRecordingExtractor.get_streams(folder_path=folder_path) @@ -37,7 +39,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, stream_name: Optional[str] = None, block_index: Optional[int] = None, stub_test: bool = False, diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py index 13fc4e96e..b47df98b9 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py @@ -1,10 +1,11 @@ from pathlib import Path from typing import Optional +from pydantic import DirectoryPath + from .openephysbinarydatainterface import OpenEphysBinaryRecordingInterface from .openephyslegacydatainterface import OpenEphysLegacyRecordingInterface from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils import FolderPathType class OpenEphysRecordingInterface(BaseRecordingExtractorInterface): @@ -26,7 +27,7 @@ def get_source_schema(cls) -> dict: def __new__( cls, - folder_path: FolderPathType, + folder_path: DirectoryPath, stream_name: Optional[str] = None, block_index: Optional[int] = None, verbose: bool = True, diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py index 5d680f93b..3403705b1 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py @@ -2,8 +2,9 @@ from typing import List, Optional from warnings import warn +from pydantic import DirectoryPath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils import FolderPathType class OpenEphysLegacyRecordingInterface(BaseRecordingExtractorInterface): @@ -18,7 +19,7 @@ class OpenEphysLegacyRecordingInterface(BaseRecordingExtractorInterface): info = "Interface for converting legacy OpenEphys recording data." @classmethod - def get_stream_names(cls, folder_path: FolderPathType) -> List[str]: + def get_stream_names(cls, folder_path: DirectoryPath) -> List[str]: from spikeinterface.extractors import OpenEphysLegacyRecordingExtractor stream_names, _ = OpenEphysLegacyRecordingExtractor.get_streams(folder_path=folder_path) @@ -36,7 +37,7 @@ def get_source_schema(cls): def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, stream_name: Optional[str] = None, block_index: Optional[int] = None, verbose: bool = True, diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py index 8ccbf2ab1..20f292c9a 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py @@ -1,5 +1,7 @@ +from pydantic import DirectoryPath + from ..basesortingextractorinterface import BaseSortingExtractorInterface -from ....utils import FolderPathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class OpenEphysSortingInterface(BaseSortingExtractorInterface): @@ -21,7 +23,7 @@ def get_source_schema(cls) -> dict: metadata_schema["additionalProperties"] = False return metadata_schema - def __init__(self, folder_path: FolderPathType, experiment_id: int = 0, recording_id: int = 0): + def __init__(self, folder_path: DirectoryPath, experiment_id: int = 0, recording_id: int = 0): from spikeextractors import OpenEphysSortingExtractor self.Extractor = OpenEphysSortingExtractor diff --git a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py index cac24faa2..e5d250a6e 100644 --- a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py @@ -1,7 +1,8 @@ from typing import Optional +from pydantic import DirectoryPath + from ..basesortingextractorinterface import BaseSortingExtractorInterface -from ....utils import FolderPathType class PhySortingInterface(BaseSortingExtractorInterface): @@ -25,7 +26,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, exclude_cluster_groups: Optional[list] = None, verbose: bool = True, ): diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py index cf10d5f0b..8b2e53338 100644 --- a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py @@ -1,9 +1,10 @@ from pathlib import Path +from pydantic import FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ..basesortingextractorinterface import BaseSortingExtractorInterface from ....utils import DeepDict -from ....utils.types import FilePathType class PlexonRecordingInterface(BaseRecordingExtractorInterface): @@ -23,7 +24,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .plx file." return source_schema - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for Plexon. @@ -67,7 +68,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .pl2 file." return source_schema - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for Plexon. @@ -118,7 +119,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the plexon spiking data (.plx file)." return source_schema - def __init__(self, file_path: FilePathType, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = True): """ Load and prepare data for Plexon. diff --git a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py index dc55b9eb4..5895dfe3c 100644 --- a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py @@ -1,8 +1,10 @@ from pathlib import Path +from pydantic import FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....tools import get_package -from ....utils import FilePathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature def _test_sonpy_installation() -> None: @@ -33,12 +35,12 @@ def get_source_schema(cls) -> dict: return source_schema @classmethod - def get_all_channels_info(cls, file_path: FilePathType): + def get_all_channels_info(cls, file_path: FilePath): """Retrieve and inspect necessary channel information prior to initialization.""" _test_sonpy_installation() return cls.get_extractor().get_all_channels_info(file_path=file_path) - def __init__(self, file_path: FilePathType, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Initialize reading of Spike2 file. diff --git a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py index 578487a5b..a5ad98d52 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py @@ -1,7 +1,9 @@ from typing import Optional +from pydantic import FilePath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils import ArrayType, FilePathType +from ....utils import ArrayType, get_json_schema_from_method_signature class SpikeGadgetsRecordingInterface(BaseRecordingExtractorInterface): @@ -16,13 +18,13 @@ class SpikeGadgetsRecordingInterface(BaseRecordingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: - source_schema = super().get_source_schema() + source_schema = get_json_schema_from_method_signature(cls, exclude=["source_data"]) source_schema["properties"]["file_path"].update(description="Path to SpikeGadgets (.rec) file.") return source_schema def __init__( self, - file_path: FilePathType, + file_path: FilePath, stream_id: str = "trodes", gains: Optional[ArrayType] = None, verbose: bool = True, diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py index e9ba6514e..ba83d296d 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py @@ -4,7 +4,7 @@ from datetime import datetime from pathlib import Path -from ....utils import FilePathType +from pydantic import FilePath def add_recording_extractor_properties(recording_extractor) -> None: @@ -55,7 +55,7 @@ def get_session_start_time(recording_metadata: dict) -> datetime: return session_start_time -def fetch_stream_id_for_spikelgx_file(file_path: FilePathType) -> str: +def fetch_stream_id_for_spikelgx_file(file_path: FilePath) -> str: """ Returns the stream_id for a spikelgx file. diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py index eccd430c4..35a4d8881 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py @@ -1,10 +1,12 @@ from pathlib import Path from typing import List, Optional +from pydantic import DirectoryPath + from .spikeglxdatainterface import SpikeGLXRecordingInterface from .spikeglxnidqinterface import SpikeGLXNIDQInterface from ....nwbconverter import ConverterPipe -from ....utils import FolderPathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class SpikeGLXConverterPipe(ConverterPipe): @@ -26,14 +28,14 @@ def get_source_schema(cls): return source_schema @classmethod - def get_streams(cls, folder_path: FolderPathType) -> List[str]: + def get_streams(cls, folder_path: DirectoryPath) -> List[str]: from spikeinterface.extractors import SpikeGLXRecordingExtractor return SpikeGLXRecordingExtractor.get_streams(folder_path=folder_path)[0] def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, streams: Optional[List[str]] = None, verbose: bool = False, ): diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 0cecb39cb..6e8c33586 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -4,6 +4,7 @@ from typing import Optional import numpy as np +from pydantic import FilePath from .spikeglx_utils import ( add_recording_extractor_properties, @@ -12,7 +13,7 @@ get_session_start_time, ) from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils import FilePathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class SpikeGLXRecordingInterface(BaseRecordingExtractorInterface): @@ -39,7 +40,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, verbose: bool = True, es_key: Optional[str] = None, ): diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 61bf6b056..7cff7f11f 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -2,11 +2,12 @@ from typing import List import numpy as np +from pydantic import FilePath from .spikeglx_utils import get_session_start_time from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....tools.signal_processing import get_rising_frames_from_ttl -from ....utils import FilePathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class SpikeGLXNIDQInterface(BaseRecordingExtractorInterface): @@ -27,7 +28,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, verbose: bool = True, load_sync_channel: bool = False, es_key: str = "ElectricalSeriesNIDQ", diff --git a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py index 82251c820..420d0f242 100644 --- a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py @@ -1,5 +1,6 @@ +from pydantic import DirectoryPath + from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils.types import FolderPathType class TdtRecordingInterface(BaseRecordingExtractorInterface): @@ -11,7 +12,7 @@ class TdtRecordingInterface(BaseRecordingExtractorInterface): def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, gain: float, stream_id: str = "0", verbose: bool = True, diff --git a/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py b/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py index 3b9457901..10ec3e8a3 100644 --- a/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py +++ b/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py @@ -4,6 +4,8 @@ from typing import List from warnings import warn +from pydantic import FilePath + from ..baseicephysinterface import BaseIcephysInterface @@ -49,17 +51,19 @@ def get_source_schema(cls) -> dict: ) return source_schema - def __init__(self, file_paths: list, icephys_metadata: dict = None, icephys_metadata_file_path: str = None): + def __init__( + self, file_paths: List[FilePath], icephys_metadata: dict = None, icephys_metadata_file_path: FilePath = None + ): """ ABF IcephysInterface based on Neo AxonIO. Parameters ---------- - file_paths : list + file_paths : list of FilePaths List of files to be converted to the same NWB file. icephys_metadata : dict, optional Dictionary containing the Icephys-specific metadata. - icephys_metadata_file_path : str, optional + icephys_metadata_file_path : FilePath, optional JSON file containing the Icephys-specific metadata. """ super().__init__(file_paths=file_paths) diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py index e6b7abf15..6f5daeb0c 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py @@ -1,5 +1,6 @@ from typing import Literal, Optional +from pydantic import DirectoryPath, FilePath from pynwb import NWBFile from ... import ( @@ -8,7 +9,7 @@ ) from ....nwbconverter import NWBConverter from ....tools.nwb_helpers import make_or_load_nwbfile -from ....utils import FolderPathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class BrukerTiffMultiPlaneConverter(NWBConverter): @@ -31,7 +32,7 @@ def get_conversion_options_schema(self): def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, plane_separation_type: Literal["disjoint", "contiguous"] = None, verbose: bool = False, ): @@ -40,7 +41,7 @@ def __init__( Parameters ---------- - folder_path : PathType + folder_path : DirectoryPath The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). plane_separation_type: {'contiguous', 'disjoint'} Defines how to write volumetric imaging data. Use 'contiguous' to create the volumetric two photon series, @@ -97,7 +98,7 @@ def add_to_nwbfile( def run_conversion( self, - nwbfile_path: Optional[str] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -137,7 +138,7 @@ def get_conversion_options_schema(self): def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, verbose: bool = False, ): """ @@ -145,7 +146,7 @@ def __init__( Parameters ---------- - folder_path : PathType + folder_path : DirectoryPath The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). verbose : bool, default: True Controls verbosity. @@ -189,7 +190,7 @@ def add_to_nwbfile( def run_conversion( self, - nwbfile_path: Optional[str] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py index 2c663d7e6..d92676219 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py @@ -1,9 +1,9 @@ from typing import List, Literal, Optional from dateutil.parser import parse +from pydantic import DirectoryPath from ..baseimagingextractorinterface import BaseImagingExtractorInterface -from ....utils import FolderPathType from ....utils.dict import DeepDict @@ -25,7 +25,7 @@ def get_source_schema(cls) -> dict: @classmethod def get_streams( cls, - folder_path: FolderPathType, + folder_path: DirectoryPath, plane_separation_type: Literal["contiguous", "disjoint"] = None, ) -> dict: from roiextractors import BrukerTiffMultiPlaneImagingExtractor @@ -40,7 +40,7 @@ def get_streams( def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, stream_name: Optional[str] = None, verbose: bool = True, ): @@ -49,7 +49,7 @@ def __init__( Parameters ---------- - folder_path : FolderPathType + folder_path : DirectoryPath The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). stream_name : str, optional The name of the recording stream (e.g. 'Ch2'). @@ -190,7 +190,7 @@ def get_source_schema(cls) -> dict: return source_schema @classmethod - def get_streams(cls, folder_path: FolderPathType) -> dict: + def get_streams(cls, folder_path: DirectoryPath) -> dict: from roiextractors import BrukerTiffMultiPlaneImagingExtractor streams = BrukerTiffMultiPlaneImagingExtractor.get_streams(folder_path=folder_path) @@ -198,7 +198,7 @@ def get_streams(cls, folder_path: FolderPathType) -> dict: def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, stream_name: Optional[str] = None, verbose: bool = True, ): @@ -207,7 +207,7 @@ def __init__( Parameters ---------- - folder_path : FolderPathType + folder_path : DirectoryPath The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). stream_name : str, optional The name of the recording stream (e.g. 'Ch2'). diff --git a/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py b/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py index 5d86695f4..386c03d3c 100644 --- a/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py @@ -1,5 +1,6 @@ +from pydantic import FilePath + from ..basesegmentationextractorinterface import BaseSegmentationExtractorInterface -from ....utils import FilePathType class CaimanSegmentationInterface(BaseSegmentationExtractorInterface): @@ -15,12 +16,12 @@ def get_source_schema(cls) -> dict: source_metadata["properties"]["file_path"]["description"] = "Path to .hdf5 file." return source_metadata - def __init__(self, file_path: FilePathType, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = True): """ Parameters ---------- - file_path : FilePathType + file_path : FilePath Path to .hdf5 file. verbose : bool, default True Whether to print progress diff --git a/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py b/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py index 8c5b93c64..4ea60c892 100644 --- a/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py @@ -1,5 +1,6 @@ +from pydantic import FilePath + from ..basesegmentationextractorinterface import BaseSegmentationExtractorInterface -from ....utils import FilePathType class CnmfeSegmentationInterface(BaseSegmentationExtractorInterface): @@ -9,6 +10,6 @@ class CnmfeSegmentationInterface(BaseSegmentationExtractorInterface): associated_suffixes = (".mat",) info = "Interface for constrained non-negative matrix factorization (CNMFE) segmentation." - def __init__(self, file_path: FilePathType, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = True): super().__init__(file_path=file_path) self.verbose = verbose diff --git a/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py b/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py index 211667001..39b3d0a79 100644 --- a/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py @@ -1,7 +1,8 @@ from typing import Optional +from pydantic import FilePath + from ..basesegmentationextractorinterface import BaseSegmentationExtractorInterface -from ....utils import FilePathType class ExtractSegmentationInterface(BaseSegmentationExtractorInterface): @@ -13,7 +14,7 @@ class ExtractSegmentationInterface(BaseSegmentationExtractorInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, sampling_frequency: float, output_struct_name: Optional[str] = None, verbose: bool = True, diff --git a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py index d30ff3161..c980fcf41 100644 --- a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py +++ b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py @@ -1,7 +1,9 @@ from typing import Literal +from pydantic import FilePath + from ..baseimagingextractorinterface import BaseImagingExtractorInterface -from ....utils import ArrayType, FilePathType +from ....utils import ArrayType class Hdf5ImagingInterface(BaseImagingExtractorInterface): @@ -13,7 +15,7 @@ class Hdf5ImagingInterface(BaseImagingExtractorInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, mov_field: str = "mov", sampling_frequency: float = None, start_time: float = None, @@ -26,7 +28,7 @@ def __init__( Parameters ---------- - file_path : FilePathType + file_path : FilePath Path to .h5 or .hdf5 file. mov_field : str, default: 'mov' sampling_frequency : float, optional diff --git a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py index 57d966da9..4e758e742 100644 --- a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py @@ -1,7 +1,7 @@ from dateutil.parser import parse +from pydantic import DirectoryPath from ..baseimagingextractorinterface import BaseImagingExtractorInterface -from ....utils import FolderPathType class MicroManagerTiffImagingInterface(BaseImagingExtractorInterface): @@ -18,7 +18,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["folder_path"]["description"] = "The folder containing the OME-TIF image files." return source_schema - def __init__(self, folder_path: FolderPathType, verbose: bool = True): + def __init__(self, folder_path: DirectoryPath, verbose: bool = True): """ Data Interface for MicroManagerTiffImagingExtractor. diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py index ab7706646..09aba9f0e 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py @@ -1,11 +1,12 @@ from typing import Optional +from pydantic import DirectoryPath from pynwb import NWBFile from ... import MiniscopeBehaviorInterface, MiniscopeImagingInterface from ....nwbconverter import NWBConverter from ....tools.nwb_helpers import make_or_load_nwbfile -from ....utils import FolderPathType, get_schema_from_method_signature +from ....utils import get_schema_from_method_signature class MiniscopeConverter(NWBConverter): @@ -22,7 +23,7 @@ def get_source_schema(cls): source_schema["properties"]["folder_path"]["description"] = "The path to the main Miniscope folder." return source_schema - def __init__(self, folder_path: FolderPathType, verbose: bool = True): + def __init__(self, folder_path: DirectoryPath, verbose: bool = True): """ Initializes the data interfaces for the Miniscope recording and behavioral data stream. diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py index 7031fd7f1..e26cc69fb 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py @@ -3,10 +3,11 @@ from typing import Literal, Optional import numpy as np +from pydantic import DirectoryPath from pynwb import NWBFile from ..baseimagingextractorinterface import BaseImagingExtractorInterface -from ....utils import DeepDict, FolderPathType, dict_deep_update +from ....utils import DeepDict, dict_deep_update class MiniscopeImagingInterface(BaseImagingExtractorInterface): @@ -25,13 +26,13 @@ def get_source_schema(cls) -> dict: return source_schema - def __init__(self, folder_path: FolderPathType): + def __init__(self, folder_path: DirectoryPath): """ Initialize reading the Miniscope imaging data. Parameters ---------- - folder_path : FolderPathType + folder_path : DirectoryPath The main Miniscope folder. The microscope movie files are expected to be in sub folders within the main folder. """ diff --git a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py index 079ad6fc9..2fa90a2bb 100644 --- a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py @@ -1,7 +1,8 @@ from typing import Literal +from pydantic import FilePath + from ..baseimagingextractorinterface import BaseImagingExtractorInterface -from ....utils import FilePathType class SbxImagingInterface(BaseImagingExtractorInterface): @@ -13,7 +14,7 @@ class SbxImagingInterface(BaseImagingExtractorInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, sampling_frequency: float = None, verbose: bool = True, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", diff --git a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py index 9c9fa7371..76a67ef9b 100644 --- a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py +++ b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py @@ -4,9 +4,9 @@ from typing import Optional from dateutil.parser import parse as dateparse +from pydantic import DirectoryPath, FilePath from ..baseimagingextractorinterface import BaseImagingExtractorInterface -from ....utils import FilePathType, FolderPathType class ScanImageImagingInterface(BaseImagingExtractorInterface): @@ -34,7 +34,7 @@ def get_source_schema(cls) -> dict: def __new__( cls, - file_path: FilePathType, + file_path: FilePath, channel_name: Optional[str] = None, plane_name: Optional[str] = None, fallback_sampling_frequency: Optional[float] = None, @@ -92,7 +92,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, fallback_sampling_frequency: Optional[float] = None, verbose: bool = True, ): @@ -102,7 +102,7 @@ def __init__( Parameters ---------- - file_path: str + file_path: FilePath Path to tiff file. fallback_sampling_frequency: float, optional The sampling frequency can usually be extracted from the scanimage metadata in @@ -170,7 +170,7 @@ def get_source_schema(cls) -> dict: def __new__( cls, - folder_path: FolderPathType, + folder_path: DirectoryPath, file_pattern: str, channel_name: Optional[str] = None, plane_name: Optional[str] = None, @@ -228,7 +228,7 @@ class ScanImageMultiPlaneImagingInterface(BaseImagingExtractorInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, channel_name: Optional[str] = None, image_metadata: Optional[dict] = None, parsed_metadata: Optional[dict] = None, @@ -239,7 +239,7 @@ def __init__( Parameters ---------- - file_path : PathType + file_path : FilePath Path to the TIFF file. channel_name : str Name of the channel for this extractor. @@ -329,7 +329,7 @@ class ScanImageMultiPlaneMultiFileImagingInterface(BaseImagingExtractorInterface def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, file_pattern: str, channel_name: Optional[str] = None, extract_all_metadata: bool = False, @@ -342,7 +342,7 @@ def __init__( Parameters ---------- - folder_path : PathType + folder_path : DirectoryPath Path to the folder containing the TIFF files. file_pattern : str Pattern for the TIFF files to read -- see pathlib.Path.glob for details. @@ -445,7 +445,7 @@ class ScanImageSinglePlaneImagingInterface(BaseImagingExtractorInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, channel_name: Optional[str] = None, plane_name: Optional[str] = None, image_metadata: Optional[dict] = None, @@ -457,7 +457,7 @@ def __init__( Parameters ---------- - file_path : PathType + file_path : FilePath Path to the TIFF file. channel_name : str The name of the channel to load, to determine what channels are available use ScanImageTiffSinglePlaneImagingExtractor.get_available_channels(file_path=...). @@ -562,7 +562,7 @@ class ScanImageSinglePlaneMultiFileImagingInterface(BaseImagingExtractorInterfac def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, file_pattern: str, channel_name: Optional[str] = None, plane_name: Optional[str] = None, @@ -576,7 +576,7 @@ def __init__( Parameters ---------- - folder_path : PathType + folder_path : DirectoryPath Path to the folder containing the TIFF files. file_pattern : str Pattern for the TIFF files to read -- see pathlib.Path.glob for details. diff --git a/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py b/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py index f89ef716c..8bf34cc04 100644 --- a/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py @@ -1,5 +1,6 @@ +from pydantic import FilePath + from ..basesegmentationextractorinterface import BaseSegmentationExtractorInterface -from ....utils import FilePathType class SimaSegmentationInterface(BaseSegmentationExtractorInterface): @@ -9,12 +10,12 @@ class SimaSegmentationInterface(BaseSegmentationExtractorInterface): associated_suffixes = (".sima",) info = "Interface for SIMA segmentation." - def __init__(self, file_path: FilePathType, sima_segmentation_label: str = "auto_ROIs"): + def __init__(self, file_path: FilePath, sima_segmentation_label: str = "auto_ROIs"): """ Parameters ---------- - file_path : FilePathType + file_path : FilePath sima_segmentation_label : str, default: "auto_ROIs" """ super().__init__(file_path=file_path, sima_segmentation_label=sima_segmentation_label) diff --git a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py index 8fa04bb5c..359dea25f 100644 --- a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py @@ -1,10 +1,11 @@ from copy import deepcopy from typing import Optional +from pydantic import DirectoryPath from pynwb import NWBFile from ..basesegmentationextractorinterface import BaseSegmentationExtractorInterface -from ....utils import DeepDict, FolderPathType +from ....utils import DeepDict def _update_metadata_links_for_plane_segmentation_name(metadata: dict, plane_segmentation_name: str) -> DeepDict: @@ -60,20 +61,20 @@ def get_source_schema(cls) -> dict: return schema @classmethod - def get_available_planes(cls, folder_path: FolderPathType) -> dict: + def get_available_planes(cls, folder_path: DirectoryPath) -> dict: from roiextractors import Suite2pSegmentationExtractor return Suite2pSegmentationExtractor.get_available_planes(folder_path=folder_path) @classmethod - def get_available_channels(cls, folder_path: FolderPathType) -> dict: + def get_available_channels(cls, folder_path: DirectoryPath) -> dict: from roiextractors import Suite2pSegmentationExtractor return Suite2pSegmentationExtractor.get_available_channels(folder_path=folder_path) def __init__( self, - folder_path: FolderPathType, + folder_path: DirectoryPath, channel_name: Optional[str] = None, plane_name: Optional[str] = None, plane_segmentation_name: Optional[str] = None, @@ -83,7 +84,7 @@ def __init__( Parameters ---------- - folder_path : FolderPathType + folder_path : DirectoryPath Path to the folder containing Suite2p segmentation data. Should contain 'plane#' sub-folders. channel_name: str, optional The name of the channel to load. diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py index 7015a7487..72c854634 100644 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py @@ -6,12 +6,13 @@ import numpy as np import pytz +from pydantic import FilePath from pynwb.file import NWBFile from neuroconv.basetemporalalignmentinterface import BaseTemporalAlignmentInterface from neuroconv.tools import get_package from neuroconv.tools.fiber_photometry import add_fiber_photometry_device -from neuroconv.utils import DeepDict, FilePathType +from neuroconv.utils import DeepDict class TDTFiberPhotometryInterface(BaseTemporalAlignmentInterface): @@ -27,12 +28,12 @@ class TDTFiberPhotometryInterface(BaseTemporalAlignmentInterface): info = "Data Interface for converting fiber photometry data from TDT files." associated_suffixes = ("Tbk", "Tdx", "tev", "tin", "tsq") - def __init__(self, folder_path: FilePathType, verbose: bool = True): + def __init__(self, folder_path: FilePath, verbose: bool = True): """Initialize the TDTFiberPhotometryInterface. Parameters ---------- - folder_path : FilePathType + folder_path : FilePath The path to the folder containing the TDT data. verbose : bool, optional Whether to print status messages, default = True. diff --git a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py index 30417f345..017aa2e98 100644 --- a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py @@ -1,7 +1,8 @@ from typing import Literal +from pydantic import FilePath + from ..baseimagingextractorinterface import BaseImagingExtractorInterface -from ....utils import FilePathType class TiffImagingInterface(BaseImagingExtractorInterface): @@ -19,7 +20,7 @@ def get_source_schema(cls) -> dict: def __init__( self, - file_path: FilePathType, + file_path: FilePath, sampling_frequency: float, verbose: bool = True, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", diff --git a/src/neuroconv/datainterfaces/text/csv/csvtimeintervalsinterface.py b/src/neuroconv/datainterfaces/text/csv/csvtimeintervalsinterface.py index 4904e1c19..cb1aecb2b 100644 --- a/src/neuroconv/datainterfaces/text/csv/csvtimeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/csv/csvtimeintervalsinterface.py @@ -1,7 +1,7 @@ import pandas as pd +from pydantic import FilePath from ..timeintervalsinterface import TimeIntervalsInterface -from ....utils.types import FilePathType class CsvTimeIntervalsInterface(TimeIntervalsInterface): @@ -11,5 +11,5 @@ class CsvTimeIntervalsInterface(TimeIntervalsInterface): associated_suffixes = (".csv",) info = "Interface for writing a time intervals table from a comma separated value (CSV) file." - def _read_file(self, file_path: FilePathType, **read_kwargs): + def _read_file(self, file_path: FilePath, **read_kwargs): return pd.read_csv(file_path, **read_kwargs) diff --git a/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py b/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py index 96a375d50..cb4ae2330 100644 --- a/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py @@ -1,9 +1,9 @@ from typing import Optional import pandas as pd +from pydantic import FilePath from ..timeintervalsinterface import TimeIntervalsInterface -from ....utils.types import FilePathType class ExcelTimeIntervalsInterface(TimeIntervalsInterface): @@ -15,7 +15,7 @@ class ExcelTimeIntervalsInterface(TimeIntervalsInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, read_kwargs: Optional[dict] = None, verbose: bool = True, ): @@ -29,5 +29,5 @@ def __init__( """ super().__init__(file_path=file_path, read_kwargs=read_kwargs, verbose=verbose) - def _read_file(self, file_path: FilePathType, **read_kwargs): + def _read_file(self, file_path: FilePath, **read_kwargs): return pd.read_excel(file_path, **read_kwargs) diff --git a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py index dd7f491f8..ad55852c5 100644 --- a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py @@ -3,12 +3,12 @@ from typing import Dict, Optional import numpy as np +from pydantic import FilePath from pynwb import NWBFile from ...basedatainterface import BaseDataInterface from ...tools.text import convert_df_to_time_intervals from ...utils.dict import load_dict_from_file -from ...utils.types import FilePathType class TimeIntervalsInterface(BaseDataInterface): @@ -18,7 +18,7 @@ class TimeIntervalsInterface(BaseDataInterface): def __init__( self, - file_path: FilePathType, + file_path: FilePath, read_kwargs: Optional[dict] = None, verbose: bool = True, ): @@ -152,5 +152,5 @@ def add_to_nwbfile( return nwbfile @abstractmethod - def _read_file(self, file_path: FilePathType, **read_kwargs): + def _read_file(self, file_path: FilePath, **read_kwargs): pass diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index 689ac6050..2f5074d6a 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -7,6 +7,7 @@ from typing import Dict, List, Literal, Optional, Tuple, Union from jsonschema import validate +from pydantic import FilePath from pynwb import NWBFile from .basedatainterface import BaseDataInterface @@ -173,7 +174,7 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optiona def run_conversion( self, - nwbfile_path: Optional[str] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, diff --git a/src/neuroconv/tools/data_transfers/_dandi.py b/src/neuroconv/tools/data_transfers/_dandi.py index 291cb5e11..f67f43197 100644 --- a/src/neuroconv/tools/data_transfers/_dandi.py +++ b/src/neuroconv/tools/data_transfers/_dandi.py @@ -7,15 +7,14 @@ from typing import List, Optional, Union from warnings import warn +from pydantic import DirectoryPath from pynwb import NWBHDF5IO -from ...utils import FolderPathType - def automatic_dandi_upload( dandiset_id: str, - nwb_folder_path: FolderPathType, - dandiset_folder_path: Optional[FolderPathType] = None, + nwb_folder_path: DirectoryPath, + dandiset_folder_path: Optional[DirectoryPath] = None, version: str = "draft", staging: bool = False, cleanup: bool = False, diff --git a/src/neuroconv/tools/neo/neo.py b/src/neuroconv/tools/neo/neo.py index bcb27a40b..8873359db 100644 --- a/src/neuroconv/tools/neo/neo.py +++ b/src/neuroconv/tools/neo/neo.py @@ -8,9 +8,9 @@ import neo.io.baseio import numpy as np import pynwb +from pydantic import FilePath from ..nwb_helpers import add_device_from_metadata -from ...utils import OptionalFilePathType response_classes = dict( voltage_clamp=pynwb.icephys.VoltageClampSeries, @@ -439,7 +439,7 @@ def add_neo_to_nwb( def write_neo_to_nwb( neo_reader: neo.io.baseio.BaseIO, - save_path: OptionalFilePathType = None, # pragma: no cover + save_path: Optional[FilePath] = None, # pragma: no cover overwrite: bool = False, nwbfile=None, metadata: dict = None, diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 3b8cac2ac..75c5a0642 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -9,6 +9,7 @@ # from hdmf.common import VectorData from hdmf.data_utils import DataChunkIterator +from pydantic import FilePath from pynwb import NWBFile from pynwb.base import Images from pynwb.device import Device @@ -35,7 +36,6 @@ from ..nwb_helpers import get_default_nwbfile_metadata, get_module, make_or_load_nwbfile from ...utils import ( DeepDict, - OptionalFilePathType, calculate_regular_series_rate, dict_deep_update, ) @@ -570,7 +570,7 @@ def add_imaging( def write_imaging( imaging: ImagingExtractor, - nwbfile_path: OptionalFilePathType = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -1447,7 +1447,7 @@ def add_segmentation( def write_segmentation( segmentation_extractor: SegmentationExtractor, - nwbfile_path: OptionalFilePathType = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 48a102bc5..c3ad28813 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -7,6 +7,7 @@ import psutil import pynwb from hdmf.data_utils import AbstractDataChunkIterator, DataChunkIterator +from pydantic import FilePath from spikeinterface import BaseRecording, BaseSorting, SortingAnalyzer from .spikeinterfacerecordingdatachunkiterator import ( @@ -15,7 +16,6 @@ from ..nwb_helpers import get_module, make_or_load_nwbfile from ...utils import ( DeepDict, - FilePathType, calculate_regular_series_rate, dict_deep_update, ) @@ -1130,7 +1130,7 @@ def add_recording_to_nwbfile( def write_recording( recording: BaseRecording, - nwbfile_path: Optional[FilePathType] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -1175,7 +1175,7 @@ def write_recording( def write_recording_to_nwbfile( recording: BaseRecording, - nwbfile_path: Optional[FilePathType] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -1196,7 +1196,7 @@ def write_recording_to_nwbfile( Parameters ---------- recording : spikeinterface.BaseRecording - nwbfile_path : FilePathType, optional + nwbfile_path : FilePath, optional Path for where to write or load (if overwrite=False) the NWBFile. If specified, the context will always write to this location. nwbfile : NWBFile, optional @@ -1730,7 +1730,7 @@ def add_sorting_to_nwbfile( def write_sorting( sorting: BaseSorting, - nwbfile_path: Optional[FilePathType] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -1777,7 +1777,7 @@ def write_sorting( def write_sorting_to_nwbfile( sorting: BaseSorting, - nwbfile_path: Optional[FilePathType] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -1799,7 +1799,7 @@ def write_sorting_to_nwbfile( Parameters ---------- sorting : spikeinterface.BaseSorting - nwbfile_path : FilePathType, optional + nwbfile_path : FilePath, optional Path for where to write or load (if overwrite=False) the NWBFile. If specified, the context will always write to this location. nwbfile : NWBFile, optional @@ -2011,7 +2011,7 @@ def add_sorting_analyzer_to_nwbfile( def write_sorting_analyzer( sorting_analyzer: SortingAnalyzer, - nwbfile_path: Optional[FilePathType] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -2056,7 +2056,7 @@ def write_sorting_analyzer( def write_sorting_analyzer_to_nwbfile( sorting_analyzer: SortingAnalyzer, - nwbfile_path: Optional[FilePathType] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, @@ -2083,7 +2083,7 @@ def write_sorting_analyzer_to_nwbfile( ---------- sorting_analyzer : spikeinterface.SortingAnalyzer The sorting analyzer object to be written to the NWBFile. - nwbfile_path : FilePathType + nwbfile_path : FilePath Path for where to write or load (if overwrite=False) the NWBFile. If specified, the context will always write to this location. nwbfile : NWBFile, optional @@ -2163,7 +2163,7 @@ def write_sorting_analyzer_to_nwbfile( # TODO: Remove February 2025 def write_waveforms( waveform_extractor, - nwbfile_path: Optional[FilePathType] = None, + nwbfile_path: Optional[FilePath] = None, nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, diff --git a/src/neuroconv/tools/testing/mock_ttl_signals.py b/src/neuroconv/tools/testing/mock_ttl_signals.py index 6b4cd5f22..936d16e7e 100644 --- a/src/neuroconv/tools/testing/mock_ttl_signals.py +++ b/src/neuroconv/tools/testing/mock_ttl_signals.py @@ -4,11 +4,12 @@ import numpy as np from numpy.typing import DTypeLike +from pydantic import DirectoryPath from pynwb import NWBHDF5IO, H5DataIO, TimeSeries from pynwb.testing.mock.file import mock_NWBFile from ..importing import is_package_installed -from ...utils import ArrayType, FolderPathType +from ...utils import ArrayType def _check_parameter_dtype_consistency( @@ -128,7 +129,7 @@ def generate_mock_ttl_signal( return trace -def regenerate_test_cases(folder_path: FolderPathType, regenerate_reference_images: bool = False): # pragma: no cover +def regenerate_test_cases(folder_path: DirectoryPath, regenerate_reference_images: bool = False): # pragma: no cover """ Regenerate the test cases of the file included in the main testing suite, which is frozen between breaking changes. diff --git a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py index 2c7e5b25c..10e33cbc8 100644 --- a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py +++ b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py @@ -5,9 +5,10 @@ import click from jsonschema import RefResolver, validate +from pydantic import DirectoryPath, FilePath from ...nwbconverter import NWBConverter -from ...utils import FilePathType, FolderPathType, dict_deep_update, load_dict_from_file +from ...utils import dict_deep_update, load_dict_from_file @click.command() @@ -45,9 +46,9 @@ def run_conversion_from_yaml_cli( def run_conversion_from_yaml( - specification_file_path: FilePathType, - data_folder_path: Optional[FolderPathType] = None, - output_folder_path: Optional[FolderPathType] = None, + specification_file_path: FilePath, + data_folder_path: Optional[DirectoryPath] = None, + output_folder_path: Optional[DirectoryPath] = None, overwrite: bool = False, ): """ diff --git a/src/neuroconv/utils/__init__.py b/src/neuroconv/utils/__init__.py index 9e80dbbac..1e2bca630 100644 --- a/src/neuroconv/utils/__init__.py +++ b/src/neuroconv/utils/__init__.py @@ -18,11 +18,6 @@ ) from .types import ( ArrayType, - FilePathType, - FloatType, - FolderPathType, IntType, OptionalArrayType, - OptionalFilePathType, - OptionalFolderPathType, ) diff --git a/src/neuroconv/utils/dict.py b/src/neuroconv/utils/dict.py index 0ab2b814e..0a92520f7 100644 --- a/src/neuroconv/utils/dict.py +++ b/src/neuroconv/utils/dict.py @@ -9,8 +9,7 @@ import numpy as np import yaml - -from .types import FilePathType +from pydantic import FilePath class NoDatesSafeLoader(yaml.SafeLoader): @@ -37,7 +36,7 @@ def remove_implicit_resolver(cls, tag_to_remove): NoDatesSafeLoader.remove_implicit_resolver("tag:yaml.org,2002:timestamp") -def load_dict_from_file(file_path: FilePathType) -> dict: +def load_dict_from_file(file_path: FilePath) -> dict: """Safely load metadata from .yml or .json files.""" file_path = Path(file_path) assert file_path.is_file(), f"{file_path} is not a file." diff --git a/src/neuroconv/utils/types.py b/src/neuroconv/utils/types.py index 67dc24c5b..fe476c8f8 100644 --- a/src/neuroconv/utils/types.py +++ b/src/neuroconv/utils/types.py @@ -1,12 +1,7 @@ -from pathlib import Path -from typing import Optional, TypeVar, Union +from typing import Optional, Union import numpy as np -FilePathType = TypeVar("FilePathType", str, Path) -FolderPathType = TypeVar("FolderPathType", str, Path) -OptionalFilePathType = Optional[FilePathType] -OptionalFolderPathType = Optional[FolderPathType] ArrayType = Union[list, np.ndarray] OptionalArrayType = Optional[ArrayType] FloatType = float diff --git a/tests/test_behavior/test_audio_interface.py b/tests/test_behavior/test_audio_interface.py index 9d61f3171..bdfcf1e52 100644 --- a/tests/test_behavior/test_audio_interface.py +++ b/tests/test_behavior/test_audio_interface.py @@ -10,17 +10,17 @@ from dateutil.tz import gettz from hdmf.testing import TestCase from numpy.testing import assert_array_equal +from pydantic import FilePath from pynwb import NWBHDF5IO from scipy.io.wavfile import read, write from neuroconv import NWBConverter from neuroconv.datainterfaces.behavior.audio.audiointerface import AudioInterface from neuroconv.tools.testing.data_interface_mixins import AudioInterfaceTestMixin -from neuroconv.utils import FilePathType def create_audio_files( - test_dir: FilePathType, + test_dir: FilePath, num_audio_files: int, sampling_rate: int, num_frames: int, From 8e20a25ad30609aefffc5ad6f9b56354ee5e6d10 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 21 Aug 2024 16:54:46 -0400 Subject: [PATCH 002/118] [Pydantic III] Use list/dict annotations (#1021) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Co-authored-by: CodyCBakerPhD --- CHANGELOG.md | 1 + src/neuroconv/basedatainterface.py | 6 +- .../behavior/audio/audiointerface.py | 8 +-- .../behavior/deeplabcut/_dlc_utils.py | 4 +- .../deeplabcut/deeplabcutdatainterface.py | 4 +- .../lightningpose/lightningposeconverter.py | 6 +- .../lightningposedatainterface.py | 4 +- .../behavior/neuralynx/nvt_utils.py | 8 +-- .../behavior/video/video_utils.py | 2 +- .../behavior/video/videodatainterface.py | 14 ++-- .../baserecordingextractorinterface.py | 10 +-- .../ecephys/basesortingextractorinterface.py | 8 +-- .../neuralynx/neuralynxdatainterface.py | 10 +-- .../neuroscope/neuroscopedatainterface.py | 4 +- .../openephys/openephysbinarydatainterface.py | 4 +- .../openephys/openephyslegacydatainterface.py | 4 +- .../ecephys/phy/phydatainterface.py | 2 +- .../ecephys/spikeglx/spikeglxconverter.py | 6 +- .../ecephys/spikeglx/spikeglxnidqinterface.py | 3 +- .../icephys/abf/abfdatainterface.py | 5 +- .../icephys/baseicephysinterface.py | 3 +- .../brukertiff/brukertiffdatainterface.py | 6 +- .../text/timeintervalsinterface.py | 6 +- src/neuroconv/nwbconverter.py | 18 ++--- .../tools/aws/_submit_aws_batch_job.py | 10 +-- src/neuroconv/tools/data_transfers/_dandi.py | 4 +- src/neuroconv/tools/data_transfers/_globus.py | 14 ++-- src/neuroconv/tools/hdmf.py | 11 ++-- src/neuroconv/tools/importing.py | 8 +-- src/neuroconv/tools/neo/neo.py | 8 +-- .../_configuration_models/_base_backend.py | 16 ++--- .../_configuration_models/_base_dataset_io.py | 24 +++---- .../_configuration_models/_hdf5_backend.py | 4 +- .../_configuration_models/_hdf5_dataset_io.py | 6 +- .../_configuration_models/_zarr_backend.py | 4 +- .../_configuration_models/_zarr_dataset_io.py | 12 ++-- src/neuroconv/tools/optogenetics.py | 4 +- src/neuroconv/tools/path_expansion.py | 6 +- .../imagingextractordatachunkiterator.py | 4 +- .../tools/spikeinterface/spikeinterface.py | 66 +++++++++---------- ...pikeinterfacerecordingdatachunkiterator.py | 6 +- .../testing/_mock/_mock_dataset_models.py | 24 +++---- .../tools/testing/data_interface_mixins.py | 6 +- .../tools/testing/mock_interfaces.py | 6 +- src/neuroconv/tools/testing/mock_probes.py | 4 +- src/neuroconv/tools/text.py | 6 +- src/neuroconv/utils/json_schema.py | 12 ++-- 47 files changed, 201 insertions(+), 210 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e0f198d0..b343771a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ * `BaseRecordingInterface` now calls default metadata when metadata is not passing mimicking `run_conversion` behavior. [PR #1012](https://github.com/catalystneuro/neuroconv/pull/1012) * Added `get_json_schema_from_method_signature` which constructs Pydantic models automatically from the signature of any function with typical annotation types used throughout NeuroConv. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016) * Replaced all interface annotations with Pydantic types. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017) +* Changed typehint collections (e.g. `List`) to standard collections (e.g. `list`). [PR #1021](https://github.com/catalystneuro/neuroconv/pull/1021) diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 0c9b9d813..4e0a0aac4 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -3,7 +3,7 @@ import uuid from abc import ABC, abstractmethod from pathlib import Path -from typing import Literal, Optional, Tuple, Union +from typing import Literal, Optional, Union from jsonschema.validators import validate from pydantic import FilePath @@ -30,8 +30,8 @@ class BaseDataInterface(ABC): """Abstract class defining the structure of all DataInterfaces.""" display_name: Union[str, None] = None - keywords: Tuple[str] = tuple() - associated_suffixes: Tuple[str] = tuple() + keywords: tuple[str] = tuple() + associated_suffixes: tuple[str] = tuple() info: Union[str, None] = None @classmethod diff --git a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py index 532ad5b42..d61cdf18b 100644 --- a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py +++ b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import List, Literal, Optional +from typing import Literal, Optional import numpy as np import scipy @@ -28,7 +28,7 @@ class AudioInterface(BaseTemporalAlignmentInterface): associated_suffixes = (".wav",) info = "Interface for writing audio recordings to an NWB file." - def __init__(self, file_paths: List[FilePath], verbose: bool = False): + def __init__(self, file_paths: list[FilePath], verbose: bool = False): """ Data interface for writing acoustic recordings to an NWB file. @@ -105,7 +105,7 @@ def get_original_timestamps(self) -> np.ndarray: def get_timestamps(self) -> Optional[np.ndarray]: raise NotImplementedError("The AudioInterface does not yet support timestamps.") - def set_aligned_timestamps(self, aligned_timestamps: List[np.ndarray]): + def set_aligned_timestamps(self, aligned_timestamps: list[np.ndarray]): raise NotImplementedError("The AudioInterface does not yet support timestamps.") def set_aligned_starting_time(self, aligned_starting_time: float): @@ -132,7 +132,7 @@ def set_aligned_starting_time(self, aligned_starting_time: float): "Please set them using 'set_aligned_segment_starting_times'." ) - def set_aligned_segment_starting_times(self, aligned_segment_starting_times: List[float]): + def set_aligned_segment_starting_times(self, aligned_segment_starting_times: list[float]): """ Align the individual starting time for each audio file in this interface relative to the common session start time. diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 39245c307..7608fffaa 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -2,7 +2,7 @@ import pickle import warnings from pathlib import Path -from typing import List, Optional, Union +from typing import Optional, Union import numpy as np import pandas as pd @@ -305,7 +305,7 @@ def add_subject_to_nwbfile( h5file: FilePath, individual_name: str, config_file: FilePath, - timestamps: Optional[Union[List, np.ndarray]] = None, + timestamps: Optional[Union[list, np.ndarray]] = None, pose_estimation_container_kwargs: Optional[dict] = None, ) -> NWBFile: """ diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index f35f3854c..c6850b555 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import List, Optional, Union +from typing import Optional, Union import numpy as np from pydantic import FilePath @@ -76,7 +76,7 @@ def get_timestamps(self) -> np.ndarray: "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface." ) - def set_aligned_timestamps(self, aligned_timestamps: Union[List, np.ndarray]): + def set_aligned_timestamps(self, aligned_timestamps: Union[list, np.ndarray]): """ Set aligned timestamps vector for DLC data with user defined timestamps diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py index 20c080a6e..114cff0dd 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import List, Optional +from typing import Optional from pydantic import FilePath from pynwb import NWBFile @@ -106,8 +106,8 @@ def add_to_nwbfile( reference_frame: Optional[str] = None, confidence_definition: Optional[str] = None, external_mode: bool = True, - starting_frames_original_videos: Optional[List[int]] = None, - starting_frames_labeled_videos: Optional[List[int]] = None, + starting_frames_original_videos: Optional[list[int]] = None, + starting_frames_labeled_videos: Optional[list[int]] = None, stub_test: bool = False, ): original_video_interface = self.data_interface_objects["OriginalVideo"] diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py index b9761b2c6..b87366109 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py @@ -2,7 +2,7 @@ from copy import deepcopy from datetime import datetime from pathlib import Path -from typing import Optional, Tuple +from typing import Optional import numpy as np from pydantic import FilePath @@ -116,7 +116,7 @@ def _load_source_data(self): pose_estimation_data = pd.read_csv(self.file_path, header=[0, 1, 2]) return pose_estimation_data - def _get_original_video_shape(self) -> Tuple[int, int]: + def _get_original_video_shape(self) -> tuple[int, int]: with self._vc(file_path=str(self.original_video_file_path)) as video: video_shape = video.get_frame_shape() # image size of the original video is in height x width diff --git a/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py b/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py index 99a22e577..8d6f4268f 100644 --- a/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py +++ b/src/neuroconv/datainterfaces/behavior/neuralynx/nvt_utils.py @@ -5,7 +5,7 @@ import os from datetime import datetime from shutil import copy -from typing import Dict, List, Union +from typing import Union import numpy as np from pydantic import FilePath @@ -26,7 +26,7 @@ ] -def read_header(filename: str) -> Dict[str, Union[str, datetime, float, int, List[int]]]: +def read_header(filename: str) -> dict[str, Union[str, datetime, float, int, list[int]]]: """ Parses a Neuralynx NVT File Header and returns it as a dictionary. @@ -83,7 +83,7 @@ def parse_bool(x): return out -def read_data(filename: str) -> Dict[str, np.ndarray]: +def read_data(filename: str) -> dict[str, np.ndarray]: """ Reads a NeuroLynx NVT file and returns its data. @@ -97,7 +97,7 @@ def read_data(filename: str) -> Dict[str, np.ndarray]: Returns ------- - Dict[str, np.ndarray] + dict[str, np.ndarray] Dictionary containing the parsed data. Raises diff --git a/src/neuroconv/datainterfaces/behavior/video/video_utils.py b/src/neuroconv/datainterfaces/behavior/video/video_utils.py index 78c66472a..a8f2412aa 100644 --- a/src/neuroconv/datainterfaces/behavior/video/video_utils.py +++ b/src/neuroconv/datainterfaces/behavior/video/video_utils.py @@ -222,7 +222,7 @@ def _get_frame_details(self): min_frame_size_mb = (math.prod(frame_shape) * self._get_dtype().itemsize) / 1e6 return min_frame_size_mb, frame_shape - def _get_data(self, selection: Tuple[slice]) -> np.ndarray: + def _get_data(self, selection: tuple[slice]) -> np.ndarray: start_frame = selection[0].start end_frame = selection[0].stop frames = np.empty(shape=[end_frame - start_frame, *self._maxshape[1:]]) diff --git a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py index ca651e597..dfb22deba 100644 --- a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py @@ -1,7 +1,7 @@ import warnings from copy import deepcopy from pathlib import Path -from typing import List, Literal, Optional +from typing import Literal, Optional import numpy as np import psutil @@ -30,7 +30,7 @@ class VideoInterface(BaseDataInterface): def __init__( self, - file_paths: List[FilePath], + file_paths: list[FilePath], verbose: bool = False, *, metadata_key_name: str = "Videos", @@ -104,7 +104,7 @@ def get_metadata(self): return metadata - def get_original_timestamps(self, stub_test: bool = False) -> List[np.ndarray]: + def get_original_timestamps(self, stub_test: bool = False) -> list[np.ndarray]: """ Retrieve the original unaltered timestamps for the data in this interface. @@ -159,7 +159,7 @@ def get_timing_type(self) -> Literal["starting_time and rate", "timestamps"]: "Please specify the temporal alignment of each video." ) - def get_timestamps(self, stub_test: bool = False) -> List[np.ndarray]: + def get_timestamps(self, stub_test: bool = False) -> list[np.ndarray]: """ Retrieve the timestamps for the data in this interface. @@ -176,7 +176,7 @@ def get_timestamps(self, stub_test: bool = False) -> List[np.ndarray]: """ return self._timestamps or self.get_original_timestamps(stub_test=stub_test) - def set_aligned_timestamps(self, aligned_timestamps: List[np.ndarray]): + def set_aligned_timestamps(self, aligned_timestamps: list[np.ndarray]): """ Replace all timestamps for this interface with those aligned to the common session start time. @@ -221,7 +221,7 @@ def set_aligned_starting_time(self, aligned_starting_time: float, stub_test: boo else: raise ValueError("There are no timestamps or starting times set to shift by a common value!") - def set_aligned_segment_starting_times(self, aligned_segment_starting_times: List[float], stub_test: bool = False): + def set_aligned_segment_starting_times(self, aligned_segment_starting_times: list[float], stub_test: bool = False): """ Align the individual starting time for each video (segment) in this interface relative to the common session start time. @@ -264,7 +264,7 @@ def add_to_nwbfile( metadata: Optional[dict] = None, stub_test: bool = False, external_mode: bool = True, - starting_frames: Optional[List[int]] = None, + starting_frames: Optional[list[int]] = None, chunk_data: bool = True, module_name: Optional[str] = None, module_description: Optional[str] = None, diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index b21210d77..354d78f80 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, Union +from typing import Literal, Optional, Union import numpy as np from pynwb import NWBFile @@ -106,7 +106,7 @@ def get_metadata(self) -> DeepDict: return metadata - def get_original_timestamps(self) -> Union[np.ndarray, List[np.ndarray]]: + def get_original_timestamps(self) -> Union[np.ndarray, list[np.ndarray]]: """ Retrieve the original unaltered timestamps for the data in this interface. @@ -128,7 +128,7 @@ def get_original_timestamps(self) -> Union[np.ndarray, List[np.ndarray]]: for segment_index in range(self._number_of_segments) ] - def get_timestamps(self) -> Union[np.ndarray, List[np.ndarray]]: + def get_timestamps(self) -> Union[np.ndarray, list[np.ndarray]]: """ Retrieve the timestamps for the data in this interface. @@ -152,7 +152,7 @@ def set_aligned_timestamps(self, aligned_timestamps: np.ndarray): self.recording_extractor.set_times(times=aligned_timestamps) - def set_aligned_segment_timestamps(self, aligned_segment_timestamps: List[np.ndarray]): + def set_aligned_segment_timestamps(self, aligned_segment_timestamps: list[np.ndarray]): """ Replace all timestamps for all segments in this interface with those aligned to the common session start time. @@ -185,7 +185,7 @@ def set_aligned_starting_time(self, aligned_starting_time: float): ] ) - def set_aligned_segment_starting_times(self, aligned_segment_starting_times: List[float]): + def set_aligned_segment_starting_times(self, aligned_segment_starting_times: list[float]): """ Align the starting time for each segment in this interface relative to the common session start time. diff --git a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py index 9c3c06849..b3cd25d24 100644 --- a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import List, Literal, Optional, Union +from typing import Literal, Optional, Union import numpy as np from pynwb import NWBFile @@ -83,7 +83,7 @@ def get_original_timestamps(self) -> np.ndarray: "Unable to fetch original timestamps for a SortingInterface since it relies upon an attached recording." ) - def get_timestamps(self) -> Union[np.ndarray, List[np.ndarray]]: + def get_timestamps(self) -> Union[np.ndarray, list[np.ndarray]]: if not self.sorting_extractor.has_recording(): raise NotImplementedError( "In order to align timestamps for a SortingInterface, it must have a recording " @@ -138,7 +138,7 @@ def set_aligned_timestamps(self, aligned_timestamps: np.ndarray): times=aligned_timestamps[segment_index], segment_index=segment_index ) - def set_aligned_segment_timestamps(self, aligned_segment_timestamps: List[np.ndarray]): + def set_aligned_segment_timestamps(self, aligned_segment_timestamps: list[np.ndarray]): """ Replace all timestamps for all segments in this interface with those aligned to the common session start time. @@ -182,7 +182,7 @@ def set_aligned_starting_time(self, aligned_starting_time: float): else: sorting_segment._t_start += aligned_starting_time - def set_aligned_segment_starting_times(self, aligned_segment_starting_times: List[float]): + def set_aligned_segment_starting_times(self, aligned_segment_starting_times: list[float]): """ Align the starting time for each segment in this interface relative to the common session start time. diff --git a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py index b999944e1..da0573249 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py @@ -1,5 +1,5 @@ import json -from typing import List, Optional +from typing import Optional import numpy as np from pydantic import DirectoryPath @@ -18,7 +18,7 @@ class NeuralynxRecordingInterface(BaseRecordingExtractorInterface): info = "Interface for Neuralynx recording data." @classmethod - def get_stream_names(cls, folder_path: DirectoryPath) -> List[str]: + def get_stream_names(cls, folder_path: DirectoryPath) -> list[str]: from spikeinterface.extractors import NeuralynxRecordingExtractor stream_names, _ = NeuralynxRecordingExtractor.get_streams(folder_path=folder_path) @@ -158,16 +158,16 @@ def extract_neo_header_metadata(neo_reader) -> dict: return common_header -def _dict_intersection(dict_list: List) -> dict: +def _dict_intersection(dict_list: list[dict]) -> dict: """ Intersect dict_list and return only common keys and values Parameters ---------- - dict_list: list of dicitionaries each representing a header + dict_list: list of dictionaries each representing a header Returns ------- dict: - Dictionary containing key-value pairs common to all input dicitionary_list + Dictionary containing key-value pairs common to all input dictionary_list """ # Collect keys appearing in all dictionaries diff --git a/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py index d3f9fdfb4..d68532a94 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py @@ -269,7 +269,7 @@ def __init__( self, folder_path: DirectoryPath, keep_mua_units: bool = True, - exclude_shanks: Optional[list] = None, + exclude_shanks: Optional[list[int]] = None, xml_file_path: Optional[FilePath] = None, verbose: bool = True, ): @@ -282,7 +282,7 @@ def __init__( Path to folder containing .clu and .res files. keep_mua_units : bool, default: True Optional. Whether to return sorted spikes from multi-unit activity. - exclude_shanks : list, optional + exclude_shanks : list of integers, optional List of indices to ignore. The set of all possible indices is chosen by default, extracted as the final integer of all the .res.%i and .clu.%i pairs. xml_file_path : FilePathType, optional diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py index ccbd3cbbd..371b96f94 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Optional from pydantic import DirectoryPath @@ -20,7 +20,7 @@ class OpenEphysBinaryRecordingInterface(BaseRecordingExtractorInterface): ExtractorName = "OpenEphysBinaryRecordingExtractor" @classmethod - def get_stream_names(cls, folder_path: DirectoryPath) -> List[str]: + def get_stream_names(cls, folder_path: DirectoryPath) -> list[str]: from spikeinterface.extractors import OpenEphysBinaryRecordingExtractor stream_names, _ = OpenEphysBinaryRecordingExtractor.get_streams(folder_path=folder_path) diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py index 3403705b1..0818257a7 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Optional +from typing import Optional from warnings import warn from pydantic import DirectoryPath @@ -19,7 +19,7 @@ class OpenEphysLegacyRecordingInterface(BaseRecordingExtractorInterface): info = "Interface for converting legacy OpenEphys recording data." @classmethod - def get_stream_names(cls, folder_path: DirectoryPath) -> List[str]: + def get_stream_names(cls, folder_path: DirectoryPath) -> list[str]: from spikeinterface.extractors import OpenEphysLegacyRecordingExtractor stream_names, _ = OpenEphysLegacyRecordingExtractor.get_streams(folder_path=folder_path) diff --git a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py index e5d250a6e..da5ebbfc2 100644 --- a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py @@ -27,7 +27,7 @@ def get_source_schema(cls) -> dict: def __init__( self, folder_path: DirectoryPath, - exclude_cluster_groups: Optional[list] = None, + exclude_cluster_groups: Optional[list[str]] = None, verbose: bool = True, ): """ diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py index 35a4d8881..9d40cde3d 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import List, Optional +from typing import Optional from pydantic import DirectoryPath @@ -28,7 +28,7 @@ def get_source_schema(cls): return source_schema @classmethod - def get_streams(cls, folder_path: DirectoryPath) -> List[str]: + def get_streams(cls, folder_path: DirectoryPath) -> list[str]: from spikeinterface.extractors import SpikeGLXRecordingExtractor return SpikeGLXRecordingExtractor.get_streams(folder_path=folder_path)[0] @@ -36,7 +36,7 @@ def get_streams(cls, folder_path: DirectoryPath) -> List[str]: def __init__( self, folder_path: DirectoryPath, - streams: Optional[List[str]] = None, + streams: Optional[list[str]] = None, verbose: bool = False, ): """ diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 7cff7f11f..3c0b886ec 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import List import numpy as np from pydantic import FilePath @@ -93,7 +92,7 @@ def get_metadata(self) -> dict: ] = "Raw acquisition traces from the NIDQ (.nidq.bin) channels." return metadata - def get_channel_names(self) -> List[str]: + def get_channel_names(self) -> list[str]: """Return a list of channel names as set in the recording extractor.""" return list(self.recording_extractor.get_channel_ids()) diff --git a/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py b/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py index 10ec3e8a3..5336b6116 100644 --- a/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py +++ b/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py @@ -1,7 +1,6 @@ import json from datetime import datetime, timedelta from pathlib import Path -from typing import List from warnings import warn from pydantic import FilePath @@ -52,7 +51,7 @@ def get_source_schema(cls) -> dict: return source_schema def __init__( - self, file_paths: List[FilePath], icephys_metadata: dict = None, icephys_metadata_file_path: FilePath = None + self, file_paths: list[FilePath], icephys_metadata: dict = None, icephys_metadata_file_path: FilePath = None ): """ ABF IcephysInterface based on Neo AxonIO. @@ -161,7 +160,7 @@ def set_aligned_starting_time(self, aligned_starting_time: float): reader._t_starts[segment_index] += aligned_starting_time def set_aligned_segment_starting_times( - self, aligned_segment_starting_times: List[List[float]], stub_test: bool = False + self, aligned_segment_starting_times: list[list[float]], stub_test: bool = False ): """ Align the individual starting time for each video in this interface relative to the common session start time. diff --git a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py index 341fc4c0d..ff82ba391 100644 --- a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py +++ b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py @@ -1,5 +1,4 @@ import importlib.util -from typing import Tuple import numpy as np from pynwb import NWBFile @@ -92,7 +91,7 @@ def add_to_nwbfile( nwbfile: NWBFile, metadata: dict = None, icephys_experiment_type: str = "voltage_clamp", - skip_electrodes: Tuple[int] = (), + skip_electrodes: tuple[int] = (), ): """ Primary function for converting raw (unprocessed) intracellular data to the NWB standard. diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py index d92676219..9742711e1 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional +from typing import Literal, Optional from dateutil.parser import parse from pydantic import DirectoryPath @@ -64,7 +64,7 @@ def __init__( self._stream_name = self.imaging_extractor.stream_name.replace("_", "") self._image_size = self.imaging_extractor.get_image_size() - def _determine_position_current(self) -> List[float]: + def _determine_position_current(self) -> list[float]: """ Returns y, x, and z position values. The unit of values is in the microscope reference frame. """ @@ -222,7 +222,7 @@ def __init__( self._stream_name = self.imaging_extractor.stream_name.replace("_", "") self._image_size = self.imaging_extractor.get_image_size() - def _determine_position_current(self) -> List[float]: + def _determine_position_current(self) -> list[float]: """ Returns y, x, and z position values. The unit of values is in the microscope reference frame. """ diff --git a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py index ad55852c5..b00779cb0 100644 --- a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py @@ -1,6 +1,6 @@ from abc import abstractmethod from pathlib import Path -from typing import Dict, Optional +from typing import Optional import numpy as np from pydantic import FilePath @@ -120,8 +120,8 @@ def add_to_nwbfile( nwbfile: NWBFile, metadata: Optional[dict] = None, tag: str = "trials", - column_name_mapping: Dict[str, str] = None, - column_descriptions: Dict[str, str] = None, + column_name_mapping: dict[str, str] = None, + column_descriptions: dict[str, str] = None, ) -> NWBFile: """ Run the NWB conversion for the instantiated data interface. diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index 2f5074d6a..ea7b7cd67 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -4,7 +4,7 @@ import warnings from collections import Counter from pathlib import Path -from typing import Dict, List, Literal, Optional, Tuple, Union +from typing import Literal, Optional, Union from jsonschema import validate from pydantic import FilePath @@ -36,8 +36,8 @@ class NWBConverter: """Primary class for all NWB conversion classes.""" display_name: Union[str, None] = None - keywords: Tuple[str] = tuple() - associated_suffixes: Tuple[str] = tuple() + keywords: tuple[str] = tuple() + associated_suffixes: tuple[str] = tuple() info: Union[str, None] = None data_interface_classes = None @@ -57,11 +57,11 @@ def get_source_schema(cls) -> dict: return source_schema @classmethod - def validate_source(cls, source_data: Dict[str, dict], verbose: bool = True): + def validate_source(cls, source_data: dict[str, dict], verbose: bool = True): """Validate source_data against Converter source_schema.""" cls._validate_source_data(source_data=source_data, verbose=verbose) - def _validate_source_data(self, source_data: Dict[str, dict], verbose: bool = True): + def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = True): encoder = NWBSourceDataEncoder() # The encoder produces a serialized object, so we deserialized it for comparison @@ -73,7 +73,7 @@ def _validate_source_data(self, source_data: Dict[str, dict], verbose: bool = Tr if verbose: print("Source data is valid!") - def __init__(self, source_data: Dict[str, dict], verbose: bool = True): + def __init__(self, source_data: dict[str, dict], verbose: bool = True): """Validate source_data against source_schema and initialize all data interfaces.""" self.verbose = verbose self._validate_source_data(source_data=source_data, verbose=self.verbose) @@ -102,7 +102,7 @@ def get_metadata(self) -> DeepDict: metadata = dict_deep_update(metadata, interface_metadata) return metadata - def validate_metadata(self, metadata: Dict[str, dict], append_mode: bool = False): + def validate_metadata(self, metadata: dict[str, dict], append_mode: bool = False): """Validate metadata against Converter metadata_schema.""" encoder = NWBMetaDataEncoder() # The encoder produces a serialized object, so we deserialized it for comparison @@ -135,7 +135,7 @@ def get_conversion_options_schema(self) -> dict: return conversion_options_schema - def validate_conversion_options(self, conversion_options: Dict[str, dict]): + def validate_conversion_options(self, conversion_options: dict[str, dict]): """Validate conversion_options against Converter conversion_options_schema.""" validate(instance=conversion_options or {}, schema=self.get_conversion_options_schema()) if self.verbose: @@ -289,7 +289,7 @@ def get_source_schema(cls) -> dict: def validate_source(cls): raise NotImplementedError("Source data not available with previously initialized classes.") - def __init__(self, data_interfaces: Union[List[BaseDataInterface], Dict[str, BaseDataInterface]], verbose=True): + def __init__(self, data_interfaces: Union[list[BaseDataInterface], dict[str, BaseDataInterface]], verbose=True): self.verbose = verbose if isinstance(data_interfaces, list): # Create unique names for each interface diff --git a/src/neuroconv/tools/aws/_submit_aws_batch_job.py b/src/neuroconv/tools/aws/_submit_aws_batch_job.py index 9b4dfe81a..0d36bee7f 100644 --- a/src/neuroconv/tools/aws/_submit_aws_batch_job.py +++ b/src/neuroconv/tools/aws/_submit_aws_batch_job.py @@ -4,7 +4,7 @@ import os import time from datetime import datetime -from typing import Dict, List, Optional +from typing import Optional from uuid import uuid4 @@ -12,9 +12,9 @@ def submit_aws_batch_job( *, job_name: str, docker_image: str, - commands: Optional[List[str]] = None, - environment_variables: Optional[Dict[str, str]] = None, - job_dependencies: Optional[List[Dict[str, str]]] = None, + commands: Optional[list[str]] = None, + environment_variables: Optional[dict[str, str]] = None, + job_dependencies: Optional[list[dict[str, str]]] = None, status_tracker_table_name: str = "neuroconv_batch_status_tracker", iam_role_name: str = "neuroconv_batch_role", compute_environment_name: str = "neuroconv_batch_environment", @@ -24,7 +24,7 @@ def submit_aws_batch_job( minimum_worker_cpus: int = 4, submission_id: Optional[str] = None, region: Optional[str] = None, -) -> Dict[str, str]: +) -> dict[str, str]: """ Submit a job to AWS Batch for processing. diff --git a/src/neuroconv/tools/data_transfers/_dandi.py b/src/neuroconv/tools/data_transfers/_dandi.py index f67f43197..11d38b0cb 100644 --- a/src/neuroconv/tools/data_transfers/_dandi.py +++ b/src/neuroconv/tools/data_transfers/_dandi.py @@ -4,7 +4,7 @@ from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -from typing import List, Optional, Union +from typing import Optional, Union from warnings import warn from pydantic import DirectoryPath @@ -20,7 +20,7 @@ def automatic_dandi_upload( cleanup: bool = False, number_of_jobs: Union[int, None] = None, number_of_threads: Union[int, None] = None, -) -> List[Path]: +) -> list[Path]: """ Fully automated upload of NWB files to a Dandiset. diff --git a/src/neuroconv/tools/data_transfers/_globus.py b/src/neuroconv/tools/data_transfers/_globus.py index 3429127f1..62bb654ed 100644 --- a/src/neuroconv/tools/data_transfers/_globus.py +++ b/src/neuroconv/tools/data_transfers/_globus.py @@ -4,7 +4,7 @@ import re from pathlib import Path from time import sleep, time -from typing import Dict, List, Tuple, Union +from typing import Union from pydantic import DirectoryPath from tqdm import tqdm @@ -15,7 +15,7 @@ def get_globus_dataset_content_sizes( globus_endpoint_id: str, path: str, recursive: bool = True, timeout: float = 120.0 -) -> Dict[str, int]: # pragma: no cover +) -> dict[str, int]: # pragma: no cover """ May require external login via 'globus login' from CLI. @@ -35,13 +35,13 @@ def get_globus_dataset_content_sizes( def transfer_globus_content( source_endpoint_id: str, - source_files: Union[str, List[List[str]]], + source_files: Union[str, list[list[str]]], destination_endpoint_id: str, destination_folder: DirectoryPath, display_progress: bool = True, progress_update_rate: float = 60.0, progress_update_timeout: float = 600.0, -) -> Tuple[bool, List[str]]: # pragma: no cover +) -> tuple[bool, list[str]]: # pragma: no cover """ Track progress for transferring content from source_endpoint_id to destination_endpoint_id:destination_folder. @@ -81,10 +81,10 @@ def transfer_globus_content( def _submit_transfer_request( source_endpoint_id: str, - source_files: Union[str, List[List[str]]], + source_files: Union[str, list[list[str]]], destination_endpoint_id: str, destination_folder_path: Path, - ) -> Dict[str, int]: + ) -> dict[str, int]: """Send transfer request to Globus.""" folder_content_sizes = dict() task_total_sizes = dict() @@ -134,7 +134,7 @@ def _submit_transfer_request( return task_total_sizes def _track_transfer( - task_total_sizes: Dict[str, int], + task_total_sizes: dict[str, int], display_progress: bool = True, progress_update_rate: float = 60.0, progress_update_timeout: float = 600.0, diff --git a/src/neuroconv/tools/hdmf.py b/src/neuroconv/tools/hdmf.py index 64db7b66e..e8dfff294 100644 --- a/src/neuroconv/tools/hdmf.py +++ b/src/neuroconv/tools/hdmf.py @@ -2,21 +2,20 @@ import math import warnings -from typing import Tuple import numpy as np from hdmf.data_utils import GenericDataChunkIterator as HDMFGenericDataChunkIterator class GenericDataChunkIterator(HDMFGenericDataChunkIterator): - def _get_default_buffer_shape(self, buffer_gb: float = 1.0) -> Tuple[int]: + def _get_default_buffer_shape(self, buffer_gb: float = 1.0) -> tuple[int]: return self.estimate_default_buffer_shape( buffer_gb=buffer_gb, chunk_shape=self.chunk_shape, maxshape=self.maxshape, dtype=self.dtype ) # TODO: move this to the core iterator in HDMF so it can be easily swapped out as well as run on its own @staticmethod - def estimate_default_chunk_shape(chunk_mb: float, maxshape: Tuple[int, ...], dtype: np.dtype) -> Tuple[int, ...]: + def estimate_default_chunk_shape(chunk_mb: float, maxshape: tuple[int, ...], dtype: np.dtype) -> tuple[int, ...]: """ Select chunk shape with size in MB less than the threshold of chunk_mb. @@ -47,8 +46,8 @@ def estimate_default_chunk_shape(chunk_mb: float, maxshape: Tuple[int, ...], dty # TODO: move this to the core iterator in HDMF so it can be easily swapped out as well as run on its own @staticmethod def estimate_default_buffer_shape( - buffer_gb: float, chunk_shape: Tuple[int, ...], maxshape: Tuple[int, ...], dtype: np.dtype - ) -> Tuple[int, ...]: + buffer_gb: float, chunk_shape: tuple[int, ...], maxshape: tuple[int, ...], dtype: np.dtype + ) -> tuple[int, ...]: # Elevate any overflow warnings to trigger error. # This is usually an indicator of something going terribly wrong with the estimation calculations and should be # avoided at all costs. @@ -149,5 +148,5 @@ def _get_dtype(self) -> np.dtype: def _get_maxshape(self) -> tuple: return self.data.shape - def _get_data(self, selection: Tuple[slice]) -> np.ndarray: + def _get_data(self, selection: tuple[slice]) -> np.ndarray: return self.data[selection] diff --git a/src/neuroconv/tools/importing.py b/src/neuroconv/tools/importing.py index 3b4e67b9d..04a3cbf21 100644 --- a/src/neuroconv/tools/importing.py +++ b/src/neuroconv/tools/importing.py @@ -6,7 +6,7 @@ from importlib.util import find_spec from platform import processor, python_version from types import ModuleType -from typing import Dict, List, Optional, Tuple, Union +from typing import Optional, Union from packaging import version @@ -41,8 +41,8 @@ def is_package_installed(package_name: str) -> bool: def get_package( package_name: str, installation_instructions: Optional[str] = None, - excluded_python_versions: Optional[List[str]] = None, - excluded_platforms_and_python_versions: Optional[Dict[str, Union[List[str], Dict[str, List[str]]]]] = None, + excluded_python_versions: Optional[list[str]] = None, + excluded_platforms_and_python_versions: Optional[dict[str, Union[list[str], dict[str, list[str]]]]] = None, ) -> ModuleType: """ Check if package is installed and return module if so. @@ -128,7 +128,7 @@ def get_package( ) -def get_format_summaries() -> Dict[str, Dict[str, Union[str, Tuple[str, ...], None]]]: +def get_format_summaries() -> dict[str, dict[str, Union[str, tuple[str, ...], None]]]: """Simple helper function for compiling high level summaries of all format interfaces and converters.""" # Local scope import to avoid circularity from ..converters import converter_list diff --git a/src/neuroconv/tools/neo/neo.py b/src/neuroconv/tools/neo/neo.py index 8873359db..220c64de0 100644 --- a/src/neuroconv/tools/neo/neo.py +++ b/src/neuroconv/tools/neo/neo.py @@ -3,7 +3,7 @@ import warnings from copy import deepcopy from pathlib import Path -from typing import Optional, Tuple +from typing import Optional import neo.io.baseio import numpy as np @@ -65,7 +65,7 @@ def get_number_of_segments(neo_reader, block: int = 0) -> int: return neo_reader.header["nb_segment"][block] -def get_command_traces(neo_reader, segment: int = 0, cmd_channel: int = 0) -> Tuple[list, str, str]: +def get_command_traces(neo_reader, segment: int = 0, cmd_channel: int = 0) -> tuple[list, str, str]: """ Get command traces (e.g. voltage clamp command traces). @@ -213,7 +213,7 @@ def add_icephys_recordings( metadata: dict = None, icephys_experiment_type: str = "voltage_clamp", stimulus_type: str = "not described", - skip_electrodes: Tuple[int] = (), + skip_electrodes: tuple[int] = (), compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 ): """ @@ -383,7 +383,7 @@ def add_neo_to_nwb( compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 icephys_experiment_type: str = "voltage_clamp", stimulus_type: Optional[str] = None, - skip_electrodes: Tuple[int] = (), + skip_electrodes: tuple[int] = (), ): """ Auxiliary static method for nwbextractor. diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_backend.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_backend.py index 2c07a1bb0..2b0cc507e 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_backend.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_backend.py @@ -1,6 +1,6 @@ """Base Pydantic models for DatasetInfo and DatasetConfiguration.""" -from typing import Any, ClassVar, Dict, Literal, Type +from typing import Any, ClassVar, Literal, Type from hdmf.container import DataIO from pydantic import BaseModel, ConfigDict, Field @@ -21,7 +21,7 @@ class BackendConfiguration(BaseModel): model_config = ConfigDict(validate_assignment=True) # Re-validate model on mutation - dataset_configurations: Dict[str, DatasetIOConfiguration] = Field( + dataset_configurations: dict[str, DatasetIOConfiguration] = Field( description=( "A mapping from object locations (e.g. `acquisition/TestElectricalSeriesAP/data`) " "to their DatasetConfiguration specification that contains all information " @@ -42,15 +42,15 @@ def __str__(self) -> str: # Pydantic models have several API calls for retrieving the schema - override all of them to work @classmethod - def schema(cls, **kwargs) -> Dict[str, Any]: + def schema(cls, **kwargs) -> dict[str, Any]: return cls.model_json_schema(**kwargs) @classmethod - def schema_json(cls, **kwargs) -> Dict[str, Any]: + def schema_json(cls, **kwargs) -> dict[str, Any]: return cls.model_json_schema(**kwargs) @classmethod - def model_json_schema(cls, **kwargs) -> Dict[str, Any]: + def model_json_schema(cls, **kwargs) -> dict[str, Any]: assert "mode" not in kwargs, "The 'mode' of this method is fixed to be 'validation' and cannot be changed." assert "schema_generator" not in kwargs, "The 'schema_generator' of this method cannot be changed." return super().model_json_schema(mode="validation", schema_generator=PureJSONSchemaGenerator, **kwargs) @@ -65,7 +65,7 @@ def from_nwbfile(cls, nwbfile: NWBFile) -> Self: return cls(dataset_configurations=dataset_configurations) - def find_locations_requiring_remapping(self, nwbfile: NWBFile) -> Dict[str, DatasetIOConfiguration]: + def find_locations_requiring_remapping(self, nwbfile: NWBFile) -> dict[str, DatasetIOConfiguration]: """ Find locations of objects with mismatched IDs in the file. @@ -80,7 +80,7 @@ def find_locations_requiring_remapping(self, nwbfile: NWBFile) -> Dict[str, Data Returns ------- - Dict[str, DatasetIOConfiguration] + dict[str, DatasetIOConfiguration] A dictionary where: * Keys: Locations in the NWB of objects with mismatched IDs. * Values: New `DatasetIOConfiguration` objects corresponding to the updated object IDs. @@ -127,7 +127,7 @@ def find_locations_requiring_remapping(self, nwbfile: NWBFile) -> Dict[str, Data def build_remapped_backend( self, - locations_to_remap: Dict[str, DatasetIOConfiguration], + locations_to_remap: dict[str, DatasetIOConfiguration], ) -> Self: """ Build a remapped backend configuration by updating mismatched object IDs. diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py index 01e291034..8b40e9a9e 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py @@ -2,7 +2,7 @@ import math from abc import ABC, abstractmethod -from typing import Any, Dict, List, Literal, Tuple, Union +from typing import Any, Literal, Union import h5py import numcodecs @@ -56,7 +56,7 @@ def _find_location_in_memory_nwbfile(neurodata_object: Container, field_name: st return _recursively_find_location_in_memory_nwbfile(current_location=field_name, neurodata_object=neurodata_object) -def _infer_dtype_of_list(list_: List[Union[int, float, list]]) -> np.dtype: +def _infer_dtype_of_list(list_: list[Union[int, float, list]]) -> np.dtype: """ Attempt to infer the dtype of values in an arbitrarily sized and nested list. @@ -103,16 +103,16 @@ class DatasetIOConfiguration(BaseModel, ABC): ) dataset_name: Literal["data", "timestamps"] = Field(description="The reference name of the dataset.", frozen=True) dtype: InstanceOf[np.dtype] = Field(description="The data type of elements of this dataset.", frozen=True) - full_shape: Tuple[int, ...] = Field(description="The maximum shape of the entire dataset.", frozen=True) + full_shape: tuple[int, ...] = Field(description="The maximum shape of the entire dataset.", frozen=True) # User specifiable fields - chunk_shape: Union[Tuple[PositiveInt, ...], None] = Field( + chunk_shape: Union[tuple[PositiveInt, ...], None] = Field( description=( "The specified shape to use when chunking the dataset. " "For optimized streaming speeds, a total size of around 10 MB is recommended." ), ) - buffer_shape: Union[Tuple[int, ...], None] = Field( + buffer_shape: Union[tuple[int, ...], None] = Field( description=( "The specified shape to use when iteratively loading data into memory while writing the dataset. " "For optimized writing speeds and minimal RAM usage, a total size of around 1 GB is recommended." @@ -123,12 +123,12 @@ class DatasetIOConfiguration(BaseModel, ABC): ] = Field( description="The specified compression method to apply to this dataset. Set to `None` to disable compression.", ) - compression_options: Union[Dict[str, Any], None] = Field( + compression_options: Union[dict[str, Any], None] = Field( default=None, description="The optional parameters to use for the specified compression method." ) @abstractmethod - def get_data_io_kwargs(self) -> Dict[str, Any]: + def get_data_io_kwargs(self) -> dict[str, Any]: """ Fetch the properly structured dictionary of input arguments. @@ -142,7 +142,7 @@ def __str__(self) -> str: Reason being two-fold; a standard `repr` is intended to be slightly more machine-readable / a more basic representation of the true object state. But then also because an iterable of these objects, such as a - `List[DatasetConfiguration]`, would print out the nested representations, which only look good when using the + `list[DatasetConfiguration]`, would print out the nested representations, which only look good when using the basic `repr` (that is, this fancy string print-out does not look good when nested in another container). """ size_in_bytes = math.prod(self.full_shape) * self.dtype.itemsize @@ -174,7 +174,7 @@ def __str__(self) -> str: return string @model_validator(mode="before") - def validate_all_shapes(cls, values: Dict[str, Any]) -> Dict[str, Any]: + def validate_all_shapes(cls, values: dict[str, Any]) -> dict[str, Any]: location_in_file = values["location_in_file"] dataset_name = values["dataset_name"] @@ -231,15 +231,15 @@ def validate_all_shapes(cls, values: Dict[str, Any]) -> Dict[str, Any]: # Pydantic models have several API calls for retrieving the schema - override all of them to work @classmethod - def schema(cls, **kwargs) -> Dict[str, Any]: + def schema(cls, **kwargs) -> dict[str, Any]: return cls.model_json_schema(**kwargs) @classmethod - def schema_json(cls, **kwargs) -> Dict[str, Any]: + def schema_json(cls, **kwargs) -> dict[str, Any]: return cls.model_json_schema(**kwargs) @classmethod - def model_json_schema(cls, **kwargs) -> Dict[str, Any]: + def model_json_schema(cls, **kwargs) -> dict[str, Any]: assert "mode" not in kwargs, "The 'mode' of this method is fixed to be 'validation' and cannot be changed." assert "schema_generator" not in kwargs, "The 'schema_generator' of this method cannot be changed." return super().model_json_schema(mode="validation", schema_generator=PureJSONSchemaGenerator, **kwargs) diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_backend.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_backend.py index f85d388b7..011b2e26d 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_backend.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_backend.py @@ -1,6 +1,6 @@ """Base Pydantic models for the HDF5DatasetConfiguration.""" -from typing import ClassVar, Dict, Literal, Type +from typing import ClassVar, Literal, Type from pydantic import Field from pynwb import H5DataIO @@ -16,7 +16,7 @@ class HDF5BackendConfiguration(BackendConfiguration): pretty_backend_name: ClassVar[Literal["HDF5"]] = "HDF5" data_io_class: ClassVar[Type[H5DataIO]] = H5DataIO - dataset_configurations: Dict[str, HDF5DatasetIOConfiguration] = Field( + dataset_configurations: dict[str, HDF5DatasetIOConfiguration] = Field( description=( "A mapping from object locations to their HDF5DatasetConfiguration specification that contains all " "information for writing the datasets to disk using the HDF5 backend." diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_dataset_io.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_dataset_io.py index 828a37998..44c7660ab 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_dataset_io.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_hdf5_dataset_io.py @@ -1,6 +1,6 @@ """Base Pydantic models for the HDF5DatasetConfiguration.""" -from typing import Any, Dict, Literal, Union +from typing import Any, Literal, Union import h5py from pydantic import Field, InstanceOf @@ -45,11 +45,11 @@ class HDF5DatasetIOConfiguration(DatasetIOConfiguration): ) # TODO: actually provide better schematic rendering of options. Only support defaults in GUIDE for now. # Looks like they'll have to be hand-typed however... Can try parsing the google docstrings - no annotation typing. - compression_options: Union[Dict[str, Any], None] = Field( + compression_options: Union[dict[str, Any], None] = Field( default=None, description="The optional parameters to use for the specified compression method." ) - def get_data_io_kwargs(self) -> Dict[str, Any]: + def get_data_io_kwargs(self) -> dict[str, Any]: if is_package_installed(package_name="hdf5plugin"): import hdf5plugin diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_backend.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_backend.py index abd5bdd67..ee26e0553 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_backend.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_backend.py @@ -1,6 +1,6 @@ """Base Pydantic models for the ZarrDatasetConfiguration.""" -from typing import ClassVar, Dict, Literal, Type +from typing import ClassVar, Literal, Type import psutil from hdmf_zarr import ZarrDataIO @@ -17,7 +17,7 @@ class ZarrBackendConfiguration(BackendConfiguration): pretty_backend_name: ClassVar[Literal["Zarr"]] = "Zarr" data_io_class: ClassVar[Type[ZarrDataIO]] = ZarrDataIO - dataset_configurations: Dict[str, ZarrDatasetIOConfiguration] = Field( + dataset_configurations: dict[str, ZarrDatasetIOConfiguration] = Field( description=( "A mapping from object locations to their ZarrDatasetConfiguration specification that contains all " "information for writing the datasets to disk using the Zarr backend." diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_dataset_io.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_dataset_io.py index c070a20e9..48b7c070b 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_dataset_io.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_zarr_dataset_io.py @@ -1,6 +1,6 @@ """Base Pydantic models for the ZarrDatasetConfiguration.""" -from typing import Any, Dict, List, Literal, Union +from typing import Any, Literal, Union import numcodecs import zarr @@ -58,11 +58,11 @@ class ZarrDatasetIOConfiguration(DatasetIOConfiguration): ) # TODO: actually provide better schematic rendering of options. Only support defaults in GUIDE for now. # Looks like they'll have to be hand-typed however... Can try parsing the numpy docstrings - no annotation typing. - compression_options: Union[Dict[str, Any], None] = Field( + compression_options: Union[dict[str, Any], None] = Field( default=None, description="The optional parameters to use for the specified compression method." ) filter_methods: Union[ - List[Union[Literal[tuple(AVAILABLE_ZARR_COMPRESSION_METHODS.keys())], InstanceOf[numcodecs.abc.Codec]]], None + list[Union[Literal[tuple(AVAILABLE_ZARR_COMPRESSION_METHODS.keys())], InstanceOf[numcodecs.abc.Codec]]], None ] = Field( default=None, description=( @@ -72,7 +72,7 @@ class ZarrDatasetIOConfiguration(DatasetIOConfiguration): "Set to `None` to disable filtering." ), ) - filter_options: Union[List[Dict[str, Any]], None] = Field( + filter_options: Union[list[dict[str, Any]], None] = Field( default=None, description="The optional parameters to use for each specified filter method." ) @@ -88,7 +88,7 @@ def __str__(self) -> str: # Inherited docstring from parent. noqa: D105 return string @model_validator(mode="before") - def validate_filter_methods_and_options_length_match(cls, values: Dict[str, Any]): + def validate_filter_methods_and_options_length_match(cls, values: dict[str, Any]): filter_methods = values.get("filter_methods", None) filter_options = values.get("filter_options", None) @@ -110,7 +110,7 @@ def validate_filter_methods_and_options_length_match(cls, values: Dict[str, Any] return values - def get_data_io_kwargs(self) -> Dict[str, Any]: + def get_data_io_kwargs(self) -> dict[str, Any]: filters = None if self.filter_methods: filters = list() diff --git a/src/neuroconv/tools/optogenetics.py b/src/neuroconv/tools/optogenetics.py index c4249f3ee..f8ed27015 100644 --- a/src/neuroconv/tools/optogenetics.py +++ b/src/neuroconv/tools/optogenetics.py @@ -1,5 +1,3 @@ -from typing import Tuple - import numpy as np @@ -10,7 +8,7 @@ def create_optogenetic_stimulation_timeseries( frequency: float, pulse_width: float, power: float, -) -> Tuple[np.ndarray, np.ndarray]: +) -> tuple[np.ndarray, np.ndarray]: """Create a continuous stimulation time series from stimulation onset times and parameters. In the resulting data array, the offset time of each pulse is represented by a 0 power value. diff --git a/src/neuroconv/tools/path_expansion.py b/src/neuroconv/tools/path_expansion.py index b2dd367f4..427a33a9e 100644 --- a/src/neuroconv/tools/path_expansion.py +++ b/src/neuroconv/tools/path_expansion.py @@ -4,7 +4,7 @@ import os from datetime import date, datetime from pathlib import Path -from typing import Dict, Iterable, List +from typing import Iterable from parse import parse from pydantic import DirectoryPath, FilePath @@ -34,7 +34,7 @@ def extract_metadata(self, base_directory: DirectoryPath, format_: str): Yields ------ - Tuple[Path, Dict[str, Any]] + tuple[Path, dict[str, Any]] A tuple containing the file path as a `Path` object and a dictionary of the named metadata extracted from the file path. """ @@ -67,7 +67,7 @@ def list_directory(self, base_directory: DirectoryPath) -> Iterable[FilePath]: """ pass - def expand_paths(self, source_data_spec: Dict[str, dict]) -> List[DeepDict]: + def expand_paths(self, source_data_spec: dict[str, dict]) -> list[DeepDict]: """ Match paths in a directory to specs and extract metadata from the paths. diff --git a/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py b/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py index 98001dd8e..0792e2caf 100644 --- a/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py +++ b/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py @@ -1,7 +1,7 @@ """General purpose iterator for all ImagingExtractor data.""" import math -from typing import Optional, Tuple +from typing import Optional import numpy as np from hdmf.data_utils import GenericDataChunkIterator @@ -138,7 +138,7 @@ def _get_maxshape(self) -> tuple: video_shape += (depth,) return video_shape - def _get_data(self, selection: Tuple[slice]) -> np.ndarray: + def _get_data(self, selection: tuple[slice]) -> np.ndarray: data = self.imaging_extractor.get_video( start_frame=selection[0].start, end_frame=selection[0].stop, diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index c3ad28813..262e1eaa8 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -1,7 +1,7 @@ import uuid import warnings from collections import defaultdict -from typing import Any, List, Literal, Optional, Union +from typing import Any, Literal, Optional, Union import numpy as np import psutil @@ -261,7 +261,7 @@ def _get_group_name(recording: BaseRecording) -> np.ndarray: return group_names -def _get_electrodes_table_global_ids(nwbfile: pynwb.NWBFile) -> List[str]: +def _get_electrodes_table_global_ids(nwbfile: pynwb.NWBFile) -> list[str]: """ Generate a list of global identifiers for channels in the electrode table of an NWB file. @@ -274,7 +274,7 @@ def _get_electrodes_table_global_ids(nwbfile: pynwb.NWBFile) -> List[str]: Returns ------- - List[str] + list[str] A list of unique keys, each representing a combination of channel name and group name from the electrodes table. If the electrodes table or the necessary columns are not present, an empty list is returned. @@ -293,7 +293,7 @@ def _get_electrodes_table_global_ids(nwbfile: pynwb.NWBFile) -> List[str]: return unique_keys -def _get_electrode_table_indices_for_recording(recording: BaseRecording, nwbfile: pynwb.NWBFile) -> List[int]: +def _get_electrode_table_indices_for_recording(recording: BaseRecording, nwbfile: pynwb.NWBFile) -> list[int]: """ Get the indices of the electrodes in the NWBFile that correspond to the channels in the recording. @@ -311,7 +311,7 @@ def _get_electrode_table_indices_for_recording(recording: BaseRecording, nwbfile Returns ------- - List[int] + list[int] A list of indices corresponding to the positions in the NWBFile's electrodes table that match the channels in the recording. """ @@ -1316,9 +1316,9 @@ def write_recording_to_nwbfile( def add_units_table( sorting: BaseSorting, nwbfile: pynwb.NWBFile, - unit_ids: Optional[List[Union[str, int]]] = None, + unit_ids: Optional[list[Union[str, int]]] = None, property_descriptions: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, units_table_name: str = "units", unit_table_description: str = "Autogenerated by neuroconv.", write_in_processing_module: bool = False, @@ -1355,9 +1355,9 @@ def add_units_table( def add_units_table_to_nwbfile( sorting: BaseSorting, nwbfile: pynwb.NWBFile, - unit_ids: Optional[List[Union[str, int]]] = None, + unit_ids: Optional[list[Union[str, int]]] = None, property_descriptions: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, units_table_name: str = "units", unit_table_description: Optional[str] = None, write_in_processing_module: bool = False, @@ -1614,10 +1614,10 @@ def add_units_table_to_nwbfile( def add_sorting( sorting: BaseSorting, nwbfile: Optional[pynwb.NWBFile] = None, - unit_ids: Optional[Union[List[str], List[int]]] = None, + unit_ids: Optional[Union[list[str], list[int]]] = None, property_descriptions: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, - skip_features: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, + skip_features: Optional[list[str]] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", units_description: str = "Autogenerated by neuroconv.", @@ -1653,10 +1653,10 @@ def add_sorting( def add_sorting_to_nwbfile( sorting: BaseSorting, nwbfile: Optional[pynwb.NWBFile] = None, - unit_ids: Optional[Union[List[str], List[int]]] = None, + unit_ids: Optional[Union[list[str], list[int]]] = None, property_descriptions: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, - skip_features: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, + skip_features: Optional[list[str]] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", units_description: str = "Autogenerated by neuroconv.", @@ -1735,10 +1735,10 @@ def write_sorting( metadata: Optional[dict] = None, overwrite: bool = False, verbose: bool = True, - unit_ids: Optional[List[Union[str, int]]] = None, + unit_ids: Optional[list[Union[str, int]]] = None, property_descriptions: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, - skip_features: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, + skip_features: Optional[list[str]] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", units_description: str = "Autogenerated by neuroconv.", @@ -1782,10 +1782,10 @@ def write_sorting_to_nwbfile( metadata: Optional[dict] = None, overwrite: bool = False, verbose: bool = True, - unit_ids: Optional[List[Union[str, int]]] = None, + unit_ids: Optional[list[Union[str, int]]] = None, property_descriptions: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, - skip_features: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, + skip_features: Optional[list[str]] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", units_description: str = "Autogenerated by neuroconv.", @@ -1868,8 +1868,8 @@ def add_sorting_analyzer( nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, recording: Optional[BaseRecording] = None, - unit_ids: Optional[Union[List[str], List[int]]] = None, - skip_properties: Optional[List[str]] = None, + unit_ids: Optional[Union[list[str], list[int]]] = None, + skip_properties: Optional[list[str]] = None, property_descriptions: Optional[dict] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", @@ -1903,8 +1903,8 @@ def add_sorting_analyzer_to_nwbfile( nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, recording: Optional[BaseRecording] = None, - unit_ids: Optional[Union[List[str], List[int]]] = None, - skip_properties: Optional[List[str]] = None, + unit_ids: Optional[Union[list[str], list[int]]] = None, + skip_properties: Optional[list[str]] = None, property_descriptions: Optional[dict] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", @@ -2017,10 +2017,10 @@ def write_sorting_analyzer( overwrite: bool = False, recording: Optional[BaseRecording] = None, verbose: bool = True, - unit_ids: Optional[Union[List[str], List[int]]] = None, + unit_ids: Optional[Union[list[str], list[int]]] = None, write_electrical_series: bool = False, add_electrical_series_kwargs: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, property_descriptions: Optional[dict] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", @@ -2062,10 +2062,10 @@ def write_sorting_analyzer_to_nwbfile( overwrite: bool = False, recording: Optional[BaseRecording] = None, verbose: bool = True, - unit_ids: Optional[Union[List[str], List[int]]] = None, + unit_ids: Optional[Union[list[str], list[int]]] = None, write_electrical_series: bool = False, add_electrical_series_kwargs: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, property_descriptions: Optional[dict] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", @@ -2169,10 +2169,10 @@ def write_waveforms( overwrite: bool = False, recording: Optional[BaseRecording] = None, verbose: bool = True, - unit_ids: Optional[List[Union[str, int]]] = None, + unit_ids: Optional[list[Union[str, int]]] = None, write_electrical_series: bool = False, add_electrical_series_kwargs: Optional[dict] = None, - skip_properties: Optional[List[str]] = None, + skip_properties: Optional[list[str]] = None, property_descriptions: Optional[dict] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", @@ -2196,8 +2196,8 @@ def add_waveforms( nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, recording: Optional[BaseRecording] = None, - unit_ids: Optional[List[Union[str, int]]] = None, - skip_properties: Optional[List[str]] = None, + unit_ids: Optional[list[Union[str, int]]] = None, + skip_properties: Optional[list[str]] = None, property_descriptions: Optional[dict] = None, write_as: Literal["units", "processing"] = "units", units_name: str = "units", diff --git a/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py b/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py index e6909a2e5..95b14fc23 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py @@ -1,4 +1,4 @@ -from typing import Iterable, Optional, Tuple +from typing import Iterable, Optional from hdmf.data_utils import GenericDataChunkIterator from spikeinterface import BaseRecording @@ -77,7 +77,7 @@ def __init__( progress_bar_options=progress_bar_options, ) - def _get_default_chunk_shape(self, chunk_mb: float = 10.0) -> Tuple[int, int]: + def _get_default_chunk_shape(self, chunk_mb: float = 10.0) -> tuple[int, int]: assert chunk_mb > 0, f"chunk_mb ({chunk_mb}) must be greater than zero!" chunk_channels = min( @@ -91,7 +91,7 @@ def _get_default_chunk_shape(self, chunk_mb: float = 10.0) -> Tuple[int, int]: return (chunk_frames, chunk_channels) - def _get_data(self, selection: Tuple[slice]) -> Iterable: + def _get_data(self, selection: tuple[slice]) -> Iterable: return self.recording.get_traces( segment_index=self.segment_index, channel_ids=self.channel_ids[selection[1]], diff --git a/src/neuroconv/tools/testing/_mock/_mock_dataset_models.py b/src/neuroconv/tools/testing/_mock/_mock_dataset_models.py index 77901f220..6f69f9e93 100644 --- a/src/neuroconv/tools/testing/_mock/_mock_dataset_models.py +++ b/src/neuroconv/tools/testing/_mock/_mock_dataset_models.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Iterable, Literal, Tuple, Union +from typing import Any, Iterable, Literal, Union import h5py import numcodecs @@ -18,14 +18,14 @@ def mock_HDF5DatasetIOConfiguration( object_id: str = "481a0860-3a0c-40ec-b931-df4a3e9b101f", location_in_file: str = "acquisition/TestElectricalSeries/data", dataset_name: Literal["data", "timestamps"] = "data", - full_shape: Tuple[int, ...] = (60 * 30_000, 384), # ~1 minute of v1 NeuroPixels probe + full_shape: tuple[int, ...] = (60 * 30_000, 384), # ~1 minute of v1 NeuroPixels probe dtype: np.dtype = np.dtype("int16"), - chunk_shape: Tuple[int, ...] = (78_125, 64), # ~10 MB - buffer_shape: Tuple[int, ...] = (1_250_000, 384), # ~1 GB + chunk_shape: tuple[int, ...] = (78_125, 64), # ~10 MB + buffer_shape: tuple[int, ...] = (1_250_000, 384), # ~1 GB compression_method: Union[ Literal[tuple(AVAILABLE_HDF5_COMPRESSION_METHODS.keys())], h5py._hl.filters.FilterRefBase, None ] = "gzip", - compression_options: Union[Dict[str, Any], None] = None, + compression_options: Union[dict[str, Any], None] = None, ) -> HDF5DatasetIOConfiguration: """Mock object of a HDF5DatasetIOConfiguration with NeuroPixel-like values to show chunk/buffer recommendations.""" return HDF5DatasetIOConfiguration( @@ -45,18 +45,18 @@ def mock_ZarrDatasetIOConfiguration( object_id: str = "481a0860-3a0c-40ec-b931-df4a3e9b101f", location_in_file: str = "acquisition/TestElectricalSeries/data", dataset_name: Literal["data", "timestamps"] = "data", - full_shape: Tuple[int, ...] = (60 * 30_000, 384), # ~1 minute of v1 NeuroPixels probe + full_shape: tuple[int, ...] = (60 * 30_000, 384), # ~1 minute of v1 NeuroPixels probe dtype: np.dtype = np.dtype("int16"), - chunk_shape: Tuple[int, ...] = (78_125, 64), # ~10 MB - buffer_shape: Tuple[int, ...] = (1_250_000, 384), # ~1 GB + chunk_shape: tuple[int, ...] = (78_125, 64), # ~10 MB + buffer_shape: tuple[int, ...] = (1_250_000, 384), # ~1 GB compression_method: Union[ Literal[tuple(AVAILABLE_ZARR_COMPRESSION_METHODS.keys())], numcodecs.abc.Codec, None ] = "gzip", - compression_options: Union[Dict[str, Any]] = None, + compression_options: Union[dict[str, Any]] = None, filter_methods: Iterable[ Union[Literal[tuple(AVAILABLE_ZARR_COMPRESSION_METHODS.keys())], numcodecs.abc.Codec, None] ] = None, - filter_options: Union[Iterable[Dict[str, Any]], None] = None, + filter_options: Union[Iterable[dict[str, Any]], None] = None, ) -> ZarrDatasetIOConfiguration: """Mock object of a ZarrDatasetIOConfiguration with NeuroPixel-like values to show chunk/buffer recommendations.""" return ZarrDatasetIOConfiguration( @@ -76,7 +76,7 @@ def mock_ZarrDatasetIOConfiguration( def mock_HDF5BackendConfiguration() -> HDF5BackendConfiguration: """Mock instance of a HDF5BackendConfiguration with two NeuroPixel-like datasets.""" - dataset_configurations: Dict[str, HDF5DatasetIOConfiguration] = { + dataset_configurations: dict[str, HDF5DatasetIOConfiguration] = { "acquisition/TestElectricalSeriesAP/data": mock_HDF5DatasetIOConfiguration( location_in_file="acquisition/TestElectricalSeriesAP/data", dataset_name="data" ), @@ -97,7 +97,7 @@ def mock_HDF5BackendConfiguration() -> HDF5BackendConfiguration: def mock_ZarrBackendConfiguration() -> ZarrBackendConfiguration: """Mock instance of a HDF5BackendConfiguration with several NeuroPixel-like datasets.""" - dataset_configurations: Dict[str, ZarrDatasetIOConfiguration] = { + dataset_configurations: dict[str, ZarrDatasetIOConfiguration] = { "acquisition/TestElectricalSeriesAP/data": mock_ZarrDatasetIOConfiguration( location_in_file="acquisition/TestElectricalSeriesAP/data", dataset_name="data", diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 578f3688c..b923851c2 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -5,7 +5,7 @@ from copy import deepcopy from datetime import datetime from pathlib import Path -from typing import List, Literal, Optional, Type, Union +from typing import Literal, Optional, Type, Union import numpy as np from hdmf.testing import TestCase as HDMFTestCase @@ -63,7 +63,7 @@ class DataInterfaceTestMixin: """ data_interface_cls: Type[BaseDataInterface] - interface_kwargs: Union[dict, List[dict]] + interface_kwargs: Union[dict, list[dict]] save_directory: Path = Path(tempfile.mkdtemp()) conversion_options: dict = dict() maxDiff = None @@ -260,7 +260,7 @@ class TemporalAlignmentMixin: """ data_interface_cls: Type[BaseDataInterface] - interface_kwargs: Union[dict, List[dict]] + interface_kwargs: Union[dict, list[dict]] maxDiff = None def setUpFreshInterface(self): diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 902e805f4..6f91e775f 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Literal, Optional +from typing import Literal, Optional import numpy as np from pynwb import NWBFile @@ -65,7 +65,7 @@ def get_source_schema(cls) -> dict: return source_schema def __init__( - self, signal_duration: float = 7.0, ttl_times: Optional[List[List[float]]] = None, ttl_duration: float = 1.0 + self, signal_duration: float = 7.0, ttl_times: Optional[list[list[float]]] = None, ttl_duration: float = 1.0 ): """ Define a mock SpikeGLXNIDQInterface by overriding the recording extractor to be a mock TTL signal. @@ -128,7 +128,7 @@ def __init__( self, num_channels: int = 4, sampling_frequency: float = 30_000.0, - # durations: Tuple[float] = (1.0,), # Uncomment when pydantic is integrated for schema validation + # durations: tuple[float] = (1.0,), # Uncomment when pydantic is integrated for schema validation durations: tuple = (1.0,), seed: int = 0, verbose: bool = True, diff --git a/src/neuroconv/tools/testing/mock_probes.py b/src/neuroconv/tools/testing/mock_probes.py index f5a3cea88..8b41d0f9c 100644 --- a/src/neuroconv/tools/testing/mock_probes.py +++ b/src/neuroconv/tools/testing/mock_probes.py @@ -1,5 +1,3 @@ -from typing import List - import numpy as np @@ -7,7 +5,7 @@ def generate_mock_probe(num_channels: int, num_shanks: int = 3): import probeinterface as pi # The shank ids will be 0, 0, 0, ..., 1, 1, 1, ..., 2, 2, 2, ... - shank_ids: List[int] = [] + shank_ids: list[int] = [] positions = np.zeros((num_channels, 2)) # ceil division channels_per_shank = (num_channels + num_shanks - 1) // num_shanks diff --git a/src/neuroconv/tools/text.py b/src/neuroconv/tools/text.py index 8c5b84410..b47ff2215 100644 --- a/src/neuroconv/tools/text.py +++ b/src/neuroconv/tools/text.py @@ -1,5 +1,3 @@ -from typing import Dict - import numpy as np import pandas as pd from pynwb.epoch import TimeIntervals @@ -9,8 +7,8 @@ def convert_df_to_time_intervals( df: pd.DataFrame, table_name: str = "trials", table_description: str = "experimental trials", - column_name_mapping: Dict[str, str] = None, - column_descriptions: Dict[str, str] = None, + column_name_mapping: dict[str, str] = None, + column_descriptions: dict[str, str] = None, ) -> TimeIntervals: """ Convert a dataframe to a TimeIntervals object. diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index e31a71c58..73aa97bdf 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -4,7 +4,7 @@ import warnings from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Optional import docstring_parser import hdmf.data_utils @@ -48,8 +48,8 @@ def get_base_schema( tag: Optional[str] = None, root: bool = False, id_: Optional[str] = None, - required: Optional[List] = None, - properties: Optional[Dict] = None, + required: Optional[list[str]] = None, + properties: Optional[dict] = None, **kwargs, ) -> dict: """Return the base schema used for all other schemas.""" @@ -69,7 +69,7 @@ def get_base_schema( return base_schema -def get_schema_from_method_signature(method: Callable, exclude: Optional[List[str]] = None) -> dict: +def get_schema_from_method_signature(method: Callable, exclude: Optional[list[str]] = None) -> dict: """Deprecated version of `get_json_schema_from_method_signature`.""" message = ( "The method `get_schema_from_method_signature` is now named `get_json_schema_from_method_signature`." @@ -80,7 +80,7 @@ def get_schema_from_method_signature(method: Callable, exclude: Optional[List[st return get_json_schema_from_method_signature(method=method, exclude=exclude) -def get_json_schema_from_method_signature(method: Callable, exclude: Optional[List[str]] = None) -> dict: +def get_json_schema_from_method_signature(method: Callable, exclude: Optional[list[str]] = None) -> dict: """ Get the equivalent JSON schema for a signature of a method. @@ -326,7 +326,7 @@ def get_metadata_schema_for_icephys(): return schema -def validate_metadata(metadata: Dict[str, dict], schema: Dict[str, dict], verbose: bool = False): +def validate_metadata(metadata: dict[str, dict], schema: dict[str, dict], verbose: bool = False): """Validate metadata against a schema.""" encoder = NWBMetaDataEncoder() # The encoder produces a serialized object, so we deserialized it for comparison From 11a45df35016d0061f5e8d76b81981421f0d65d8 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 22 Aug 2024 21:24:54 -0400 Subject: [PATCH 003/118] Remove more old typings (#1025) --- docs/user_guide/expand_path.rst | 1 - tests/test_minimal/test_tools/test_expand_paths.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/user_guide/expand_path.rst b/docs/user_guide/expand_path.rst index 0b746bee6..a27033d6f 100644 --- a/docs/user_guide/expand_path.rst +++ b/docs/user_guide/expand_path.rst @@ -17,7 +17,6 @@ expander will find all matching paths and automatically extract the specified me .. code-block:: python from pathlib import Path - from typing import Dict from neuroconv.tools.path_expansion import LocalPathExpander diff --git a/tests/test_minimal/test_tools/test_expand_paths.py b/tests/test_minimal/test_tools/test_expand_paths.py index ec1b15389..2667602d5 100644 --- a/tests/test_minimal/test_tools/test_expand_paths.py +++ b/tests/test_minimal/test_tools/test_expand_paths.py @@ -2,7 +2,6 @@ import unittest from datetime import datetime from pathlib import Path -from typing import List, Tuple import pytest from parse import parse @@ -14,7 +13,7 @@ def create_test_directories_and_files( - base_directory: Path, directories_and_files: List[Tuple[List[str], List[str]]] + base_directory: Path, directories_and_files: list[tuple[list[str], list[str]]] ) -> None: """ Create test directories and files in a way that is compatible across different @@ -24,7 +23,7 @@ def create_test_directories_and_files( ---------- base_directory : Path The base directory under which all subdirectories and files will be created. - directories_and_files : List[Tuple[List[str], List[str]]] + directories_and_files : list[tuple[list[str], list[str]]] A list where each element is a tuple. The first element of the tuple is a list of directory components, and the second element is a list of file names to be created in that directory. From bd1493cead2e781595d984fb3523fff63f59e872 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 23 Aug 2024 15:12:39 -0600 Subject: [PATCH 004/118] [testing] Re-organize data tests around the one dataset per test (#1026) --- CHANGELOG.md | 1 + .../tools/testing/data_interface_mixins.py | 297 +++++------ tests/test_behavior/test_audio_interface.py | 64 ++- .../test_mock_recording_interface.py | 8 +- .../test_on_data/test_behavior_interfaces.py | 265 +++++----- .../test_miniscope_converter.py | 16 +- tests/test_on_data/test_imaging_interfaces.py | 229 +++++---- .../test_on_data/test_recording_interfaces.py | 481 ++++++++++-------- .../test_segmentation_interfaces.py | 225 +++++--- tests/test_on_data/test_sorting_interfaces.py | 185 ++++--- 10 files changed, 970 insertions(+), 801 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b343771a8..1edae1bb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ * Added `get_json_schema_from_method_signature` which constructs Pydantic models automatically from the signature of any function with typical annotation types used throughout NeuroConv. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016) * Replaced all interface annotations with Pydantic types. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017) * Changed typehint collections (e.g. `List`) to standard collections (e.g. `list`). [PR #1021](https://github.com/catalystneuro/neuroconv/pull/1021) +* Testing now is only one dataset per test [PR #1026](https://github.com/catalystneuro/neuroconv/pull/1026) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index b923851c2..07e25bede 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -8,7 +8,7 @@ from typing import Literal, Optional, Type, Union import numpy as np -from hdmf.testing import TestCase as HDMFTestCase +import pytest from hdmf_zarr import NWBZarrIO from jsonschema.validators import Draft7Validator, validate from numpy.testing import assert_array_equal @@ -35,14 +35,11 @@ ) from neuroconv.utils import NWBMetaDataEncoder -from .mock_probes import generate_mock_probe - class DataInterfaceTestMixin: """ Generic class for testing DataInterfaces. - This mixin must be paired with unittest.TestCase. Several of these tests are required to be run in a specific order. In this case, there is a `test_conversion_as_lone_interface` that calls the `check` functions in @@ -63,11 +60,26 @@ class DataInterfaceTestMixin: """ data_interface_cls: Type[BaseDataInterface] - interface_kwargs: Union[dict, list[dict]] + interface_kwargs: dict save_directory: Path = Path(tempfile.mkdtemp()) - conversion_options: dict = dict() + conversion_options: Optional[dict] = None maxDiff = None + @pytest.fixture + def setup_interface(self, request): + + self.test_name: str = "" + self.conversion_options = self.conversion_options or dict() + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + + @pytest.fixture(scope="class", autouse=True) + def setup_default_conversion_options(self, request): + cls = request.cls + cls.conversion_options = cls.conversion_options or dict() + return cls.conversion_options + def test_source_schema_valid(self): schema = self.data_interface_cls.get_source_schema() Draft7Validator.check_schema(schema=schema) @@ -150,8 +162,7 @@ def check_run_conversion_in_nwbconverter_with_backend( class TestNWBConverter(NWBConverter): data_interface_classes = dict(Test=type(self.interface)) - test_kwargs = self.test_kwargs[0] if isinstance(self.test_kwargs, list) else self.test_kwargs - source_data = dict(Test=test_kwargs) + source_data = dict(Test=self.interface_kwargs) converter = TestNWBConverter(source_data=source_data) metadata = converter.get_metadata() @@ -174,8 +185,7 @@ def check_run_conversion_in_nwbconverter_with_backend_configuration( class TestNWBConverter(NWBConverter): data_interface_classes = dict(Test=type(self.interface)) - test_kwargs = self.test_kwargs[0] if isinstance(self.test_kwargs, list) else self.test_kwargs - source_data = dict(Test=test_kwargs) + source_data = dict(Test=self.interface_kwargs) converter = TestNWBConverter(source_data=source_data) metadata = converter.get_metadata() @@ -213,59 +223,59 @@ def run_custom_checks(self): """Override this in child classes to inject additional custom checks.""" pass - def test_all_conversion_checks(self): - interface_kwargs = self.interface_kwargs - if isinstance(interface_kwargs, dict): - interface_kwargs = [interface_kwargs] - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs - self.interface = self.data_interface_cls(**self.test_kwargs) + def test_all_conversion_checks(self, setup_interface, tmp_path): + interface, test_name = setup_interface - self.check_metadata_schema_valid() - self.check_conversion_options_schema_valid() - self.check_metadata() - self.nwbfile_path = str(self.save_directory / f"{self.__class__.__name__}_{num}.nwb") + # Create a unique test name and file path + nwbfile_path = str(tmp_path / f"{self.__class__.__name__}_{self.test_name}.nwb") + self.nwbfile_path = nwbfile_path - self.check_no_metadata_mutation() - - self.check_configure_backend_for_equivalent_nwbfiles() - - self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5") - self.check_run_conversion_in_nwbconverter_with_backend_configuration( - nwbfile_path=self.nwbfile_path, backend="hdf5" - ) + # Now run the checks using the setup objects + self.check_metadata_schema_valid() + self.check_conversion_options_schema_valid() + self.check_metadata() + self.check_no_metadata_mutation() + self.check_configure_backend_for_equivalent_nwbfiles() - self.check_run_conversion_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5") - self.check_run_conversion_with_backend_configuration(nwbfile_path=self.nwbfile_path, backend="hdf5") + self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") + self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - self.check_read_nwb(nwbfile_path=self.nwbfile_path) + self.check_run_conversion_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") + self.check_run_conversion_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - # TODO: enable when all H5DataIO prewraps are gone - # self.nwbfile_path = str(self.save_directory / f"{self.__class__.__name__}_{num}.nwb.zarr") - # self.check_run_conversion(nwbfile_path=self.nwbfile_path, backend="zarr") - # self.check_run_conversion_custom_backend(nwbfile_path=self.nwbfile_path, backend="zarr") - # self.check_basic_zarr_read(nwbfile_path=self.nwbfile_path) + self.check_read_nwb(nwbfile_path=nwbfile_path) - # Any extra custom checks to run - self.run_custom_checks() + # Any extra custom checks to run + self.run_custom_checks() class TemporalAlignmentMixin: """ Generic class for testing temporal alignment methods. - - This mixin must be paired with a unittest.TestCase class. """ data_interface_cls: Type[BaseDataInterface] - interface_kwargs: Union[dict, list[dict]] + interface_kwargs: dict + save_directory: Path = Path(tempfile.mkdtemp()) + conversion_options: Optional[dict] = None maxDiff = None + @pytest.fixture + def setup_interface(self, request): + + self.test_name: str = "" + self.interface = self.data_interface_cls(**self.interface_kwargs) + return self.interface, self.test_name + + @pytest.fixture(scope="class", autouse=True) + def setup_default_conversion_options(self, request): + cls = request.cls + cls.conversion_options = cls.conversion_options or dict() + return cls.conversion_options + def setUpFreshInterface(self): """Protocol for creating a fresh instance of the interface.""" - self.interface = self.data_interface_cls(**self.test_kwargs) + self.interface = self.data_interface_cls(**self.interface_kwargs) def check_interface_get_original_timestamps(self): """ @@ -330,22 +340,17 @@ def check_nwbfile_temporal_alignment(self): """Check the temporally aligned timing information makes it into the NWB file.""" pass # TODO: will be easier to add when interface have 'add' methods separate from .run_conversion() - def test_interface_alignment(self): - interface_kwargs = self.interface_kwargs - if isinstance(interface_kwargs, dict): - interface_kwargs = [interface_kwargs] - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs + def test_interface_alignment(self, setup_interface): - self.check_interface_get_original_timestamps() - self.check_interface_get_timestamps() - self.check_interface_set_aligned_timestamps() - self.check_shift_timestamps_by_start_time() - self.check_interface_original_timestamps_inmutability() + interface, test_name = setup_interface - self.check_nwbfile_temporal_alignment() + self.check_interface_get_original_timestamps() + self.check_interface_get_timestamps() + self.check_interface_set_aligned_timestamps() + self.check_shift_timestamps_by_start_time() + self.check_interface_original_timestamps_inmutability() + + self.check_nwbfile_temporal_alignment() class ImagingExtractorInterfaceTestMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): @@ -366,10 +371,11 @@ def check_read_nwb(self, nwbfile_path: str): def check_nwbfile_temporal_alignment(self): nwbfile_path = str( - self.save_directory / f"{self.data_interface_cls.__name__}_{self.case}_test_starting_time_alignment.nwb" + self.save_directory + / f"{self.data_interface_cls.__name__}_{self.test_name}_test_starting_time_alignment.nwb" ) - interface = self.data_interface_cls(**self.test_kwargs) + interface = self.data_interface_cls(**self.interface_kwargs) aligned_starting_time = 1.23 interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) @@ -399,10 +405,6 @@ def check_read(self, nwbfile_path: str): class RecordingExtractorInterfaceTestMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): """ Generic class for testing any recording interface. - - Runs all the basic DataInterface tests as well as temporal alignment tests. - - This mixin must be paired with a hdmf.testing.TestCase class. """ data_interface_cls: Type[BaseRecordingExtractorInterface] @@ -482,12 +484,9 @@ def check_interface_set_aligned_timestamps(self): retrieved_aligned_timestamps = self.interface.get_timestamps() assert_array_equal(x=retrieved_aligned_timestamps, y=aligned_timestamps) else: - assert isinstance( - self, HDMFTestCase - ), "The RecordingExtractorInterfaceTestMixin must be mixed-in with the TestCase from hdmf.testing!" - with self.assertRaisesWith( - exc_type=AssertionError, - exc_msg="This recording has multiple segments; please use 'align_segment_timestamps' instead.", + with pytest.raises( + AssertionError, + match="This recording has multiple segments; please use 'align_segment_timestamps' instead.", ): all_unaligned_timestamps = self.interface.get_timestamps() @@ -590,74 +589,32 @@ def check_interface_original_timestamps_inmutability(self): post_alignment_original_timestamps = self.interface.get_original_timestamps() assert_array_equal(x=post_alignment_original_timestamps, y=pre_alignment_original_timestamps) else: - assert isinstance( - self, HDMFTestCase - ), "The RecordingExtractorInterfaceTestMixin must be mixed-in with the TestCase from hdmf.testing!" - with self.assertRaisesWith( - exc_type=AssertionError, - exc_msg="This recording has multiple segments; please use 'align_segment_timestamps' instead.", + with pytest.raises( + AssertionError, + match="This recording has multiple segments; please use 'align_segment_timestamps' instead.", ): - all_pre_alignement_timestamps = self.interface.get_original_timestamps() + all_pre_alignment_timestamps = self.interface.get_original_timestamps() all_aligned_timestamps = [ - unaligned_timestamps + 1.23 for unaligned_timestamps in all_pre_alignement_timestamps + unaligned_timestamps + 1.23 for unaligned_timestamps in all_pre_alignment_timestamps ] self.interface.set_aligned_timestamps(aligned_timestamps=all_aligned_timestamps) - def test_interface_alignment(self): - interface_kwargs = self.interface_kwargs - if isinstance(interface_kwargs, dict): - interface_kwargs = [interface_kwargs] - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs + def test_interface_alignment(self, setup_interface): - self.check_interface_get_original_timestamps() - self.check_interface_get_timestamps() - self.check_interface_set_aligned_timestamps() - self.check_interface_set_aligned_segment_timestamps() - self.check_shift_timestamps_by_start_time() - self.check_shift_segment_timestamps_by_starting_times() - self.check_interface_original_timestamps_inmutability() + interface, test_name = setup_interface - self.check_nwbfile_temporal_alignment() + self.check_interface_get_original_timestamps() + self.check_interface_get_timestamps() + self.check_interface_set_aligned_timestamps() + self.check_shift_timestamps_by_start_time() + self.check_interface_original_timestamps_inmutability() - def test_all_conversion_checks(self): - interface_kwargs = self.interface_kwargs - if isinstance(interface_kwargs, dict): - interface_kwargs = [interface_kwargs] - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs - self.interface = self.data_interface_cls(**self.test_kwargs) - assert isinstance(self.interface, BaseRecordingExtractorInterface) - if not self.interface.has_probe(): - self.interface.set_probe( - generate_mock_probe(num_channels=self.interface.recording_extractor.get_num_channels()), - group_mode="by_shank", - ) + self.check_interface_set_aligned_segment_timestamps() + self.check_shift_timestamps_by_start_time() + self.check_shift_segment_timestamps_by_starting_times() - self.check_metadata_schema_valid() - self.check_conversion_options_schema_valid() - self.check_metadata() - self.nwbfile_path = str(self.save_directory / f"{self.__class__.__name__}_{num}.nwb") - - self.check_no_metadata_mutation() - - self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5") - self.check_run_conversion_in_nwbconverter_with_backend_configuration( - nwbfile_path=self.nwbfile_path, backend="hdf5" - ) - - self.check_run_conversion_with_backend(nwbfile_path=self.nwbfile_path, backend="hdf5") - self.check_run_conversion_with_backend_configuration(nwbfile_path=self.nwbfile_path, backend="hdf5") - - self.check_read_nwb(nwbfile_path=self.nwbfile_path) - - # Any extra custom checks to run - self.run_custom_checks() + self.check_nwbfile_temporal_alignment() class SortingExtractorInterfaceTestMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): @@ -666,7 +623,7 @@ class SortingExtractorInterfaceTestMixin(DataInterfaceTestMixin, TemporalAlignme associated_recording_kwargs: Optional[dict] = None def setUpFreshInterface(self): - self.interface = self.data_interface_cls(**self.test_kwargs) + self.interface = self.data_interface_cls(**self.interface_kwargs) recording_interface = self.associated_recording_cls(**self.associated_recording_kwargs) self.interface.register_recording(recording_interface=recording_interface) @@ -768,26 +725,45 @@ def check_shift_segment_timestamps_by_starting_times(self): ): assert_array_equal(x=retrieved_aligned_timestamps, y=expected_aligned_timestamps) - def test_interface_alignment(self): - interface_kwargs = self.interface_kwargs - if isinstance(interface_kwargs, dict): - interface_kwargs = [interface_kwargs] - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs + def test_all_conversion_checks(self, setup_interface, tmp_path): + # The fixture `setup_interface` sets up the necessary objects + interface, test_name = setup_interface - if self.associated_recording_cls is None: - continue + # Create a unique test name and file path + nwbfile_path = str(tmp_path / f"{self.__class__.__name__}_{self.test_name}.nwb") - # Skip get_original_timestamps() checks since unsupported - self.check_interface_get_timestamps() - self.check_interface_set_aligned_timestamps() - self.check_interface_set_aligned_segment_timestamps() - self.check_shift_timestamps_by_start_time() - self.check_shift_segment_timestamps_by_starting_times() + # Now run the checks using the setup objects + self.check_metadata_schema_valid() + self.check_conversion_options_schema_valid() + self.check_metadata() + self.check_no_metadata_mutation() + self.check_configure_backend_for_equivalent_nwbfiles() - self.check_nwbfile_temporal_alignment() + self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") + self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") + + self.check_run_conversion_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") + self.check_run_conversion_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") + + self.check_read_nwb(nwbfile_path=nwbfile_path) + + # Any extra custom checks to run + self.run_custom_checks() + + def test_interface_alignment(self, setup_interface): + + # TODO sorting can have times without associated recordings, test this later + if self.associated_recording_cls is None: + return None + + # Skip get_original_timestamps() checks since unsupported + self.check_interface_get_timestamps() + self.check_interface_set_aligned_timestamps() + self.check_interface_set_aligned_segment_timestamps() + self.check_shift_timestamps_by_start_time() + self.check_shift_segment_timestamps_by_starting_times() + + self.check_nwbfile_temporal_alignment() class AudioInterfaceTestMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): @@ -824,7 +800,7 @@ class VideoInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() - video_type = Path(self.test_kwargs["file_paths"][0]).suffix[1:] + video_type = Path(self.interface_kwargs["file_paths"][0]).suffix[1:] assert f"Video: video_{video_type}" in nwbfile.acquisition def check_interface_set_aligned_timestamps(self): @@ -858,7 +834,9 @@ def check_shift_timestamps_by_start_time(self): def check_set_aligned_segment_starting_times(self): self.setUpFreshInterface() - aligned_segment_starting_times = [1.23 * file_path_index for file_path_index in range(len(self.test_kwargs))] + aligned_segment_starting_times = [ + 1.23 * file_path_index for file_path_index in range(len(self.interface_kwargs)) + ] self.interface.set_aligned_segment_starting_times(aligned_segment_starting_times=aligned_segment_starting_times) all_aligned_timestamps = self.interface.get_timestamps() @@ -887,27 +865,6 @@ def check_interface_original_timestamps_inmutability(self): ): assert_array_equal(x=post_alignment_original_timestamps, y=pre_alignment_original_timestamps) - def check_nwbfile_temporal_alignment(self): - pass # TODO in separate PR - - def test_interface_alignment(self): - interface_kwargs = self.interface_kwargs - if isinstance(interface_kwargs, dict): - interface_kwargs = [interface_kwargs] - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs - - self.check_interface_get_original_timestamps() - self.check_interface_get_timestamps() - self.check_interface_set_aligned_timestamps() - self.check_shift_timestamps_by_start_time() - self.check_interface_original_timestamps_inmutability() - self.check_set_aligned_segment_starting_times() - - self.check_nwbfile_temporal_alignment() - class MedPCInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): def check_no_metadata_mutation(self, metadata: dict): diff --git a/tests/test_behavior/test_audio_interface.py b/tests/test_behavior/test_audio_interface.py index bdfcf1e52..4fad4fe3d 100644 --- a/tests/test_behavior/test_audio_interface.py +++ b/tests/test_behavior/test_audio_interface.py @@ -1,14 +1,12 @@ -import shutil +import re from copy import deepcopy from datetime import datetime from pathlib import Path -from tempfile import mkdtemp -from warnings import warn import jsonschema import numpy as np +import pytest from dateutil.tz import gettz -from hdmf.testing import TestCase from numpy.testing import assert_array_equal from pydantic import FilePath from pynwb import NWBHDF5IO @@ -38,38 +36,39 @@ def create_audio_files( return audio_file_names -class TestAudioInterface(AudioInterfaceTestMixin, TestCase): - @classmethod - def setUpClass(cls): +class TestAudioInterface(AudioInterfaceTestMixin): + + data_interface_cls = AudioInterface + + @pytest.fixture(scope="class", autouse=True) + def setup_test(self, request, tmp_path_factory): + + cls = request.cls + cls.session_start_time = datetime.now(tz=gettz(name="US/Pacific")) cls.num_frames = int(1e7) cls.num_audio_files = 3 cls.sampling_rate = 500 cls.aligned_segment_starting_times = [0.0, 20.0, 40.0] - cls.test_dir = Path(mkdtemp()) + class_tmp_dir = tmp_path_factory.mktemp("class_tmp_dir") + cls.test_dir = Path(class_tmp_dir) cls.file_paths = create_audio_files( test_dir=cls.test_dir, num_audio_files=cls.num_audio_files, sampling_rate=cls.sampling_rate, num_frames=cls.num_frames, ) - cls.data_interface_cls = AudioInterface cls.interface_kwargs = dict(file_paths=[cls.file_paths[0]]) - def setUp(self): + @pytest.fixture(scope="function", autouse=True) + def setup_converter(self): + self.nwbfile_path = str(self.test_dir / "audio_test.nwb") self.create_audio_converter() self.metadata = self.nwb_converter.get_metadata() self.metadata["NWBFile"].update(session_start_time=self.session_start_time) - @classmethod - def tearDownClass(cls): - try: - shutil.rmtree(cls.test_dir) - except PermissionError: # Windows CI bug - warn(f"Unable to fully clean the temporary directory: {cls.test_dir}\n\nPlease remove it manually.") - def create_audio_converter(self): class AudioTestNWBConverter(NWBConverter): data_interface_classes = dict(Audio=AudioInterface) @@ -83,7 +82,7 @@ class AudioTestNWBConverter(NWBConverter): def test_unsupported_format(self): exc_msg = "The currently supported file format for audio is WAV file. Some of the provided files does not match this format: ['.test']." - with self.assertRaisesWith(ValueError, exc_msg=exc_msg): + with pytest.raises(ValueError, match=re.escape(exc_msg)): AudioInterface(file_paths=["test.test"]) def test_get_metadata(self): @@ -91,10 +90,10 @@ def test_get_metadata(self): metadata = audio_interface.get_metadata() audio_metadata = metadata["Behavior"]["Audio"] - self.assertEqual(len(audio_metadata), self.num_audio_files) + assert len(audio_metadata) == self.num_audio_files def test_incorrect_write_as(self): - with self.assertRaises(jsonschema.exceptions.ValidationError): + with pytest.raises(jsonschema.exceptions.ValidationError): self.nwb_converter.run_conversion( nwbfile_path=self.nwbfile_path, metadata=self.metadata, @@ -125,7 +124,7 @@ def test_incomplete_metadata(self): expected_error_message = ( "The Audio metadata is incomplete (1 entry)! Expected 3 (one for each entry of 'file_paths')." ) - with self.assertRaisesWith(exc_type=AssertionError, exc_msg=expected_error_message): + with pytest.raises(AssertionError, match=re.escape(expected_error_message)): self.nwb_converter.run_conversion(nwbfile_path=self.nwbfile_path, metadata=metadata, overwrite=True) def test_metadata_update(self): @@ -137,7 +136,7 @@ def test_metadata_update(self): nwbfile = io.read() container = nwbfile.stimulus audio_name = metadata["Behavior"]["Audio"][0]["name"] - self.assertEqual("New description for Acoustic waveform series.", container[audio_name].description) + assert container[audio_name].description == "New description for Acoustic waveform series." def test_not_all_metadata_are_unique(self): metadata = deepcopy(self.metadata) @@ -149,21 +148,18 @@ def test_not_all_metadata_are_unique(self): ], ) expected_error_message = "Some of the names for Audio metadata are not unique." - with self.assertRaisesWith(exc_type=AssertionError, exc_msg=expected_error_message): + with pytest.raises(AssertionError, match=re.escape(expected_error_message)): self.interface.run_conversion(nwbfile_path=self.nwbfile_path, metadata=metadata, overwrite=True) def test_segment_starting_times_are_floats(self): - with self.assertRaisesWith( - exc_type=AssertionError, exc_msg="Argument 'aligned_segment_starting_times' must be a list of floats." - ): + with pytest.raises(AssertionError, match="Argument 'aligned_segment_starting_times' must be a list of floats."): self.interface.set_aligned_segment_starting_times(aligned_segment_starting_times=[0, 1, 2]) def test_segment_starting_times_length_mismatch(self): - with self.assertRaisesWith( - exc_type=AssertionError, - exc_msg="The number of entries in 'aligned_segment_starting_times' (4) must be equal to the number of audio file paths (3).", - ): + with pytest.raises(AssertionError) as exc_info: self.interface.set_aligned_segment_starting_times(aligned_segment_starting_times=[0.0, 1.0, 2.0, 4.0]) + exc_msg = "The number of entries in 'aligned_segment_starting_times' (4) must be equal to the number of audio file paths (3)." + assert str(exc_info.value) == exc_msg def test_set_aligned_segment_starting_times(self): fresh_interface = AudioInterface(file_paths=self.file_paths[:2]) @@ -210,12 +206,10 @@ def test_run_conversion(self): nwbfile = io.read() container = nwbfile.stimulus metadata = self.nwb_converter.get_metadata() - self.assertEqual(3, len(container)) + assert len(container) == 3 for audio_ind, audio_metadata in enumerate(metadata["Behavior"]["Audio"]): audio_interface_name = audio_metadata["name"] assert audio_interface_name in container - self.assertEqual( - self.aligned_segment_starting_times[audio_ind], container[audio_interface_name].starting_time - ) - self.assertEqual(self.sampling_rate, container[audio_interface_name].rate) + assert self.aligned_segment_starting_times[audio_ind] == container[audio_interface_name].starting_time + assert self.sampling_rate == container[audio_interface_name].rate assert_array_equal(audio_test_data[audio_ind], container[audio_interface_name].data) diff --git a/tests/test_ecephys/test_mock_recording_interface.py b/tests/test_ecephys/test_mock_recording_interface.py index d7dfc5714..a33f3acd1 100644 --- a/tests/test_ecephys/test_mock_recording_interface.py +++ b/tests/test_ecephys/test_mock_recording_interface.py @@ -1,13 +1,9 @@ -import unittest - from neuroconv.tools.testing.data_interface_mixins import ( RecordingExtractorInterfaceTestMixin, ) from neuroconv.tools.testing.mock_interfaces import MockRecordingInterface -class TestMockRecordingInterface(unittest.TestCase, RecordingExtractorInterfaceTestMixin): +class TestMockRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MockRecordingInterface - interface_kwargs = [ - dict(durations=[0.100]), - ] + interface_kwargs = dict(durations=[0.100]) diff --git a/tests/test_on_data/test_behavior_interfaces.py b/tests/test_on_data/test_behavior_interfaces.py index 1d25aaf36..33d0d468b 100644 --- a/tests/test_on_data/test_behavior_interfaces.py +++ b/tests/test_on_data/test_behavior_interfaces.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +import pytest import sleap_io from hdmf.testing import TestCase from natsort import natsorted @@ -41,7 +42,7 @@ from setup_paths import BEHAVIOR_DATA_PATH, OUTPUT_PATH -class TestLightningPoseDataInterface(DataInterfaceTestMixin, TemporalAlignmentMixin, unittest.TestCase): +class TestLightningPoseDataInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): data_interface_cls = LightningPoseDataInterface interface_kwargs = dict( file_path=str(BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.csv"), @@ -52,8 +53,11 @@ class TestLightningPoseDataInterface(DataInterfaceTestMixin, TemporalAlignmentMi conversion_options = dict(reference_frame="(0,0) corresponds to the top left corner of the video.") save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls): + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(self, request): + + cls = request.cls + cls.pose_estimation_name = "PoseEstimation" cls.original_video_height = 406 cls.original_video_width = 396 @@ -97,47 +101,61 @@ def setUpClass(cls): cls.test_data = pd.read_csv(cls.interface_kwargs["file_path"], header=[0, 1, 2])["heatmap_tracker"] def check_extracted_metadata(self, metadata: dict): - self.assertEqual( - metadata["NWBFile"]["session_start_time"], - datetime(2023, 11, 9, 10, 14, 37, 0), - ) - self.assertIn(self.pose_estimation_name, metadata["Behavior"]) - self.assertEqual( - metadata["Behavior"][self.pose_estimation_name], self.expected_metadata[self.pose_estimation_name] - ) + assert metadata["NWBFile"]["session_start_time"] == datetime(2023, 11, 9, 10, 14, 37, 0) + assert self.pose_estimation_name in metadata["Behavior"] + assert metadata["Behavior"][self.pose_estimation_name] == self.expected_metadata[self.pose_estimation_name] def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() - self.assertIn("behavior", nwbfile.processing) - self.assertIn(self.pose_estimation_name, nwbfile.processing["behavior"].data_interfaces) + + # Replacing assertIn with pytest-style assert + assert "behavior" in nwbfile.processing + assert self.pose_estimation_name in nwbfile.processing["behavior"].data_interfaces + pose_estimation_container = nwbfile.processing["behavior"].data_interfaces[self.pose_estimation_name] - self.assertIsInstance(pose_estimation_container, PoseEstimation) + + # Replacing assertIsInstance with pytest-style assert + assert isinstance(pose_estimation_container, PoseEstimation) pose_estimation_metadata = self.expected_metadata[self.pose_estimation_name] - self.assertEqual(pose_estimation_container.description, pose_estimation_metadata["description"]) - self.assertEqual(pose_estimation_container.scorer, pose_estimation_metadata["scorer"]) - self.assertEqual(pose_estimation_container.source_software, pose_estimation_metadata["source_software"]) + + # Replacing assertEqual with pytest-style assert + assert pose_estimation_container.description == pose_estimation_metadata["description"] + assert pose_estimation_container.scorer == pose_estimation_metadata["scorer"] + assert pose_estimation_container.source_software == pose_estimation_metadata["source_software"] + + # Using numpy's assert_array_equal assert_array_equal( pose_estimation_container.dimensions[:], [[self.original_video_height, self.original_video_width]] ) - self.assertEqual(len(pose_estimation_container.pose_estimation_series), len(self.expected_keypoint_names)) + # Replacing assertEqual with pytest-style assert + assert len(pose_estimation_container.pose_estimation_series) == len(self.expected_keypoint_names) + for keypoint_name in self.expected_keypoint_names: series_metadata = pose_estimation_metadata[keypoint_name] - self.assertIn(series_metadata["name"], pose_estimation_container.pose_estimation_series) + + # Replacing assertIn with pytest-style assert + assert series_metadata["name"] in pose_estimation_container.pose_estimation_series + pose_estimation_series = pose_estimation_container.pose_estimation_series[series_metadata["name"]] - self.assertIsInstance(pose_estimation_series, PoseEstimationSeries) - self.assertEqual(pose_estimation_series.unit, "px") - self.assertEqual(pose_estimation_series.description, series_metadata["description"]) - self.assertEqual(pose_estimation_series.reference_frame, self.conversion_options["reference_frame"]) + + # Replacing assertIsInstance with pytest-style assert + assert isinstance(pose_estimation_series, PoseEstimationSeries) + + # Replacing assertEqual with pytest-style assert + assert pose_estimation_series.unit == "px" + assert pose_estimation_series.description == series_metadata["description"] + assert pose_estimation_series.reference_frame == self.conversion_options["reference_frame"] test_data = self.test_data[keypoint_name] + + # Using numpy's assert_array_equal assert_array_equal(pose_estimation_series.data[:], test_data[["x", "y"]].values) - assert_array_equal(pose_estimation_series.confidence[:], test_data["likelihood"].values) -class TestLightningPoseDataInterfaceWithStubTest(DataInterfaceTestMixin, TemporalAlignmentMixin, unittest.TestCase): +class TestLightningPoseDataInterfaceWithStubTest(DataInterfaceTestMixin, TemporalAlignmentMixin): data_interface_cls = LightningPoseDataInterface interface_kwargs = dict( file_path=str(BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.csv"), @@ -145,6 +163,7 @@ class TestLightningPoseDataInterfaceWithStubTest(DataInterfaceTestMixin, Tempora BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.mp4" ), ) + conversion_options = dict(stub_test=True) save_directory = OUTPUT_PATH @@ -153,18 +172,16 @@ def check_read_nwb(self, nwbfile_path: str): nwbfile = io.read() pose_estimation_container = nwbfile.processing["behavior"].data_interfaces["PoseEstimation"] for pose_estimation_series in pose_estimation_container.pose_estimation_series.values(): - self.assertEqual(pose_estimation_series.data.shape[0], 10) - self.assertEqual(pose_estimation_series.confidence.shape[0], 10) + assert pose_estimation_series.data.shape[0] == 10 + assert pose_estimation_series.confidence.shape[0] == 10 -class TestFicTracDataInterface(DataInterfaceTestMixin, unittest.TestCase): +class TestFicTracDataInterface(DataInterfaceTestMixin): data_interface_cls = FicTracDataInterface - interface_kwargs = [ - dict( - file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "sample-20230724_113055.dat"), - configuration_file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "config.txt"), - ), - ] + interface_kwargs = dict( + file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "sample-20230724_113055.dat"), + configuration_file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "config.txt"), + ) save_directory = OUTPUT_PATH @@ -228,11 +245,11 @@ def check_read_nwb(self, nwbfile_path: str): # This is currently structured to assert spatial_series.timestamps[0] == 0.0 -class TestFicTracDataInterfaceWithRadius(DataInterfaceTestMixin, unittest.TestCase): +class TestFicTracDataInterfaceWithRadius(DataInterfaceTestMixin): data_interface_cls = FicTracDataInterface - interface_kwargs = [ - dict(file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "sample-20230724_113055.dat"), radius=1.0), - ] + interface_kwargs = dict( + file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "sample-20230724_113055.dat"), radius=1.0 + ) save_directory = OUTPUT_PATH @@ -296,74 +313,25 @@ def check_read_nwb(self, nwbfile_path: str): # This is currently structured to assert spatial_series.timestamps[0] == 0.0 -class TestFicTracDataInterfaceTiming(TemporalAlignmentMixin, unittest.TestCase): +class TestFicTracDataInterfaceTiming(TemporalAlignmentMixin): data_interface_cls = FicTracDataInterface - interface_kwargs = [dict(file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "sample-20230724_113055.dat"))] + interface_kwargs = dict(file_path=str(BEHAVIOR_DATA_PATH / "FicTrac" / "sample" / "sample-20230724_113055.dat")) save_directory = OUTPUT_PATH -class TestVideoInterface(VideoInterfaceMixin, unittest.TestCase): - data_interface_cls = VideoInterface - interface_kwargs = [ - dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_avi.avi")]), - dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_flv.flv")]), - dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_mov.mov")]), - dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_mp4.mp4")]), - dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_wmv.wmv")]), - ] - save_directory = OUTPUT_PATH - - -class TestDeepLabCutInterface(DeepLabCutInterfaceMixin, unittest.TestCase): +class TestDeepLabCutInterface(DeepLabCutInterfaceMixin): data_interface_cls = DeepLabCutInterface - interface_kwargs_item = dict( + interface_kwargs = dict( file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5"), config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "config.yaml"), subject_name="ind1", ) - # intentional duplicate to workaround 2 tests with changes after interface construction - interface_kwargs = [ - interface_kwargs_item, # this is case=0, no custom timestamp - interface_kwargs_item, # this is case=1, with custom timestamp - ] - - # custom timestamps only for case 1 - _custom_timestamps_case_1 = np.concatenate( - (np.linspace(10, 110, 1000), np.linspace(150, 250, 1000), np.linspace(300, 400, 330)) - ) - save_directory = OUTPUT_PATH def run_custom_checks(self): - self.check_custom_timestamps(nwbfile_path=self.nwbfile_path) self.check_renaming_instance(nwbfile_path=self.nwbfile_path) - def check_custom_timestamps(self, nwbfile_path: str): - # TODO: Peel out into separate test class and replace this part with check_read_nwb - if self.case != 1: # set custom timestamps - return - - metadata = self.interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - self.interface.set_aligned_timestamps(self._custom_timestamps_case_1) - assert len(self.interface._timestamps) == 2330 - - self.interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) - - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - assert "behavior" in nwbfile.processing - processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces - assert "PoseEstimation" in processing_module_interfaces - - pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series - - for pose_estimation in pose_estimation_series_in_nwb.values(): - pose_timestamps = pose_estimation.timestamps - np.testing.assert_array_equal(pose_timestamps, self._custom_timestamps_case_1) - def check_renaming_instance(self, nwbfile_path: str): custom_container_name = "TestPoseEstimation" @@ -381,7 +349,6 @@ def check_renaming_instance(self, nwbfile_path: str): assert custom_container_name in nwbfile.processing["behavior"].data_interfaces def check_read_nwb(self, nwbfile_path: str): - # TODO: move this to the upstream mixin with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() assert "behavior" in nwbfile.processing @@ -398,7 +365,50 @@ def check_read_nwb(self, nwbfile_path: str): assert all(expected_pose_estimation_series_are_in_nwb_file) -class TestSLEAPInterface(DataInterfaceTestMixin, TemporalAlignmentMixin, unittest.TestCase): +class TestDeepLabCutInterfaceSetTimestamps(DeepLabCutInterfaceMixin): + data_interface_cls = DeepLabCutInterface + interface_kwargs = dict( + file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5"), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "config.yaml"), + subject_name="ind1", + ) + + save_directory = OUTPUT_PATH + + def run_custom_checks(self): + self.check_custom_timestamps(nwbfile_path=self.nwbfile_path) + + def check_custom_timestamps(self, nwbfile_path: str): + custom_timestamps = np.concatenate( + (np.linspace(10, 110, 1000), np.linspace(150, 250, 1000), np.linspace(300, 400, 330)) + ) + + metadata = self.interface.get_metadata() + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + self.interface.set_aligned_timestamps(custom_timestamps) + assert len(self.interface._timestamps) == 2330 + + self.interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) + + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces + assert "PoseEstimation" in processing_module_interfaces + + pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series + + for pose_estimation in pose_estimation_series_in_nwb.values(): + pose_timestamps = pose_estimation.timestamps + np.testing.assert_array_equal(pose_timestamps, custom_timestamps) + + # This was tested in the other test + def check_read_nwb(self, nwbfile_path: str): + pass + + +class TestSLEAPInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): data_interface_cls = SLEAPInterface interface_kwargs = dict( file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "predictions_1.2.7_provenance_and_tracking.slp"), @@ -429,16 +439,18 @@ def check_read_nwb(self, nwbfile_path: str): # This is currently structured to "wingL", "wingR", ] - self.assertCountEqual(first=pose_estimation_series_in_nwb, second=expected_pose_estimation_series) + + assert set(pose_estimation_series_in_nwb) == set(expected_pose_estimation_series) -class TestMiniscopeInterface(DataInterfaceTestMixin, unittest.TestCase): +class TestMiniscopeInterface(DataInterfaceTestMixin): data_interface_cls = MiniscopeBehaviorInterface interface_kwargs = dict(folder_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Miniscope" / "C6-J588_Disc5")) save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls) -> None: + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(self, request): + cls = request.cls folder_path = Path(OPHYS_DATA_PATH / "imaging_datasets" / "Miniscope" / "C6-J588_Disc5") cls.device_name = "BehavCam2" cls.image_series_name = "BehavCamImageSeries" @@ -455,45 +467,42 @@ def setUpClass(cls) -> None: cls.timestamps = get_timestamps(folder_path=str(folder_path), file_pattern="BehavCam*/timeStamps.csv") def check_extracted_metadata(self, metadata: dict): - self.assertEqual( - metadata["NWBFile"]["session_start_time"], - datetime(2021, 10, 7, 15, 3, 28, 635), - ) - self.assertEqual(metadata["Behavior"]["Device"][0], self.device_metadata) + assert metadata["NWBFile"]["session_start_time"] == datetime(2021, 10, 7, 15, 3, 28, 635) + assert metadata["Behavior"]["Device"][0] == self.device_metadata image_series_metadata = metadata["Behavior"]["ImageSeries"][0] - self.assertEqual(image_series_metadata["name"], self.image_series_name) - self.assertEqual(image_series_metadata["device"], self.device_name) - self.assertEqual(image_series_metadata["unit"], "px") - self.assertEqual(image_series_metadata["dimension"], [1280, 720]) # width x height + assert image_series_metadata["name"] == self.image_series_name + assert image_series_metadata["device"] == self.device_name + assert image_series_metadata["unit"] == "px" + assert image_series_metadata["dimension"] == [1280, 720] # width x height def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(nwbfile_path, "r") as io: nwbfile = io.read() # Check device metadata - self.assertIn(self.device_name, nwbfile.devices) + assert self.device_name in nwbfile.devices device = nwbfile.devices[self.device_name] - self.assertIsInstance(device, Miniscope) - self.assertEqual(device.compression, self.device_metadata["compression"]) - self.assertEqual(device.deviceType, self.device_metadata["deviceType"]) - self.assertEqual(device.framesPerFile, self.device_metadata["framesPerFile"]) + assert isinstance(device, Miniscope) + assert device.compression == self.device_metadata["compression"] + assert device.deviceType == self.device_metadata["deviceType"] + assert device.framesPerFile == self.device_metadata["framesPerFile"] roi = [self.device_metadata["ROI"]["height"], self.device_metadata["ROI"]["width"]] assert_array_equal(device.ROI[:], roi) # Check ImageSeries - self.assertIn(self.image_series_name, nwbfile.acquisition) + assert self.image_series_name in nwbfile.acquisition image_series = nwbfile.acquisition[self.image_series_name] - self.assertEqual(image_series.format, "external") + assert image_series.format == "external" assert_array_equal(image_series.starting_frame, self.starting_frames) assert_array_equal(image_series.dimension[:], [1280, 720]) - self.assertEqual(image_series.unit, "px") - self.assertEqual(device, nwbfile.acquisition[self.image_series_name].device) + assert image_series.unit == "px" + assert device == nwbfile.acquisition[self.image_series_name].device assert_array_equal(image_series.timestamps[:], self.timestamps) assert_array_equal(image_series.external_file[:], self.external_files) -class TestNeuralynxNvtInterface(DataInterfaceTestMixin, TemporalAlignmentMixin, unittest.TestCase): +class TestNeuralynxNvtInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): data_interface_cls = NeuralynxNvtInterface interface_kwargs = dict(file_path=str(BEHAVIOR_DATA_PATH / "neuralynx" / "test.nvt")) conversion_options = dict(add_angle=True) @@ -628,6 +637,30 @@ def test_sleap_interface_timestamps_propagation(self, data_interface, interface_ assert set(extracted_timestamps).issubset(expected_timestamps) +class TestVideoInterface(VideoInterfaceMixin): + data_interface_cls = VideoInterface + save_directory = OUTPUT_PATH + + @pytest.fixture( + params=[ + (dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_avi.avi")])), + (dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_flv.flv")])), + (dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_mov.mov")])), + (dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_mp4.mp4")])), + (dict(file_paths=[str(BEHAVIOR_DATA_PATH / "videos" / "CFR" / "video_wmv.wmv")])), + ], + ids=["avi", "flv", "mov", "mp4", "wmv"], + ) + def setup_interface(self, request): + + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + + class TestVideoConversions(TestCase): @classmethod def setUpClass(cls): diff --git a/tests/test_on_data/test_format_converters/test_miniscope_converter.py b/tests/test_on_data/test_format_converters/test_miniscope_converter.py index 813008455..a1e02ac1d 100644 --- a/tests/test_on_data/test_format_converters/test_miniscope_converter.py +++ b/tests/test_on_data/test_format_converters/test_miniscope_converter.py @@ -56,19 +56,9 @@ def tearDownClass(cls) -> None: def test_converter_metadata(self): metadata = self.converter.get_metadata() - self.assertEqual( - metadata["NWBFile"]["session_start_time"], - datetime(2021, 10, 7, 15, 3, 28, 635), - ) - self.assertDictEqual( - metadata["Ophys"]["Device"][0], - self.device_metadata, - ) - - self.assertDictEqual( - metadata["Behavior"]["Device"][0], - self.behavcam_metadata, - ) + assert metadata["NWBFile"]["session_start_time"] == datetime(2021, 10, 7, 15, 3, 28, 635) + assert metadata["Ophys"]["Device"][0] == self.device_metadata + assert metadata["Behavior"]["Device"][0] == self.behavcam_metadata def test_run_conversion(self): nwbfile_path = str(self.test_dir / "test_miniscope_converter.nwb") diff --git a/tests/test_on_data/test_imaging_interfaces.py b/tests/test_on_data/test_imaging_interfaces.py index 28f3d43ac..1a5328e52 100644 --- a/tests/test_on_data/test_imaging_interfaces.py +++ b/tests/test_on_data/test_imaging_interfaces.py @@ -1,9 +1,10 @@ import platform from datetime import datetime from pathlib import Path -from unittest import TestCase, skipIf +from unittest import skipIf import numpy as np +import pytest from dateutil.tz import tzoffset from hdmf.testing import TestCase as hdmf_TestCase from numpy.testing import assert_array_equal @@ -40,7 +41,7 @@ from setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH -class TestTiffImagingInterface(ImagingExtractorInterfaceTestMixin, TestCase): +class TestTiffImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = TiffImagingInterface interface_kwargs = dict( file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "demoMovie.tif"), @@ -69,7 +70,7 @@ class TestTiffImagingInterface(ImagingExtractorInterfaceTestMixin, TestCase): }, ], ) -class TestScanImageImagingInterfaceMultiPlaneCase(ScanImageMultiPlaneImagingInterfaceMixin, TestCase): +class TestScanImageImagingInterfaceMultiPlaneCase(ScanImageMultiPlaneImagingInterfaceMixin): data_interface_cls = ScanImageImagingInterface interface_kwargs = dict( file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "ScanImage" / "scanimage_20220923_roi.tif"), @@ -127,7 +128,7 @@ def check_extracted_metadata(self, metadata: dict): }, ], ) -class TestScanImageImagingInterfaceSinglePlaneCase(ScanImageSinglePlaneImagingInterfaceMixin, TestCase): +class TestScanImageImagingInterfaceSinglePlaneCase(ScanImageSinglePlaneImagingInterfaceMixin): data_interface_cls = ScanImageImagingInterface save_directory = OUTPUT_PATH interface_kwargs = dict( @@ -203,7 +204,7 @@ def test_non_volumetric_data(self): @skipIf(platform.machine() == "arm64", "Interface not supported on arm64 architecture") -class TestScanImageLegacyImagingInterface(ImagingExtractorInterfaceTestMixin, TestCase): +class TestScanImageLegacyImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = ScanImageImagingInterface interface_kwargs = dict(file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "sample_scanimage.tiff")) save_directory = OUTPUT_PATH @@ -238,7 +239,7 @@ def check_extracted_metadata(self, metadata: dict): }, ], ) -class TestScanImageMultiFileImagingInterfaceMultiPlaneCase(ScanImageMultiPlaneImagingInterfaceMixin, TestCase): +class TestScanImageMultiFileImagingInterfaceMultiPlaneCase(ScanImageMultiPlaneImagingInterfaceMixin): data_interface_cls = ScanImageMultiFileImagingInterface interface_kwargs = dict( folder_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "ScanImage"), @@ -279,7 +280,7 @@ def check_extracted_metadata(self, metadata: dict): }, ], ) -class TestScanImageMultiFileImagingInterfaceSinglePlaneCase(ScanImageSinglePlaneImagingInterfaceMixin, TestCase): +class TestScanImageMultiFileImagingInterfaceSinglePlaneCase(ScanImageSinglePlaneImagingInterfaceMixin): data_interface_cls = ScanImageMultiFileImagingInterface interface_kwargs = dict( folder_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "ScanImage"), @@ -353,22 +354,26 @@ def test_plane_name_not_specified(self): ScanImageSinglePlaneMultiFileImagingInterface(folder_path=folder_path, file_pattern=file_pattern) -class TestHdf5ImagingInterface(ImagingExtractorInterfaceTestMixin, TestCase): +class TestHdf5ImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = Hdf5ImagingInterface interface_kwargs = dict(file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "hdf5" / "demoMovie.hdf5")) save_directory = OUTPUT_PATH -class TestSbxImagingInterface(ImagingExtractorInterfaceTestMixin, TestCase): +class TestSbxImagingInterfaceMat(ImagingExtractorInterfaceTestMixin): data_interface_cls = SbxImagingInterface - interface_kwargs = [ - dict(file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Scanbox" / "sample.mat")), - dict(file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Scanbox" / "sample.sbx")), - ] + interface_kwargs = dict(file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Scanbox" / "sample.mat")) save_directory = OUTPUT_PATH -class TestBrukerTiffImagingInterface(ImagingExtractorInterfaceTestMixin, TestCase): +class TestSbxImagingInterfaceSBX(ImagingExtractorInterfaceTestMixin): + data_interface_cls = SbxImagingInterface + interface_kwargs = dict(file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Scanbox" / "sample.sbx")) + + save_directory = OUTPUT_PATH + + +class TestBrukerTiffImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = BrukerTiffSinglePlaneImagingInterface interface_kwargs = dict( folder_path=str( @@ -377,8 +382,11 @@ class TestBrukerTiffImagingInterface(ImagingExtractorInterfaceTestMixin, TestCas ) save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls) -> None: + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(cls, request): + + cls = request.cls + cls.device_metadata = dict(name="BrukerFluorescenceMicroscope", description="Version 5.6.64.400") cls.optical_channel_metadata = dict( name="Ch2", @@ -397,7 +405,6 @@ def setUpClass(cls) -> None: grid_spacing=[1.1078125e-06, 1.1078125e-06], origin_coords=[0.0, 0.0], ) - cls.two_photon_series_metadata = dict( name="TwoPhotonSeries", description="Imaging data acquired from the Bruker Two-Photon Microscope.", @@ -407,7 +414,6 @@ def setUpClass(cls) -> None: scan_line_rate=15840.580398865815, field_of_view=[0.0005672, 0.0005672], ) - cls.ophys_metadata = dict( Device=[cls.device_metadata], ImagingPlane=[cls.imaging_plane_metadata], @@ -415,8 +421,8 @@ def setUpClass(cls) -> None: ) def check_extracted_metadata(self, metadata: dict): - self.assertEqual(metadata["NWBFile"]["session_start_time"], datetime(2023, 2, 20, 15, 58, 25)) - self.assertDictEqual(metadata["Ophys"], self.ophys_metadata) + assert metadata["NWBFile"]["session_start_time"] == datetime(2023, 2, 20, 15, 58, 25) + assert metadata["Ophys"] == self.ophys_metadata def check_read_nwb(self, nwbfile_path: str): """Check the ophys metadata made it to the NWB file""" @@ -424,29 +430,27 @@ def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(nwbfile_path, "r") as io: nwbfile = io.read() - self.assertIn(self.device_metadata["name"], nwbfile.devices) - self.assertEqual( - nwbfile.devices[self.device_metadata["name"]].description, self.device_metadata["description"] - ) - self.assertIn(self.imaging_plane_metadata["name"], nwbfile.imaging_planes) + assert self.device_metadata["name"] in nwbfile.devices + assert nwbfile.devices[self.device_metadata["name"]].description == self.device_metadata["description"] + assert self.imaging_plane_metadata["name"] in nwbfile.imaging_planes imaging_plane = nwbfile.imaging_planes[self.imaging_plane_metadata["name"]] optical_channel = imaging_plane.optical_channel[0] - self.assertEqual(optical_channel.name, self.optical_channel_metadata["name"]) - self.assertEqual(optical_channel.description, self.optical_channel_metadata["description"]) - self.assertEqual(imaging_plane.description, self.imaging_plane_metadata["description"]) - self.assertEqual(imaging_plane.imaging_rate, self.imaging_plane_metadata["imaging_rate"]) + assert optical_channel.name == self.optical_channel_metadata["name"] + assert optical_channel.description == self.optical_channel_metadata["description"] + assert imaging_plane.description == self.imaging_plane_metadata["description"] + assert imaging_plane.imaging_rate == self.imaging_plane_metadata["imaging_rate"] assert_array_equal(imaging_plane.grid_spacing[:], self.imaging_plane_metadata["grid_spacing"]) - self.assertIn(self.two_photon_series_metadata["name"], nwbfile.acquisition) + assert self.two_photon_series_metadata["name"] in nwbfile.acquisition two_photon_series = nwbfile.acquisition[self.two_photon_series_metadata["name"]] - self.assertEqual(two_photon_series.description, self.two_photon_series_metadata["description"]) - self.assertEqual(two_photon_series.unit, self.two_photon_series_metadata["unit"]) - self.assertEqual(two_photon_series.scan_line_rate, self.two_photon_series_metadata["scan_line_rate"]) + assert two_photon_series.description == self.two_photon_series_metadata["description"] + assert two_photon_series.unit == self.two_photon_series_metadata["unit"] + assert two_photon_series.scan_line_rate == self.two_photon_series_metadata["scan_line_rate"] assert_array_equal(two_photon_series.field_of_view[:], self.two_photon_series_metadata["field_of_view"]) super().check_read_nwb(nwbfile_path=nwbfile_path) -class TestBrukerTiffImagingInterfaceDualPlaneCase(ImagingExtractorInterfaceTestMixin, TestCase): +class TestBrukerTiffImagingInterfaceDualPlaneCase(ImagingExtractorInterfaceTestMixin): data_interface_cls = BrukerTiffMultiPlaneImagingInterface interface_kwargs = dict( folder_path=str( @@ -455,8 +459,10 @@ class TestBrukerTiffImagingInterfaceDualPlaneCase(ImagingExtractorInterfaceTestM ) save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls) -> None: + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(self, request): + cls = request.cls + cls.photon_series_name = "TwoPhotonSeries" cls.num_frames = 5 cls.image_shape = (512, 512, 2) @@ -501,22 +507,23 @@ def run_custom_checks(self): streams = self.data_interface_cls.get_streams( folder_path=self.interface_kwargs["folder_path"], plane_separation_type="contiguous" ) - self.assertEqual(streams, self.available_streams) + + assert streams == self.available_streams def check_extracted_metadata(self, metadata: dict): - self.assertEqual(metadata["NWBFile"]["session_start_time"], datetime(2022, 11, 3, 11, 20, 34)) - self.assertDictEqual(metadata["Ophys"], self.ophys_metadata) + assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 11, 3, 11, 20, 34) + assert metadata["Ophys"] == self.ophys_metadata def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() photon_series = nwbfile.acquisition[self.photon_series_name] - self.assertEqual(photon_series.data.shape, (self.num_frames, *self.image_shape)) - assert_array_equal(photon_series.dimension[:], self.image_shape) - self.assertEqual(photon_series.rate, 20.629515014336377) + assert photon_series.data.shape == (self.num_frames, *self.image_shape) + np.testing.assert_array_equal(photon_series.dimension[:], self.image_shape) + assert photon_series.rate == 20.629515014336377 -class TestBrukerTiffImagingInterfaceDualPlaneDisjointCase(ImagingExtractorInterfaceTestMixin, TestCase): +class TestBrukerTiffImagingInterfaceDualPlaneDisjointCase(ImagingExtractorInterfaceTestMixin): data_interface_cls = BrukerTiffSinglePlaneImagingInterface interface_kwargs = dict( folder_path=str( @@ -526,8 +533,11 @@ class TestBrukerTiffImagingInterfaceDualPlaneDisjointCase(ImagingExtractorInterf ) save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls) -> None: + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(cls, request): + + cls = request.cls + cls.photon_series_name = "TwoPhotonSeriesCh2000002" cls.num_frames = 5 cls.image_shape = (512, 512) @@ -570,18 +580,19 @@ def setUpClass(cls) -> None: def run_custom_checks(self): # check stream names streams = self.data_interface_cls.get_streams(folder_path=self.interface_kwargs["folder_path"]) - self.assertEqual(streams, self.available_streams) + assert streams == self.available_streams def check_extracted_metadata(self, metadata: dict): - self.assertEqual(metadata["NWBFile"]["session_start_time"], datetime(2022, 11, 3, 11, 20, 34)) - self.assertDictEqual(metadata["Ophys"], self.ophys_metadata) + assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 11, 3, 11, 20, 34) + assert metadata["Ophys"] == self.ophys_metadata def check_nwbfile_temporal_alignment(self): nwbfile_path = str( - self.save_directory / f"{self.data_interface_cls.__name__}_{self.case}_test_starting_time_alignment.nwb" + self.save_directory + / f"{self.data_interface_cls.__name__}_{self.test_name}_test_starting_time_alignment.nwb" ) - interface = self.data_interface_cls(**self.test_kwargs) + interface = self.data_interface_cls(**self.interface_kwargs) aligned_starting_time = 1.23 interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) @@ -598,12 +609,12 @@ def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() photon_series = nwbfile.acquisition[self.photon_series_name] - self.assertEqual(photon_series.data.shape, (self.num_frames, *self.image_shape)) - assert_array_equal(photon_series.dimension[:], self.image_shape) - self.assertEqual(photon_series.rate, 10.314757507168189) + assert photon_series.data.shape == (self.num_frames, *self.image_shape) + np.testing.assert_array_equal(photon_series.dimension[:], self.image_shape) + assert photon_series.rate == 10.314757507168189 -class TestBrukerTiffImagingInterfaceDualColorCase(ImagingExtractorInterfaceTestMixin, TestCase): +class TestBrukerTiffImagingInterfaceDualColorCase(ImagingExtractorInterfaceTestMixin): data_interface_cls = BrukerTiffSinglePlaneImagingInterface interface_kwargs = dict( folder_path=str( @@ -613,8 +624,10 @@ class TestBrukerTiffImagingInterfaceDualColorCase(ImagingExtractorInterfaceTestM ) save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls) -> None: + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(cls, request): + + cls = request.cls cls.photon_series_name = "TwoPhotonSeriesCh2" cls.num_frames = 10 cls.image_shape = (512, 512) @@ -657,26 +670,27 @@ def setUpClass(cls) -> None: def run_custom_checks(self): # check stream names streams = self.data_interface_cls.get_streams(folder_path=self.interface_kwargs["folder_path"]) - self.assertEqual(streams, self.available_streams) + assert streams == self.available_streams def check_extracted_metadata(self, metadata: dict): - self.assertEqual(metadata["NWBFile"]["session_start_time"], datetime(2023, 7, 6, 15, 13, 58)) - self.assertDictEqual(metadata["Ophys"], self.ophys_metadata) + assert metadata["NWBFile"]["session_start_time"] == datetime(2023, 7, 6, 15, 13, 58) + assert metadata["Ophys"] == self.ophys_metadata def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() photon_series = nwbfile.acquisition[self.photon_series_name] - self.assertEqual(photon_series.data.shape, (self.num_frames, *self.image_shape)) - assert_array_equal(photon_series.dimension[:], self.image_shape) - self.assertEqual(photon_series.rate, 29.873615189896864) + assert photon_series.data.shape == (self.num_frames, *self.image_shape) + np.testing.assert_array_equal(photon_series.dimension[:], self.image_shape) + assert photon_series.rate == 29.873615189896864 def check_nwbfile_temporal_alignment(self): nwbfile_path = str( - self.save_directory / f"{self.data_interface_cls.__name__}_{self.case}_test_starting_time_alignment.nwb" + self.save_directory + / f"{self.data_interface_cls.__name__}_{self.test_name}_test_starting_time_alignment.nwb" ) - interface = self.data_interface_cls(**self.test_kwargs) + interface = self.data_interface_cls(**self.interface_kwargs) aligned_starting_time = 1.23 interface.set_aligned_starting_time(aligned_starting_time=aligned_starting_time) @@ -690,15 +704,16 @@ def check_nwbfile_temporal_alignment(self): assert nwbfile.acquisition[self.photon_series_name].starting_time == aligned_starting_time -class TestMicroManagerTiffImagingInterface(ImagingExtractorInterfaceTestMixin, TestCase): +class TestMicroManagerTiffImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = MicroManagerTiffImagingInterface interface_kwargs = dict( folder_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "MicroManagerTif" / "TS12_20220407_20hz_noteasy_1") ) save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls) -> None: + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(self, request): + cls = request.cls cls.device_metadata = dict(name="Microscope") cls.optical_channel_metadata = dict( name="OpticalChannelDefault", @@ -731,45 +746,47 @@ def setUpClass(cls) -> None: ) def check_extracted_metadata(self, metadata: dict): - self.assertEqual( - metadata["NWBFile"]["session_start_time"], - datetime(2022, 4, 7, 15, 6, 56, 842000, tzinfo=tzoffset(None, -18000)), + + assert metadata["NWBFile"]["session_start_time"] == datetime( + 2022, 4, 7, 15, 6, 56, 842000, tzinfo=tzoffset(None, -18000) ) - self.assertDictEqual(metadata["Ophys"], self.ophys_metadata) + assert metadata["Ophys"] == self.ophys_metadata def check_read_nwb(self, nwbfile_path: str): """Check the ophys metadata made it to the NWB file""" - with NWBHDF5IO(nwbfile_path, "r") as io: + # Assuming you would create and write an NWB file here before reading it back + + with NWBHDF5IO(str(nwbfile_path), "r") as io: nwbfile = io.read() - self.assertIn(self.imaging_plane_metadata["name"], nwbfile.imaging_planes) + assert self.imaging_plane_metadata["name"] in nwbfile.imaging_planes imaging_plane = nwbfile.imaging_planes[self.imaging_plane_metadata["name"]] optical_channel = imaging_plane.optical_channel[0] - self.assertEqual(optical_channel.name, self.optical_channel_metadata["name"]) - self.assertEqual(optical_channel.description, self.optical_channel_metadata["description"]) - self.assertEqual(imaging_plane.description, self.imaging_plane_metadata["description"]) - self.assertEqual(imaging_plane.imaging_rate, self.imaging_plane_metadata["imaging_rate"]) - self.assertIn(self.two_photon_series_metadata["name"], nwbfile.acquisition) + assert optical_channel.name == self.optical_channel_metadata["name"] + assert optical_channel.description == self.optical_channel_metadata["description"] + assert imaging_plane.description == self.imaging_plane_metadata["description"] + assert imaging_plane.imaging_rate == self.imaging_plane_metadata["imaging_rate"] + assert self.two_photon_series_metadata["name"] in nwbfile.acquisition two_photon_series = nwbfile.acquisition[self.two_photon_series_metadata["name"]] - self.assertEqual(two_photon_series.description, self.two_photon_series_metadata["description"]) - self.assertEqual(two_photon_series.unit, self.two_photon_series_metadata["unit"]) - self.assertEqual(two_photon_series.format, self.two_photon_series_metadata["format"]) + assert two_photon_series.description == self.two_photon_series_metadata["description"] + assert two_photon_series.unit == self.two_photon_series_metadata["unit"] + assert two_photon_series.format == self.two_photon_series_metadata["format"] assert_array_equal(two_photon_series.dimension[:], self.two_photon_series_metadata["dimension"]) super().check_read_nwb(nwbfile_path=nwbfile_path) -class TestMiniscopeImagingInterface(MiniscopeImagingInterfaceMixin, hdmf_TestCase): +class TestMiniscopeImagingInterface(MiniscopeImagingInterfaceMixin): data_interface_cls = MiniscopeImagingInterface interface_kwargs = dict(folder_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Miniscope" / "C6-J588_Disc5")) save_directory = OUTPUT_PATH - @classmethod - def setUpClass(cls) -> None: + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(cls, request): + cls = request.cls + cls.device_name = "Miniscope" - cls.imaging_plane_name = "ImagingPlane" - cls.photon_series_name = "OnePhotonSeries" cls.device_metadata = dict( name=cls.device_name, @@ -781,27 +798,35 @@ def setUpClass(cls) -> None: led0=47, ) - def check_extracted_metadata(self, metadata: dict): - self.assertEqual( - metadata["NWBFile"]["session_start_time"], - datetime(2021, 10, 7, 15, 3, 28, 635), + cls.imaging_plane_name = "ImagingPlane" + cls.imaging_plane_metadata = dict( + name=cls.imaging_plane_name, + device=cls.device_name, + imaging_rate=15.0, ) - self.assertEqual(metadata["Ophys"]["Device"][0], self.device_metadata) + + cls.photon_series_name = "OnePhotonSeries" + cls.photon_series_metadata = dict( + name=cls.photon_series_name, + unit="px", + ) + + def check_extracted_metadata(self, metadata: dict): + assert metadata["NWBFile"]["session_start_time"] == datetime(2021, 10, 7, 15, 3, 28, 635) + assert metadata["Ophys"]["Device"][0] == self.device_metadata + imaging_plane_metadata = metadata["Ophys"]["ImagingPlane"][0] - self.assertEqual(imaging_plane_metadata["name"], self.imaging_plane_name) - self.assertEqual(imaging_plane_metadata["device"], self.device_name) - self.assertEqual(imaging_plane_metadata["imaging_rate"], 15.0) + assert imaging_plane_metadata["name"] == self.imaging_plane_metadata["name"] + assert imaging_plane_metadata["device"] == self.imaging_plane_metadata["device"] + assert imaging_plane_metadata["imaging_rate"] == self.imaging_plane_metadata["imaging_rate"] one_photon_series_metadata = metadata["Ophys"]["OnePhotonSeries"][0] - self.assertEqual(one_photon_series_metadata["name"], self.photon_series_name) - self.assertEqual(one_photon_series_metadata["unit"], "px") - - def run_custom_checks(self): - self.check_incorrect_folder_structure_raises() + assert one_photon_series_metadata["name"] == self.photon_series_metadata["name"] + assert one_photon_series_metadata["unit"] == self.photon_series_metadata["unit"] - def check_incorrect_folder_structure_raises(self): + def test_incorrect_folder_structure_raises(self): folder_path = Path(self.interface_kwargs["folder_path"]) / "15_03_28/BehavCam_2/" - with self.assertRaisesWith( - exc_type=AssertionError, exc_msg="The main folder should contain at least one subfolder named 'Miniscope'." + with pytest.raises( + AssertionError, match="The main folder should contain at least one subfolder named 'Miniscope'." ): self.data_interface_cls(folder_path=folder_path) diff --git a/tests/test_on_data/test_recording_interfaces.py b/tests/test_on_data/test_recording_interfaces.py index 6c3a410e2..cf838cadc 100644 --- a/tests/test_on_data/test_recording_interfaces.py +++ b/tests/test_on_data/test_recording_interfaces.py @@ -2,10 +2,9 @@ from platform import python_version from sys import platform from typing import Literal -from unittest import skip, skipIf import numpy as np -from hdmf.testing import TestCase +import pytest from numpy.testing import assert_array_equal from packaging import version from pynwb import NWBHDF5IO @@ -47,7 +46,7 @@ this_python_version = version.parse(python_version()) -class TestAlphaOmegaRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestAlphaOmegaRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = AlphaOmegaRecordingInterface interface_kwargs = dict(folder_path=str(DATA_PATH / "alphaomega" / "mpx_map_version4")) save_directory = OUTPUT_PATH @@ -56,49 +55,73 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2021, 11, 19, 15, 23, 15) -class TestAxonRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestAxonRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = AxonaRecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "axona" / "axona_raw.bin")) save_directory = OUTPUT_PATH -class TestBiocamRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestBiocamRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = BiocamRecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "biocam" / "biocam_hw3.0_fw1.6.brw")) save_directory = OUTPUT_PATH -class TestBlackrockRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestBlackrockRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = BlackrockRecordingInterface - interface_kwargs = [ - dict(file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns5")), - dict(file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.ns5")), - dict(file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns2")), - ] save_directory = OUTPUT_PATH + @pytest.fixture( + params=[ + dict(file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns5")), + dict(file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.ns5")), + dict(file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns2")), + ], + ids=["blackrock_ns5_v1", "blackrock_ns5_v2", "blackrock_ns2"], + ) + def setup_interface(self, request): + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + -@skipIf( +@pytest.mark.skipif( platform == "darwin" or this_python_version > version.parse("3.9"), - reason="Interface unsupported for OSX. Interface only runs on python 3.9", + reason="Interface unsupported for OSX. Interface only runs on Python 3.9", ) -class TestSpike2RecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestSpike2RecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = Spike2RecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "spike2" / "m365_1sec.smrx")) save_directory = OUTPUT_PATH -class TestCellExplorerRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestCellExplorerRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = CellExplorerRecordingInterface - interface_kwargs = [ - dict(folder_path=str(DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed")), - dict( - folder_path=str(DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed_hdf5") - ), - ] save_directory = OUTPUT_PATH - def test_add_channel_metadata_to_nwb(self): + @pytest.fixture( + params=[ + dict(folder_path=str(DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed")), + dict( + folder_path=str( + DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed_hdf5" + ) + ), + ], + ids=["matlab", "hdf5"], + ) + def setup_interface(self, request): + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + + def test_add_channel_metadata_to_nwb(self, setup_interface): channel_id = "1" expected_channel_properties_recorder = { "location": np.array([791.5, -160.0]), @@ -112,42 +135,41 @@ def test_add_channel_metadata_to_nwb(self): "group_name": "Group 5", } - interface_kwargs = self.interface_kwargs - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs - self.interface = self.data_interface_cls(**self.test_kwargs) - self.nwbfile_path = str(self.save_directory / f"{self.data_interface_cls.__name__}_{num}_channel.nwb") - - metadata = self.interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - self.interface.run_conversion( - nwbfile_path=self.nwbfile_path, - overwrite=True, - metadata=metadata, - ) + self.nwbfile_path = str( + self.save_directory / f"{self.data_interface_cls.__name__}_{self.test_name}_channel.nwb" + ) + + metadata = self.interface.get_metadata() + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + self.interface.run_conversion( + nwbfile_path=self.nwbfile_path, + overwrite=True, + metadata=metadata, + ) + + # Test addition to recording extractor + recording_extractor = self.interface.recording_extractor + for key, expected_value in expected_channel_properties_recorder.items(): + extracted_value = recording_extractor.get_channel_property(channel_id=channel_id, key=key) + if key == "location": + assert np.allclose(expected_value, extracted_value) + else: + assert expected_value == extracted_value + + # Test addition to electrodes table + with NWBHDF5IO(self.nwbfile_path, "r") as io: + nwbfile = io.read() + electrode_table = nwbfile.electrodes.to_dataframe() + electrode_table_row = electrode_table.query(f"channel_name=='{channel_id}'").iloc[0] + for key, value in expected_channel_properties_electrodes.items(): + assert electrode_table_row[key] == value + - # Test addition to recording extractor - recording_extractor = self.interface.recording_extractor - for key, expected_value in expected_channel_properties_recorder.items(): - extracted_value = recording_extractor.get_channel_property(channel_id=channel_id, key=key) - if key == "location": - assert np.allclose(expected_value, extracted_value) - else: - assert expected_value == extracted_value - - # Test addition to electrodes table - with NWBHDF5IO(self.nwbfile_path, "r") as io: - nwbfile = io.read() - electrode_table = nwbfile.electrodes.to_dataframe() - electrode_table_row = electrode_table.query(f"channel_name=='{channel_id}'").iloc[0] - for key, value in expected_channel_properties_electrodes.items(): - assert electrode_table_row[key] == value - - -@skipIf(platform == "darwin", reason="Not supported for OSX.") -class TestEDFRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +@pytest.mark.skipif( + platform == "darwin", + reason="Interface unsupported for OSX.", +) +class TestEDFRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = EDFRecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "edf" / "edf+C.edf")) save_directory = OUTPUT_PATH @@ -157,23 +179,17 @@ def check_extracted_metadata(self, metadata: dict): def test_interface_alignment(self): interface_kwargs = self.interface_kwargs - if isinstance(interface_kwargs, dict): - interface_kwargs = [interface_kwargs] - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs - - # TODO - debug hanging I/O from pyedflib - # self.check_interface_get_original_timestamps() - # self.check_interface_get_timestamps() - # self.check_align_starting_time_internal() - # self.check_align_starting_time_external() - # self.check_interface_align_timestamps() - # self.check_shift_timestamps_by_start_time() - # self.check_interface_original_timestamps_inmutability() - - self.check_nwbfile_temporal_alignment() + + # TODO - debug hanging I/O from pyedflib + # self.check_interface_get_original_timestamps() + # self.check_interface_get_timestamps() + # self.check_align_starting_time_internal() + # self.check_align_starting_time_external() + # self.check_interface_align_timestamps() + # self.check_shift_timestamps_by_start_time() + # self.check_interface_original_timestamps_inmutability() + + self.check_nwbfile_temporal_alignment() # EDF has simultaneous access issues; can't have multiple interfaces open on the same file at once... def check_run_conversion_in_nwbconverter_with_backend( @@ -190,17 +206,30 @@ def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal[ pass -class TestIntanRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestIntanRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = IntanRecordingInterface - interface_kwargs = [ - dict(file_path=str(DATA_PATH / "intan" / "intan_rhd_test_1.rhd")), - dict(file_path=str(DATA_PATH / "intan" / "intan_rhs_test_1.rhs")), - ] + interface_kwargs = [] save_directory = OUTPUT_PATH + @pytest.fixture( + params=[ + dict(file_path=str(DATA_PATH / "intan" / "intan_rhd_test_1.rhd")), + dict(file_path=str(DATA_PATH / "intan" / "intan_rhs_test_1.rhs")), + ], + ids=["rhd", "rhs"], + ) + def setup_interface(self, request): + + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + -@skip(reason="This interface fails to load the necessary plugin sometimes.") -class TestMaxOneRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +@pytest.mark.skip(reason="This interface fails to load the necessary plugin sometimes.") +class TestMaxOneRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MaxOneRecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "maxwell" / "MaxOne_data" / "Record" / "000011" / "data.raw.h5")) save_directory = OUTPUT_PATH @@ -211,13 +240,13 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["Ecephys"]["Device"][0]["description"] == "Recorded using Maxwell version '20190530'." -class TestMCSRawRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestMCSRawRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MCSRawRecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "rawmcs" / "raw_mcs_with_header_1.raw")) save_directory = OUTPUT_PATH -class TestMEArecRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestMEArecRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MEArecRecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "mearec" / "mearec_test_10s.h5")) save_directory = OUTPUT_PATH @@ -245,74 +274,86 @@ def check_extracted_metadata(self, metadata: dict): ) -class TestNeuralynxRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestNeuralynxRecordingInterfaceV574: data_interface_cls = NeuralynxRecordingInterface - interface_kwargs = [ - dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.7.4" / "original_data")), - dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data")), - dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.4.0" / "original_data")), - ] + interface_kwargs = (dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.7.4" / "original_data")),) + save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): file_metadata = metadata["NWBFile"] + assert metadata["NWBFile"]["session_start_time"] == datetime(2017, 2, 16, 17, 56, 4) + assert metadata["NWBFile"]["session_id"] == "d8ba8eef-8d11-4cdc-86dc-05f50d4ba13d" + assert '"FileType": "NCS"' in file_metadata["notes"] + assert '"recording_closed": "2017-02-16 18:01:18"' in file_metadata["notes"] + assert '"ADMaxValue": "32767"' in file_metadata["notes"] + assert '"sampling_rate": "32000.0"' in file_metadata["notes"] + assert metadata["Ecephys"]["Device"][-1] == { + "name": "AcqSystem1 DigitalLynxSX", + "description": "Cheetah 5.7.4", + } + + def check_read(self, nwbfile_path): + super().check_read(nwbfile_path) + expected_single_channel_props = { + "DSPLowCutFilterEnabled": "True", + "DspLowCutFrequency": "10", + "DspLowCutNumTaps": "0", + "DspLowCutFilterType": "DCO", + "DSPHighCutFilterEnabled": "True", + "DspHighCutFrequency": "9000", + "DspHighCutNumTaps": "64", + "DspHighCutFilterType": "FIR", + "DspDelayCompensation": "Enabled", + } - if self.case == 0: - assert metadata["NWBFile"]["session_start_time"] == datetime(2017, 2, 16, 17, 56, 4) - assert metadata["NWBFile"]["session_id"] == "d8ba8eef-8d11-4cdc-86dc-05f50d4ba13d" - assert '"FileType": "NCS"' in file_metadata["notes"] - assert '"recording_closed": "2017-02-16 18:01:18"' in file_metadata["notes"] - assert '"ADMaxValue": "32767"' in file_metadata["notes"] - assert '"sampling_rate": "32000.0"' in file_metadata["notes"] - assert metadata["Ecephys"]["Device"][-1] == { - "name": "AcqSystem1 DigitalLynxSX", - "description": "Cheetah 5.7.4", - } - - elif self.case == 1: - assert file_metadata["session_start_time"] == datetime(2016, 11, 28, 21, 50, 33, 322000) - # Metadata extracted directly from file header (neo >= 0.11) - assert '"FileType": "CSC"' in file_metadata["notes"] - assert '"recording_closed": "2016-11-28 22:44:41.145000"' in file_metadata["notes"] - assert '"ADMaxValue": "32767"' in file_metadata["notes"] - assert '"sampling_rate": "2000.0"' in file_metadata["notes"] - assert metadata["Ecephys"]["Device"][-1] == {"name": "DigitalLynxSX", "description": "Cheetah 5.6.3"} - - elif self.case == 2: - assert file_metadata["session_start_time"] == datetime(2001, 1, 1, 0, 0) - assert '"recording_closed": "2001-01-01 00:00:00"' in file_metadata["notes"] - assert '"ADMaxValue": "32767"' in file_metadata["notes"] - assert '"sampling_rate": "1017.375"' in file_metadata["notes"] - assert metadata["Ecephys"]["Device"][-1] == {"name": "DigitalLynx", "description": "Cheetah 5.4.0"} + n_channels = self.interface.recording_extractor.get_num_channels() + + for key, exp_value in expected_single_channel_props.items(): + extracted_value = self.interface.recording_extractor.get_property(key) + assert len(extracted_value) == n_channels + assert exp_value == extracted_value[0] + + +class TestNeuralynxRecordingInterfaceV563: + data_interface_cls = NeuralynxRecordingInterface + interface_kwargs = (dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data")),) + + save_directory = OUTPUT_PATH + + def check_extracted_metadata(self, metadata: dict): + file_metadata = metadata["NWBFile"] + assert file_metadata["session_start_time"] == datetime(2016, 11, 28, 21, 50, 33, 322000) + assert '"FileType": "CSC"' in file_metadata["notes"] + assert '"recording_closed": "2016-11-28 22:44:41.145000"' in file_metadata["notes"] + assert '"ADMaxValue": "32767"' in file_metadata["notes"] + assert '"sampling_rate": "2000.0"' in file_metadata["notes"] + assert metadata["Ecephys"]["Device"][-1] == {"name": "DigitalLynxSX", "description": "Cheetah 5.6.3"} + + def check_read(self, nwbfile_path): + super().check_read(nwbfile_path) + # Add any specific checks for Cheetah_v5.6.3 if needed + + +class TestNeuralynxRecordingInterfaceV540: + data_interface_cls = NeuralynxRecordingInterface + interface_kwargs = (dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.4.0" / "original_data")),) + save_directory = OUTPUT_PATH + + def check_extracted_metadata(self, metadata: dict): + file_metadata = metadata["NWBFile"] + assert file_metadata["session_start_time"] == datetime(2001, 1, 1, 0, 0) + assert '"recording_closed": "2001-01-01 00:00:00"' in file_metadata["notes"] + assert '"ADMaxValue": "32767"' in file_metadata["notes"] + assert '"sampling_rate": "1017.375"' in file_metadata["notes"] + assert metadata["Ecephys"]["Device"][-1] == {"name": "DigitalLynx", "description": "Cheetah 5.4.0"} def check_read(self, nwbfile_path): super().check_read(nwbfile_path) - if self.case == 0: - expected_single_channel_props = { - "DSPLowCutFilterEnabled": "True", - "DspLowCutFrequency": "10", - "DspLowCutNumTaps": "0", - "DspLowCutFilterType": "DCO", - "DSPHighCutFilterEnabled": "True", - "DspHighCutFrequency": "9000", - "DspHighCutNumTaps": "64", - "DspHighCutFilterType": "FIR", - "DspDelayCompensation": "Enabled", - # don't check for filter delay as the unit might be differently parsed - # "DspFilterDelay_µs": "984" - } - - n_channels = self.interface.recording_extractor.get_num_channels() - - for key, exp_value in expected_single_channel_props.items(): - extracted_value = self.interface.recording_extractor.get_property(key) - # check consistency of number of entries - assert len(extracted_value) == n_channels - # check values for first channel - assert exp_value == extracted_value[0] - - -class TestMultiStreamNeuralynxRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): + # Add any specific checks for Cheetah_v5.4.0 if need + + +class TestMultiStreamNeuralynxRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = NeuralynxRecordingInterface interface_kwargs = dict( folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v6.4.1dev" / "original_data"), @@ -334,57 +375,44 @@ def check_extracted_metadata(self, metadata: dict): } -class TestNeuroScopeRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestNeuroScopeRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = NeuroScopeRecordingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "neuroscope" / "test1" / "test1.dat")) save_directory = OUTPUT_PATH -class TestOpenEphysBinaryRecordingInterfaceClassMethodsAndAssertions(RecordingExtractorInterfaceTestMixin, TestCase): +class TestOpenEphysBinaryRecordingInterfaceClassMethodsAndAssertions: + data_interface_cls = OpenEphysBinaryRecordingInterface - interface_kwargs = [] - save_directory = OUTPUT_PATH def test_get_stream_names(self): - self.assertCountEqual( - first=self.data_interface_cls.get_stream_names( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream") - ), - second=["Record_Node_107#Neuropix-PXI-116.0", "Record_Node_107#Neuropix-PXI-116.1"], + + stream_names = self.data_interface_cls.get_stream_names( + folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107") ) + assert stream_names == ["Record_Node_107#Neuropix-PXI-116.0", "Record_Node_107#Neuropix-PXI-116.1"] + def test_folder_structure_assertion(self): - with self.assertRaisesWith( - exc_type=ValueError, - exc_msg=( - "Unable to identify the OpenEphys folder structure! Please check that your `folder_path` contains a " - "settings.xml file and sub-folders of the following form: 'experiment' -> 'recording' ->" - " 'continuous'." - ), + with pytest.raises( + ValueError, + match=r"Unable to identify the OpenEphys folder structure! Please check that your `folder_path` contains a settings.xml file and sub-folders of the following form: 'experiment' -> 'recording' -> 'continuous'.", ): OpenEphysBinaryRecordingInterface(folder_path=str(DATA_PATH / "openephysbinary")) def test_stream_name_missing_assertion(self): - with self.assertRaisesWith( - exc_type=ValueError, - exc_msg=( - "More than one stream is detected! " - "Please specify which stream you wish to load with the `stream_name` argument. " - "To see what streams are available, call " - " `OpenEphysRecordingInterface.get_stream_names(folder_path=...)`." - ), + with pytest.raises( + ValueError, + match=r"More than one stream is detected! Please specify which stream you wish to load with the `stream_name` argument. To see what streams are available, call\s+`OpenEphysRecordingInterface.get_stream_names\(folder_path=\.\.\.\)`.", ): OpenEphysBinaryRecordingInterface( folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107") ) def test_stream_name_not_available_assertion(self): - with self.assertRaisesWith( - exc_type=ValueError, - exc_msg=( - "The selected stream 'not_a_stream' is not in the available streams " - "'['Record_Node_107#Neuropix-PXI-116.0', 'Record_Node_107#Neuropix-PXI-116.1']'!" - ), + with pytest.raises( + ValueError, + match=r"The selected stream 'not_a_stream' is not in the available streams '\['Record_Node_107#Neuropix-PXI-116.0', 'Record_Node_107#Neuropix-PXI-116.1'\]'!", ): OpenEphysBinaryRecordingInterface( folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), @@ -392,7 +420,7 @@ def test_stream_name_not_available_assertion(self): ) -class TestOpenEphysBinaryRecordingInterfaceVersion0_4_4(RecordingExtractorInterfaceTestMixin, TestCase): +class TestOpenEphysBinaryRecordingInterfaceVersion0_4_4(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysBinaryRecordingInterface interface_kwargs = dict(folder_path=str(DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking")) save_directory = OUTPUT_PATH @@ -401,7 +429,7 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2021, 2, 15, 17, 20, 4) -class TestOpenEphysBinaryRecordingInterfaceVersion0_5_3_Stream1(RecordingExtractorInterfaceTestMixin, TestCase): +class TestOpenEphysBinaryRecordingInterfaceVersion0_5_3_Stream1(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysBinaryRecordingInterface interface_kwargs = dict( folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), @@ -413,7 +441,7 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2020, 11, 24, 15, 46, 56) -class TestOpenEphysBinaryRecordingInterfaceVersion0_5_3_Stream2(RecordingExtractorInterfaceTestMixin, TestCase): +class TestOpenEphysBinaryRecordingInterfaceVersion0_5_3_Stream2(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysBinaryRecordingInterface interface_kwargs = dict( folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), @@ -426,7 +454,7 @@ def check_extracted_metadata(self, metadata: dict): class TestOpenEphysBinaryRecordingInterfaceWithBlocks_version_0_6_block_1_stream_1( - RecordingExtractorInterfaceTestMixin, TestCase + RecordingExtractorInterfaceTestMixin ): """From Issue #695, exposed `block_index` argument and added tests on data that include multiple blocks.""" @@ -442,7 +470,7 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 5, 3, 10, 52, 24) -class TestOpenEphysLegacyRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestOpenEphysLegacyRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysLegacyRecordingInterface interface_kwargs = dict(folder_path=str(DATA_PATH / "openephys" / "OpenEphys_SampleData_1")) save_directory = OUTPUT_PATH @@ -451,37 +479,84 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2018, 10, 3, 13, 16, 50) -class TestOpenEphysRecordingInterfaceRouter(RecordingExtractorInterfaceTestMixin, TestCase): +class TestOpenEphysRecordingInterfaceRouter(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysRecordingInterface - interface_kwargs = [ - dict(folder_path=str(DATA_PATH / "openephys" / "OpenEphys_SampleData_1")), - dict(folder_path=str(DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking")), - dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), - stream_name="Record_Node_107#Neuropix-PXI-116.0", - ), - dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), - stream_name="Record_Node_107#Neuropix-PXI-116.1", - ), - ] save_directory = OUTPUT_PATH + @pytest.fixture( + params=[ + dict(folder_path=str(DATA_PATH / "openephys" / "OpenEphys_SampleData_1")), + dict(folder_path=str(DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking")), + dict( + folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), + stream_name="Record_Node_107#Neuropix-PXI-116.0", + ), + dict( + folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), + stream_name="Record_Node_107#Neuropix-PXI-116.1", + ), + ], + ids=[ + "OpenEphys_SampleData_1", + "v0.4.4.1_with_video_tracking", + "Record_Node_107_Neuropix-PXI-116.0", + "Record_Node_107_Neuropix-PXI-116.1", + ], + ) + def setup_interface(self, request): + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + + def test_interface_extracted_metadata(self, setup_interface): + interface, test_name = setup_interface + metadata = interface.get_metadata() + assert "NWBFile" in metadata # Example assertion + # Additional assertions specific to the metadata can be added here + -class TestSpikeGadgetsRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestSpikeGadgetsRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = SpikeGadgetsRecordingInterface - interface_kwargs = [ - dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec")), - dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.195]), - dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.385] * 512), - dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec")), - dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.195]), - dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.385] * 128), - ] save_directory = OUTPUT_PATH + @pytest.fixture( + params=[ + dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec")), + dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.195]), + dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.385] * 512), + dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec")), + dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.195]), + dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.385] * 128), + ], + ids=[ + "20210225_em8_minirec2_ac_default_gains", + "20210225_em8_minirec2_ac_gains_0.195", + "20210225_em8_minirec2_ac_gains_0.385x512", + "W122_06_09_2019_1_fromSD_default_gains", + "W122_06_09_2019_1_fromSD_gains_0.195", + "W122_06_09_2019_1_fromSD_gains_0.385x128", + ], + ) + def setup_interface(self, request): + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + + def test_extracted_metadata(self, setup_interface): + interface, test_name = setup_interface + metadata = interface.get_metadata() + # Example assertion + assert "NWBFile" in metadata + # Additional assertions specific to the metadata can be added here + -class TestSpikeGLXRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestSpikeGLXRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = SpikeGLXRecordingInterface interface_kwargs = dict( file_path=str(DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.ap.bin") @@ -502,7 +577,7 @@ def check_extracted_metadata(self, metadata: dict): ) -class TestSpikeGLXRecordingInterfaceLongNHP(RecordingExtractorInterfaceTestMixin, TestCase): +class TestSpikeGLXRecordingInterfaceLongNHP(RecordingExtractorInterfaceTestMixin): data_interface_cls = SpikeGLXRecordingInterface interface_kwargs = dict( file_path=str( @@ -530,7 +605,7 @@ def check_extracted_metadata(self, metadata: dict): ) -class TestTdtRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestTdtRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = TdtRecordingInterface test_gain_value = 0.195 # arbitrary value to test gain interface_kwargs = dict(folder_path=str(DATA_PATH / "tdt" / "aep_05"), gain=test_gain_value) @@ -555,7 +630,7 @@ def check_read_nwb(self, nwbfile_path: str): return super().check_read_nwb(nwbfile_path=nwbfile_path) -class TestPlexonRecordingInterface(RecordingExtractorInterfaceTestMixin, TestCase): +class TestPlexonRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = PlexonRecordingInterface interface_kwargs = dict( # Only File_plexon_3.plx has an ecephys recording stream diff --git a/tests/test_on_data/test_segmentation_interfaces.py b/tests/test_on_data/test_segmentation_interfaces.py index 4caf9c48e..3d2547df8 100644 --- a/tests/test_on_data/test_segmentation_interfaces.py +++ b/tests/test_on_data/test_segmentation_interfaces.py @@ -1,6 +1,4 @@ -from unittest import TestCase - -from parameterized import parameterized_class +import pytest from neuroconv.datainterfaces import ( CaimanSegmentationInterface, @@ -18,26 +16,44 @@ from setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH -@parameterized_class( - [ - {"conversion_options": {"mask_type": "image", "include_background_segmentation": True}}, - {"conversion_options": {"mask_type": "pixel", "include_background_segmentation": True}}, - {"conversion_options": {"mask_type": "voxel", "include_background_segmentation": True}}, - # {"conversion_options": {"mask_type": None, "include_background_segmentation": True}}, # Uncomment when https://github.com/catalystneuro/neuroconv/issues/530 is resolved - {"conversion_options": {"include_roi_centroids": False, "include_background_segmentation": True}}, - {"conversion_options": {"include_roi_acceptance": False, "include_background_segmentation": True}}, - {"conversion_options": {"include_background_segmentation": False}}, - ] -) -class TestCaimanSegmentationInterface(SegmentationExtractorInterfaceTestMixin, TestCase): +class TestCaimanSegmentationInterface(SegmentationExtractorInterfaceTestMixin): data_interface_cls = CaimanSegmentationInterface interface_kwargs = dict( file_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "caiman" / "caiman_analysis.hdf5") ) save_directory = OUTPUT_PATH + @pytest.fixture( + params=[ + {"mask_type": "image", "include_background_segmentation": True}, + {"mask_type": "pixel", "include_background_segmentation": True}, + {"mask_type": "voxel", "include_background_segmentation": True}, + # {"mask_type": None, "include_background_segmentation": True}, # Uncomment when https://github.com/catalystneuro/neuroconv/issues/530 is resolved + {"include_roi_centroids": False, "include_background_segmentation": True}, + {"include_roi_acceptance": False, "include_background_segmentation": True}, + {"include_background_segmentation": False}, + ], + ids=[ + "mask_type_image", + "mask_type_pixel", + "mask_type_voxel", + "exclude_roi_centroids", + "exclude_roi_acceptance", + "exclude_background_segmentation", + ], + ) + def setup_interface(self, request): + + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = self.interface_kwargs + self.conversion_options = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + -class TestCnmfeSegmentationInterface(SegmentationExtractorInterfaceTestMixin, TestCase): +class TestCnmfeSegmentationInterface(SegmentationExtractorInterfaceTestMixin): data_interface_cls = CnmfeSegmentationInterface interface_kwargs = dict( file_path=str( @@ -47,84 +63,139 @@ class TestCnmfeSegmentationInterface(SegmentationExtractorInterfaceTestMixin, Te save_directory = OUTPUT_PATH -class TestExtractSegmentationInterface(SegmentationExtractorInterfaceTestMixin, TestCase): +class TestExtractSegmentationInterface(SegmentationExtractorInterfaceTestMixin): data_interface_cls = ExtractSegmentationInterface - interface_kwargs = [ - dict( - file_path=str( - OPHYS_DATA_PATH - / "segmentation_datasets" - / "extract" - / "2014_04_01_p203_m19_check01_extractAnalysis.mat" + save_directory = OUTPUT_PATH + + @pytest.fixture( + params=[ + dict( + file_path=str( + OPHYS_DATA_PATH + / "segmentation_datasets" + / "extract" + / "2014_04_01_p203_m19_check01_extractAnalysis.mat" + ), + sampling_frequency=15.0, # typically provided by user + ), + dict( + file_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "extract" / "extract_public_output.mat"), + sampling_frequency=15.0, # typically provided by user ), - sampling_frequency=15.0, # typically provided by user - ), - dict( - file_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "extract" / "extract_public_output.mat"), - sampling_frequency=15.0, # typically provided by user - ), - ] + ], + ids=["dataset_1", "dataset_2"], + ) + def setup_interface(self, request): + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + + +def test_extract_segmentation_interface_non_default_output_struct_name(): + """Test that the value for 'output_struct_name' is propagated to the extractor level + where an error is raised.""" + file_path = OPHYS_DATA_PATH / "segmentation_datasets" / "extract" / "extract_public_output.mat" + + with pytest.raises(AssertionError, match="Output struct name 'not_output' not found in file."): + ExtractSegmentationInterface( + file_path=str(file_path), + sampling_frequency=15.0, + output_struct_name="not_output", + ) + + +class TestSuite2pSegmentationInterfaceChan1Plane0(SegmentationExtractorInterfaceTestMixin): + data_interface_cls = Suite2pSegmentationInterface save_directory = OUTPUT_PATH + interface_kwargs = dict( + folder_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "suite2p"), + channel_name="chan1", + plane_name="plane0", + ) + + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(self, request): + cls = request.cls + plane_suffix = "Chan1Plane0" + cls.imaging_plane_names = "ImagingPlane" + plane_suffix + cls.plane_segmentation_names = "PlaneSegmentation" + plane_suffix + cls.mean_image_names = "MeanImage" + plane_suffix + cls.correlation_image_names = "CorrelationImage" + plane_suffix + cls.raw_traces_names = "RoiResponseSeries" + plane_suffix + cls.neuropil_traces_names = "Neuropil" + plane_suffix + cls.deconvolved_trace_name = "Deconvolved" + plane_suffix + + def test_check_extracted_metadata(self): + self.interface = self.data_interface_cls(**self.interface_kwargs) + + metadata = self.interface.get_metadata() + + assert metadata["Ophys"]["ImagingPlane"][0]["name"] == self.imaging_plane_names + plane_segmentation_metadata = metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][0] + plane_segmentation_name = self.plane_segmentation_names + assert plane_segmentation_metadata["name"] == plane_segmentation_name + summary_images_metadata = metadata["Ophys"]["SegmentationImages"][plane_segmentation_name] + assert summary_images_metadata["correlation"]["name"] == self.correlation_image_names + assert summary_images_metadata["mean"]["name"] == self.mean_image_names + + raw_traces_metadata = metadata["Ophys"]["Fluorescence"][plane_segmentation_name]["raw"] + assert raw_traces_metadata["name"] == self.raw_traces_names + neuropil_traces_metadata = metadata["Ophys"]["Fluorescence"][plane_segmentation_name]["neuropil"] + assert neuropil_traces_metadata["name"] == self.neuropil_traces_names - def test_extract_segmentation_interface_non_default_output_struct_name(self): - """Test that the value for 'output_struct_name' is propagated to the extractor level - where an error is raised.""" - file_path = OPHYS_DATA_PATH / "segmentation_datasets" / "extract" / "extract_public_output.mat" - with self.assertRaisesRegex(AssertionError, "Output struct name 'not_output' not found in file."): - ExtractSegmentationInterface( - file_path=str(file_path), - sampling_frequency=15.0, - output_struct_name="not_output", - ) + deconvolved_trace_metadata = metadata["Ophys"]["Fluorescence"][plane_segmentation_name]["deconvolved"] + assert deconvolved_trace_metadata["name"] == self.deconvolved_trace_name -class TestSuite2pSegmentationInterface(SegmentationExtractorInterfaceTestMixin, TestCase): +class TestSuite2pSegmentationInterfaceChan2Plane0(SegmentationExtractorInterfaceTestMixin): data_interface_cls = Suite2pSegmentationInterface - interface_kwargs = [ - dict( - folder_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "suite2p"), - channel_name="chan1", - plane_name="plane0", - ), - dict( - folder_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "suite2p"), - channel_name="chan2", - plane_name="plane0", - ), - ] save_directory = OUTPUT_PATH + interface_kwargs = dict( + folder_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "suite2p"), + channel_name="chan2", + plane_name="plane0", + ) - @classmethod - def setUpClass(cls) -> None: - plane_suffices = ["Chan1Plane0", "Chan2Plane0"] - cls.imaging_plane_names = ["ImagingPlane" + plane_suffix for plane_suffix in plane_suffices] - cls.plane_segmentation_names = ["PlaneSegmentation" + plane_suffix for plane_suffix in plane_suffices] - cls.mean_image_names = ["MeanImage" + plane_suffix for plane_suffix in plane_suffices] - cls.correlation_image_names = ["CorrelationImage" + plane_suffix for plane_suffix in plane_suffices] - cls.raw_traces_names = ["RoiResponseSeries" + plane_suffix for plane_suffix in plane_suffices] - cls.neuropil_traces_names = ["Neuropil" + plane_suffix for plane_suffix in plane_suffices] - cls.deconvolved_trace_name = "Deconvolved" + plane_suffices[0] - - def check_extracted_metadata(self, metadata: dict): - """Check extracted metadata is adjusted correctly for each plane and channel combination.""" - self.assertEqual(metadata["Ophys"]["ImagingPlane"][0]["name"], self.imaging_plane_names[self.case]) + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(self, request): + cls = request.cls + + plane_suffix = "Chan2Plane0" + cls.imaging_plane_names = "ImagingPlane" + plane_suffix + cls.plane_segmentation_names = "PlaneSegmentation" + plane_suffix + cls.mean_image_names = "MeanImage" + plane_suffix + cls.correlation_image_names = "CorrelationImage" + plane_suffix + cls.raw_traces_names = "RoiResponseSeries" + plane_suffix + cls.neuropil_traces_names = "Neuropil" + plane_suffix + cls.deconvolved_trace_name = None + + def test_check_extracted_metadata(self): + self.interface = self.data_interface_cls(**self.interface_kwargs) + + metadata = self.interface.get_metadata() + + assert metadata["Ophys"]["ImagingPlane"][0]["name"] == self.imaging_plane_names plane_segmentation_metadata = metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][0] - plane_segmentation_name = self.plane_segmentation_names[self.case] - self.assertEqual(plane_segmentation_metadata["name"], plane_segmentation_name) + plane_segmentation_name = self.plane_segmentation_names + assert plane_segmentation_metadata["name"] == plane_segmentation_name summary_images_metadata = metadata["Ophys"]["SegmentationImages"][plane_segmentation_name] - self.assertEqual(summary_images_metadata["correlation"]["name"], self.correlation_image_names[self.case]) - self.assertEqual(summary_images_metadata["mean"]["name"], self.mean_image_names[self.case]) + assert summary_images_metadata["correlation"]["name"] == self.correlation_image_names + assert summary_images_metadata["mean"]["name"] == self.mean_image_names raw_traces_metadata = metadata["Ophys"]["Fluorescence"][plane_segmentation_name]["raw"] - self.assertEqual(raw_traces_metadata["name"], self.raw_traces_names[self.case]) + assert raw_traces_metadata["name"] == self.raw_traces_names neuropil_traces_metadata = metadata["Ophys"]["Fluorescence"][plane_segmentation_name]["neuropil"] - self.assertEqual(neuropil_traces_metadata["name"], self.neuropil_traces_names[self.case]) - if self.case == 0: + assert neuropil_traces_metadata["name"] == self.neuropil_traces_names + + if self.deconvolved_trace_name: deconvolved_trace_metadata = metadata["Ophys"]["Fluorescence"][plane_segmentation_name]["deconvolved"] - self.assertEqual(deconvolved_trace_metadata["name"], self.deconvolved_trace_name) + assert deconvolved_trace_metadata["name"] == self.deconvolved_trace_name -class TestSuite2pSegmentationInterfaceWithStubTest(SegmentationExtractorInterfaceTestMixin, TestCase): +class TestSuite2pSegmentationInterfaceWithStubTest(SegmentationExtractorInterfaceTestMixin): data_interface_cls = Suite2pSegmentationInterface interface_kwargs = dict( folder_path=str(OPHYS_DATA_PATH / "segmentation_datasets" / "suite2p"), diff --git a/tests/test_on_data/test_sorting_interfaces.py b/tests/test_on_data/test_sorting_interfaces.py index 8898d780b..dfb4ff599 100644 --- a/tests/test_on_data/test_sorting_interfaces.py +++ b/tests/test_on_data/test_sorting_interfaces.py @@ -1,5 +1,4 @@ from datetime import datetime -from unittest import TestCase import numpy as np from pynwb import NWBHDF5IO @@ -25,7 +24,7 @@ from setup_paths import OUTPUT_PATH -class TestBlackrockSortingInterface(SortingExtractorInterfaceTestMixin, TestCase): +class TestBlackrockSortingInterface(SortingExtractorInterfaceTestMixin): data_interface_cls = BlackrockSortingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.nev")) @@ -35,51 +34,81 @@ class TestBlackrockSortingInterface(SortingExtractorInterfaceTestMixin, TestCase save_directory = OUTPUT_PATH -class TestCellExplorerSortingInterfaceBuzCode(SortingExtractorInterfaceTestMixin, TestCase): +import pytest + + +class TestCellExplorerSortingInterfaceBuzCode(SortingExtractorInterfaceTestMixin): """This corresponds to the Buzsaki old CellExplorerFormat or Buzcode format.""" data_interface_cls = CellExplorerSortingInterface - interface_kwargs = [ - dict( - file_path=str( - DATA_PATH / "cellexplorer" / "dataset_1" / "20170311_684um_2088um_170311_134350.spikes.cellinfo.mat" - ) - ), - dict(file_path=str(DATA_PATH / "cellexplorer" / "dataset_2" / "20170504_396um_0um_merge.spikes.cellinfo.mat")), - dict( - file_path=str(DATA_PATH / "cellexplorer" / "dataset_3" / "20170519_864um_900um_merge.spikes.cellinfo.mat") - ), - ] save_directory = OUTPUT_PATH + @pytest.fixture( + params=[ + dict( + file_path=str( + DATA_PATH / "cellexplorer" / "dataset_1" / "20170311_684um_2088um_170311_134350.spikes.cellinfo.mat" + ) + ), + dict( + file_path=str(DATA_PATH / "cellexplorer" / "dataset_2" / "20170504_396um_0um_merge.spikes.cellinfo.mat") + ), + dict( + file_path=str( + DATA_PATH / "cellexplorer" / "dataset_3" / "20170519_864um_900um_merge.spikes.cellinfo.mat" + ) + ), + ], + ids=["dataset_1", "dataset_2", "dataset_3"], + ) + def setup_interface(self, request): + test_id = request.node.callspec.id + self.test_name = test_id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) -class TestCellEploreSortingInterface(SortingExtractorInterfaceTestMixin, TestCase): + return self.interface, self.test_name + + +class TestCellExplorerSortingInterface(SortingExtractorInterfaceTestMixin): """This corresponds to the Buzsaki new CellExplorerFormat where a session.mat file with rich metadata is provided.""" data_interface_cls = CellExplorerSortingInterface - interface_kwargs = [ - dict( - file_path=str( - DATA_PATH - / "cellexplorer" - / "dataset_4" - / "Peter_MS22_180629_110319_concat_stubbed" - / "Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat" - ) - ), - dict( - file_path=str( - DATA_PATH - / "cellexplorer" - / "dataset_4" - / "Peter_MS22_180629_110319_concat_stubbed_hdf5" - / "Peter_MS22_180629_110319_concat_stubbed_hdf5.spikes.cellinfo.mat" - ) - ), - ] save_directory = OUTPUT_PATH - def test_writing_channel_metadata(self): + @pytest.fixture( + params=[ + dict( + file_path=str( + DATA_PATH + / "cellexplorer" + / "dataset_4" + / "Peter_MS22_180629_110319_concat_stubbed" + / "Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat" + ) + ), + dict( + file_path=str( + DATA_PATH + / "cellexplorer" + / "dataset_4" + / "Peter_MS22_180629_110319_concat_stubbed_hdf5" + / "Peter_MS22_180629_110319_concat_stubbed_hdf5.spikes.cellinfo.mat" + ) + ), + ], + ids=["mat", "hdf5"], + ) + def setup_interface(self, request): + self.test_name = request.node.callspec.id + self.interface_kwargs = request.param + self.interface = self.data_interface_cls(**self.interface_kwargs) + + return self.interface, self.test_name + + def test_writing_channel_metadata(self, setup_interface): + interface, test_name = setup_interface + channel_id = "1" expected_channel_properties_recorder = { "location": np.array([791.5, -160.0]), @@ -93,51 +122,49 @@ def test_writing_channel_metadata(self): "group_name": "Group 5", } - interface_kwargs = self.interface_kwargs - for num, kwargs in enumerate(interface_kwargs): - with self.subTest(str(num)): - self.case = num - self.test_kwargs = kwargs - self.interface = self.data_interface_cls(**self.test_kwargs) - self.nwbfile_path = str(self.save_directory / f"{self.data_interface_cls.__name__}_{num}_channel.nwb") - - metadata = self.interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - self.interface.run_conversion( - nwbfile_path=self.nwbfile_path, - overwrite=True, - metadata=metadata, - write_ecephys_metadata=True, - ) + self.nwbfile_path = str(self.save_directory / f"{self.data_interface_cls.__name__}_{test_name}_channel.nwb") + + metadata = interface.get_metadata() + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + interface.run_conversion( + nwbfile_path=self.nwbfile_path, + overwrite=True, + metadata=metadata, + write_ecephys_metadata=True, + ) + + # Test that the registered recording has the expected channel properties + recording_extractor = interface.generate_recording_with_channel_metadata() + for key, expected_value in expected_channel_properties_recorder.items(): + extracted_value = recording_extractor.get_channel_property(channel_id=channel_id, key=key) + if key == "location": + assert np.allclose(expected_value, extracted_value) + else: + assert expected_value == extracted_value + + # Test that the electrode table has the expected values + with NWBHDF5IO(self.nwbfile_path, "r") as io: + nwbfile = io.read() + electrode_table = nwbfile.electrodes.to_dataframe() + electrode_table_row = electrode_table.query(f"channel_name=='{channel_id}'").iloc[0] + for key, value in expected_channel_properties_electrodes.items(): + assert electrode_table_row[key] == value + + +class TestNeuralynxSortingInterfaceCheetahV551(SortingExtractorInterfaceTestMixin): + data_interface_cls = NeuralynxSortingInterface + interface_kwargs = dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.5.1" / "original_data")) + save_directory = OUTPUT_PATH - # Test that the registered recording has the `` - recording_extractor = self.interface.generate_recording_with_channel_metadata() - for key, expected_value in expected_channel_properties_recorder.items(): - extracted_value = recording_extractor.get_channel_property(channel_id=channel_id, key=key) - if key == "location": - assert np.allclose(expected_value, extracted_value) - else: - assert expected_value == extracted_value - - # Test that the electrode table has the expected values - with NWBHDF5IO(self.nwbfile_path, "r") as io: - nwbfile = io.read() - electrode_table = nwbfile.electrodes.to_dataframe() - electrode_table_row = electrode_table.query(f"channel_name=='{channel_id}'").iloc[0] - for key, value in expected_channel_properties_electrodes.items(): - assert electrode_table_row[key] == value - - -class TestNeuralynxSortingInterface(SortingExtractorInterfaceTestMixin, TestCase): + +class TestNeuralynxSortingInterfaceCheetah563(SortingExtractorInterfaceTestMixin): data_interface_cls = NeuralynxSortingInterface - interface_kwargs = [ - dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.5.1" / "original_data")), - dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data")), - ] + interface_kwargs = dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data")) + save_directory = OUTPUT_PATH -class TestNeuroScopeSortingInterface(SortingExtractorInterfaceTestMixin, TestCase): +class TestNeuroScopeSortingInterface(SortingExtractorInterfaceTestMixin): data_interface_cls = NeuroScopeSortingInterface interface_kwargs = dict( folder_path=str(DATA_PATH / "neuroscope" / "dataset_1"), @@ -149,7 +176,7 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2015, 8, 31, 0, 0) -class TestNeuroScopeSortingInterfaceNoXMLSpecified(SortingExtractorInterfaceTestMixin, TestCase): +class TestNeuroScopeSortingInterfaceNoXMLSpecified(SortingExtractorInterfaceTestMixin): """Corresponding to issue https://github.com/NeurodataWithoutBorders/nwb-guide/issues/881.""" data_interface_cls = NeuroScopeSortingInterface @@ -161,13 +188,13 @@ def check_extracted_metadata(self, metadata: dict): pass -class TestPhySortingInterface(SortingExtractorInterfaceTestMixin, TestCase): +class TestPhySortingInterface(SortingExtractorInterfaceTestMixin): data_interface_cls = PhySortingInterface interface_kwargs = dict(folder_path=str(DATA_PATH / "phy" / "phy_example_0")) save_directory = OUTPUT_PATH -class TestPlexonSortingInterface(SortingExtractorInterfaceTestMixin, TestCase): +class TestPlexonSortingInterface(SortingExtractorInterfaceTestMixin): data_interface_cls = PlexonSortingInterface interface_kwargs = dict(file_path=str(DATA_PATH / "plexon" / "File_plexon_2.plx")) save_directory = OUTPUT_PATH From 727228c8bd871d0ffdcca48ac76a2010f293652d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sun, 25 Aug 2024 15:33:32 -0600 Subject: [PATCH 005/118] Add to_nwbfile to methods in roi extractors (#1027) --- CHANGELOG.md | 2 + .../ophys/baseimagingextractorinterface.py | 4 +- .../basesegmentationextractorinterface.py | 4 +- .../miniscopeimagingdatainterface.py | 4 +- src/neuroconv/tools/roiextractors/__init__.py | 17 +- .../tools/roiextractors/roiextractors.py | 597 +++++++++++++++--- tests/test_ophys/test_tools_roiextractors.py | 252 ++++---- 7 files changed, 645 insertions(+), 235 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1edae1bb0..50287da81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Deprecated use of `compression` and `compression_options` in `VideoInterface` [PR #1005](https://github.com/catalystneuro/neuroconv/pull/1005) * `get_schema_from_method_signature` has been deprecated; please use `get_json_schema_from_method_signature` instead. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016) * `neuroconv.utils.FilePathType` and `neuroconv.utils.FolderPathType` have been deprecated; please use `pydantic.FilePath` and `pydantic.DirectoryPath` instead. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017) +* Changed the roiextractors.tool function (e.g. `add_imaging` and `add_segmentation`) to have the `_to_nwbfile` suffix [PR #1027][PR #1017](https://github.com/catalystneuro/neuroconv/pull/1027) + ### Features * Added MedPCInterface for operant behavioral output files. [PR #883](https://github.com/catalystneuro/neuroconv/pull/883) diff --git a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py index e00bb0771..548b57d0c 100644 --- a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py @@ -147,7 +147,7 @@ def add_to_nwbfile( stub_test: bool = False, stub_frames: int = 100, ): - from ...tools.roiextractors import add_imaging + from ...tools.roiextractors import add_imaging_to_nwbfile if stub_test: stub_frames = min([stub_frames, self.imaging_extractor.get_num_frames()]) @@ -155,7 +155,7 @@ def add_to_nwbfile( else: imaging_extractor = self.imaging_extractor - add_imaging( + add_imaging_to_nwbfile( imaging=imaging_extractor, nwbfile=nwbfile, metadata=metadata, diff --git a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py index 5fa6217cc..6b55b5afb 100644 --- a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py @@ -165,7 +165,7 @@ def add_to_nwbfile( ------- """ - from ...tools.roiextractors import add_segmentation + from ...tools.roiextractors import add_segmentation_to_nwbfile if stub_test: stub_frames = min([stub_frames, self.segmentation_extractor.get_num_frames()]) @@ -173,7 +173,7 @@ def add_to_nwbfile( else: segmentation_extractor = self.segmentation_extractor - add_segmentation( + add_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile, metadata=metadata, diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py index e26cc69fb..594bf00dc 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py @@ -93,7 +93,7 @@ def add_to_nwbfile( ): from ndx_miniscope.utils import add_miniscope_device - from ....tools.roiextractors import add_photon_series + from ....tools.roiextractors import add_photon_series_to_nwbfile miniscope_timestamps = self.get_original_timestamps() imaging_extractor = self.imaging_extractor @@ -108,7 +108,7 @@ def add_to_nwbfile( device_metadata = metadata["Ophys"]["Device"][0] add_miniscope_device(nwbfile=nwbfile, device_metadata=device_metadata) - add_photon_series( + add_photon_series_to_nwbfile( imaging=imaging_extractor, nwbfile=nwbfile, metadata=metadata, diff --git a/src/neuroconv/tools/roiextractors/__init__.py b/src/neuroconv/tools/roiextractors/__init__.py index 1f45f9b89..5e009fe6d 100644 --- a/src/neuroconv/tools/roiextractors/__init__.py +++ b/src/neuroconv/tools/roiextractors/__init__.py @@ -1,6 +1,20 @@ from .roiextractors import ( + check_if_imaging_fits_into_memory, + get_nwb_imaging_metadata, + get_nwb_segmentation_metadata, add_background_fluorescence_traces, add_background_plane_segmentation, + add_devices_to_nwbfile, + add_fluorescence_traces_to_nwbfile, + add_image_segmentation_to_nwbfile, + add_imaging_to_nwbfile, + add_imaging_plane_to_nwbfile, + add_photon_series_to_nwbfile, + add_plane_segmentation_to_nwbfile, + add_segmentation_to_nwbfile, + add_summary_images_to_nwbfile, + write_imaging_to_nwbfile, + write_segmentation_to_nwbfile, add_devices, add_fluorescence_traces, add_image_segmentation, @@ -10,9 +24,6 @@ add_plane_segmentation, add_segmentation, add_summary_images, - check_if_imaging_fits_into_memory, - get_nwb_imaging_metadata, - get_nwb_segmentation_metadata, write_imaging, write_segmentation, ) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 75c5a0642..4da660914 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -1,8 +1,8 @@ import math +import warnings from collections import defaultdict from copy import deepcopy from typing import Literal, Optional -from warnings import warn import numpy as np import psutil @@ -42,7 +42,7 @@ from ...utils.str_utils import human_readable_size -def get_default_ophys_metadata() -> DeepDict: +def _get_default_ophys_metadata() -> DeepDict: """Fill default metadata for Device and ImagingPlane.""" metadata = get_default_nwbfile_metadata() @@ -74,9 +74,9 @@ def get_default_ophys_metadata() -> DeepDict: return metadata -def get_default_segmentation_metadata() -> DeepDict: +def _get_default_segmentation_metadata() -> DeepDict: """Fill default metadata for segmentation.""" - metadata = get_default_ophys_metadata() + metadata = _get_default_ophys_metadata() default_fluorescence_roi_response_series = dict( name="RoiResponseSeries", description="Array of raw fluorescence traces.", unit="n.a." @@ -153,7 +153,7 @@ def get_nwb_imaging_metadata( imgextractor : ImagingExtractor photon_series_type : {'OnePhotonSeries', 'TwoPhotonSeries'}, optional """ - metadata = get_default_ophys_metadata() + metadata = _get_default_ophys_metadata() channel_name_list = imgextractor.get_channel_names() or ( ["OpticalChannel"] @@ -189,6 +189,28 @@ def get_nwb_imaging_metadata( def add_devices(nwbfile: NWBFile, metadata: Optional[dict] = None) -> NWBFile: + """ + Deprecated function. Use 'add_devices_to_nwbfile' instead. + """ + + message = ( + "Function 'add_devices' is deprecated and will be removed on or after March 2025. " + "Use 'add_devices_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_devices_to_nwbfile( + nwbfile=nwbfile, + metadata=metadata, + ) + + +def add_devices_to_nwbfile(nwbfile: NWBFile, metadata: Optional[dict] = None) -> NWBFile: """ Add optical physiology devices from metadata. The metadata concerning the optical physiology should be stored in metadata["Ophys]["Device"] @@ -196,7 +218,7 @@ def add_devices(nwbfile: NWBFile, metadata: Optional[dict] = None) -> NWBFile: """ metadata_copy = {} if metadata is None else deepcopy(metadata) - default_metadata = get_default_ophys_metadata() + default_metadata = _get_default_ophys_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) device_metadata = metadata_copy["Ophys"]["Device"] @@ -243,7 +265,33 @@ def add_imaging_plane( nwbfile: NWBFile, metadata: dict, imaging_plane_name: Optional[str] = None, - imaging_plane_index: Optional[int] = None, +): + """ + Deprecated function. Use 'add_imaging_plane_to_nwbfile' instead. + """ + + message = ( + "Function 'add_imaging_plane' is deprecated and will be removed on or after March 2025. " + "Use 'add_imaging_plane_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_imaging_plane_to_nwbfile( + nwbfile=nwbfile, + metadata=metadata, + imaging_plane_name=imaging_plane_name, + ) + + +def add_imaging_plane_to_nwbfile( + nwbfile: NWBFile, + metadata: dict, + imaging_plane_name: Optional[str] = None, ) -> NWBFile: """ Adds the imaging plane specified by the metadata to the nwb file. @@ -257,27 +305,18 @@ def add_imaging_plane( The metadata in the nwb conversion tools format. imaging_plane_name: str The name of the imaging plane to be added. - imaging_plane_index: int, optional - Returns ------- NWBFile The nwbfile passed as an input with the imaging plane added. """ - if imaging_plane_index is not None: - warn( - message="Keyword argument 'imaging_plane_index' is deprecated and will be removed on or after Dec 1st, 2023. " - "Use 'imaging_plane_name' to specify which imaging plane to add by its name.", - category=DeprecationWarning, - ) - imaging_plane_name = metadata["Ophys"]["ImagingPlane"][imaging_plane_index]["name"] # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_ophys_metadata() + default_metadata = _get_default_ophys_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) - add_devices(nwbfile=nwbfile, metadata=metadata_copy) + add_devices_to_nwbfile(nwbfile=nwbfile, metadata=metadata_copy) default_imaging_plane_name = default_metadata["Ophys"]["ImagingPlane"][0]["name"] imaging_plane_name = imaging_plane_name or default_imaging_plane_name @@ -306,6 +345,28 @@ def add_imaging_plane( def add_image_segmentation(nwbfile: NWBFile, metadata: dict) -> NWBFile: + """ + Deprecated function. Use 'add_image_segmentation_to_nwbfile' instead. + """ + + message = ( + "Function 'add_image_segmentation' is deprecated and will be removed on or after March 2025. " + "Use 'add_image_segmentation_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_image_segmentation_to_nwbfile( + nwbfile=nwbfile, + metadata=metadata, + ) + + +def add_image_segmentation_to_nwbfile(nwbfile: NWBFile, metadata: dict) -> NWBFile: """ Adds the image segmentation specified by the metadata to the nwb file. @@ -323,7 +384,7 @@ def add_image_segmentation(nwbfile: NWBFile, metadata: dict) -> NWBFile: """ # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_segmentation_metadata() + default_metadata = _get_default_segmentation_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) image_segmentation_metadata = metadata_copy["Ophys"]["ImageSegmentation"] @@ -345,7 +406,43 @@ def add_photon_series( photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", photon_series_index: int = 0, parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", - two_photon_series_index: Optional[int] = None, # TODO: to be removed + iterator_type: Optional[str] = "v2", + iterator_options: Optional[dict] = None, +) -> NWBFile: + """ + Deprecated function. Use 'add_photon_series_to_nwbfile' instead. + """ + + message = ( + "Function 'add_photon_series' is deprecated and will be removed on or after March 2025. " + "Use 'add_photon_series_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_photon_series_to_nwbfile( + imaging=imaging, + nwbfile=nwbfile, + metadata=metadata, + photon_series_type=photon_series_type, + photon_series_index=photon_series_index, + parent_container=parent_container, + iterator_type=iterator_type, + iterator_options=iterator_options, + ) + + +def add_photon_series_to_nwbfile( + imaging: ImagingExtractor, + nwbfile: NWBFile, + metadata: Optional[dict] = None, + photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", + photon_series_index: int = 0, + parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, ) -> NWBFile: @@ -382,12 +479,6 @@ def add_photon_series( The NWBFile passed as an input with the photon series added. """ - if two_photon_series_index: - warn( - "Keyword argument 'two_photon_series_index' is deprecated and it will be removed on 2024-04-16. Use 'photon_series_index' instead." - ) - photon_series_index = two_photon_series_index - iterator_options = iterator_options or dict() metadata_copy = {} if metadata is None else deepcopy(metadata) @@ -400,7 +491,7 @@ def add_photon_series( ) if photon_series_type == "TwoPhotonSeries" and "OnePhotonSeries" in metadata_copy["Ophys"]: - warn( + warnings.warn( "Received metadata for both 'OnePhotonSeries' and 'TwoPhotonSeries', make sure photon_series_type is specified correctly." ) @@ -422,7 +513,7 @@ def add_photon_series( # Add the image plane to nwb imaging_plane_name = photon_series_metadata["imaging_plane"] - add_imaging_plane(nwbfile=nwbfile, metadata=metadata_copy, imaging_plane_name=imaging_plane_name) + add_imaging_plane_to_nwbfile(nwbfile=nwbfile, metadata=metadata_copy, imaging_plane_name=imaging_plane_name) imaging_plane = nwbfile.get_imaging_plane(name=imaging_plane_name) photon_series_kwargs = deepcopy(photon_series_metadata) photon_series_kwargs.update(imaging_plane=imaging_plane) @@ -554,9 +645,46 @@ def add_imaging( iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", +) -> NWBFile: + """ + Deprecated function. Use 'add_imaging_to_nwbfile' instead. + """ + + message = ( + "Function 'add_imaging' is deprecated and will be removed on or after March 2025. " + "Use 'add_imaging_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_imaging_to_nwbfile( + imaging=imaging, + nwbfile=nwbfile, + metadata=metadata, + photon_series_type=photon_series_type, + photon_series_index=photon_series_index, + iterator_type=iterator_type, + iterator_options=iterator_options, + parent_container=parent_container, + ) + + +def add_imaging_to_nwbfile( + imaging: ImagingExtractor, + nwbfile: NWBFile, + metadata: Optional[dict] = None, + photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", + photon_series_index: int = 0, + iterator_type: Optional[str] = "v2", + iterator_options: Optional[dict] = None, + parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", ): - add_devices(nwbfile=nwbfile, metadata=metadata) - add_photon_series( + add_devices_to_nwbfile(nwbfile=nwbfile, metadata=metadata) + add_photon_series_to_nwbfile( imaging=imaging, nwbfile=nwbfile, metadata=metadata, @@ -578,7 +706,45 @@ def write_imaging( iterator_type: str = "v2", iterator_options: Optional[dict] = None, photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", - buffer_size: Optional[int] = None, # TODO: to be removed +): + """ + Deprecated function. Use 'write_imaging_to_nwbfile' instead. + """ + + message = ( + "Function 'write_imaging' is deprecated and will be removed on or after March 2025. " + "Use 'write_imaging_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return write_imaging_to_nwbfile( + imaging=imaging, + nwbfile_path=nwbfile_path, + nwbfile=nwbfile, + metadata=metadata, + overwrite=overwrite, + verbose=verbose, + iterator_type=iterator_type, + iterator_options=iterator_options, + photon_series_type=photon_series_type, + ) + + +def write_imaging_to_nwbfile( + imaging: ImagingExtractor, + nwbfile_path: Optional[FilePath] = None, + nwbfile: Optional[NWBFile] = None, + metadata: Optional[dict] = None, + overwrite: bool = False, + verbose: bool = True, + iterator_type: str = "v2", + iterator_options: Optional[dict] = None, + photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", ): """ Primary method for writing an ImagingExtractor object to an NWBFile. @@ -626,11 +792,6 @@ def write_imaging( assert isinstance(nwbfile, NWBFile), "'nwbfile' should be of type pynwb.NWBFile" iterator_options = iterator_options or dict() - if buffer_size: - warn( - "Keyword argument 'buffer_size' is deprecated and will be removed on or after September 1st, 2022." - "Specify as a key in the new 'iterator_options' dictionary instead." - ) if metadata is None: metadata = dict() @@ -640,7 +801,7 @@ def write_imaging( with make_or_load_nwbfile( nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=verbose ) as nwbfile_out: - add_imaging( + add_imaging_to_nwbfile( imaging=imaging, nwbfile=nwbfile, metadata=metadata, @@ -659,7 +820,7 @@ def get_nwb_segmentation_metadata(sgmextractor: SegmentationExtractor) -> dict: ---------- sgmextractor: SegmentationExtractor """ - metadata = get_default_segmentation_metadata() + metadata = _get_default_segmentation_metadata() # Optical Channel name: for i in range(sgmextractor.get_num_channels()): ch_name = sgmextractor.get_channel_names()[i] @@ -693,7 +854,45 @@ def add_plane_segmentation( nwbfile: NWBFile, metadata: Optional[dict], plane_segmentation_name: Optional[str] = None, - plane_segmentation_index: Optional[int] = None, # TODO: to be removed + include_roi_centroids: bool = True, + include_roi_acceptance: bool = True, + mask_type: Optional[str] = "image", # Optional[Literal["image", "pixel"]] + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 +) -> NWBFile: + """ + Deprecated function. Use 'add_plane_segmentation_to_nwbfile' instead. + """ + + message = ( + "Function 'add_plane_segmentation' is deprecated and will be removed on or after March 2025. " + "Use 'add_plane_segmentation_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_plane_segmentation_to_nwbfile( + segmentation_extractor=segmentation_extractor, + nwbfile=nwbfile, + metadata=metadata, + plane_segmentation_name=plane_segmentation_name, + include_roi_centroids=include_roi_centroids, + include_roi_acceptance=include_roi_acceptance, + mask_type=mask_type, + iterator_options=iterator_options, + compression_options=compression_options, + ) + + +def add_plane_segmentation_to_nwbfile( + segmentation_extractor: SegmentationExtractor, + nwbfile: NWBFile, + metadata: Optional[dict], + plane_segmentation_name: Optional[str] = None, include_roi_centroids: bool = True, include_roi_acceptance: bool = True, mask_type: Optional[str] = "image", # Optional[Literal["image", "pixel"]] @@ -745,7 +944,7 @@ def add_plane_segmentation( """ # TODO: remove completely after 10/1/2024 if compression_options is not None: - warn( + warnings.warn( message=( "Specifying compression methods and their options at the level of tool functions has been deprecated. " "Please use the `configure_backend` tool function for this purpose." @@ -777,6 +976,7 @@ def add_plane_segmentation( roi_locations = segmentation_extractor.get_roi_locations()[tranpose_image_convention, :].T else: roi_locations = None + nwbfile = _add_plane_segmentation( background_or_roi_ids=roi_ids, image_or_pixel_masks=image_or_pixel_masks, @@ -787,7 +987,6 @@ def add_plane_segmentation( nwbfile=nwbfile, metadata=metadata, plane_segmentation_name=plane_segmentation_name, - plane_segmentation_index=plane_segmentation_index, include_roi_centroids=include_roi_centroids, include_roi_acceptance=include_roi_acceptance, mask_type=mask_type, @@ -803,7 +1002,6 @@ def _add_plane_segmentation( nwbfile: NWBFile, metadata: Optional[dict], plane_segmentation_name: Optional[str] = None, - plane_segmentation_index: Optional[int] = None, # TODO: to be removed include_roi_centroids: bool = False, roi_locations: Optional[np.ndarray] = None, include_roi_acceptance: bool = False, @@ -812,11 +1010,12 @@ def _add_plane_segmentation( mask_type: Optional[str] = "image", # Optional[Literal["image", "pixel"]] iterator_options: Optional[dict] = None, ) -> NWBFile: + iterator_options = iterator_options or dict() # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_segmentation_metadata() + default_metadata = _get_default_segmentation_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) image_segmentation_metadata = metadata_copy["Ophys"]["ImageSegmentation"] @@ -826,11 +1025,7 @@ def _add_plane_segmentation( "name" ] ) - if plane_segmentation_index: - warn( - "Keyword argument 'plane_segmentation_index' is deprecated and it will be removed on 2024-04-16. Use 'plane_segmentation_name' instead." - ) - plane_segmentation_name = image_segmentation_metadata["plane_segmentations"][plane_segmentation_index]["name"] + plane_segmentation_metadata = next( ( plane_segmentation_metadata @@ -845,8 +1040,8 @@ def _add_plane_segmentation( ) imaging_plane_name = plane_segmentation_metadata["imaging_plane"] - add_imaging_plane(nwbfile=nwbfile, metadata=metadata_copy, imaging_plane_name=imaging_plane_name) - add_image_segmentation(nwbfile=nwbfile, metadata=metadata_copy) + add_imaging_plane_to_nwbfile(nwbfile=nwbfile, metadata=metadata_copy, imaging_plane_name=imaging_plane_name) + add_image_segmentation_to_nwbfile(nwbfile=nwbfile, metadata=metadata_copy) ophys = get_module(nwbfile, "ophys") image_segmentation_name = image_segmentation_metadata["name"] @@ -877,13 +1072,13 @@ def _add_plane_segmentation( "Please open a ticket with https://github.com/catalystneuro/roiextractors/issues" ) if mask_type == "pixel" and num_pixel_dims == 4: - warn( + warnings.warn( "Specified mask_type='pixel', but ROIExtractors returned 4-dimensional masks. " "Using mask_type='voxel' instead." ) mask_type = "voxel" if mask_type == "voxel" and num_pixel_dims == 3: - warn( + warnings.warn( "Specified mask_type='voxel', but ROIExtractors returned 3-dimensional masks. " "Using mask_type='pixel' instead." ) @@ -928,9 +1123,46 @@ def add_background_plane_segmentation( iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: + """ + Deprecated function. Use 'add_background_plane_segmentation_to_nwbfile' instead. + """ + + message = ( + "Function 'add_background_plane_segmentation' is deprecated and will be removed on or after March 2025. " + "Use 'add_background_plane_segmentation_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_background_plane_segmentation_to_nwbfile( + segmentation_extractor=segmentation_extractor, + nwbfile=nwbfile, + metadata=metadata, + background_plane_segmentation_name=background_plane_segmentation_name, + mask_type=mask_type, + iterator_options=iterator_options, + compression_options=compression_options, + ) + + +def add_background_plane_segmentation_to_nwbfile( + segmentation_extractor: SegmentationExtractor, + nwbfile: NWBFile, + metadata: Optional[dict], + background_plane_segmentation_name: Optional[str] = None, + mask_type: Optional[str] = "image", # Optional[Literal["image", "pixel"]] + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 +) -> NWBFile: + # TODO needs docstring + # TODO: remove completely after 10/1/2024 if compression_options is not None: - warn( + warnings.warn( message=( "Specifying compression methods and their options at the level of tool functions has been deprecated. " "Please use the `configure_backend` tool function for this purpose." @@ -971,7 +1203,41 @@ def add_fluorescence_traces( metadata: Optional[dict], plane_segmentation_name: Optional[str] = None, include_background_segmentation: bool = False, - plane_index: Optional[int] = None, # TODO: to be removed + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 +) -> NWBFile: + """ + Deprecated function. Use 'add_fluorescence_traces_to_nwbfile' instead. + """ + + message = ( + "Function 'add_fluorescence_traces' is deprecated and will be removed on or after March 2025. " + "Use 'add_fluorescence_traces_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_fluorescence_traces_to_nwbfile( + segmentation_extractor=segmentation_extractor, + nwbfile=nwbfile, + metadata=metadata, + plane_segmentation_name=plane_segmentation_name, + include_background_segmentation=include_background_segmentation, + iterator_options=iterator_options, + compression_options=compression_options, + ) + + +def add_fluorescence_traces_to_nwbfile( + segmentation_extractor: SegmentationExtractor, + nwbfile: NWBFile, + metadata: Optional[dict], + plane_segmentation_name: Optional[str] = None, + include_background_segmentation: bool = False, iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: @@ -1002,7 +1268,7 @@ def add_fluorescence_traces( """ # TODO: remove completely after 10/1/2024 if compression_options is not None: - warn( + warnings.warn( message=( "Specifying compression methods and their options at the level of tool functions has been deprecated. " "Please use the `configure_backend` tool function for this purpose." @@ -1024,7 +1290,7 @@ def add_fluorescence_traces( return nwbfile roi_ids = segmentation_extractor.get_roi_ids() - nwbfile = _add_fluorescence_traces( + nwbfile = _add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, traces_to_add=traces_to_add, background_or_roi_ids=roi_ids, @@ -1032,13 +1298,12 @@ def add_fluorescence_traces( metadata=metadata, default_plane_segmentation_index=default_plane_segmentation_index, plane_segmentation_name=plane_segmentation_name, - plane_index=plane_index, iterator_options=iterator_options, ) return nwbfile -def _add_fluorescence_traces( +def _add_fluorescence_traces_to_nwbfile( segmentation_extractor: SegmentationExtractor, traces_to_add: dict, background_or_roi_ids: list, @@ -1046,23 +1311,15 @@ def _add_fluorescence_traces( metadata: Optional[dict], default_plane_segmentation_index: int, plane_segmentation_name: Optional[str] = None, - plane_index: Optional[int] = None, # TODO: to be removed iterator_options: Optional[dict] = None, ): iterator_options = iterator_options or dict() # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_segmentation_metadata() + default_metadata = _get_default_segmentation_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) - if plane_index: - warn( - "Keyword argument 'plane_index' is deprecated and it will be removed on 2024-04-16. Use 'plane_segmentation_name' instead." - ) - plane_segmentation_name = metadata_copy["Ophys"]["ImageSegmentation"]["plane_segmentations"][plane_index][ - "name" - ] plane_segmentation_name = ( plane_segmentation_name or default_metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][default_plane_segmentation_index][ @@ -1154,7 +1411,6 @@ def _create_roi_table_region( nwbfile: NWBFile, metadata: dict, plane_segmentation_name: Optional[str] = None, - plane_index: Optional[int] = None, ): """Private method to create ROI table region. @@ -1171,13 +1427,7 @@ def _create_roi_table_region( """ image_segmentation_metadata = metadata["Ophys"]["ImageSegmentation"] - if plane_index: - warn( - "Keyword argument 'plane_index' is deprecated and it will be removed on 2024-04-16. Use 'plane_segmentation_name' instead." - ) - plane_segmentation_name = image_segmentation_metadata["plane_segmentations"][plane_index]["name"] - - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile, metadata=metadata, @@ -1227,7 +1477,39 @@ def add_background_fluorescence_traces( nwbfile: NWBFile, metadata: Optional[dict], background_plane_segmentation_name: Optional[str] = None, - plane_index: Optional[int] = None, # TODO: to be removed + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 +) -> NWBFile: + """ + Deprecated function. Use 'add_background_fluorescence_traces_to_nwbfile' instead. + """ + + message = ( + "Function 'add_background_fluorescence_traces' is deprecated and will be removed on or after March 2025. " + "Use 'add_background_fluorescence_traces_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_background_fluorescence_traces_to_nwbfile( + segmentation_extractor=segmentation_extractor, + nwbfile=nwbfile, + metadata=metadata, + background_plane_segmentation_name=background_plane_segmentation_name, + iterator_options=iterator_options, + compression_options=compression_options, + ) + + +def add_background_fluorescence_traces_to_nwbfile( + segmentation_extractor: SegmentationExtractor, + nwbfile: NWBFile, + metadata: Optional[dict], + background_plane_segmentation_name: Optional[str] = None, iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: @@ -1255,7 +1537,7 @@ def add_background_fluorescence_traces( """ # TODO: remove completely after 10/1/2024 if compression_options is not None: - warn( + warnings.warn( message=( "Specifying compression methods and their options at the level of tool functions has been deprecated. " "Please use the `configure_backend` tool function for this purpose." @@ -1277,7 +1559,7 @@ def add_background_fluorescence_traces( return nwbfile background_ids = segmentation_extractor.get_background_ids() - nwbfile = _add_fluorescence_traces( + nwbfile = _add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, traces_to_add=traces_to_add, background_or_roi_ids=background_ids, @@ -1285,18 +1567,45 @@ def add_background_fluorescence_traces( metadata=metadata, default_plane_segmentation_index=default_plane_segmentation_index, plane_segmentation_name=background_plane_segmentation_name, - plane_index=plane_index, iterator_options=iterator_options, ) return nwbfile def add_summary_images( + segmentation_extractor: SegmentationExtractor, + nwbfile: NWBFile, + metadata: Optional[dict] = None, + plane_segmentation_name: Optional[str] = None, +) -> NWBFile: + """ + Deprecated function. Use 'add_summary_images_to_nwbfile' instead. + """ + + message = ( + "Function 'add_summary_images' is deprecated and will be removed on or after March 2025. " + "Use 'add_summary_images_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_summary_images_to_nwbfile( + nwbfile=nwbfile, + segmentation_extractor=segmentation_extractor, + metadata=metadata, + plane_segmentation_name=plane_segmentation_name, + ) + + +def add_summary_images_to_nwbfile( nwbfile: NWBFile, segmentation_extractor: SegmentationExtractor, metadata: Optional[dict] = None, plane_segmentation_name: Optional[str] = None, - images_set_name: Optional[str] = None, # TODO: to be removed ) -> NWBFile: """ Adds summary images (i.e. mean and correlation) to the nwbfile using an image container object pynwb.Image @@ -1317,23 +1626,14 @@ def add_summary_images( NWBFile The nwbfile passed as an input with the summary images added. """ - if metadata is None: - metadata = dict() + metadata = metadata or dict() # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_segmentation_metadata() + default_metadata = _get_default_segmentation_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) segmentation_images_metadata = metadata_copy["Ophys"]["SegmentationImages"] - - if images_set_name is not None: - warn( - "Keyword argument 'images_set_name' is deprecated it will be removed on 2024-04-16." - "Specify the name of the Images container in metadata['Ophys']['SegmentationImages'] instead." - ) - segmentation_images_metadata["name"] = images_set_name - images_container_name = segmentation_images_metadata["name"] images_dict = segmentation_extractor.get_images_dict() @@ -1375,18 +1675,61 @@ def add_segmentation( plane_segmentation_name: Optional[str] = None, background_plane_segmentation_name: Optional[str] = None, include_background_segmentation: bool = False, - plane_num: Optional[int] = None, # TODO: to be removed include_roi_centroids: bool = True, include_roi_acceptance: bool = True, mask_type: Optional[str] = "image", # Literal["image", "pixel"] iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 -): +) -> NWBFile: + """ + Deprecated function. Use 'add_segmentation_to_nwbfile' instead. + """ + + message = ( + "Function 'add_segmentation' is deprecated and will be removed on or after March 2025. " + "Use 'add_segmentation_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return add_segmentation_to_nwbfile( + segmentation_extractor=segmentation_extractor, + nwbfile=nwbfile, + metadata=metadata, + plane_segmentation_name=plane_segmentation_name, + background_plane_segmentation_name=background_plane_segmentation_name, + include_background_segmentation=include_background_segmentation, + include_roi_centroids=include_roi_centroids, + include_roi_acceptance=include_roi_acceptance, + mask_type=mask_type, + iterator_options=iterator_options, + compression_options=compression_options, + ) + + +def add_segmentation_to_nwbfile( + segmentation_extractor: SegmentationExtractor, + nwbfile: NWBFile, + metadata: Optional[dict] = None, + plane_segmentation_name: Optional[str] = None, + background_plane_segmentation_name: Optional[str] = None, + include_background_segmentation: bool = False, + include_roi_centroids: bool = True, + include_roi_acceptance: bool = True, + mask_type: Optional[str] = "image", # Literal["image", "pixel"] + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 +) -> NWBFile: # TODO: remove completely after 10/1/2024 if compression_options is not None: - warn( + warnings.warn( message=( "Specifying compression methods and their options at the level of tool functions has been deprecated. " + "The option will be removed after 2024-10-01. " "Please use the `configure_backend` tool function for this purpose." ), category=DeprecationWarning, @@ -1394,11 +1737,10 @@ def add_segmentation( ) # Add device: - add_devices(nwbfile=nwbfile, metadata=metadata) + add_devices_to_nwbfile(nwbfile=nwbfile, metadata=metadata) - # `add_imaging_plane` is also called from `add_plane_segmentation` so no need to call it explicitly here # Add PlaneSegmentation: - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile, metadata=metadata, @@ -1409,7 +1751,7 @@ def add_segmentation( iterator_options=iterator_options, ) if include_background_segmentation: - add_background_plane_segmentation( + add_background_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile, metadata=metadata, @@ -1419,7 +1761,7 @@ def add_segmentation( ) # Add fluorescence traces: - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile, metadata=metadata, @@ -1427,8 +1769,9 @@ def add_segmentation( include_background_segmentation=include_background_segmentation, iterator_options=iterator_options, ) + if include_background_segmentation: - add_background_fluorescence_traces( + add_background_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile, metadata=metadata, @@ -1437,13 +1780,15 @@ def add_segmentation( ) # Adding summary images (mean and correlation) - add_summary_images( + add_summary_images_to_nwbfile( nwbfile=nwbfile, segmentation_extractor=segmentation_extractor, metadata=metadata, plane_segmentation_name=plane_segmentation_name, ) + return nwbfile + def write_segmentation( segmentation_extractor: SegmentationExtractor, @@ -1458,6 +1803,51 @@ def write_segmentation( mask_type: Optional[str] = "image", # Literal["image", "pixel"] iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 +) -> NWBFile: + """ + Deprecated function. Use 'write_segmentation_to_nwbfile' instead. + """ + + message = ( + "Function 'write_segmentation' is deprecated and will be removed on or after March 2025. " + "Use 'write_segmentation_to_nwbfile' instead." + ) + + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=2, + ) + + return write_segmentation_to_nwbfile( + segmentation_extractor=segmentation_extractor, + nwbfile_path=nwbfile_path, + nwbfile=nwbfile, + metadata=metadata, + overwrite=overwrite, + verbose=verbose, + include_background_segmentation=include_background_segmentation, + include_roi_centroids=include_roi_centroids, + include_roi_acceptance=include_roi_acceptance, + mask_type=mask_type, + iterator_options=iterator_options, + compression_options=compression_options, + ) + + +def write_segmentation_to_nwbfile( + segmentation_extractor: SegmentationExtractor, + nwbfile_path: Optional[FilePath] = None, + nwbfile: Optional[NWBFile] = None, + metadata: Optional[dict] = None, + overwrite: bool = False, + verbose: bool = True, + include_background_segmentation: bool = False, + include_roi_centroids: bool = True, + include_roi_acceptance: bool = True, + mask_type: Optional[str] = "image", # Literal["image", "pixel"] + iterator_options: Optional[dict] = None, + compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: """ Primary method for writing an SegmentationExtractor object to an NWBFile. @@ -1516,9 +1906,10 @@ def write_segmentation( # TODO: remove completely after 10/1/2024 if compression_options is not None: - warn( + warnings.warn( message=( "Specifying compression methods and their options at the level of tool functions has been deprecated. " + "They will be removed on or after 2024-10-01. " "Please use the `configure_backend` tool function for this purpose." ), category=DeprecationWarning, @@ -1560,7 +1951,7 @@ def write_segmentation( for plane_no_loop, (segmentation_extractor, metadata) in enumerate( zip(segmentation_extractors, metadata_base_list) ): - add_segmentation( + add_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=nwbfile_out, metadata=metadata, diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index f535b6469..60162527b 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -27,20 +27,20 @@ from neuroconv.tools.nwb_helpers import get_module from neuroconv.tools.roiextractors import ( - add_devices, - add_fluorescence_traces, - add_image_segmentation, - add_imaging_plane, - add_photon_series, - add_plane_segmentation, - add_summary_images, + add_devices_to_nwbfile, + add_fluorescence_traces_to_nwbfile, + add_image_segmentation_to_nwbfile, + add_imaging_plane_to_nwbfile, + add_photon_series_to_nwbfile, + add_plane_segmentation_to_nwbfile, + add_summary_images_to_nwbfile, check_if_imaging_fits_into_memory, ) from neuroconv.tools.roiextractors.imagingextractordatachunkiterator import ( ImagingExtractorDataChunkIterator, ) from neuroconv.tools.roiextractors.roiextractors import ( - get_default_segmentation_metadata, + _get_default_segmentation_metadata, ) from neuroconv.utils import dict_deep_update @@ -60,7 +60,7 @@ def test_add_device(self): device_name = "new_device" device_list = [dict(name=device_name)] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -74,7 +74,7 @@ def test_add_device_with_further_metadata(self): device_list = [dict(name=device_name, description=description, manufacturer=manufacturer)] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices device = devices["new_device"] @@ -88,7 +88,7 @@ def test_add_two_devices(self): device_name_list = ["device1", "device2"] device_list = [dict(name=device_name) for device_name in device_name_list] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -99,12 +99,12 @@ def test_add_one_device_and_then_another(self): device_name1 = "new_device" device_list = [dict(name=device_name1)] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) device_name2 = "another_device" device_list = [dict(name=device_name2)] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -116,12 +116,12 @@ def test_not_overwriting_devices(self): device_name1 = "same_device" device_list = [dict(name=device_name1)] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) device_name2 = "same_device" device_list = [dict(name=device_name2)] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -129,7 +129,7 @@ def test_not_overwriting_devices(self): assert device_name1 in devices def test_add_device_defaults(self): - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -139,7 +139,7 @@ def test_add_device_defaults(self): def test_add_empty_device_list_in_metadata(self): device_list = [] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -150,7 +150,7 @@ def test_device_object(self): device_object = Device(name=device_name) device_list = [device_object] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -162,7 +162,7 @@ def test_device_object_and_metadata_mix(self): device_metadata = dict(name="device_metadata") device_list = [device_object, device_metadata] self.metadata["Ophys"].update(Device=device_list) - add_devices(self.nwbfile, metadata=self.metadata) + add_devices_to_nwbfile(self.nwbfile, metadata=self.metadata) devices = self.nwbfile.devices @@ -206,8 +206,10 @@ def setUp(self): self.metadata["Ophys"].update(ImagingPlane=[self.imaging_plane_metadata]) - def test_add_imaging_plane(self): - add_imaging_plane(nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=self.imaging_plane_name) + def test_add_imaging_plane_to_nwbfile(self): + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=self.imaging_plane_name + ) imaging_planes = self.nwbfile.imaging_planes assert len(imaging_planes) == 1 @@ -217,10 +219,14 @@ def test_add_imaging_plane(self): assert imaging_plane.description == self.imaging_plane_description def test_not_overwriting_imaging_plane_if_same_name(self): - add_imaging_plane(nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=self.imaging_plane_name) + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=self.imaging_plane_name + ) self.imaging_plane_metadata["description"] = "modified description" - add_imaging_plane(nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=self.imaging_plane_name) + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=self.imaging_plane_name + ) imaging_planes = self.nwbfile.imaging_planes assert len(imaging_planes) == 1 @@ -232,14 +238,18 @@ def test_add_two_imaging_planes(self): first_imaging_plane_description = "first_imaging_plane_description" self.imaging_plane_metadata["name"] = first_imaging_plane_name self.imaging_plane_metadata["description"] = first_imaging_plane_description - add_imaging_plane(nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=first_imaging_plane_name) + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=first_imaging_plane_name + ) # Add the second imaging plane second_imaging_plane_name = "second_imaging_plane_name" second_imaging_plane_description = "second_imaging_plane_description" self.imaging_plane_metadata["name"] = second_imaging_plane_name self.imaging_plane_metadata["description"] = second_imaging_plane_description - add_imaging_plane(nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=second_imaging_plane_name) + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=second_imaging_plane_name + ) # Test expected values imaging_planes = self.nwbfile.imaging_planes @@ -253,14 +263,16 @@ def test_add_two_imaging_planes(self): assert second_imaging_plane.name == second_imaging_plane_name assert second_imaging_plane.description == second_imaging_plane_description - def test_add_imaging_plane_raises_when_name_not_found_in_metadata(self): + def test_add_imaging_plane_to_nwbfile_raises_when_name_not_found_in_metadata(self): """Test adding an imaging plane raises an error when the name is not found in the metadata.""" imaging_plane_name = "imaging_plane_non_existing_in_the_metadata" with self.assertRaisesWith( exc_type=ValueError, exc_msg=f"Metadata for Imaging Plane '{imaging_plane_name}' not found in metadata['Ophys']['ImagingPlane'].", ): - add_imaging_plane(nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=imaging_plane_name) + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_name=imaging_plane_name + ) def test_add_two_imaging_planes_from_metadata(self): """Test adding two imaging planes when there are multiple imaging plane metadata.""" @@ -271,8 +283,12 @@ def test_add_two_imaging_planes_from_metadata(self): second_imaging_plane_metadata = deepcopy(metadata["Ophys"]["ImagingPlane"][0]) second_imaging_plane_metadata.update(name="second_imaging_plane_name") imaging_planes_metadata.append(second_imaging_plane_metadata) - add_imaging_plane(nwbfile=self.nwbfile, metadata=metadata, imaging_plane_name=self.imaging_plane_name) - add_imaging_plane(nwbfile=self.nwbfile, metadata=metadata, imaging_plane_name="second_imaging_plane_name") + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=metadata, imaging_plane_name=self.imaging_plane_name + ) + add_imaging_plane_to_nwbfile( + nwbfile=self.nwbfile, metadata=metadata, imaging_plane_name="second_imaging_plane_name" + ) # Test expected values imaging_planes = self.nwbfile.imaging_planes @@ -284,18 +300,6 @@ def test_add_two_imaging_planes_from_metadata(self): second_imaging_plane = imaging_planes[second_imaging_plane_name] assert second_imaging_plane.name == second_imaging_plane_name - def test_add_imaging_plane_warns_when_index_is_used(self): - """Test adding an imaging plane with the index specified warns with DeprecationWarning.""" - exc_msg = "Keyword argument 'imaging_plane_index' is deprecated and will be removed on or after Dec 1st, 2023. Use 'imaging_plane_name' to specify which imaging plane to add by its name." - with self.assertWarnsWith(warn_type=DeprecationWarning, exc_msg=exc_msg): - add_imaging_plane(nwbfile=self.nwbfile, metadata=self.metadata, imaging_plane_index=0) - # Test expected values - imaging_planes = self.nwbfile.imaging_planes - assert len(imaging_planes) == 1 - - imaging_plane = imaging_planes[self.imaging_plane_name] - assert imaging_plane.name == self.imaging_plane_name - class TestAddImageSegmentation(unittest.TestCase): def setUp(self): @@ -313,13 +317,13 @@ def setUp(self): self.metadata["Ophys"].update(image_segmentation_metadata) - def test_add_image_segmentation(self): + def test_add_image_segmentation_to_nwbfile(self): """ - Test that add_image_segmentation method adds an image segmentation to the nwbfile + Test that add_image_segmentation_to_nwbfile method adds an image segmentation to the nwbfile specified by the metadata. """ - add_image_segmentation(nwbfile=self.nwbfile, metadata=self.metadata) + add_image_segmentation_to_nwbfile(nwbfile=self.nwbfile, metadata=self.metadata) ophys = get_module(self.nwbfile, "ophys") @@ -395,10 +399,10 @@ def setUp(self): self.metadata["Ophys"].update(image_segmentation_metadata) - def test_add_plane_segmentation(self): - """Test that add_plane_segmentation method adds a plane segmentation to the nwbfile + def test_add_plane_segmentation_to_nwbfile(self): + """Test that add_plane_segmentation_to_nwbfile method adds a plane segmentation to the nwbfile specified by the metadata.""" - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -429,7 +433,7 @@ def test_add_plane_segmentation(self): def test_do_not_include_roi_centroids(self): """Test that setting `include_roi_centroids=False` prevents the centroids from being calculated and added.""" - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -445,7 +449,7 @@ def test_do_not_include_roi_centroids(self): def test_do_not_include_acceptance(self): """Test that setting `include_roi_acceptance=False` prevents the boolean acceptance columns from being added.""" - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -502,7 +506,7 @@ def test_rejected_roi_ids(self, rejected_list, expected_rejected_roi_ids): rejected_list=rejected_list, ) - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -537,7 +541,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd segmentation_extractor.get_roi_pixel_masks = MethodType(get_roi_pixel_masks, segmentation_extractor) - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -569,7 +573,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd segmentation_extractor.get_roi_pixel_masks = MethodType(get_roi_pixel_masks, segmentation_extractor) - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -594,7 +598,7 @@ def test_none_masks(self): num_columns=self.num_columns, ) - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -623,7 +627,7 @@ def test_invalid_mask_type(self): "None (to not write any masks)! Received 'invalid'." ) with pytest.raises(AssertionError, match=expected_error_message): - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -653,7 +657,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd "Using mask_type='pixel' instead." ), ): - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -691,7 +695,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd "Using mask_type='voxel' instead." ), ): - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -711,7 +715,7 @@ def test_not_overwriting_plane_segmentation_if_same_name(self): """Test that adding a plane segmentation with the same name will not overwrite the existing plane segmentation.""" - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -720,7 +724,7 @@ def test_not_overwriting_plane_segmentation_if_same_name(self): self.plane_segmentation_metadata["description"] = "modified description" - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -744,7 +748,7 @@ def test_add_two_plane_segmentation(self): first_plane_segmentation_description = "first_plane_segmentation_description" self.plane_segmentation_metadata["name"] = first_plane_segmentation_name self.plane_segmentation_metadata["description"] = first_plane_segmentation_description - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -756,7 +760,7 @@ def test_add_two_plane_segmentation(self): second_plane_segmentation_description = "second_plane_segmentation_description" self.plane_segmentation_metadata["name"] = second_plane_segmentation_name self.plane_segmentation_metadata["description"] = second_plane_segmentation_description - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -778,14 +782,14 @@ def test_add_two_plane_segmentation(self): assert second_plane_segmentation.name == second_plane_segmentation_name assert second_plane_segmentation.description == second_plane_segmentation_description - def test_add_plane_segmentation_raises_when_name_not_found_in_metadata(self): + def test_add_plane_segmentation_to_nwbfile_raises_when_name_not_found_in_metadata(self): """Test adding a plane segmentation raises an error when the name is not found in the metadata.""" plane_segmentation_name = "plane_segmentation_non_existing_in_the_metadata" with self.assertRaisesWith( exc_type=ValueError, exc_msg=f"Metadata for Plane Segmentation '{plane_segmentation_name}' not found in metadata['Ophys']['ImageSegmentation']['plane_segmentations'].", ): - add_plane_segmentation( + add_plane_segmentation_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -865,10 +869,10 @@ def setUp(self): self.metadata["Ophys"].update(fluorescence_metadata) self.metadata["Ophys"].update(dff_metadata) - def test_add_fluorescence_traces(self): + def test_add_fluorescence_traces_to_nwbfile(self): """Test fluorescence traces are added correctly to the nwbfile.""" - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -939,7 +943,7 @@ def test_add_df_over_f_trace(self): ) segmentation_extractor._roi_response_raw = None - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -986,7 +990,7 @@ def test_add_fluorescence_one_of_the_traces_is_none(self): has_neuropil_signal=False, ) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1004,7 +1008,7 @@ def test_add_fluorescence_one_of_the_traces_is_empty(self): self.segmentation_extractor._roi_response_deconvolved = np.empty((self.num_frames, 0)) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1022,7 +1026,7 @@ def test_add_fluorescence_one_of_the_traces_is_all_zeros(self): self.segmentation_extractor._roi_response_deconvolved = np.zeros((self.num_rois, self.num_frames)) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1050,7 +1054,7 @@ def test_no_traces_are_added(self): segmentation_extractor._roi_response_raw = None - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1064,7 +1068,7 @@ def test_not_overwriting_fluorescence_if_same_name(self): """Test that adding fluorescence traces container with the same name will not overwrite the existing fluorescence container in nwbfile.""" - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1072,7 +1076,7 @@ def test_not_overwriting_fluorescence_if_same_name(self): self.deconvolved_roi_response_series_metadata["description"] = "second description" - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1083,7 +1087,7 @@ def test_not_overwriting_fluorescence_if_same_name(self): self.assertNotEqual(roi_response_series["Deconvolved"].description, "second description") - def test_add_fluorescence_traces_to_existing_container(self): + def test_add_fluorescence_traces_to_nwbfile_to_existing_container(self): """Test that new traces can be added to an existing fluorescence container.""" segmentation_extractor = generate_dummy_segmentation_extractor( @@ -1097,7 +1101,7 @@ def test_add_fluorescence_traces_to_existing_container(self): has_neuropil_signal=False, ) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1117,7 +1121,7 @@ def test_add_fluorescence_traces_to_existing_container(self): num_columns=self.num_columns, ) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1131,7 +1135,7 @@ def test_add_fluorescence_traces_to_existing_container(self): # check that raw traces are not overwritten self.assertNotEqual(roi_response_series["RoiResponseSeries"].description, "second description") - def test_add_fluorescence_traces_irregular_timestamps(self): + def test_add_fluorescence_traces_to_nwbfile_irregular_timestamps(self): """Test adding traces with irregular timestamps.""" times = [0.0, 0.12, 0.15, 0.19, 0.1] @@ -1143,7 +1147,7 @@ def test_add_fluorescence_traces_irregular_timestamps(self): ) segmentation_extractor.set_times(times) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1156,7 +1160,7 @@ def test_add_fluorescence_traces_irregular_timestamps(self): self.assertEqual(roi_response_series[series_name].starting_time, None) assert_array_equal(roi_response_series[series_name].timestamps.data, times) - def test_add_fluorescence_traces_regular_timestamps(self): + def test_add_fluorescence_traces_to_nwbfile_regular_timestamps(self): """Test that adding traces with regular timestamps, the 'timestamps' are not added to the NWB file, instead 'rate' and 'starting_time' is used.""" @@ -1169,7 +1173,7 @@ def test_add_fluorescence_traces_regular_timestamps(self): ) segmentation_extractor.set_times(times) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1182,7 +1186,7 @@ def test_add_fluorescence_traces_regular_timestamps(self): self.assertEqual(roi_response_series[series_name].starting_time, times[0]) self.assertEqual(roi_response_series[series_name].timestamps, None) - def test_add_fluorescence_traces_regular_timestamps_with_metadata(self): + def test_add_fluorescence_traces_to_nwbfile_regular_timestamps_with_metadata(self): """Test adding traces with regular timestamps and also metadata-specified rate.""" times = np.arange(0, 5) segmentation_extractor = generate_dummy_segmentation_extractor( @@ -1197,7 +1201,7 @@ def test_add_fluorescence_traces_regular_timestamps_with_metadata(self): metadata["Ophys"]["Fluorescence"]["PlaneSegmentation"]["raw"].update(rate=1.23) metadata["Ophys"]["DfOverF"]["PlaneSegmentation"]["dff"].update(rate=1.23) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=metadata, @@ -1210,7 +1214,7 @@ def test_add_fluorescence_traces_regular_timestamps_with_metadata(self): self.assertEqual(roi_response_series[series_name].starting_time, 0) self.assertEqual(roi_response_series[series_name].timestamps, None) - def test_add_fluorescence_traces_irregular_timestamps_with_metadata(self): + def test_add_fluorescence_traces_to_nwbfile_irregular_timestamps_with_metadata(self): """Test adding traces with default timestamps and metadata rates (auto included in current segmentation interfaces).""" times = [0.0, 0.12, 0.15, 0.19, 0.1] segmentation_extractor = generate_dummy_segmentation_extractor( @@ -1224,7 +1228,7 @@ def test_add_fluorescence_traces_irregular_timestamps_with_metadata(self): metadata = deepcopy(self.metadata) metadata["Ophys"]["Fluorescence"]["PlaneSegmentation"]["raw"].update(rate=1.23) - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=segmentation_extractor, nwbfile=self.nwbfile, metadata=metadata, @@ -1237,9 +1241,9 @@ def test_add_fluorescence_traces_irregular_timestamps_with_metadata(self): self.assertEqual(roi_response_series[series_name].starting_time, None) assert_array_equal(roi_response_series[series_name].timestamps.data, times) - def test_add_fluorescence_traces_with_plane_segmentation_name_specified(self): + def test_add_fluorescence_traces_to_nwbfile_with_plane_segmentation_name_specified(self): plane_segmentation_name = "plane_segmentation_name" - metadata = get_default_segmentation_metadata() + metadata = _get_default_segmentation_metadata() metadata = dict_deep_update(metadata, self.metadata) metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][0].update(name=plane_segmentation_name) @@ -1248,7 +1252,7 @@ def test_add_fluorescence_traces_with_plane_segmentation_name_specified(self): ) metadata["Ophys"]["DfOverF"][plane_segmentation_name] = metadata["Ophys"]["DfOverF"].pop("PlaneSegmentation") - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor, nwbfile=self.nwbfile, metadata=metadata, @@ -1273,7 +1277,7 @@ def setUpClass(cls): cls.session_start_time = datetime.now().astimezone() - cls.metadata = get_default_segmentation_metadata() + cls.metadata = _get_default_segmentation_metadata() cls.plane_segmentation_first_plane_name = "PlaneSegmentationFirstPlane" cls.metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][0].update( @@ -1342,8 +1346,8 @@ def setUp(self): session_start_time=self.session_start_time, ) - def test_add_fluorescence_traces_for_two_plane_segmentations(self): - add_fluorescence_traces( + def test_add_fluorescence_traces_to_nwbfile_for_two_plane_segmentations(self): + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor_first_plane, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1378,7 +1382,7 @@ def test_add_fluorescence_traces_for_two_plane_segmentations(self): metadata["Ophys"]["Fluorescence"][second_plane_segmentation_name]["neuropil"].update(name="NeuropilSecondPlane") metadata["Ophys"]["DfOverF"][second_plane_segmentation_name]["dff"].update(name="RoiResponseSeriesSecondPlane") - add_fluorescence_traces( + add_fluorescence_traces_to_nwbfile( segmentation_extractor=self.segmentation_extractor_second_plane, nwbfile=self.nwbfile, metadata=metadata, @@ -1475,7 +1479,7 @@ def setUp(self): def test_default_values(self): """Test adding two photon series with default values.""" - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata ) @@ -1508,7 +1512,7 @@ def test_invalid_iterator_type_raises_error(self): AssertionError, "'iterator_type' must be either 'v1', 'v2' (recommended), or None.", ): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1539,7 +1543,7 @@ def test_non_iterative_write_assertion(self): def test_non_iterative_two_photon(self): """Test adding two photon series with using DataChunkIterator as iterator type.""" - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1559,7 +1563,7 @@ def test_non_iterative_two_photon(self): def test_v1_iterator(self): """Test adding two photon series with using DataChunkIterator as iterator type.""" - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1584,7 +1588,7 @@ def test_iterator_options_propagation(self): """Test that iterator options are propagated to the data chunk iterator.""" buffer_shape = (20, 5, 5) chunk_shape = (10, 5, 5) - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1601,7 +1605,7 @@ def test_iterator_options_propagation(self): def test_iterator_options_chunk_mb_propagation(self): """Test that chunk_mb is propagated to the data chunk iterator and the chunk shape is correctly set to fit.""" chunk_mb = 10.0 - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1618,7 +1622,7 @@ def test_iterator_options_chunk_mb_propagation(self): def test_iterator_options_chunk_shape_is_at_least_one(self): """Test that when a small chunk_mb is selected the chunk shape is guaranteed to include at least one frame.""" chunk_mb = 1.0 - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1634,7 +1638,7 @@ def test_iterator_options_chunk_shape_is_at_least_one(self): def test_iterator_options_chunk_shape_does_not_exceed_maxshape(self): """Test that when a large chunk_mb is selected the chunk shape is guaranteed to not exceed maxshape.""" chunk_mb = 1000.0 - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1648,7 +1652,7 @@ def test_iterator_options_chunk_shape_does_not_exceed_maxshape(self): assert_array_equal(chunk_shape, data_chunk_iterator.maxshape) def test_add_two_photon_series_roundtrip(self): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata ) @@ -1685,14 +1689,14 @@ def test_add_invalid_photon_series_type(self): AssertionError, "'photon_series_type' must be either 'OnePhotonSeries' or 'TwoPhotonSeries'.", ): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, photon_series_type="invalid", ) - def test_add_photon_series_inconclusive_metadata(self): + def test_add_photon_series_to_nwbfile_inconclusive_metadata(self): """Test warning is raised when `photon_series_type` specifies 'TwoPhotonSeries' but metadata contains also 'OnePhotonSeries'.""" exc_msg = "Received metadata for both 'OnePhotonSeries' and 'TwoPhotonSeries', make sure photon_series_type is specified correctly." @@ -1702,7 +1706,7 @@ def test_add_photon_series_inconclusive_metadata(self): ) with self.assertWarnsWith(warn_type=UserWarning, exc_msg=exc_msg): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=photon_series_metadata, @@ -1719,7 +1723,7 @@ def test_add_one_photon_series(self): binning=2, power=500.0, ) - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=metadata, @@ -1734,7 +1738,7 @@ def test_add_one_photon_series(self): self.assertEqual(one_photon_series.unit, "n.a.") def test_add_one_photon_series_roundtrip(self): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.one_photon_series_metadata, @@ -1758,13 +1762,13 @@ def test_add_one_photon_series_roundtrip(self): expected_one_photon_series_shape = (self.num_frames, self.num_columns, self.num_rows) assert one_photon_series.shape == expected_one_photon_series_shape - def test_add_photon_series_invalid_module_name_raises(self): + def test_add_photon_series_to_nwbfile_invalid_module_name_raises(self): """Test that adding photon series with invalid module name raises error.""" with self.assertRaisesWith( exc_type=AssertionError, exc_msg="'parent_container' must be either 'acquisition' or 'processing/ophys'.", ): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1776,7 +1780,7 @@ def test_add_one_photon_series_to_processing(self): metadata = self.one_photon_series_metadata metadata["Ophys"]["OnePhotonSeries"][0].update(name="OnePhotonSeriesProcessed") - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.one_photon_series_metadata, @@ -1793,12 +1797,12 @@ def test_photon_series_not_added_to_acquisition_with_same_name(self): with self.assertRaisesWith( exc_type=ValueError, exc_msg=f"{self.two_photon_series_name} already added to nwbfile.acquisition." ): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, ) - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1812,13 +1816,13 @@ def test_photon_series_not_added_to_processing_with_same_name(self): exc_type=ValueError, exc_msg=f"{self.two_photon_series_name} already added to nwbfile.processing['ophys'].", ): - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, parent_container="processing/ophys", ) - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1828,7 +1832,7 @@ def test_photon_series_not_added_to_processing_with_same_name(self): def test_ophys_module_not_created_when_photon_series_added_to_acquisition(self): """Test that ophys module is not created when photon series are added to nwbfile.acquisition.""" - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata, @@ -1844,7 +1848,7 @@ def test_add_multiple_one_photon_series_with_same_imaging_plane(self): shared_photon_series_metadata["Ophys"]["ImagingPlane"][0]["name"] = shared_imaging_plane_name shared_photon_series_metadata["Ophys"]["OnePhotonSeries"][0]["imaging_plane"] = shared_imaging_plane_name - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=shared_photon_series_metadata, @@ -1852,7 +1856,7 @@ def test_add_multiple_one_photon_series_with_same_imaging_plane(self): ) shared_photon_series_metadata["Ophys"]["OnePhotonSeries"][0]["name"] = "second_photon_series" - add_photon_series( + add_photon_series_to_nwbfile( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=shared_photon_series_metadata, @@ -1892,10 +1896,10 @@ def setUp(self): session_start_time=self.session_start_time, ) - def test_add_summary_images(self): + def test_add_summary_images_to_nwbfile(self): segmentation_extractor = generate_dummy_segmentation_extractor(num_rows=10, num_columns=15) - add_summary_images( + add_summary_images_to_nwbfile( nwbfile=self.nwbfile, segmentation_extractor=segmentation_extractor, metadata=self.metadata, @@ -1923,7 +1927,7 @@ def test_extractor_with_one_summary_image_suppressed(self): segmentation_extractor = generate_dummy_segmentation_extractor(num_rows=10, num_columns=15) segmentation_extractor._image_correlation = None - add_summary_images( + add_summary_images_to_nwbfile( nwbfile=self.nwbfile, segmentation_extractor=segmentation_extractor, metadata=self.metadata, @@ -1945,7 +1949,7 @@ def test_extractor_with_no_summary_images(self): self.nwbfile.create_processing_module("ophys", "contains optical physiology processed data") - add_summary_images( + add_summary_images_to_nwbfile( nwbfile=self.nwbfile, segmentation_extractor=segmentation_extractor, metadata=self.metadata, @@ -1959,26 +1963,28 @@ def test_extractor_with_no_summary_images_and_no_ophys_module(self): num_rows=10, num_columns=15, has_summary_images=False ) - add_summary_images(nwbfile=self.nwbfile, segmentation_extractor=segmentation_extractor, metadata=self.metadata) + add_summary_images_to_nwbfile( + nwbfile=self.nwbfile, segmentation_extractor=segmentation_extractor, metadata=self.metadata + ) assert len(self.nwbfile.processing) == 0 - def test_add_summary_images_invalid_plane_segmentation_name(self): + def test_add_summary_images_to_nwbfile_invalid_plane_segmentation_name(self): with self.assertRaisesWith( exc_type=AssertionError, exc_msg="Plane segmentation 'invalid_plane_segmentation_name' not found in metadata['Ophys']['SegmentationImages']", ): - add_summary_images( + add_summary_images_to_nwbfile( nwbfile=self.nwbfile, segmentation_extractor=generate_dummy_segmentation_extractor(num_rows=10, num_columns=15), metadata=self.metadata, plane_segmentation_name="invalid_plane_segmentation_name", ) - def test_add_summary_images_from_two_planes(self): + def test_add_summary_images_to_nwbfile_from_two_planes(self): segmentation_extractor_first_plane = generate_dummy_segmentation_extractor(num_rows=10, num_columns=15) - add_summary_images( + add_summary_images_to_nwbfile( nwbfile=self.nwbfile, segmentation_extractor=segmentation_extractor_first_plane, metadata=self.metadata, @@ -1997,7 +2003,7 @@ def test_add_summary_images_from_two_planes(self): segmentation_extractor_second_plane = generate_dummy_segmentation_extractor(num_rows=10, num_columns=15) - add_summary_images( + add_summary_images_to_nwbfile( nwbfile=self.nwbfile, segmentation_extractor=segmentation_extractor_second_plane, metadata=metadata, From f62b8a8a642e6ebc0fd9c4da9b68854ad096a6a4 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:13:46 -0400 Subject: [PATCH 006/118] [Pydantic IV] Restore soft deprecation of old types (#1033) --- src/neuroconv/utils/__init__.py | 41 +++++++++++++++++++++++++++++++++ src/neuroconv/utils/types.py | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/src/neuroconv/utils/__init__.py b/src/neuroconv/utils/__init__.py index 1e2bca630..1670eb60f 100644 --- a/src/neuroconv/utils/__init__.py +++ b/src/neuroconv/utils/__init__.py @@ -21,3 +21,44 @@ IntType, OptionalArrayType, ) + + +# TODO: remove after 3/1/2025 +def __getattr__(name): + from warnings import warn + from typing import Optional + + from pydantic import FilePath, DirectoryPath + + if name == "FilePathType": + message = ( + "The 'FilePathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `pydantic.FilePath` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return FilePath + if name == "OptionalFilePathType": + message = ( + "The 'OptionalFilePathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `typing.Optional[pydantic.FilePath]` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return Optional[FilePath] + if name == "FolderPathType": + message = ( + "The 'FolderPathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `pydantic.DirectoryPath` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return DirectoryPath + if name == "OptionalFolderPathType": + message = ( + "The 'OptionalFolderPathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `typing.Optional[pydantic.DirectoryPath]` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return Optional[DirectoryPath] diff --git a/src/neuroconv/utils/types.py b/src/neuroconv/utils/types.py index fe476c8f8..95ea27524 100644 --- a/src/neuroconv/utils/types.py +++ b/src/neuroconv/utils/types.py @@ -6,3 +6,44 @@ OptionalArrayType = Optional[ArrayType] FloatType = float IntType = Union[int, np.integer] + + +# TODO: remove after 3/1/2025 +def __getattr__(name): + from typing import Optional + from warnings import warn + + from pydantic import DirectoryPath, FilePath + + if name == "FilePathType": + message = ( + "The 'FilePathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `pydantic.FilePath` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return FilePath + if name == "OptionalFilePathType": + message = ( + "The 'OptionalFilePathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `typing.Optional[pydantic.FilePath]` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return Optional[FilePath] + if name == "FolderPathType": + message = ( + "The 'FolderPathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `pydantic.DirectoryPath` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return DirectoryPath + if name == "OptionalFolderPathType": + message = ( + "The 'OptionalFolderPathType' type has been deprecated and will be removed after 3/1/2025. " + "Please use `typing.Optional[pydantic.DirectoryPath]` instead." + ) + warn(message=message, category=DeprecationWarning, stacklevel=2) + + return Optional[DirectoryPath] From 4fa9816b1192402b8be71069d352f8b68d268901 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 27 Aug 2024 15:55:36 -0600 Subject: [PATCH 007/118] Update CHANGELOG.md for release --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50287da81..b846a15ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Upcoming +### Deprecations + +### Features + +### Bug fixes + +### Improvements + + +## V0.6.0 (August 27, 2024) + ### Deprecations * Deprecated `WaveformExtractor` usage. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821) * Changed the spikeinterface.tool functions (e.g. `add_recording`, `add_sorting`) to have `_to_nwbfile` as suffix [PR #1015](https://github.com/catalystneuro/neuroconv/pull/1015) From d10a172549a5a078b6d66127f26d4c08f40ed33e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 27 Aug 2024 15:56:24 -0600 Subject: [PATCH 008/118] bump version for release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9b60372ab..158fca482 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.5.1" +version = "0.6.0" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ From 70e4f7b0138c56267c97093197fe93a06b1456be Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 27 Aug 2024 16:11:03 -0600 Subject: [PATCH 009/118] Release v0.6.1 --- CHANGELOG.md | 2 +- docs/developer_guide/making_a_release.rst | 2 +- pyproject.toml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b846a15ba..6abcde80e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ### Improvements -## V0.6.0 (August 27, 2024) +## v0.6.0 (August 27, 2024) ### Deprecations * Deprecated `WaveformExtractor` usage. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821) diff --git a/docs/developer_guide/making_a_release.rst b/docs/developer_guide/making_a_release.rst index 57d2597aa..ab6b428fb 100644 --- a/docs/developer_guide/making_a_release.rst +++ b/docs/developer_guide/making_a_release.rst @@ -23,7 +23,7 @@ A simple to-do list for the Neuroconv release process: - The title and tag should be the release version. - The changelog should be copied correspondingly. - - Check the hashes in the markdown to ensure they match with the format of previous releases. + - Check the hashes in the markdown to ensure they match with the format of previous releases. This can be done efficiently by searching for `@ git` in an IDE. 5. **Release**: diff --git a/pyproject.toml b/pyproject.toml index 158fca482..f1ef72886 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ "Operating System :: POSIX :: Linux", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS", - "License :: BSD-3-Clause ", + "License :: OSI Approved :: BSD License", + ] requires-python = ">=3.9" dependencies = [ From 039530d450138bc2fd3f459043bde366d07e3c15 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 27 Aug 2024 16:15:02 -0600 Subject: [PATCH 010/118] Release v0.6.0 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f1ef72886..0ccde2682 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ authors = [ {name = "Cody Baker"}, {name = "Szonja Weigl"}, {name = "Heberto Mayorquin"}, + {name = "Paul Adkisson"}, {name = "Luiz Tauffer"}, {name = "Ben Dichter", email = "ben.dichter@catalystneuro.com"} ] From 2a78eb8f088fdc18570ab7e42b7eb18559ca8502 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 27 Aug 2024 16:18:21 -0600 Subject: [PATCH 011/118] post release version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0ccde2682..ba04e5c0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.0" +version = "0.6.1" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ From 8522241af1a39d90a29431cdb6997a881dc6f6ba Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 27 Aug 2024 21:17:15 -0600 Subject: [PATCH 012/118] Remove some duplicate ecephys tests (#1032) --- .../test_gin_ecephys/test_openephys.py | 34 ---- .../test_gin_ecephys/test_raw_recordings.py | 171 ------------------ .../test_on_data/test_recording_interfaces.py | 142 ++++++++++----- 3 files changed, 94 insertions(+), 253 deletions(-) delete mode 100644 tests/test_on_data/test_gin_ecephys/test_openephys.py diff --git a/tests/test_on_data/test_gin_ecephys/test_openephys.py b/tests/test_on_data/test_gin_ecephys/test_openephys.py deleted file mode 100644 index 5ea61adbb..000000000 --- a/tests/test_on_data/test_gin_ecephys/test_openephys.py +++ /dev/null @@ -1,34 +0,0 @@ -from hdmf.testing import TestCase - -from neuroconv.datainterfaces import ( - OpenEphysBinaryRecordingInterface, - OpenEphysLegacyRecordingInterface, - OpenEphysRecordingInterface, -) - -from ..setup_paths import ECEPHY_DATA_PATH - - -class TestOpenEphysRecordingInterfaceRedirects(TestCase): - def test_legacy_format(self): - folder_path = ECEPHY_DATA_PATH / "openephys" / "OpenEphys_SampleData_1" - - interface = OpenEphysRecordingInterface(folder_path=folder_path) - self.assertIsInstance(interface, OpenEphysLegacyRecordingInterface) - - def test_propagate_stream_name(self): - folder_path = ECEPHY_DATA_PATH / "openephys" / "OpenEphys_SampleData_1" - exc_msg = "The selected stream 'AUX' is not in the available streams '['Signals CH']'!" - with self.assertRaisesWith(ValueError, exc_msg=exc_msg): - OpenEphysRecordingInterface(folder_path=folder_path, stream_name="AUX") - - def test_binary_format(self): - folder_path = ECEPHY_DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking" - interface = OpenEphysRecordingInterface(folder_path=folder_path) - self.assertIsInstance(interface, OpenEphysBinaryRecordingInterface) - - def test_unrecognized_format(self): - folder_path = ECEPHY_DATA_PATH / "plexon" - exc_msg = "The Open Ephys data must be in 'legacy' (.continuous) or in 'binary' (.dat) format." - with self.assertRaisesWith(AssertionError, exc_msg=exc_msg): - OpenEphysRecordingInterface(folder_path=folder_path) diff --git a/tests/test_on_data/test_gin_ecephys/test_raw_recordings.py b/tests/test_on_data/test_gin_ecephys/test_raw_recordings.py index 92b23f554..7388845cc 100644 --- a/tests/test_on_data/test_gin_ecephys/test_raw_recordings.py +++ b/tests/test_on_data/test_gin_ecephys/test_raw_recordings.py @@ -1,8 +1,5 @@ -import itertools -import os import unittest from datetime import datetime -from platform import system import pytest from jsonschema.validators import Draft7Validator @@ -13,24 +10,8 @@ from neuroconv import NWBConverter from neuroconv.datainterfaces import ( - AlphaOmegaRecordingInterface, - AxonaRecordingInterface, - BiocamRecordingInterface, - BlackrockRecordingInterface, - EDFRecordingInterface, IntanRecordingInterface, - MaxOneRecordingInterface, - MCSRawRecordingInterface, - MEArecRecordingInterface, - NeuralynxRecordingInterface, - NeuroScopeRecordingInterface, - OpenEphysBinaryRecordingInterface, - OpenEphysLegacyRecordingInterface, Plexon2RecordingInterface, - PlexonRecordingInterface, - SpikeGadgetsRecordingInterface, - SpikeGLXRecordingInterface, - TdtRecordingInterface, ) # enable to run locally in interactive mode @@ -60,142 +41,14 @@ class TestEcephysRawRecordingsNwbConversions(unittest.TestCase): savedir = OUTPUT_PATH parameterized_recording_list = [ - param( - data_interface=AxonaRecordingInterface, - interface_kwargs=dict(file_path=str(DATA_PATH / "axona" / "axona_raw.bin")), - ), - param( - data_interface=EDFRecordingInterface, - interface_kwargs=dict(file_path=str(DATA_PATH / "edf" / "edf+C.edf")), - case_name="artificial_data", - ), - param( - data_interface=TdtRecordingInterface, - interface_kwargs=dict(folder_path=str(DATA_PATH / "tdt" / "aep_05"), gain=1.0), - case_name="multi_segment", - ), - param( - data_interface=BlackrockRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns5"), - ), - case_name="multi_stream_case_ns5", - ), - param( - data_interface=BlackrockRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns2"), - ), - case_name="multi_stream_case_ns2", - ), - param( - data_interface=PlexonRecordingInterface, - interface_kwargs=dict( - # Only File_plexon_3.plx has an ecephys recording stream - file_path=str(DATA_PATH / "plexon" / "File_plexon_3.plx"), - ), - case_name="plexon_recording", - ), param( data_interface=Plexon2RecordingInterface, interface_kwargs=dict( file_path=str(DATA_PATH / "plexon" / "4chDemoPL2.pl2"), ), ), - param( - data_interface=BiocamRecordingInterface, - interface_kwargs=dict(file_path=str(DATA_PATH / "biocam" / "biocam_hw3.0_fw1.6.brw")), - case_name="biocam", - ), - param( - data_interface=AlphaOmegaRecordingInterface, - interface_kwargs=dict( - folder_path=str(DATA_PATH / "alphaomega" / "mpx_map_version4"), - ), - case_name="alphaomega", - ), - param( - data_interface=MEArecRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "mearec" / "mearec_test_10s.h5"), - ), - case_name="mearec", - ), - param( - data_interface=MCSRawRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "rawmcs" / "raw_mcs_with_header_1.raw"), - ), - case_name="rawmcs", - ), - param( - data_interface=SpikeGLXRecordingInterface, - interface_kwargs=dict( - file_path=str( - DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.ap.bin" - ), - ), - ), - param( - data_interface=OpenEphysLegacyRecordingInterface, - interface_kwargs=dict( - folder_path=str(DATA_PATH / "openephys" / "OpenEphys_SampleData_1"), - ), - case_name="openephyslegacy", - ), ] - if system() == "Linux" and "CI" not in os.environ: - parameterized_recording_list.append( - param( - data_interface=MaxOneRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "maxwell" / "MaxOne_data" / "Record" / "000011" / "data.raw.h5"), - ), - ), - ) - - parameterized_recording_list.extend( - [ - param( - data_interface=NeuralynxRecordingInterface, - interface_kwargs=dict( - folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.7.4" / "original_data"), - ), - case_name="neuralynx", - ), - param( - data_interface=OpenEphysBinaryRecordingInterface, - interface_kwargs=dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking"), - ), - ), - param( - data_interface=BlackrockRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.ns5"), - ), - ), - param( - data_interface=NeuroScopeRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "neuroscope" / "test1" / "test1.dat"), - ), - ), - ] - ) - - for suffix in ["rhd", "rhs"]: - parameterized_recording_list.append( - param( - data_interface=IntanRecordingInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "intan" / f"intan_{suffix}_test_1.{suffix}"), - ), - case_name=suffix, - ) - ) - # Intan multiple files format parameterized_recording_list.append( param( @@ -213,30 +66,6 @@ class TestEcephysRawRecordingsNwbConversions(unittest.TestCase): ) ) - file_name_list = ["20210225_em8_minirec2_ac", "W122_06_09_2019_1_fromSD"] - num_channels_list = [512, 128] - file_name_num_channels_pairs = zip(file_name_list, num_channels_list) - gains_list = [None, [0.195], [0.385]] - for iteration in itertools.product(file_name_num_channels_pairs, gains_list): - (file_name, num_channels), gains = iteration - - interface_kwargs = dict( - file_path=str(DATA_PATH / "spikegadgets" / f"{file_name}.rec"), - ) - - if gains is not None: - if gains[0] == 0.385: - gains = gains * num_channels - interface_kwargs.update(gains=gains) - gain_string = gains[0] - else: - gain_string = None - - case_name = f"{file_name}, num_channels={num_channels}, gains={gain_string}, " - parameterized_recording_list.append( - param(data_interface=SpikeGadgetsRecordingInterface, interface_kwargs=interface_kwargs, case_name=case_name) - ) - @parameterized.expand(input=parameterized_recording_list, name_func=custom_name_func) def test_recording_extractor_to_nwb(self, data_interface, interface_kwargs, case_name=""): nwbfile_path = str(self.savedir / f"{data_interface.__name__}_{case_name}.nwb") diff --git a/tests/test_on_data/test_recording_interfaces.py b/tests/test_on_data/test_recording_interfaces.py index cf838cadc..14b8e87ea 100644 --- a/tests/test_on_data/test_recording_interfaces.py +++ b/tests/test_on_data/test_recording_interfaces.py @@ -5,6 +5,7 @@ import numpy as np import pytest +from hdmf.testing import TestCase from numpy.testing import assert_array_equal from packaging import version from pynwb import NWBHDF5IO @@ -36,11 +37,9 @@ ) try: - from .setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from .setup_paths import OUTPUT_PATH + from .setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH except ImportError: - from setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from setup_paths import OUTPUT_PATH + from setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH this_python_version = version.parse(python_version()) @@ -48,7 +47,7 @@ class TestAlphaOmegaRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = AlphaOmegaRecordingInterface - interface_kwargs = dict(folder_path=str(DATA_PATH / "alphaomega" / "mpx_map_version4")) + interface_kwargs = dict(folder_path=str(ECEPHY_DATA_PATH / "alphaomega" / "mpx_map_version4")) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): @@ -57,13 +56,13 @@ def check_extracted_metadata(self, metadata: dict): class TestAxonRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = AxonaRecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "axona" / "axona_raw.bin")) + interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "axona" / "axona_raw.bin")) save_directory = OUTPUT_PATH class TestBiocamRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = BiocamRecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "biocam" / "biocam_hw3.0_fw1.6.brw")) + interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "biocam" / "biocam_hw3.0_fw1.6.brw")) save_directory = OUTPUT_PATH @@ -73,11 +72,11 @@ class TestBlackrockRecordingInterface(RecordingExtractorInterfaceTestMixin): @pytest.fixture( params=[ - dict(file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns5")), - dict(file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.ns5")), - dict(file_path=str(DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns2")), + dict(file_path=str(ECEPHY_DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns5")), + dict(file_path=str(ECEPHY_DATA_PATH / "blackrock" / "FileSpec2.3001.ns5")), + dict(file_path=str(ECEPHY_DATA_PATH / "blackrock" / "blackrock_2_1" / "l101210-001.ns2")), ], - ids=["blackrock_ns5_v1", "blackrock_ns5_v2", "blackrock_ns2"], + ids=["multi_stream_case_ns5", "blackrock_ns5_v2", "multi_stream_case_ns2"], ) def setup_interface(self, request): test_id = request.node.callspec.id @@ -94,7 +93,7 @@ def setup_interface(self, request): ) class TestSpike2RecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = Spike2RecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "spike2" / "m365_1sec.smrx")) + interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "spike2" / "m365_1sec.smrx")) save_directory = OUTPUT_PATH @@ -104,10 +103,14 @@ class TestCellExplorerRecordingInterface(RecordingExtractorInterfaceTestMixin): @pytest.fixture( params=[ - dict(folder_path=str(DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed")), dict( folder_path=str( - DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed_hdf5" + ECEPHY_DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed" + ) + ), + dict( + folder_path=str( + ECEPHY_DATA_PATH / "cellexplorer" / "dataset_4" / "Peter_MS22_180629_110319_concat_stubbed_hdf5" ) ), ], @@ -171,7 +174,7 @@ def test_add_channel_metadata_to_nwb(self, setup_interface): ) class TestEDFRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = EDFRecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "edf" / "edf+C.edf")) + interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "edf" / "edf+C.edf")) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): @@ -213,8 +216,8 @@ class TestIntanRecordingInterface(RecordingExtractorInterfaceTestMixin): @pytest.fixture( params=[ - dict(file_path=str(DATA_PATH / "intan" / "intan_rhd_test_1.rhd")), - dict(file_path=str(DATA_PATH / "intan" / "intan_rhs_test_1.rhs")), + dict(file_path=str(ECEPHY_DATA_PATH / "intan" / "intan_rhd_test_1.rhd")), + dict(file_path=str(ECEPHY_DATA_PATH / "intan" / "intan_rhs_test_1.rhs")), ], ids=["rhd", "rhs"], ) @@ -231,7 +234,9 @@ def setup_interface(self, request): @pytest.mark.skip(reason="This interface fails to load the necessary plugin sometimes.") class TestMaxOneRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MaxOneRecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "maxwell" / "MaxOne_data" / "Record" / "000011" / "data.raw.h5")) + interface_kwargs = dict( + file_path=str(ECEPHY_DATA_PATH / "maxwell" / "MaxOne_data" / "Record" / "000011" / "data.raw.h5") + ) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): @@ -242,13 +247,13 @@ def check_extracted_metadata(self, metadata: dict): class TestMCSRawRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MCSRawRecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "rawmcs" / "raw_mcs_with_header_1.raw")) + interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "rawmcs" / "raw_mcs_with_header_1.raw")) save_directory = OUTPUT_PATH class TestMEArecRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MEArecRecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "mearec" / "mearec_test_10s.h5")) + interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "mearec" / "mearec_test_10s.h5")) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): @@ -276,7 +281,7 @@ def check_extracted_metadata(self, metadata: dict): class TestNeuralynxRecordingInterfaceV574: data_interface_cls = NeuralynxRecordingInterface - interface_kwargs = (dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.7.4" / "original_data")),) + interface_kwargs = (dict(folder_path=str(ECEPHY_DATA_PATH / "neuralynx" / "Cheetah_v5.7.4" / "original_data")),) save_directory = OUTPUT_PATH @@ -317,7 +322,7 @@ def check_read(self, nwbfile_path): class TestNeuralynxRecordingInterfaceV563: data_interface_cls = NeuralynxRecordingInterface - interface_kwargs = (dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data")),) + interface_kwargs = (dict(folder_path=str(ECEPHY_DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data")),) save_directory = OUTPUT_PATH @@ -337,7 +342,7 @@ def check_read(self, nwbfile_path): class TestNeuralynxRecordingInterfaceV540: data_interface_cls = NeuralynxRecordingInterface - interface_kwargs = (dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.4.0" / "original_data")),) + interface_kwargs = (dict(folder_path=str(ECEPHY_DATA_PATH / "neuralynx" / "Cheetah_v5.4.0" / "original_data")),) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): @@ -356,7 +361,7 @@ def check_read(self, nwbfile_path): class TestMultiStreamNeuralynxRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = NeuralynxRecordingInterface interface_kwargs = dict( - folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v6.4.1dev" / "original_data"), + folder_path=str(ECEPHY_DATA_PATH / "neuralynx" / "Cheetah_v6.4.1dev" / "original_data"), stream_name="Stream (rate,#packet,t0): (32000.0, 31, 1614363777985169)", ) save_directory = OUTPUT_PATH @@ -377,7 +382,7 @@ def check_extracted_metadata(self, metadata: dict): class TestNeuroScopeRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = NeuroScopeRecordingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "neuroscope" / "test1" / "test1.dat")) + interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "neuroscope" / "test1" / "test1.dat")) save_directory = OUTPUT_PATH @@ -388,7 +393,7 @@ class TestOpenEphysBinaryRecordingInterfaceClassMethodsAndAssertions: def test_get_stream_names(self): stream_names = self.data_interface_cls.get_stream_names( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107") + folder_path=str(ECEPHY_DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107") ) assert stream_names == ["Record_Node_107#Neuropix-PXI-116.0", "Record_Node_107#Neuropix-PXI-116.1"] @@ -398,7 +403,7 @@ def test_folder_structure_assertion(self): ValueError, match=r"Unable to identify the OpenEphys folder structure! Please check that your `folder_path` contains a settings.xml file and sub-folders of the following form: 'experiment' -> 'recording' -> 'continuous'.", ): - OpenEphysBinaryRecordingInterface(folder_path=str(DATA_PATH / "openephysbinary")) + OpenEphysBinaryRecordingInterface(folder_path=str(ECEPHY_DATA_PATH / "openephysbinary")) def test_stream_name_missing_assertion(self): with pytest.raises( @@ -406,7 +411,9 @@ def test_stream_name_missing_assertion(self): match=r"More than one stream is detected! Please specify which stream you wish to load with the `stream_name` argument. To see what streams are available, call\s+`OpenEphysRecordingInterface.get_stream_names\(folder_path=\.\.\.\)`.", ): OpenEphysBinaryRecordingInterface( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107") + folder_path=str( + ECEPHY_DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107" + ) ) def test_stream_name_not_available_assertion(self): @@ -415,14 +422,16 @@ def test_stream_name_not_available_assertion(self): match=r"The selected stream 'not_a_stream' is not in the available streams '\['Record_Node_107#Neuropix-PXI-116.0', 'Record_Node_107#Neuropix-PXI-116.1'\]'!", ): OpenEphysBinaryRecordingInterface( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), + folder_path=str( + ECEPHY_DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107" + ), stream_name="not_a_stream", ) class TestOpenEphysBinaryRecordingInterfaceVersion0_4_4(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysBinaryRecordingInterface - interface_kwargs = dict(folder_path=str(DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking")) + interface_kwargs = dict(folder_path=str(ECEPHY_DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking")) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): @@ -432,7 +441,7 @@ def check_extracted_metadata(self, metadata: dict): class TestOpenEphysBinaryRecordingInterfaceVersion0_5_3_Stream1(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysBinaryRecordingInterface interface_kwargs = dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), + folder_path=str(ECEPHY_DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), stream_name="Record_Node_107#Neuropix-PXI-116.0", ) save_directory = OUTPUT_PATH @@ -444,7 +453,7 @@ def check_extracted_metadata(self, metadata: dict): class TestOpenEphysBinaryRecordingInterfaceVersion0_5_3_Stream2(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysBinaryRecordingInterface interface_kwargs = dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), + folder_path=str(ECEPHY_DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), stream_name="Record_Node_107#Neuropix-PXI-116.1", ) save_directory = OUTPUT_PATH @@ -460,7 +469,9 @@ class TestOpenEphysBinaryRecordingInterfaceWithBlocks_version_0_6_block_1_stream data_interface_cls = OpenEphysBinaryRecordingInterface interface_kwargs = dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.6.x_neuropixels_multiexp_multistream" / "Record Node 101"), + folder_path=str( + ECEPHY_DATA_PATH / "openephysbinary" / "v0.6.x_neuropixels_multiexp_multistream" / "Record Node 101" + ), stream_name="Record Node 101#NI-DAQmx-103.PXIe-6341", block_index=1, ) @@ -472,7 +483,7 @@ def check_extracted_metadata(self, metadata: dict): class TestOpenEphysLegacyRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = OpenEphysLegacyRecordingInterface - interface_kwargs = dict(folder_path=str(DATA_PATH / "openephys" / "OpenEphys_SampleData_1")) + interface_kwargs = dict(folder_path=str(ECEPHY_DATA_PATH / "openephys" / "OpenEphys_SampleData_1")) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): @@ -485,14 +496,18 @@ class TestOpenEphysRecordingInterfaceRouter(RecordingExtractorInterfaceTestMixin @pytest.fixture( params=[ - dict(folder_path=str(DATA_PATH / "openephys" / "OpenEphys_SampleData_1")), - dict(folder_path=str(DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking")), + dict(folder_path=str(ECEPHY_DATA_PATH / "openephys" / "OpenEphys_SampleData_1")), + dict(folder_path=str(ECEPHY_DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking")), dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), + folder_path=str( + ECEPHY_DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107" + ), stream_name="Record_Node_107#Neuropix-PXI-116.0", ), dict( - folder_path=str(DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107"), + folder_path=str( + ECEPHY_DATA_PATH / "openephysbinary" / "v0.5.3_two_neuropixels_stream" / "Record_Node_107" + ), stream_name="Record_Node_107#Neuropix-PXI-116.1", ), ], @@ -518,18 +533,47 @@ def test_interface_extracted_metadata(self, setup_interface): # Additional assertions specific to the metadata can be added here +class TestOpenEphysRecordingInterfaceRedirects(TestCase): + def test_legacy_format(self): + folder_path = ECEPHY_DATA_PATH / "openephys" / "OpenEphys_SampleData_1" + + interface = OpenEphysRecordingInterface(folder_path=folder_path) + self.assertIsInstance(interface, OpenEphysLegacyRecordingInterface) + + def test_propagate_stream_name(self): + folder_path = ECEPHY_DATA_PATH / "openephys" / "OpenEphys_SampleData_1" + exc_msg = "The selected stream 'AUX' is not in the available streams '['Signals CH']'!" + with self.assertRaisesWith(ValueError, exc_msg=exc_msg): + OpenEphysRecordingInterface(folder_path=folder_path, stream_name="AUX") + + def test_binary_format(self): + folder_path = ECEPHY_DATA_PATH / "openephysbinary" / "v0.4.4.1_with_video_tracking" + interface = OpenEphysRecordingInterface(folder_path=folder_path) + self.assertIsInstance(interface, OpenEphysBinaryRecordingInterface) + + def test_unrecognized_format(self): + folder_path = ECEPHY_DATA_PATH / "plexon" + exc_msg = "The Open Ephys data must be in 'legacy' (.continuous) or in 'binary' (.dat) format." + with self.assertRaisesWith(AssertionError, exc_msg=exc_msg): + OpenEphysRecordingInterface(folder_path=folder_path) + + class TestSpikeGadgetsRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = SpikeGadgetsRecordingInterface save_directory = OUTPUT_PATH @pytest.fixture( params=[ - dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec")), - dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.195]), - dict(file_path=str(DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.385] * 512), - dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec")), - dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.195]), - dict(file_path=str(DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.385] * 128), + dict(file_path=str(ECEPHY_DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec")), + dict(file_path=str(ECEPHY_DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.195]), + dict( + file_path=str(ECEPHY_DATA_PATH / "spikegadgets" / "20210225_em8_minirec2_ac.rec"), gains=[0.385] * 512 + ), + dict(file_path=str(ECEPHY_DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec")), + dict(file_path=str(ECEPHY_DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.195]), + dict( + file_path=str(ECEPHY_DATA_PATH / "spikegadgets" / "W122_06_09_2019_1_fromSD.rec"), gains=[0.385] * 128 + ), ], ids=[ "20210225_em8_minirec2_ac_default_gains", @@ -559,7 +603,9 @@ def test_extracted_metadata(self, setup_interface): class TestSpikeGLXRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = SpikeGLXRecordingInterface interface_kwargs = dict( - file_path=str(DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.ap.bin") + file_path=str( + ECEPHY_DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.ap.bin" + ) ) save_directory = OUTPUT_PATH @@ -581,7 +627,7 @@ class TestSpikeGLXRecordingInterfaceLongNHP(RecordingExtractorInterfaceTestMixin data_interface_cls = SpikeGLXRecordingInterface interface_kwargs = dict( file_path=str( - DATA_PATH + ECEPHY_DATA_PATH / "spikeglx" / "long_nhp_stubbed" / "snippet_g0" @@ -608,7 +654,7 @@ def check_extracted_metadata(self, metadata: dict): class TestTdtRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = TdtRecordingInterface test_gain_value = 0.195 # arbitrary value to test gain - interface_kwargs = dict(folder_path=str(DATA_PATH / "tdt" / "aep_05"), gain=test_gain_value) + interface_kwargs = dict(folder_path=str(ECEPHY_DATA_PATH / "tdt" / "aep_05"), gain=test_gain_value) save_directory = OUTPUT_PATH def run_custom_checks(self): @@ -634,7 +680,7 @@ class TestPlexonRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = PlexonRecordingInterface interface_kwargs = dict( # Only File_plexon_3.plx has an ecephys recording stream - file_path=str(DATA_PATH / "plexon" / "File_plexon_3.plx"), + file_path=str(ECEPHY_DATA_PATH / "plexon" / "File_plexon_3.plx"), ) save_directory = OUTPUT_PATH From 63a927f7efdd999e51a0b7cb8c88746898af203d Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 28 Aug 2024 19:02:13 -0400 Subject: [PATCH 013/118] changelog formatting (#1036) --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6abcde80e..4a8533ca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,20 +13,20 @@ ### Deprecations * Deprecated `WaveformExtractor` usage. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821) -* Changed the spikeinterface.tool functions (e.g. `add_recording`, `add_sorting`) to have `_to_nwbfile` as suffix [PR #1015](https://github.com/catalystneuro/neuroconv/pull/1015) +* Changed the `tools.spikeinterface` functions (e.g. `add_recording`, `add_sorting`) to have `_to_nwbfile` as suffix [PR #1015](https://github.com/catalystneuro/neuroconv/pull/1015) * Deprecated use of `compression` and `compression_options` in `VideoInterface` [PR #1005](https://github.com/catalystneuro/neuroconv/pull/1005) * `get_schema_from_method_signature` has been deprecated; please use `get_json_schema_from_method_signature` instead. [PR #1016](https://github.com/catalystneuro/neuroconv/pull/1016) * `neuroconv.utils.FilePathType` and `neuroconv.utils.FolderPathType` have been deprecated; please use `pydantic.FilePath` and `pydantic.DirectoryPath` instead. [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1017) -* Changed the roiextractors.tool function (e.g. `add_imaging` and `add_segmentation`) to have the `_to_nwbfile` suffix [PR #1027][PR #1017](https://github.com/catalystneuro/neuroconv/pull/1027) +* Changed the `tools.roiextractors` function (e.g. `add_imaging` and `add_segmentation`) to have the `_to_nwbfile` suffix [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1027) ### Features -* Added MedPCInterface for operant behavioral output files. [PR #883](https://github.com/catalystneuro/neuroconv/pull/883) +* Added `MedPCInterface` for operant behavioral output files. [PR #883](https://github.com/catalystneuro/neuroconv/pull/883) * Support `SortingAnalyzer` in the `SpikeGLXConverterPipe`. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821) * Added `TDTFiberPhotometryInterface` data interface, for converting fiber photometry data from TDT file formats. [PR #920](https://github.com/catalystneuro/neuroconv/pull/920) * Add argument to `add_electrodes` that grants fine control of what to do with the missing values. As a side effect this drops the implicit casting to int when writing int properties to the electrodes table [PR #985](https://github.com/catalystneuro/neuroconv/pull/985) * Add Plexon2 support [PR #918](https://github.com/catalystneuro/neuroconv/pull/918) -* Converter working with multiple VideoInterface instances [PR #914](https://github.com/catalystneuro/neuroconv/pull/914) +* Converter working with multiple `VideoInterface` instances [PR #914](https://github.com/catalystneuro/neuroconv/pull/914) * Added helper function `neuroconv.tools.data_transfers.submit_aws_batch_job` for basic automated submission of AWS batch jobs. [PR #384](https://github.com/catalystneuro/neuroconv/pull/384) * Data interfaces `run_conversion` method now performs metadata validation before running the conversion. [PR #949](https://github.com/catalystneuro/neuroconv/pull/949) * Introduced `null_values_for_properties` to `add_units_table` to give user control over null values behavior [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) From a8d702dcfcb57b1d533233bb07632b6f91870cd1 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:35:22 -0400 Subject: [PATCH 014/118] Schema inference enhancements (#1037) Co-authored-by: Heberto Mayorquin --- CHANGELOG.md | 2 + .../baserecordingextractorinterface.py | 1 - src/neuroconv/nwbconverter.py | 6 +- .../tools/testing/mock_interfaces.py | 3 +- src/neuroconv/utils/json_schema.py | 10 +- ...t_get_json_schema_from_method_signature.py | 116 +++++++++++++++++- 6 files changed, 128 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a8533ca6..1068f6943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Features ### Bug fixes +* Fixed the JSON schema inference warning on excluded fields; also improved error message reporting of which method + triggered the error. [PR #1037](https://github.com/catalystneuro/neuroconv/pull/1037) ### Improvements diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 354d78f80..620b86224 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -315,7 +315,6 @@ def add_to_nwbfile( metadata['Ecephys']['ElectricalSeries'] = dict(name=my_name, description=my_description) - The default is False (append mode). starting_time : float, optional Sets the starting time of the ElectricalSeries to a manually set value. stub_test : bool, default: False diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index ea7b7cd67..cb62e149d 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -319,7 +319,7 @@ def get_conversion_options_schema(self) -> dict: version="0.1.0", ) for interface_name, data_interface in self.data_interface_objects.items(): - conversion_options_schema["properties"].update( - {interface_name: unroot_schema(data_interface.get_conversion_options_schema())} - ) + + schema = data_interface.get_conversion_options_schema() + conversion_options_schema["properties"].update({interface_name: unroot_schema(schema)}) return conversion_options_schema diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 6f91e775f..9b115c1e9 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -128,8 +128,7 @@ def __init__( self, num_channels: int = 4, sampling_frequency: float = 30_000.0, - # durations: tuple[float] = (1.0,), # Uncomment when pydantic is integrated for schema validation - durations: tuple = (1.0,), + durations: tuple[float] = (1.0,), seed: int = 0, verbose: bool = True, es_key: str = "ElectricalSeries", diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index 73aa97bdf..bef733d4b 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -102,6 +102,9 @@ def get_json_schema_from_method_signature(method: Callable, exclude: Optional[li exclude = exclude or [] exclude += ["self", "cls"] + split_qualname = method.__qualname__.split(".")[-2:] + method_display = ".".join(split_qualname) if "<" not in split_qualname[0] else method.__name__ + signature = inspect.signature(obj=method) parameters = signature.parameters additional_properties = False @@ -140,10 +143,13 @@ def get_json_schema_from_method_signature(method: Callable, exclude: Optional[li # Attempt to find descriptions within the docstring of the method parsed_docstring = docstring_parser.parse(method.__doc__) for parameter_in_docstring in parsed_docstring.params: + if parameter_in_docstring.arg_name in exclude: + continue + if parameter_in_docstring.arg_name not in json_schema["properties"]: message = ( - f"The argument_name '{parameter_in_docstring.arg_name}' from the docstring not occur in the " - "method signature, possibly due to a typo." + f"The argument_name '{parameter_in_docstring.arg_name}' from the docstring of method " + f"'{method_display}' does not occur in the signature, possibly due to a typo." ) warnings.warn(message=message, stacklevel=2) continue diff --git a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py index ca0e6fe91..10139f7e1 100644 --- a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py +++ b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py @@ -4,9 +4,10 @@ import pytest from jsonschema import validate from pydantic import DirectoryPath, FilePath +from pynwb import NWBFile from neuroconv.datainterfaces import AlphaOmegaRecordingInterface -from neuroconv.utils import ArrayType, get_json_schema_from_method_signature +from neuroconv.utils import ArrayType, DeepDict, get_json_schema_from_method_signature def test_get_json_schema_from_method_signature_basic(): @@ -299,8 +300,119 @@ def method_with_typo_in_docstring(integer: int): with pytest.warns(expected_warning=UserWarning) as warning_info: test_json_schema = get_json_schema_from_method_signature(method=method_with_typo_in_docstring) + assert len(warning_info) == 1 + + expected_warning_message = ( + "The argument_name 'integ' from the docstring of method 'method_with_typo_in_docstring' does not occur in " + "the signature, possibly due to a typo." + ) + assert warning_info[0].message.args[0] == expected_warning_message + + expected_json_schema = { + "properties": {"integer": {"type": "integer"}}, + "required": ["integer"], + "type": "object", + "additionalProperties": False, + } + + assert test_json_schema == expected_json_schema + + +def test_get_json_schema_from_method_signature_docstring_warning_with_exclusions(): + def method_with_typo_in_docstring_and_exclusions(integer: int, nwbfile: NWBFile, metadata: DeepDict): + """ + This is a docstring with a typo in the argument name. + + Parameters + ---------- + integ : int + This is an integer. + nwbfile : pynwb.NWBFile + An in-memory NWBFile object. + metadata : neuroconv.utils.DeepDict + A dictionary-like object that allows for deep access and modification. + """ + pass + + with pytest.warns(expected_warning=UserWarning) as warning_info: + test_json_schema = get_json_schema_from_method_signature( + method=method_with_typo_in_docstring_and_exclusions, exclude=["nwbfile", "metadata"] + ) + + assert len(warning_info) == 1 + + expected_warning_message = ( + "The argument_name 'integ' from the docstring of method 'method_with_typo_in_docstring_and_exclusions' " + "does not occur in the signature, possibly due to a typo." + ) + assert warning_info[0].message.args[0] == expected_warning_message + + expected_json_schema = { + "properties": {"integer": {"type": "integer"}}, + "required": ["integer"], + "type": "object", + "additionalProperties": False, + } + + assert test_json_schema == expected_json_schema + + +def test_get_json_schema_from_method_signature_docstring_warning_from_bound_method(): + class TestClass: + def test_bound_method(self, integer: int): + """ + This is a docstring with a typo in the argument name. + + Parameters + ---------- + integ : int + This is an integer. + """ + pass + + with pytest.warns(expected_warning=UserWarning) as warning_info: + test_json_schema = get_json_schema_from_method_signature(method=TestClass.test_bound_method) + + assert len(warning_info) == 1 + + expected_warning_message = ( + "The argument_name 'integ' from the docstring of method 'TestClass.test_bound_method' does not occur in the " + "signature, possibly due to a typo." + ) + assert warning_info[0].message.args[0] == expected_warning_message + + expected_json_schema = { + "properties": {"integer": {"type": "integer"}}, + "required": ["integer"], + "type": "object", + "additionalProperties": False, + } + + assert test_json_schema == expected_json_schema + + +def test_get_json_schema_from_method_signature_docstring_warning_from_class_method(): + class TestClass: + @classmethod + def test_class_method(self, integer: int): + """ + This is a docstring with a typo in the argument name. + + Parameters + ---------- + integ : int + This is an integer. + """ + pass + + with pytest.warns(expected_warning=UserWarning) as warning_info: + test_json_schema = get_json_schema_from_method_signature(method=TestClass.test_class_method) + + assert len(warning_info) == 1 + expected_warning_message = ( - "The argument_name 'integ' from the docstring not occur in the method signature, possibly due to a typo." + "The argument_name 'integ' from the docstring of method 'TestClass.test_class_method' does not occur in the " + "signature, possibly due to a typo." ) assert warning_info[0].message.args[0] == expected_warning_message From b5ca011f6eff32dc41e738153d0e4a6e6c8c7422 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 29 Aug 2024 14:18:12 -0600 Subject: [PATCH 015/118] Temporary supress failing doctest for plexon2 (#1043) --- .../recording/plexon2.rst | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/conversion_examples_gallery/recording/plexon2.rst b/docs/conversion_examples_gallery/recording/plexon2.rst index e07cfe6ed..35d24727b 100644 --- a/docs/conversion_examples_gallery/recording/plexon2.rst +++ b/docs/conversion_examples_gallery/recording/plexon2.rst @@ -14,22 +14,22 @@ Convert Plexon2 recording data to NWB using :py:class:`~neuroconv.datainterfaces .. code-block:: python - >>> from datetime import datetime - >>> from zoneinfo import ZoneInfo - >>> from pathlib import Path - >>> from neuroconv.datainterfaces import Plexon2RecordingInterface - >>> - >>> file_path = f"{ECEPHY_DATA_PATH}/plexon/4chDemoPL2.pl2" - >>> # Change the file_path to the location in your system - >>> interface = Plexon2RecordingInterface(file_path=file_path, verbose=False) - >>> - >>> # Extract what metadata we can from the source files - >>> metadata = interface.get_metadata() - >>> # For data provenance we add the time zone information to the conversion - >>> tzinfo = ZoneInfo("US/Pacific") - >>> session_start_time = metadata["NWBFile"]["session_start_time"] - >>> metadata["NWBFile"].update(session_start_time=session_start_time.replace(tzinfo=tzinfo)) - >>> - >>> # Choose a path for saving the nwb file and run the conversion - >>> nwbfile_path = f"{path_to_save_nwbfile}" - >>> interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) + from datetime import datetime + from zoneinfo import ZoneInfo + from pathlib import Path + from neuroconv.datainterfaces import Plexon2RecordingInterface + + file_path = f"{ECEPHY_DATA_PATH}/plexon/4chDemoPL2.pl2" + # Change the file_path to the location in your system + interface = Plexon2RecordingInterface(file_path=file_path, verbose=False) + + # Extract what metadata we can from the source files + metadata = interface.get_metadata() + # For data provenance we add the time zone information to the conversion + tzinfo = ZoneInfo("US/Pacific") + session_start_time = metadata["NWBFile"]["session_start_time"] + metadata["NWBFile"].update(session_start_time=session_start_time.replace(tzinfo=tzinfo)) + + # Choose a path for saving the nwb file and run the conversion + nwbfile_path = f"{path_to_save_nwbfile}" + interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) From d99fef162f0ee965fda1d0dcff0763ad6d9455ba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 20:58:50 +0000 Subject: [PATCH 016/118] [pre-commit.ci] pre-commit autoupdate (#1001) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 046e6d289..7b5895c35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.2 hooks: - id: ruff args: [ --fix ] From 9bef89f11cb11872ae3ffbe1968cdd4382316ec9 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 29 Aug 2024 16:15:48 -0600 Subject: [PATCH 017/118] Update setuptools version on build (#1040) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ba04e5c0a..ae7d6aba3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=52", "wheel"] +requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [project] From 66ab8b8673cd2282b804ab632d354b0c1c31ba31 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 30 Aug 2024 08:59:51 -0600 Subject: [PATCH 018/118] Release v0.6.1 --- CHANGELOG.md | 10 ++-------- README.md | 3 ++- docs/developer_guide/making_a_release.rst | 10 +++++----- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1068f6943..a62b16f0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,9 @@ # Upcoming -### Deprecations - -### Features +## v0.6.1 (August 30, 2024) ### Bug fixes -* Fixed the JSON schema inference warning on excluded fields; also improved error message reporting of which method - triggered the error. [PR #1037](https://github.com/catalystneuro/neuroconv/pull/1037) - -### Improvements - +* Fixed the JSON schema inference warning on excluded fields; also improved error message reporting of which method triggered the error. [PR #1037](https://github.com/catalystneuro/neuroconv/pull/1037) ## v0.6.0 (August 27, 2024) diff --git a/README.md b/README.md index f90813b71..169ca8a14 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ [![License](https://img.shields.io/pypi/l/neuroconv.svg)](https://github.com/catalystneuro/neuroconv/license.txt)

- NeuroConv logo + NeuroConv logo

Automatically convert neurophysiology data to NWB

+

Explore our documentation »

diff --git a/docs/developer_guide/making_a_release.rst b/docs/developer_guide/making_a_release.rst index ab6b428fb..cb06a14d9 100644 --- a/docs/developer_guide/making_a_release.rst +++ b/docs/developer_guide/making_a_release.rst @@ -7,23 +7,23 @@ A simple to-do list for the Neuroconv release process: - Format and update the changelog. - Ensure correct formatting and add a header to indicate that the changes belong to a specific release and not upcoming ones. - - Example: `Commit Example `_ + - Example: `Commit Example ` 2. **Set the Correct Version for Release**: - The development version (the current code on `main`) should be one patch version ahead of the latest PyPI release and therefore ready for the next step. - If a minor version bump is necessary, change it accordingly. - - Example: `Version Change Example `_ + - Example: `Version Change Example ` 3. **Perform Checks**: - - Ensure that no requirement files include pointers to `git`-based dependencies (including specific branches or commit hashes). All dependencies for a PyPI release should point to the released package versions that are available on conda-forge or PyPI. + - Ensure that no requirement files include pointers to `git`-based dependencies (including specific branches or commit hashes). All dependencies for a PyPI release should point to the released package versions that are available on conda-forge or PyPI. This can be done efficiently by searching for `@ git` in an IDE. 4. **Tag on GitHub**: - The title and tag should be the release version. - The changelog should be copied correspondingly. - - Check the hashes in the markdown to ensure they match with the format of previous releases. This can be done efficiently by searching for `@ git` in an IDE. + - Check the hashes in the markdown to ensure they match with the format of previous releases. 5. **Release**: @@ -32,4 +32,4 @@ A simple to-do list for the Neuroconv release process: 6. **Bump Version Post-Release**: - To comply with the one patch version ahead policy, bump the version after the release. - - Example: `Post-Release Version Bump `_ + - Example: `Post-Release Version Bump ` From d1e38154f02b8d500a1c04b73b2d4a1eac32b5bc Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 30 Aug 2024 09:12:52 -0600 Subject: [PATCH 019/118] version bumpb --- CHANGELOG.md | 8 ++++++++ docs/developer_guide/making_a_release.rst | 6 +++--- pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a62b16f0d..29f5f6848 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Upcoming +### Deprecations + +### Features + +### Bug fixes + +### Improvements + ## v0.6.1 (August 30, 2024) ### Bug fixes diff --git a/docs/developer_guide/making_a_release.rst b/docs/developer_guide/making_a_release.rst index cb06a14d9..412a7ad9a 100644 --- a/docs/developer_guide/making_a_release.rst +++ b/docs/developer_guide/making_a_release.rst @@ -7,13 +7,13 @@ A simple to-do list for the Neuroconv release process: - Format and update the changelog. - Ensure correct formatting and add a header to indicate that the changes belong to a specific release and not upcoming ones. - - Example: `Commit Example ` + - Example: `Format Changelog Example `_ 2. **Set the Correct Version for Release**: - The development version (the current code on `main`) should be one patch version ahead of the latest PyPI release and therefore ready for the next step. - If a minor version bump is necessary, change it accordingly. - - Example: `Version Change Example ` + - Example: `Version Change Example `_ 3. **Perform Checks**: @@ -32,4 +32,4 @@ A simple to-do list for the Neuroconv release process: 6. **Bump Version Post-Release**: - To comply with the one patch version ahead policy, bump the version after the release. - - Example: `Post-Release Version Bump ` + - Example: `Post-Release Version Bump `_ diff --git a/pyproject.toml b/pyproject.toml index ae7d6aba3..b40294d25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.1" +version = "0.6.2" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ From 39c1c2afd1d71503e82962eba6b1ebfecd96d593 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 30 Aug 2024 12:43:57 -0600 Subject: [PATCH 020/118] Add pydoc with ruff check that public classes should have documentation. (#1034) Co-authored-by: Paul Adkisson --- CHANGELOG.md | 2 ++ pyproject.toml | 11 +++---- .../ecephys/axona/axonadatainterface.py | 3 ++ .../cellexplorer/cellexplorerdatainterface.py | 8 +++++ .../neuralynx/neuralynxdatainterface.py | 5 ++++ .../ecephys/spikeglx/spikeglxdatainterface.py | 4 +++ .../ophys/brukertiff/brukertiffconverter.py | 8 +++++ src/neuroconv/tools/hdmf.py | 4 ++- src/neuroconv/tools/path_expansion.py | 17 ++++++++++- .../tools/testing/data_interface_mixins.py | 30 +++++++++++++++++++ .../tools/testing/mock_interfaces.py | 12 ++++++++ src/neuroconv/utils/json_schema.py | 16 ++++++++++ 12 files changed, 113 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f5f6848..16a8ef737 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Bug fixes ### Improvements +* Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) ## v0.6.1 (August 30, 2024) @@ -54,6 +55,7 @@ + ## v0.5.0 (July 17, 2024) ### Deprecations diff --git a/pyproject.toml b/pyproject.toml index b40294d25..0eacf483b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,20 +123,21 @@ extend-exclude = ''' [tool.ruff] -exclude = [ - "*/__init__.py" -] [tool.ruff.lint] -select = ["F401", "I"] # TODO: eventually, expand to other 'F' linting +select = ["F401", "I", "D101"] # TODO: eventually, expand to other 'F' linting fixable = ["ALL"] +[tool.ruff.lint.per-file-ignores] +"**__init__.py" = ["F401", "I"] +"tests/**" = ["D"] # We are not enforcing docstrings in tests +"src/neuroconv/tools/testing/data_interface_mixins.py" = ["D"] # We are not enforcing docstrings in the interface mixings + [tool.ruff.lint.isort] relative-imports-order = "closest-to-furthest" known-first-party = ["neuroconv"] - [tool.codespell] skip = '.git*,*.pdf,*.css' check-hidden = true diff --git a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py index 69612c85c..731aec168 100644 --- a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py @@ -132,6 +132,9 @@ def __init__(self, file_path: FilePath, noise_std: float = 3.5): class AxonaLFPDataInterface(BaseLFPExtractorInterface): + """ + Primary data interface class for converting Axona LFP data. + """ display_name = "Axona LFP" associated_suffixes = (".bin", ".set") diff --git a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py index 731ce981b..e388e5e44 100644 --- a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py @@ -361,6 +361,14 @@ def get_original_timestamps(self): class CellExplorerLFPInterface(CellExplorerRecordingInterface): + """ + Adds lfp data from binary files with the new CellExplorer format: + + https://cellexplorer.org/ + + See the `CellExplorerRecordingInterface` class for more information. + """ + display_name = "CellExplorer LFP" keywords = BaseRecordingExtractorInterface.keywords + ( "extracellular electrophysiology", diff --git a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py index da0573249..d4ceb6b38 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py @@ -94,6 +94,11 @@ def get_metadata(self) -> dict: class NeuralynxSortingInterface(BaseSortingExtractorInterface): + """ + Primary data interface for converting Neuralynx sorting data. Uses + :py:class:`~spikeinterface.extractors.NeuralynxSortingExtractor`. + """ + display_name = "Neuralynx Sorting" associated_suffixes = (".nse", ".ntt", ".nse", ".nev") info = "Interface for Neuralynx sorting data." diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 6e8c33586..7d97f7d25 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -17,6 +17,10 @@ class SpikeGLXRecordingInterface(BaseRecordingExtractorInterface): + """ + Primary SpikeGLX interface for converting raw SpikeGLX data using a :py:class:`~spikeinterface.extractors.SpikeGLXRecordingExtractor`. + """ + display_name = "SpikeGLX Recording" keywords = BaseRecordingExtractorInterface.keywords + ("Neuropixels",) associated_suffixes = (".imec{probe_index}", ".ap", ".lf", ".meta", ".bin") diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py index 6f5daeb0c..86e8edc1f 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py @@ -13,6 +13,10 @@ class BrukerTiffMultiPlaneConverter(NWBConverter): + """ + Converter class for Bruker imaging data with multiple channels and multiple planes. + """ + display_name = "Bruker TIFF Imaging (multiple channels, multiple planes)" keywords = BrukerTiffMultiPlaneImagingInterface.keywords associated_suffixes = BrukerTiffMultiPlaneImagingInterface.associated_suffixes @@ -123,6 +127,10 @@ def run_conversion( class BrukerTiffSinglePlaneConverter(NWBConverter): + """ + Primary data interface class for converting Bruker imaging data with multiple channels and a single plane. + """ + display_name = "Bruker TIFF Imaging (multiple channels, single plane)" keywords = BrukerTiffMultiPlaneImagingInterface.keywords associated_suffixes = BrukerTiffMultiPlaneImagingInterface.associated_suffixes diff --git a/src/neuroconv/tools/hdmf.py b/src/neuroconv/tools/hdmf.py index e8dfff294..660971df5 100644 --- a/src/neuroconv/tools/hdmf.py +++ b/src/neuroconv/tools/hdmf.py @@ -7,7 +7,9 @@ from hdmf.data_utils import GenericDataChunkIterator as HDMFGenericDataChunkIterator -class GenericDataChunkIterator(HDMFGenericDataChunkIterator): +class GenericDataChunkIterator(HDMFGenericDataChunkIterator): # noqa: D101 + # TODO Should this be added to the API? + def _get_default_buffer_shape(self, buffer_gb: float = 1.0) -> tuple[int]: return self.estimate_default_buffer_shape( buffer_gb=buffer_gb, chunk_shape=self.chunk_shape, maxshape=self.maxshape, dtype=self.dtype diff --git a/src/neuroconv/tools/path_expansion.py b/src/neuroconv/tools/path_expansion.py index 427a33a9e..4ab839c0f 100644 --- a/src/neuroconv/tools/path_expansion.py +++ b/src/neuroconv/tools/path_expansion.py @@ -13,6 +13,15 @@ class AbstractPathExpander(abc.ABC): + """ + Abstract base class for expanding file paths and extracting metadata. + + This class provides methods to extract metadata from file paths within a directory + and to expand paths based on a specified data specification. It is designed to be + subclassed, with the `list_directory` method needing to be implemented by any + subclass to provide the specific logic for listing files in a directory. + """ + def extract_metadata(self, base_directory: DirectoryPath, format_: str): """ Uses the parse library to extract metadata from file paths in the base_directory. @@ -128,7 +137,13 @@ def expand_paths(self, source_data_spec: dict[str, dict]) -> list[DeepDict]: class LocalPathExpander(AbstractPathExpander): - def list_directory(self, base_directory: DirectoryPath) -> Iterable[FilePath]: + """ + Class for expanding file paths and extracting metadata on a local filesystem. + + See https://neuroconv.readthedocs.io/en/main/user_guide/expand_path.html for more information. + """ + + def list_directory(self, base_directory: DirectoryPath) -> Iterable[FilePath]: # noqa: D101 base_directory = Path(base_directory) assert base_directory.is_dir(), f"The specified 'base_directory' ({base_directory}) is not a directory!" return (str(path.relative_to(base_directory)) for path in base_directory.rglob("*")) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 07e25bede..9f94091d4 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -767,6 +767,10 @@ def test_interface_alignment(self, setup_interface): class AudioInterfaceTestMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """ + A mixin for testing Audio interfaces. + """ + # Currently asserted in the downstream testing suite; could be refactored in future PR def check_read_nwb(self, nwbfile_path: str): pass @@ -777,6 +781,10 @@ def test_interface_alignment(self): class DeepLabCutInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """ + A mixin for testing DeepLabCut interfaces. + """ + def check_interface_get_original_timestamps(self): pass # TODO in separate PR @@ -797,6 +805,10 @@ def check_nwbfile_temporal_alignment(self): class VideoInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """ + A mixin for testing Video interfaces. + """ + def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() @@ -867,6 +879,10 @@ def check_interface_original_timestamps_inmutability(self): class MedPCInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """ + A mixin for testing MedPC interfaces. + """ + def check_no_metadata_mutation(self, metadata: dict): """Ensure the metadata object was not altered by `add_to_nwbfile` method.""" @@ -1101,6 +1117,10 @@ def test_interface_alignment(self, medpc_name_to_info_dict: dict): class MiniscopeImagingInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """ + A mixin for testing Miniscope Imaging interfaces. + """ + def check_read_nwb(self, nwbfile_path: str): from ndx_miniscope import Miniscope @@ -1129,6 +1149,10 @@ def check_read_nwb(self, nwbfile_path: str): class ScanImageSinglePlaneImagingInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """ + A mixing for testing ScanImage Single Plane Imaging interfaces. + """ + def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(nwbfile_path, "r") as io: nwbfile = io.read() @@ -1160,6 +1184,10 @@ def check_read_nwb(self, nwbfile_path: str): class ScanImageMultiPlaneImagingInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """ + A mixin for testing ScanImage MultiPlane Imaging interfaces. + """ + def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(nwbfile_path, "r") as io: nwbfile = io.read() @@ -1190,6 +1218,8 @@ def check_read_nwb(self, nwbfile_path: str): class TDTFiberPhotometryInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): + """Mixin for testing TDT Fiber Photometry interfaces.""" + def check_no_metadata_mutation(self, metadata: dict): """Ensure the metadata object was not altered by `add_to_nwbfile` method.""" diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 9b115c1e9..f05228b34 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -18,6 +18,10 @@ class MockBehaviorEventInterface(BaseTemporalAlignmentInterface): + """ + A mock behavior event interface for testing purposes. + """ + @classmethod def get_source_schema(cls) -> dict: source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["event_times"]) @@ -56,6 +60,10 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): class MockSpikeGLXNIDQInterface(SpikeGLXNIDQInterface): + """ + A mock SpikeGLX interface for testing purposes. + """ + ExtractorName = "NumpyRecording" @classmethod @@ -150,6 +158,10 @@ def get_metadata(self) -> dict: class MockImagingInterface(BaseImagingExtractorInterface): + """ + A mock imaging interface for testing purposes. + """ + def __init__( self, num_frames: int = 30, diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index bef733d4b..1e8d5d4d4 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -17,7 +17,17 @@ class NWBMetaDataEncoder(json.JSONEncoder): + """ + Custom JSON encoder for NWB metadata. + + This encoder extends the default JSONEncoder class and provides custom serialization + for certain data types commonly used in NWB metadata. + """ + def default(self, obj): + """ + Serialize custom data types to JSON. This overwrites the default method of the JSONEncoder class. + """ # Over-write behaviors for datetime object if isinstance(obj, datetime): return obj.isoformat() @@ -34,6 +44,12 @@ def default(self, obj): class NWBSourceDataEncoder(NWBMetaDataEncoder): + """ + Custom JSON encoder for data interface source data (i.e. kwargs). + + This encoder extends the default JSONEncoder class and provides custom serialization + for certain data types commonly used in interface source data. + """ def default(self, obj): From 905dc73b549ce877487401d9dbcb67ebebff4e8d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 30 Aug 2024 18:49:01 -0600 Subject: [PATCH 021/118] Fix broken link to hdf5 documentation about IO optimization (#1044) --- .../tools/roiextractors/imagingextractordatachunkiterator.py | 4 ++-- .../spikeinterfacerecordingdatachunkiterator.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py b/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py index 0792e2caf..506967be0 100644 --- a/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py +++ b/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py @@ -44,8 +44,8 @@ def __init__( The upper bound on size in megabytes (MB) of the internal chunk for the HDF5 dataset. The chunk_shape will be set implicitly by this argument. Cannot be set if `chunk_shape` is also specified. - The default is 10MB. For more details, see - https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-ImprovingIOPerformanceCompressedDatasets.pdf + The default is 10MB, as recommended by the HDF5 group. + For more details, search the hdf5 documentation for "Improving IO Performance Compressed Datasets". chunk_shape : tuple, optional Manual specification of the internal chunk shape for the HDF5 dataset. Cannot be set if `chunk_mb` is also specified. diff --git a/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py b/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py index 95b14fc23..5d6407f19 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py @@ -48,8 +48,8 @@ def __init__( The upper bound on size in megabytes (MB) of the internal chunk for the HDF5 dataset. The chunk_shape will be set implicitly by this argument. Cannot be set if `chunk_shape` is also specified. - The default is 1MB, as recommended by the HDF5 group. For more details, see - https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-ImprovingIOPerformanceCompressedDatasets.pdf + The default is 10MB, as recommended by the HDF5 group. + For more details, search the hdf5 documentation for "Improving IO Performance Compressed Datasets". chunk_shape : tuple, optional Manual specification of the internal chunk shape for the HDF5 dataset. Cannot be set if `chunk_mb` is also specified. From 925f183caa58046a4df5f146157f67f0e0ee56a8 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sun, 1 Sep 2024 19:49:20 -0600 Subject: [PATCH 022/118] Re-organize gin test by modality (#1049) --- CHANGELOG.md | 1 + .../fiberphotometry/tdt_fp.rst | 2 +- .../test_schemas.py | 0 .../behavior}/__init__.py | 0 .../test_behavior_interfaces.py | 2 +- .../test_lightningpose_converter.py | 0 .../__init__.py | 0 .../spikeglx_multi_probe_metadata.json | 0 .../spikeglx_single_probe_metadata.json | 0 .../test_aux_interfaces.py | 0 .../{test_gin_ecephys => ecephys}/test_lfp.py | 0 .../test_openephyslegacy.py | 0 .../test_plexon_metadata.py | 0 .../test_raw_recordings.py | 4 +- .../test_recording_interfaces.py | 4 +- .../{ => ecephys}/test_sorting_interfaces.py | 4 +- .../test_spikeglx_converter.py | 0 .../test_spikeglx_metadata.py | 0 .../{ => icephys}/test_gin_icephys.py | 2 +- .../{test_gin_ecephys => ophys}/__init__.py | 0 .../fiber_photometry_metadata.yaml | 0 .../test_brukertiff_converter.py | 0 .../test_fiber_photometry_interfaces.py | 2 +- .../{ => ophys}/test_imaging_interfaces.py | 2 +- .../{ => ophys}/test_metadata_combinations.py | 2 +- .../test_miniscope_converter.py | 0 .../test_neuralynx_nvt_metadata.py | 0 .../test_segmentation_interfaces.py | 4 +- .../test_metadata/test_maxwell_metadata.py | 55 -------- .../test_metadata/test_neuroscope.py | 130 ------------------ .../{test_metadata => test_yaml}/__init__.py | 0 .../GIN_conversion_specification.yml | 0 ...on_specification_missing_nwbfile_names.yml | 0 ...tion_no_nwbfile_name_or_other_metadata.yml | 0 .../GIN_conversion_specification_videos.yml | 0 .../test_yaml_conversion_specification.py | 6 +- .../test_yaml_conversion_specification_cli.py | 4 +- 37 files changed, 20 insertions(+), 204 deletions(-) rename tests/{test_internals => test_minimal}/test_schemas.py (100%) rename tests/{test_internals => test_on_data/behavior}/__init__.py (100%) rename tests/test_on_data/{ => behavior}/test_behavior_interfaces.py (99%) rename tests/test_on_data/{test_format_converters => behavior}/test_lightningpose_converter.py (100%) rename tests/test_on_data/{test_format_converters => ecephys}/__init__.py (100%) rename tests/test_on_data/{test_format_converters => ecephys}/spikeglx_multi_probe_metadata.json (100%) rename tests/test_on_data/{test_format_converters => ecephys}/spikeglx_single_probe_metadata.json (100%) rename tests/test_on_data/{test_gin_ecephys => ecephys}/test_aux_interfaces.py (100%) rename tests/test_on_data/{test_gin_ecephys => ecephys}/test_lfp.py (100%) rename tests/test_on_data/{test_gin_ecephys => ecephys}/test_openephyslegacy.py (100%) rename tests/test_on_data/{test_metadata => ecephys}/test_plexon_metadata.py (100%) rename tests/test_on_data/{test_gin_ecephys => ecephys}/test_raw_recordings.py (98%) rename tests/test_on_data/{ => ecephys}/test_recording_interfaces.py (99%) rename tests/test_on_data/{ => ecephys}/test_sorting_interfaces.py (98%) rename tests/test_on_data/{test_format_converters => ecephys}/test_spikeglx_converter.py (100%) rename tests/test_on_data/{test_metadata => ecephys}/test_spikeglx_metadata.py (100%) rename tests/test_on_data/{ => icephys}/test_gin_icephys.py (98%) rename tests/test_on_data/{test_gin_ecephys => ophys}/__init__.py (100%) rename tests/test_on_data/{ => ophys}/fiber_photometry_metadata.yaml (100%) rename tests/test_on_data/{test_format_converters => ophys}/test_brukertiff_converter.py (100%) rename tests/test_on_data/{ => ophys}/test_fiber_photometry_interfaces.py (99%) rename tests/test_on_data/{ => ophys}/test_imaging_interfaces.py (99%) rename tests/test_on_data/{ => ophys}/test_metadata_combinations.py (98%) rename tests/test_on_data/{test_format_converters => ophys}/test_miniscope_converter.py (100%) rename tests/test_on_data/{test_metadata => ophys}/test_neuralynx_nvt_metadata.py (100%) rename tests/test_on_data/{ => ophys}/test_segmentation_interfaces.py (98%) delete mode 100644 tests/test_on_data/test_metadata/test_maxwell_metadata.py delete mode 100644 tests/test_on_data/test_metadata/test_neuroscope.py rename tests/test_on_data/{test_metadata => test_yaml}/__init__.py (100%) rename tests/test_on_data/{ => test_yaml}/conversion_specifications/GIN_conversion_specification.yml (100%) rename tests/test_on_data/{ => test_yaml}/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml (100%) rename tests/test_on_data/{ => test_yaml}/conversion_specifications/GIN_conversion_specification_no_nwbfile_name_or_other_metadata.yml (100%) rename tests/test_on_data/{ => test_yaml}/conversion_specifications/GIN_conversion_specification_videos.yml (100%) rename tests/test_on_data/{ => test_yaml}/test_yaml_conversion_specification.py (97%) rename tests/test_on_data/{ => test_yaml}/test_yaml_conversion_specification_cli.py (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16a8ef737..5fe29befe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) +* Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049) ## v0.6.1 (August 30, 2024) diff --git a/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst b/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst index 7d6ebd48c..b82e4a801 100644 --- a/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst +++ b/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst @@ -205,7 +205,7 @@ Convert TDT Fiber Photometry data to NWB using >>> folder_path = OPHYS_DATA_PATH / "fiber_photometry_datasets" / "TDT" / "Photo_249_391-200721-120136_stubbed" >>> LOCAL_PATH = Path(".") # Path to neuroconv - >>> editable_metadata_path = LOCAL_PATH / "tests" / "test_on_data" / "fiber_photometry_metadata.yaml" + >>> editable_metadata_path = LOCAL_PATH / "tests" / "test_on_data" / "ophys" / "fiber_photometry_metadata.yaml" >>> interface = TDTFiberPhotometryInterface(folder_path=folder_path, verbose=True) >>> metadata = interface.get_metadata() diff --git a/tests/test_internals/test_schemas.py b/tests/test_minimal/test_schemas.py similarity index 100% rename from tests/test_internals/test_schemas.py rename to tests/test_minimal/test_schemas.py diff --git a/tests/test_internals/__init__.py b/tests/test_on_data/behavior/__init__.py similarity index 100% rename from tests/test_internals/__init__.py rename to tests/test_on_data/behavior/__init__.py diff --git a/tests/test_on_data/test_behavior_interfaces.py b/tests/test_on_data/behavior/test_behavior_interfaces.py similarity index 99% rename from tests/test_on_data/test_behavior_interfaces.py rename to tests/test_on_data/behavior/test_behavior_interfaces.py index 33d0d468b..2ec43f96e 100644 --- a/tests/test_on_data/test_behavior_interfaces.py +++ b/tests/test_on_data/behavior/test_behavior_interfaces.py @@ -37,7 +37,7 @@ from neuroconv.utils import DeepDict try: - from .setup_paths import BEHAVIOR_DATA_PATH, OPHYS_DATA_PATH, OUTPUT_PATH + from ..setup_paths import BEHAVIOR_DATA_PATH, OPHYS_DATA_PATH, OUTPUT_PATH except ImportError: from setup_paths import BEHAVIOR_DATA_PATH, OUTPUT_PATH diff --git a/tests/test_on_data/test_format_converters/test_lightningpose_converter.py b/tests/test_on_data/behavior/test_lightningpose_converter.py similarity index 100% rename from tests/test_on_data/test_format_converters/test_lightningpose_converter.py rename to tests/test_on_data/behavior/test_lightningpose_converter.py diff --git a/tests/test_on_data/test_format_converters/__init__.py b/tests/test_on_data/ecephys/__init__.py similarity index 100% rename from tests/test_on_data/test_format_converters/__init__.py rename to tests/test_on_data/ecephys/__init__.py diff --git a/tests/test_on_data/test_format_converters/spikeglx_multi_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_multi_probe_metadata.json similarity index 100% rename from tests/test_on_data/test_format_converters/spikeglx_multi_probe_metadata.json rename to tests/test_on_data/ecephys/spikeglx_multi_probe_metadata.json diff --git a/tests/test_on_data/test_format_converters/spikeglx_single_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json similarity index 100% rename from tests/test_on_data/test_format_converters/spikeglx_single_probe_metadata.json rename to tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json diff --git a/tests/test_on_data/test_gin_ecephys/test_aux_interfaces.py b/tests/test_on_data/ecephys/test_aux_interfaces.py similarity index 100% rename from tests/test_on_data/test_gin_ecephys/test_aux_interfaces.py rename to tests/test_on_data/ecephys/test_aux_interfaces.py diff --git a/tests/test_on_data/test_gin_ecephys/test_lfp.py b/tests/test_on_data/ecephys/test_lfp.py similarity index 100% rename from tests/test_on_data/test_gin_ecephys/test_lfp.py rename to tests/test_on_data/ecephys/test_lfp.py diff --git a/tests/test_on_data/test_gin_ecephys/test_openephyslegacy.py b/tests/test_on_data/ecephys/test_openephyslegacy.py similarity index 100% rename from tests/test_on_data/test_gin_ecephys/test_openephyslegacy.py rename to tests/test_on_data/ecephys/test_openephyslegacy.py diff --git a/tests/test_on_data/test_metadata/test_plexon_metadata.py b/tests/test_on_data/ecephys/test_plexon_metadata.py similarity index 100% rename from tests/test_on_data/test_metadata/test_plexon_metadata.py rename to tests/test_on_data/ecephys/test_plexon_metadata.py diff --git a/tests/test_on_data/test_gin_ecephys/test_raw_recordings.py b/tests/test_on_data/ecephys/test_raw_recordings.py similarity index 98% rename from tests/test_on_data/test_gin_ecephys/test_raw_recordings.py rename to tests/test_on_data/ecephys/test_raw_recordings.py index 7388845cc..97ca75976 100644 --- a/tests/test_on_data/test_gin_ecephys/test_raw_recordings.py +++ b/tests/test_on_data/ecephys/test_raw_recordings.py @@ -19,8 +19,8 @@ from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH from ..setup_paths import OUTPUT_PATH except ImportError: - from setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from setup_paths import OUTPUT_PATH + from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH + from ..setup_paths import OUTPUT_PATH if not DATA_PATH.exists(): pytest.fail(f"No folder found in location: {DATA_PATH}!") diff --git a/tests/test_on_data/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py similarity index 99% rename from tests/test_on_data/test_recording_interfaces.py rename to tests/test_on_data/ecephys/test_recording_interfaces.py index 14b8e87ea..66e6f2b7b 100644 --- a/tests/test_on_data/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -37,9 +37,9 @@ ) try: - from .setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH + from ..setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH except ImportError: - from setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH + from ..setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH this_python_version = version.parse(python_version()) diff --git a/tests/test_on_data/test_sorting_interfaces.py b/tests/test_on_data/ecephys/test_sorting_interfaces.py similarity index 98% rename from tests/test_on_data/test_sorting_interfaces.py rename to tests/test_on_data/ecephys/test_sorting_interfaces.py index dfb4ff599..7c572c269 100644 --- a/tests/test_on_data/test_sorting_interfaces.py +++ b/tests/test_on_data/ecephys/test_sorting_interfaces.py @@ -17,8 +17,8 @@ ) try: - from .setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from .setup_paths import OUTPUT_PATH + from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH + from ..setup_paths import OUTPUT_PATH except ImportError: from setup_paths import ECEPHY_DATA_PATH as DATA_PATH from setup_paths import OUTPUT_PATH diff --git a/tests/test_on_data/test_format_converters/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py similarity index 100% rename from tests/test_on_data/test_format_converters/test_spikeglx_converter.py rename to tests/test_on_data/ecephys/test_spikeglx_converter.py diff --git a/tests/test_on_data/test_metadata/test_spikeglx_metadata.py b/tests/test_on_data/ecephys/test_spikeglx_metadata.py similarity index 100% rename from tests/test_on_data/test_metadata/test_spikeglx_metadata.py rename to tests/test_on_data/ecephys/test_spikeglx_metadata.py diff --git a/tests/test_on_data/test_gin_icephys.py b/tests/test_on_data/icephys/test_gin_icephys.py similarity index 98% rename from tests/test_on_data/test_gin_icephys.py rename to tests/test_on_data/icephys/test_gin_icephys.py index 31d40105a..2d71e0636 100644 --- a/tests/test_on_data/test_gin_icephys.py +++ b/tests/test_on_data/icephys/test_gin_icephys.py @@ -20,7 +20,7 @@ except ImportError: HAVE_PARAMETERIZED = False # Load the configuration for the data tests -test_config_dict = load_dict_from_file(Path(__file__).parent / "gin_test_config.json") +test_config_dict = load_dict_from_file(Path(__file__).parent.parent / "gin_test_config.json") # GIN dataset: https://gin.g-node.org/NeuralEnsemble/ephy_testing_data if os.getenv("CI"): diff --git a/tests/test_on_data/test_gin_ecephys/__init__.py b/tests/test_on_data/ophys/__init__.py similarity index 100% rename from tests/test_on_data/test_gin_ecephys/__init__.py rename to tests/test_on_data/ophys/__init__.py diff --git a/tests/test_on_data/fiber_photometry_metadata.yaml b/tests/test_on_data/ophys/fiber_photometry_metadata.yaml similarity index 100% rename from tests/test_on_data/fiber_photometry_metadata.yaml rename to tests/test_on_data/ophys/fiber_photometry_metadata.yaml diff --git a/tests/test_on_data/test_format_converters/test_brukertiff_converter.py b/tests/test_on_data/ophys/test_brukertiff_converter.py similarity index 100% rename from tests/test_on_data/test_format_converters/test_brukertiff_converter.py rename to tests/test_on_data/ophys/test_brukertiff_converter.py diff --git a/tests/test_on_data/test_fiber_photometry_interfaces.py b/tests/test_on_data/ophys/test_fiber_photometry_interfaces.py similarity index 99% rename from tests/test_on_data/test_fiber_photometry_interfaces.py rename to tests/test_on_data/ophys/test_fiber_photometry_interfaces.py index 125d892a5..b9121f08b 100644 --- a/tests/test_on_data/test_fiber_photometry_interfaces.py +++ b/tests/test_on_data/ophys/test_fiber_photometry_interfaces.py @@ -14,7 +14,7 @@ from neuroconv.utils import dict_deep_update, load_dict_from_file try: - from .setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH + from ..setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH except ImportError: from setup_paths import OUTPUT_PATH diff --git a/tests/test_on_data/test_imaging_interfaces.py b/tests/test_on_data/ophys/test_imaging_interfaces.py similarity index 99% rename from tests/test_on_data/test_imaging_interfaces.py rename to tests/test_on_data/ophys/test_imaging_interfaces.py index 1a5328e52..8bfc8c85c 100644 --- a/tests/test_on_data/test_imaging_interfaces.py +++ b/tests/test_on_data/ophys/test_imaging_interfaces.py @@ -36,7 +36,7 @@ ) try: - from .setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH + from ..setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH except ImportError: from setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH diff --git a/tests/test_on_data/test_metadata_combinations.py b/tests/test_on_data/ophys/test_metadata_combinations.py similarity index 98% rename from tests/test_on_data/test_metadata_combinations.py rename to tests/test_on_data/ophys/test_metadata_combinations.py index 2ba80f586..68441ae70 100644 --- a/tests/test_on_data/test_metadata_combinations.py +++ b/tests/test_on_data/ophys/test_metadata_combinations.py @@ -6,7 +6,7 @@ from neuroconv import NWBConverter from neuroconv.datainterfaces import Suite2pSegmentationInterface, TiffImagingInterface -from .setup_paths import OPHYS_DATA_PATH +from ..setup_paths import OPHYS_DATA_PATH TiffImagingInterface_source_data = dict( file_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "demoMovie.tif"), sampling_frequency=15.0 diff --git a/tests/test_on_data/test_format_converters/test_miniscope_converter.py b/tests/test_on_data/ophys/test_miniscope_converter.py similarity index 100% rename from tests/test_on_data/test_format_converters/test_miniscope_converter.py rename to tests/test_on_data/ophys/test_miniscope_converter.py diff --git a/tests/test_on_data/test_metadata/test_neuralynx_nvt_metadata.py b/tests/test_on_data/ophys/test_neuralynx_nvt_metadata.py similarity index 100% rename from tests/test_on_data/test_metadata/test_neuralynx_nvt_metadata.py rename to tests/test_on_data/ophys/test_neuralynx_nvt_metadata.py diff --git a/tests/test_on_data/test_segmentation_interfaces.py b/tests/test_on_data/ophys/test_segmentation_interfaces.py similarity index 98% rename from tests/test_on_data/test_segmentation_interfaces.py rename to tests/test_on_data/ophys/test_segmentation_interfaces.py index 3d2547df8..b2c6428f1 100644 --- a/tests/test_on_data/test_segmentation_interfaces.py +++ b/tests/test_on_data/ophys/test_segmentation_interfaces.py @@ -11,9 +11,9 @@ ) try: - from .setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH + from ..setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH except ImportError: - from setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH + from ..setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH class TestCaimanSegmentationInterface(SegmentationExtractorInterfaceTestMixin): diff --git a/tests/test_on_data/test_metadata/test_maxwell_metadata.py b/tests/test_on_data/test_metadata/test_maxwell_metadata.py deleted file mode 100644 index b66653b61..000000000 --- a/tests/test_on_data/test_metadata/test_maxwell_metadata.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -from datetime import datetime -from pathlib import Path -from platform import system -from shutil import rmtree -from tempfile import mkdtemp -from zoneinfo import ZoneInfo - -import pytest -from hdmf.testing import TestCase - -from neuroconv.datainterfaces import MaxOneRecordingInterface - -from ..setup_paths import ECEPHY_DATA_PATH - - -@pytest.mark.skipif(system() == "Linux", reason="Specific tests for raising assertion on non-linux systems.") -class TestMaxOneAssertion(TestCase): - def test_max_one_usage_assertion(self): - with self.assertRaisesWith( - exc_type=NotImplementedError, - exc_msg="The MaxOneRecordingInterface has not yet been implemented for systems other than Linux.", - ): - file_path = ECEPHY_DATA_PATH / "maxwell" / "MaxOne_data" / "Record" / "000011" / "data.raw.h5" - MaxOneRecordingInterface(file_path=file_path) - - -@pytest.mark.skip(reason="Stochastically fails to download compression library.") -# @pytest.mark.skipif(system() != "Linux", reason="MaxOne only works on Linux at the moment.") -class TestMaxOneMetadata(TestCase): - @classmethod - def setUpClass(cls): - file_path = ECEPHY_DATA_PATH / "maxwell" / "MaxOne_data" / "Record" / "000011" / "data.raw.h5" - cls.interface = MaxOneRecordingInterface(file_path=file_path) - - cls.tmpdir = Path(mkdtemp()) - cls.nwbfile_path = cls.tmpdir / "maxone_meadata_test.nwb" - cls.metadata = cls.interface.get_metadata() - cls.metadata["NWBFile"].update( - session_start_time=datetime(2020, 1, 1, 12, 30, 0, tzinfo=ZoneInfo("US/Pacific")) - ) - cls.interface.run_conversion(nwbfile_path=cls.nwbfile_path, metadata=cls.metadata) - - @classmethod - def tearDownClass(cls): - rmtree(cls.tmpdir) - - def test_neuroconv_metadata(self): - assert len(self.metadata["Ecephys"]["Device"]) == 1 - assert self.metadata["Ecephys"]["Device"][0]["name"] == "DeviceEcephys" - assert self.metadata["Ecephys"]["Device"][0]["description"] == "Recorded using Maxwell version '20190530'." - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_on_data/test_metadata/test_neuroscope.py b/tests/test_on_data/test_metadata/test_neuroscope.py deleted file mode 100644 index 576a8a019..000000000 --- a/tests/test_on_data/test_metadata/test_neuroscope.py +++ /dev/null @@ -1,130 +0,0 @@ -import unittest -from datetime import datetime - -import numpy as np -import numpy.testing as npt -import pytest -from parameterized import param, parameterized -from pynwb import NWBHDF5IO -from spikeinterface.extractors import NwbRecordingExtractor - -from neuroconv import NWBConverter -from neuroconv.datainterfaces import NeuroScopeRecordingInterface - -# enable to run locally in interactive mode -try: - from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from ..setup_paths import OUTPUT_PATH -except ImportError: - from setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from setup_paths import OUTPUT_PATH - -if not DATA_PATH.exists(): - pytest.fail(f"No folder found in location: {DATA_PATH}!") - - -def custom_name_func(testcase_func, param_num, param): - interface_name = param.kwargs["data_interface"].__name__ - reduced_interface_name = interface_name.replace("Recording", "").replace("Interface", "").replace("Sorting", "") - - return ( - f"{testcase_func.__name__}_{param_num}_" - f"{parameterized.to_safe_name(reduced_interface_name)}" - f"_{param.kwargs.get('case_name', '')}" - ) - - -class TestNeuroscopeNwbConversions(unittest.TestCase): - savedir = OUTPUT_PATH - - @parameterized.expand( - input=[ - param( - name="complete", - conversion_options=None, - ), - param(name="stub", conversion_options=dict(TestRecording=dict(stub_test=True))), - ] - ) - def test_neuroscope_gains(self, name, conversion_options): - input_gain = 2.0 - interface_kwargs = dict(file_path=str(DATA_PATH / "neuroscope" / "test1" / "test1.dat"), gain=input_gain) - - nwbfile_path = str(self.savedir / f"test_neuroscope_gains_{name}.nwb") - - class TestConverter(NWBConverter): - data_interface_classes = dict(TestRecording=NeuroScopeRecordingInterface) - - converter = TestConverter(source_data=dict(TestRecording=interface_kwargs)) - metadata = converter.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - converter.run_conversion( - nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata, conversion_options=conversion_options - ) - - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - # output_channel_conversion = nwbfile.acquisition["ElectricalSeriesRaw"].channel_conversion[:] - # input_gain_array = np.ones_like(output_channel_conversion) * input_gain - # np.testing.assert_array_almost_equal(input_gain_array, output_channel_conversion) - assert nwbfile.acquisition["ElectricalSeries"].channel_conversion is None - - nwb_recording = NwbRecordingExtractor(file_path=nwbfile_path) - nwb_recording_gains = nwb_recording.get_channel_gains() - npt.assert_almost_equal(input_gain * np.ones_like(nwb_recording_gains), nwb_recording_gains) - - @parameterized.expand( - input=[ - param( - name="complete", - conversion_options=None, - ), - param(name="stub", conversion_options=dict(TestRecording=dict(stub_test=True))), - ] - ) - def test_neuroscope_dtype(self, name, conversion_options): - interface_kwargs = dict(file_path=str(DATA_PATH / "neuroscope" / "test1" / "test1.dat"), gain=2.0) - - nwbfile_path = str(self.savedir / f"test_neuroscope_dtype_{name}.nwb") - - class TestConverter(NWBConverter): - data_interface_classes = dict(TestRecording=NeuroScopeRecordingInterface) - - converter = TestConverter(source_data=dict(TestRecording=interface_kwargs)) - metadata = converter.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - converter.run_conversion( - nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata, conversion_options=conversion_options - ) - - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - output_dtype = nwbfile.acquisition["ElectricalSeries"].data.dtype - self.assertEqual(first=output_dtype, second=np.dtype("int16")) - - def test_neuroscope_starting_time(self): - nwbfile_path = str(self.savedir / "testing_start_time.nwb") - - class TestConverter(NWBConverter): - data_interface_classes = dict(TestRecording=NeuroScopeRecordingInterface) - - converter = TestConverter( - source_data=dict(TestRecording=dict(file_path=str(DATA_PATH / "neuroscope" / "test1" / "test1.dat"))) - ) - metadata = converter.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - starting_time = 123.0 - converter.run_conversion( - nwbfile_path=nwbfile_path, - overwrite=True, - metadata=metadata, - conversion_options=dict(TestRecording=dict(starting_time=starting_time)), - ) - - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - nwbfile = io.read() - self.assertEqual(first=starting_time, second=nwbfile.acquisition["ElectricalSeries"].starting_time) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_on_data/test_metadata/__init__.py b/tests/test_on_data/test_yaml/__init__.py similarity index 100% rename from tests/test_on_data/test_metadata/__init__.py rename to tests/test_on_data/test_yaml/__init__.py diff --git a/tests/test_on_data/conversion_specifications/GIN_conversion_specification.yml b/tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification.yml similarity index 100% rename from tests/test_on_data/conversion_specifications/GIN_conversion_specification.yml rename to tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification.yml diff --git a/tests/test_on_data/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml b/tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml similarity index 100% rename from tests/test_on_data/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml rename to tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml diff --git a/tests/test_on_data/conversion_specifications/GIN_conversion_specification_no_nwbfile_name_or_other_metadata.yml b/tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_no_nwbfile_name_or_other_metadata.yml similarity index 100% rename from tests/test_on_data/conversion_specifications/GIN_conversion_specification_no_nwbfile_name_or_other_metadata.yml rename to tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_no_nwbfile_name_or_other_metadata.yml diff --git a/tests/test_on_data/conversion_specifications/GIN_conversion_specification_videos.yml b/tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_videos.yml similarity index 100% rename from tests/test_on_data/conversion_specifications/GIN_conversion_specification_videos.yml rename to tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_videos.yml diff --git a/tests/test_on_data/test_yaml_conversion_specification.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py similarity index 97% rename from tests/test_on_data/test_yaml_conversion_specification.py rename to tests/test_on_data/test_yaml/test_yaml_conversion_specification.py index 56fd7f6c3..61c71cf86 100644 --- a/tests/test_on_data/test_yaml_conversion_specification.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py @@ -11,8 +11,8 @@ from neuroconv import run_conversion_from_yaml from neuroconv.utils import load_dict_from_file -from .setup_paths import BEHAVIOR_DATA_PATH, OUTPUT_PATH -from .setup_paths import ECEPHY_DATA_PATH as DATA_PATH +from ..setup_paths import BEHAVIOR_DATA_PATH, OUTPUT_PATH +from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH @pytest.mark.parametrize( @@ -26,7 +26,7 @@ ) def test_validate_example_specifications(fname): path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" - schema_folder = path_to_test_yml_files.parent.parent.parent / "src" / "neuroconv" / "schemas" + schema_folder = path_to_test_yml_files.parent.parent.parent.parent / "src" / "neuroconv" / "schemas" specification_schema = load_dict_from_file(file_path=schema_folder / "yaml_conversion_specification_schema.json") sys_uri_base = "file://" if sys.platform.startswith("win32"): diff --git a/tests/test_on_data/test_yaml_conversion_specification_cli.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py similarity index 98% rename from tests/test_on_data/test_yaml_conversion_specification_cli.py rename to tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py index 1d6758fac..cc4391623 100644 --- a/tests/test_on_data/test_yaml_conversion_specification_cli.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification_cli.py @@ -7,8 +7,8 @@ from neuroconv.tools import deploy_process -from .setup_paths import ECEPHY_DATA_PATH as DATA_PATH -from .setup_paths import OUTPUT_PATH +from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH +from ..setup_paths import OUTPUT_PATH class TestYAMLConversionSpecification(TestCase): From e73745a0f32799394e805170cfc645a9c6b21fd6 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 2 Sep 2024 00:09:00 -0600 Subject: [PATCH 023/118] Make some methods private (#1050) Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- CHANGELOG.md | 1 + src/neuroconv/basedatainterface.py | 4 ++-- .../behavior/neuralynx/neuralynx_nvt_interface.py | 6 +++--- .../datainterfaces/ecephys/mearec/mearecdatainterface.py | 4 ++-- src/neuroconv/nwbconverter.py | 6 +++--- src/neuroconv/tools/roiextractors/__init__.py | 2 +- src/neuroconv/tools/roiextractors/roiextractors.py | 4 ++-- src/neuroconv/tools/testing/data_interface_mixins.py | 4 ++-- src/neuroconv/utils/__init__.py | 2 +- src/neuroconv/utils/dict.py | 6 +++--- src/neuroconv/utils/json_schema.py | 6 +++--- tests/test_minimal/test_tools/test_expand_paths.py | 4 ++-- tests/test_minimal/test_utils/test_json_schema_utils.py | 4 ++-- tests/test_ophys/test_tools_roiextractors.py | 4 ++-- 14 files changed, 29 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fe29befe..52c1ebb9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Upcoming ### Deprecations +* The following classes and objects are now private `NWBMetaDataEncoder`, `NWBMetaDataEncoder`, `check_if_imaging_fits_into_memory`, `NoDatesSafeLoader` [PR #1050](https://github.com/catalystneuro/neuroconv/pull/1050) ### Features diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 4e0a0aac4..f8fce62b0 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -19,7 +19,7 @@ ) from .tools.nwb_helpers._metadata_and_file_helpers import _resolve_backend from .utils import ( - NWBMetaDataEncoder, + _NWBMetaDataEncoder, get_json_schema_from_method_signature, load_dict_from_file, ) @@ -63,7 +63,7 @@ def get_metadata(self) -> DeepDict: def validate_metadata(self, metadata: dict, append_mode: bool = False) -> None: """Validate the metadata against the schema.""" - encoder = NWBMetaDataEncoder() + encoder = _NWBMetaDataEncoder() # The encoder produces a serialized object, so we deserialized it for comparison serialized_metadata = encoder.encode(metadata) diff --git a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py index 51e04821d..f01d11e53 100644 --- a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py +++ b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py @@ -8,7 +8,7 @@ from .nvt_utils import read_data, read_header from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface -from ....utils import DeepDict, NWBMetaDataEncoder, get_base_schema +from ....utils import DeepDict, _NWBMetaDataEncoder, get_base_schema from ....utils.path import infer_path @@ -136,7 +136,7 @@ def add_to_nwbfile( unit="pixels", conversion=1.0, timestamps=self.get_timestamps(), - description=f"Pixel x and y coordinates from the .nvt file with header data: {json.dumps(self.header, cls=NWBMetaDataEncoder)}", + description=f"Pixel x and y coordinates from the .nvt file with header data: {json.dumps(self.header, cls=_NWBMetaDataEncoder)}", ) nwbfile.add_acquisition(Position([spatial_series], name="NvtPosition")) @@ -151,7 +151,7 @@ def add_to_nwbfile( unit="degrees", conversion=1.0, timestamps=spatial_series if add_position else self.get_timestamps(), - description=f"Angle from the .nvt file with header data: {json.dumps(self.header, cls=NWBMetaDataEncoder)}", + description=f"Angle from the .nvt file with header data: {json.dumps(self.header, cls=_NWBMetaDataEncoder)}", ), name="NvtCompassDirection", ) diff --git a/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py b/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py index 9d3797e0f..7a82025ca 100644 --- a/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py @@ -3,7 +3,7 @@ from pydantic import FilePath from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils.json_schema import NWBMetaDataEncoder +from ....utils.json_schema import _NWBMetaDataEncoder class MEArecRecordingInterface(BaseRecordingExtractorInterface): @@ -61,7 +61,7 @@ def get_metadata(self) -> dict: for unneeded_key in ["fs", "dtype"]: recording_metadata.pop(unneeded_key) metadata["Ecephys"].update( - {self.es_key: dict(name=self.es_key, description=json.dumps(recording_metadata, cls=NWBMetaDataEncoder))} + {self.es_key: dict(name=self.es_key, description=json.dumps(recording_metadata, cls=_NWBMetaDataEncoder))} ) return metadata diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index cb62e149d..eabf6d772 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -29,7 +29,7 @@ unroot_schema, ) from .utils.dict import DeepDict -from .utils.json_schema import NWBMetaDataEncoder, NWBSourceDataEncoder +from .utils.json_schema import _NWBMetaDataEncoder, _NWBSourceDataEncoder class NWBConverter: @@ -63,7 +63,7 @@ def validate_source(cls, source_data: dict[str, dict], verbose: bool = True): def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = True): - encoder = NWBSourceDataEncoder() + encoder = _NWBSourceDataEncoder() # The encoder produces a serialized object, so we deserialized it for comparison serialized_source_data = encoder.encode(source_data) @@ -104,7 +104,7 @@ def get_metadata(self) -> DeepDict: def validate_metadata(self, metadata: dict[str, dict], append_mode: bool = False): """Validate metadata against Converter metadata_schema.""" - encoder = NWBMetaDataEncoder() + encoder = _NWBMetaDataEncoder() # The encoder produces a serialized object, so we deserialized it for comparison serialized_metadata = encoder.encode(metadata) decoded_metadata = json.loads(serialized_metadata) diff --git a/src/neuroconv/tools/roiextractors/__init__.py b/src/neuroconv/tools/roiextractors/__init__.py index 5e009fe6d..181bbe38c 100644 --- a/src/neuroconv/tools/roiextractors/__init__.py +++ b/src/neuroconv/tools/roiextractors/__init__.py @@ -1,5 +1,5 @@ from .roiextractors import ( - check_if_imaging_fits_into_memory, + _check_if_imaging_fits_into_memory, get_nwb_imaging_metadata, get_nwb_segmentation_metadata, add_background_fluorescence_traces, diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 4da660914..618d30b4a 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -558,7 +558,7 @@ def add_photon_series_to_nwbfile( return nwbfile -def check_if_imaging_fits_into_memory(imaging: ImagingExtractor) -> None: +def _check_if_imaging_fits_into_memory(imaging: ImagingExtractor) -> None: """ Raise an error if the full traces of an imaging extractor are larger than available memory. @@ -625,7 +625,7 @@ def data_generator(imaging): iterator_options = dict() if iterator_options is None else iterator_options if iterator_type is None: - check_if_imaging_fits_into_memory(imaging=imaging) + _check_if_imaging_fits_into_memory(imaging=imaging) return imaging.get_video().transpose((0, 2, 1)) if iterator_type == "v1": diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 9f94091d4..24042feee 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -33,7 +33,7 @@ configure_backend, get_default_backend_configuration, ) -from neuroconv.utils import NWBMetaDataEncoder +from neuroconv.utils import _NWBMetaDataEncoder class DataInterfaceTestMixin: @@ -98,7 +98,7 @@ def check_metadata(self): if "session_start_time" not in metadata["NWBFile"]: metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) # handle json encoding of datetimes and other tricky types - metadata_for_validation = json.loads(json.dumps(metadata, cls=NWBMetaDataEncoder)) + metadata_for_validation = json.loads(json.dumps(metadata, cls=_NWBMetaDataEncoder)) validate(metadata_for_validation, schema) self.check_extracted_metadata(metadata) diff --git a/src/neuroconv/utils/__init__.py b/src/neuroconv/utils/__init__.py index 1670eb60f..f59cf59c5 100644 --- a/src/neuroconv/utils/__init__.py +++ b/src/neuroconv/utils/__init__.py @@ -7,7 +7,7 @@ load_dict_from_file, ) from .json_schema import ( - NWBMetaDataEncoder, + _NWBMetaDataEncoder, fill_defaults, get_base_schema, get_metadata_schema_for_icephys, diff --git a/src/neuroconv/utils/dict.py b/src/neuroconv/utils/dict.py index 0a92520f7..f0507b653 100644 --- a/src/neuroconv/utils/dict.py +++ b/src/neuroconv/utils/dict.py @@ -12,7 +12,7 @@ from pydantic import FilePath -class NoDatesSafeLoader(yaml.SafeLoader): +class _NoDatesSafeLoader(yaml.SafeLoader): """Custom override of yaml Loader class for datetime considerations.""" @classmethod @@ -33,7 +33,7 @@ def remove_implicit_resolver(cls, tag_to_remove): ] -NoDatesSafeLoader.remove_implicit_resolver("tag:yaml.org,2002:timestamp") +_NoDatesSafeLoader.remove_implicit_resolver("tag:yaml.org,2002:timestamp") def load_dict_from_file(file_path: FilePath) -> dict: @@ -44,7 +44,7 @@ def load_dict_from_file(file_path: FilePath) -> dict: if file_path.suffix in (".yml", ".yaml"): with open(file=file_path, mode="r") as stream: - dictionary = yaml.load(stream=stream, Loader=NoDatesSafeLoader) + dictionary = yaml.load(stream=stream, Loader=_NoDatesSafeLoader) elif file_path.suffix == ".json": with open(file=file_path, mode="r") as fp: dictionary = json.load(fp=fp) diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index 1e8d5d4d4..6c1ba7245 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -16,7 +16,7 @@ from pynwb.icephys import IntracellularElectrode -class NWBMetaDataEncoder(json.JSONEncoder): +class _NWBMetaDataEncoder(json.JSONEncoder): """ Custom JSON encoder for NWB metadata. @@ -43,7 +43,7 @@ def default(self, obj): return super().default(obj) -class NWBSourceDataEncoder(NWBMetaDataEncoder): +class _NWBSourceDataEncoder(_NWBMetaDataEncoder): """ Custom JSON encoder for data interface source data (i.e. kwargs). @@ -350,7 +350,7 @@ def get_metadata_schema_for_icephys(): def validate_metadata(metadata: dict[str, dict], schema: dict[str, dict], verbose: bool = False): """Validate metadata against a schema.""" - encoder = NWBMetaDataEncoder() + encoder = _NWBMetaDataEncoder() # The encoder produces a serialized object, so we deserialized it for comparison serialized_metadata = encoder.encode(metadata) diff --git a/tests/test_minimal/test_tools/test_expand_paths.py b/tests/test_minimal/test_tools/test_expand_paths.py index 2667602d5..9e7f03631 100644 --- a/tests/test_minimal/test_tools/test_expand_paths.py +++ b/tests/test_minimal/test_tools/test_expand_paths.py @@ -9,7 +9,7 @@ from neuroconv.tools import LocalPathExpander from neuroconv.tools.path_expansion import construct_path_template from neuroconv.tools.testing import generate_path_expander_demo_ibl -from neuroconv.utils import NWBMetaDataEncoder +from neuroconv.utils import _NWBMetaDataEncoder def create_test_directories_and_files( @@ -409,7 +409,7 @@ def test_expand_paths_ibl(tmpdir): ), ), ) - path_expansion_results = json.loads(json.dumps(path_expansion_results, cls=NWBMetaDataEncoder)) + path_expansion_results = json.loads(json.dumps(path_expansion_results, cls=_NWBMetaDataEncoder)) # build expected output from file expected_file_path = Path(__file__).parent / "expand_paths_ibl_expected.json" diff --git a/tests/test_minimal/test_utils/test_json_schema_utils.py b/tests/test_minimal/test_utils/test_json_schema_utils.py index be03a2699..4edf1e724 100644 --- a/tests/test_minimal/test_utils/test_json_schema_utils.py +++ b/tests/test_minimal/test_utils/test_json_schema_utils.py @@ -6,7 +6,7 @@ from pynwb.ophys import ImagingPlane, TwoPhotonSeries from neuroconv.utils import ( - NWBMetaDataEncoder, + _NWBMetaDataEncoder, dict_deep_update, fill_defaults, get_schema_from_hdmf_class, @@ -204,5 +204,5 @@ def test_get_schema_from_TwoPhotonSeries_array_type(): def test_np_array_encoding(): np_array = np.array([1, 2, 3]) - encoded = json.dumps(np_array, cls=NWBMetaDataEncoder) + encoded = json.dumps(np_array, cls=_NWBMetaDataEncoder) assert encoded == "[1, 2, 3]" diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index 60162527b..f750a2a40 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -27,6 +27,7 @@ from neuroconv.tools.nwb_helpers import get_module from neuroconv.tools.roiextractors import ( + _check_if_imaging_fits_into_memory, add_devices_to_nwbfile, add_fluorescence_traces_to_nwbfile, add_image_segmentation_to_nwbfile, @@ -34,7 +35,6 @@ add_photon_series_to_nwbfile, add_plane_segmentation_to_nwbfile, add_summary_images_to_nwbfile, - check_if_imaging_fits_into_memory, ) from neuroconv.tools.roiextractors.imagingextractordatachunkiterator import ( ImagingExtractorDataChunkIterator, @@ -1539,7 +1539,7 @@ def test_non_iterative_write_assertion(self): reg_expression = "Memory error, full TwoPhotonSeries data is (.*?) are available! Please use iterator_type='v2'" with self.assertRaisesRegex(MemoryError, reg_expression): - check_if_imaging_fits_into_memory(imaging=mock_imaging) + _check_if_imaging_fits_into_memory(imaging=mock_imaging) def test_non_iterative_two_photon(self): """Test adding two photon series with using DataChunkIterator as iterator type.""" From 77d2252088b61bf922aa95eb97e2cbf5cfb470aa Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 2 Sep 2024 03:27:00 -0600 Subject: [PATCH 024/118] Refactor DLC (#1047) Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- .../behavior/deeplabcut/_dlc_utils.py | 142 ++++++++++++------ 1 file changed, 93 insertions(+), 49 deletions(-) diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 7608fffaa..3704e859e 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -163,40 +163,23 @@ def _ensure_individuals_in_header(df, individual_name): return df -def _get_pes_args( - *, - config_file: Path, - h5file: Path, - individual_name: str, - timestamps_available: bool = False, - infer_timestamps: bool = True, -): - config_file = Path(config_file) - h5file = Path(h5file) - - if "DLC" not in h5file.name or not h5file.suffix == ".h5": - raise IOError("The file passed in is not a DeepLabCut h5 data file.") - - cfg = _read_config(config_file) - - vidname, scorer = h5file.stem.split("DLC") - scorer = "DLC" + scorer - video = None +def _get_graph_edges(metadata_file_path: Path): + """ + Extracts the part affinity field graph from the metadata pickle file. - df = _ensure_individuals_in_header(pd.read_hdf(h5file), individual_name) + Parameters + ---------- + metadata_file_path : Path + The path to the metadata pickle file. - # Fetch the corresponding metadata pickle file + Returns + ------- + list + The part affinity field graph, which defines the edges between the keypoints in the pose estimation. + """ paf_graph = [] - filename = str(h5file.parent / h5file.stem) - for i, c in enumerate(filename[::-1]): - if c.isnumeric(): - break - if i > 0: - filename = filename[:-i] - metadata_file = Path(filename + "_meta.pickle") - - if metadata_file.exists(): - with open(metadata_file, "rb") as file: + if metadata_file_path.exists(): + with open(metadata_file_path, "rb") as file: metadata = pickle.load(file) test_cfg = metadata["data"]["DLC-model-config file"] @@ -208,25 +191,64 @@ def _get_pes_args( else: warnings.warn("Metadata not found...") + return paf_graph + + +def _get_video_info_from_config_file(config_file_path: Path, vidname: str): + """ + Get the video information from the project config file. + + Parameters + ---------- + config_file_path : Path + The path to the project config file. + vidname : str + The name of the video. + + Returns + ------- + tuple + A tuple containing the video file path and the image shape. + """ + config_file_path = Path(config_file_path) + cfg = _read_config(config_file_path) + + video = None for video_path, params in cfg["video_sets"].items(): if vidname in video_path: video = video_path, params["crop"] break - # find timestamps only if required: - if timestamps_available: - timestamps = None - else: - if video is None: - timestamps = df.index.tolist() # setting timestamps to dummy TODO: extract timestamps in DLC? - else: - timestamps = _get_movie_timestamps(video[0], infer_timestamps=infer_timestamps) - if video is None: - warnings.warn(f"The video file corresponding to {h5file} could not be found...") - video = "fake_path", "0, 0, 0, 0" + warnings.warn(f"The corresponding video file could not be found...") + video = None, "0, 0, 0, 0" + + # The video in the config_file looks like this: + # video_sets: + # /Data/openfield-Pranav-2018-08-20/videos/m1s1.mp4: + # crop: 0, 640, 0, 480 + + video_file_path, image_shape = video + + return video_file_path, image_shape + + +def _get_pes_args( + *, + h5file: Path, + individual_name: str, +): + h5file = Path(h5file) + + if "DLC" not in h5file.name or not h5file.suffix == ".h5": + raise IOError("The file passed in is not a DeepLabCut h5 data file.") + + _, scorer = h5file.stem.split("DLC") + scorer = "DLC" + scorer + + df = _ensure_individuals_in_header(pd.read_hdf(h5file), individual_name) - return scorer, df, video, paf_graph, timestamps, cfg + return scorer, df def _write_pes_to_nwbfile( @@ -332,15 +354,37 @@ def add_subject_to_nwbfile( nwbfile : pynwb.NWBFile nwbfile with pes written in the behavior module """ - timestamps_available = timestamps is not None - scorer, df, video, paf_graph, dlc_timestamps, _ = _get_pes_args( - config_file=config_file, + h5file = Path(h5file) + + scorer, df = _get_pes_args( h5file=h5file, individual_name=individual_name, - timestamps_available=timestamps_available, ) - if timestamps is None: - timestamps = dlc_timestamps + + # Note the video here is a tuple of the video path and the image shape + vidname, scorer = h5file.stem.split("DLC") + video = _get_video_info_from_config_file(config_file_path=config_file, vidname=vidname) + + # find timestamps only if required:`` + timestamps_available = timestamps is not None + video_file_path = video[0] + if not timestamps_available: + if video_file_path is None: + timestamps = df.index.tolist() # setting timestamps to dummy + else: + timestamps = _get_movie_timestamps(video_file_path, infer_timestamps=True) + + # Fetch the corresponding metadata pickle file, we extract the edges graph from here + # TODO: This is the original implementation way to extract the file name but looks very brittle + filename = str(h5file.parent / h5file.stem) + for i, c in enumerate(filename[::-1]): + if c.isnumeric(): + break + if i > 0: + filename = filename[:-i] + + metadata_file_path = Path(filename + "_meta.pickle") + paf_graph = _get_graph_edges(metadata_file_path=metadata_file_path) df_animal = df.xs(individual_name, level="individuals", axis=1) From 0e242b52eb82eda943d34b8a90203e46fa2ebd7d Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Tue, 3 Sep 2024 03:45:59 +1000 Subject: [PATCH 025/118] added get_stream_names (#1039) Co-authored-by: Heberto Mayorquin --- CHANGELOG.md | 1 + .../ecephys/openephys/openephysdatainterface.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c1ebb9a..beee4e54b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * The following classes and objects are now private `NWBMetaDataEncoder`, `NWBMetaDataEncoder`, `check_if_imaging_fits_into_memory`, `NoDatesSafeLoader` [PR #1050](https://github.com/catalystneuro/neuroconv/pull/1050) ### Features +* Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) ### Bug fixes diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py index b47df98b9..81b84c36c 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py @@ -25,6 +25,15 @@ def get_source_schema(cls) -> dict: ] = "Path to OpenEphys directory (.continuous or .dat files)." return source_schema + @classmethod + def get_stream_names(cls, folder_path: DirectoryPath) -> list[str]: + if any(Path(folder_path).rglob("*.continuous")): + return OpenEphysLegacyRecordingInterface.get_stream_names(folder_path=folder_path) + elif any(Path(folder_path).rglob("*.dat")): + return OpenEphysBinaryRecordingInterface.get_stream_names(folder_path=folder_path) + else: + raise AssertionError("The Open Ephys data must be in 'legacy' (.continuous) or in 'binary' (.dat) format.") + def __new__( cls, folder_path: DirectoryPath, From 58afd3b7bc0fed4d3848f62f637355bbd2cc840d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 2 Sep 2024 14:39:31 -0600 Subject: [PATCH 026/118] Remove broken links from developer docs (#1051) --- docs/developer_guide/testing_suite.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/developer_guide/testing_suite.rst b/docs/developer_guide/testing_suite.rst index 4eeeda89a..9a49a65ea 100644 --- a/docs/developer_guide/testing_suite.rst +++ b/docs/developer_guide/testing_suite.rst @@ -39,12 +39,9 @@ Minimal These test internal functionality using only minimal dependencies or pre-downloaded data. -Sub-folders: `tests/test_minimal `_ and -`tests/test_internals `_ - -These can be run using only ``pip install -e neuroconv[test]`` and calling ``pytest tests/test_minimal`` and -``pytest tests/test_internal``. +Sub-folders: `tests/test_minimal ` +These can be run using only ``pip install -e neuroconv[test]`` and calling ``pytest tests/test_minimal`` Modality From a9b993ac7b8f913c9a9d1822c7b212ed5ee04590 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:06:17 -0600 Subject: [PATCH 027/118] [pre-commit.ci] pre-commit autoupdate (#1052) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b5895c35..a8ee7373a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.2 + rev: v0.6.3 hooks: - id: ruff args: [ --fix ] From ee70ba3cd11cd4928f9f71cf29cb3e6a14b57dd1 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:11:40 -0400 Subject: [PATCH 028/118] [Pydantic IVa] Add validation to most interfaces and converters (#1022) Co-authored-by: CodyCBakerPhD --- CHANGELOG.md | 7 +++++-- src/neuroconv/basedatainterface.py | 3 ++- .../datainterfaces/behavior/audio/audiointerface.py | 3 ++- .../behavior/deeplabcut/deeplabcutdatainterface.py | 3 ++- .../behavior/fictrac/fictracdatainterface.py | 3 ++- .../behavior/lightningpose/lightningposeconverter.py | 3 ++- .../behavior/lightningpose/lightningposedatainterface.py | 3 ++- .../datainterfaces/behavior/medpc/medpcdatainterface.py | 3 ++- .../behavior/miniscope/miniscopedatainterface.py | 3 ++- .../behavior/neuralynx/neuralynx_nvt_interface.py | 3 ++- .../datainterfaces/behavior/sleap/sleapdatainterface.py | 3 ++- .../datainterfaces/behavior/video/videodatainterface.py | 3 ++- .../ecephys/openephys/openephyslegacydatainterface.py | 3 ++- .../ecephys/openephys/openephyssortingdatainterface.py | 3 ++- .../datainterfaces/ecephys/phy/phydatainterface.py | 3 ++- .../datainterfaces/ecephys/plexon/plexondatainterface.py | 5 ++++- .../datainterfaces/ecephys/spike2/spike2datainterface.py | 3 ++- .../ecephys/spikegadgets/spikegadgetsdatainterface.py | 1 + .../datainterfaces/ecephys/spikeglx/spikeglxconverter.py | 3 ++- .../ecephys/spikeglx/spikeglxdatainterface.py | 3 ++- .../ecephys/spikeglx/spikeglxnidqinterface.py | 1 + .../datainterfaces/ecephys/tdt/tdtdatainterface.py | 3 ++- .../datainterfaces/icephys/abf/abfdatainterface.py | 9 +++++++-- .../datainterfaces/icephys/baseicephysinterface.py | 4 +++- .../datainterfaces/ophys/hdf5/hdf5datainterface.py | 1 + .../micromanagertiff/micromanagertiffdatainterface.py | 3 ++- .../datainterfaces/ophys/miniscope/miniscopeconverter.py | 3 ++- .../ophys/miniscope/miniscopeimagingdatainterface.py | 3 ++- .../datainterfaces/ophys/sbx/sbxdatainterface.py | 3 ++- .../ophys/scanimage/scanimageimaginginterfaces.py | 9 ++++++++- .../datainterfaces/ophys/sima/simadatainterface.py | 3 ++- .../datainterfaces/ophys/suite2p/suite2pdatainterface.py | 3 ++- .../ophys/tdt_fp/tdtfiberphotometrydatainterface.py | 5 +++-- .../datainterfaces/ophys/tiff/tiffdatainterface.py | 3 ++- .../text/excel/exceltimeintervalsinterface.py | 3 ++- .../datainterfaces/text/timeintervalsinterface.py | 3 ++- src/neuroconv/nwbconverter.py | 3 ++- tests/test_behavior/test_audio_interface.py | 5 ----- .../ophys/test_fiber_photometry_interfaces.py | 5 ----- 39 files changed, 89 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beee4e54b..6d61f7762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,17 +6,19 @@ ### Features * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) -### Bug fixes - ### Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) * Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049) + + ## v0.6.1 (August 30, 2024) ### Bug fixes * Fixed the JSON schema inference warning on excluded fields; also improved error message reporting of which method triggered the error. [PR #1037](https://github.com/catalystneuro/neuroconv/pull/1037) + + ## v0.6.0 (August 27, 2024) ### Deprecations @@ -38,6 +40,7 @@ * Added helper function `neuroconv.tools.data_transfers.submit_aws_batch_job` for basic automated submission of AWS batch jobs. [PR #384](https://github.com/catalystneuro/neuroconv/pull/384) * Data interfaces `run_conversion` method now performs metadata validation before running the conversion. [PR #949](https://github.com/catalystneuro/neuroconv/pull/949) * Introduced `null_values_for_properties` to `add_units_table` to give user control over null values behavior [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) +* Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) ### Bug fixes * Fixed the default naming of multiple electrical series in the `SpikeGLXConverterPipe`. [PR #957](https://github.com/catalystneuro/neuroconv/pull/957) diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index f8fce62b0..59415b043 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -6,7 +6,7 @@ from typing import Literal, Optional, Union from jsonschema.validators import validate -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from .tools.nwb_helpers import ( @@ -39,6 +39,7 @@ def get_source_schema(cls) -> dict: """Infer the JSON schema for the source_data from the method signature (annotation typing).""" return get_json_schema_from_method_signature(cls, exclude=["source_data"]) + @validate_call def __init__(self, verbose: bool = False, **source_data): self.verbose = verbose self.source_data = source_data diff --git a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py index d61cdf18b..fc3f08fb8 100644 --- a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py +++ b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py @@ -4,7 +4,7 @@ import numpy as np import scipy -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -28,6 +28,7 @@ class AudioInterface(BaseTemporalAlignmentInterface): associated_suffixes = (".wav",) info = "Interface for writing audio recordings to an NWB file." + @validate_call def __init__(self, file_paths: list[FilePath], verbose: bool = False): """ Data interface for writing acoustic recordings to an NWB file. diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index c6850b555..a0719470b 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -2,7 +2,7 @@ from typing import Optional, Union import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb.file import NWBFile from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -25,6 +25,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["config_file_path"]["description"] = "Path to .yml config file" return source_schema + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index 13136b691..1b9686fd1 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -6,7 +6,7 @@ from typing import Optional, Union import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb.behavior import Position, SpatialSeries from pynwb.file import NWBFile @@ -154,6 +154,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .dat file (the output of fictrac)" return source_schema + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py index 114cff0dd..dee848f19 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py @@ -1,7 +1,7 @@ from copy import deepcopy from typing import Optional -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from neuroconv import NWBConverter @@ -26,6 +26,7 @@ class LightningPoseConverter(NWBConverter): def get_source_schema(cls): return get_schema_from_method_signature(cls) + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py index b87366109..f103b7c9a 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py @@ -5,7 +5,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -58,6 +58,7 @@ def get_metadata_schema(self) -> dict: return metadata_schema + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py index 0c109f559..6a4127663 100644 --- a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py @@ -1,7 +1,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb.behavior import BehavioralEpochs, IntervalSeries from pynwb.file import NWBFile @@ -41,6 +41,7 @@ class MedPCInterface(BaseTemporalAlignmentInterface): info = "Interface for handling MedPC output files." associated_suffixes = (".txt",) + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py b/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py index 6a46e84fe..ecb763523 100644 --- a/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from pynwb import NWBFile from .... import BaseDataInterface @@ -24,6 +24,7 @@ def get_source_schema(cls) -> dict: ] = "The main Miniscope folder. The movie files are expected to be in sub folders within the main folder." return source_schema + @validate_call def __init__(self, folder_path: DirectoryPath): """ Initialize reading recordings from the Miniscope behavioral camera. diff --git a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py index f01d11e53..e161387f0 100644 --- a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py +++ b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py @@ -2,7 +2,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from pynwb.behavior import CompassDirection, Position, SpatialSeries @@ -20,6 +20,7 @@ class NeuralynxNvtInterface(BaseTemporalAlignmentInterface): associated_suffixes = (".nvt",) info = "Interface for writing Neuralynx position tracking .nvt files to NWB." + @validate_call def __init__(self, file_path: FilePath, verbose: bool = True): """ Interface for writing Neuralynx .nvt files to nwb. diff --git a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py index a74b90a67..713b21c98 100644 --- a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py @@ -2,7 +2,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb.file import NWBFile from .sleap_utils import extract_timestamps @@ -27,6 +27,7 @@ def get_source_schema(cls) -> dict: ] = "Path of the video for extracting timestamps (optional)." return source_schema + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py index dfb22deba..7a28c2d2f 100644 --- a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py @@ -6,7 +6,7 @@ import numpy as np import psutil from hdmf.data_utils import DataChunkIterator -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from pynwb.image import ImageSeries from tqdm import tqdm @@ -28,6 +28,7 @@ class VideoInterface(BaseDataInterface): # Other suffixes, while they can be opened by OpenCV, are not supported by DANDI so should probably not list here info = "Interface for handling standard video file formats." + @validate_call def __init__( self, file_paths: list[FilePath], diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py index 0818257a7..b3392d2db 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py @@ -2,7 +2,7 @@ from typing import Optional from warnings import warn -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from ..baserecordingextractorinterface import BaseRecordingExtractorInterface @@ -35,6 +35,7 @@ def get_source_schema(cls): return source_schema + @validate_call def __init__( self, folder_path: DirectoryPath, diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py index 20f292c9a..2d53e6331 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py @@ -1,4 +1,4 @@ -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from ..basesortingextractorinterface import BaseSortingExtractorInterface from ....utils import get_schema_from_method_signature @@ -23,6 +23,7 @@ def get_source_schema(cls) -> dict: metadata_schema["additionalProperties"] = False return metadata_schema + @validate_call def __init__(self, folder_path: DirectoryPath, experiment_id: int = 0, recording_id: int = 0): from spikeextractors import OpenEphysSortingExtractor diff --git a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py index da5ebbfc2..07324b602 100644 --- a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from ..basesortingextractorinterface import BaseSortingExtractorInterface @@ -24,6 +24,7 @@ def get_source_schema(cls) -> dict: ] = "Path to the output Phy folder (containing the params.py)." return source_schema + @validate_call def __init__( self, folder_path: DirectoryPath, diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py index 8b2e53338..4a6a50fa5 100644 --- a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import FilePath +from pydantic import FilePath, validate_call from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ..basesortingextractorinterface import BaseSortingExtractorInterface @@ -24,6 +24,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .plx file." return source_schema + @validate_call def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for Plexon. @@ -68,6 +69,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .pl2 file." return source_schema + @validate_call def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for Plexon. @@ -119,6 +121,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the plexon spiking data (.plx file)." return source_schema + @validate_call def __init__(self, file_path: FilePath, verbose: bool = True): """ Load and prepare data for Plexon. diff --git a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py index 5895dfe3c..ccd98a369 100644 --- a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import FilePath +from pydantic import FilePath, validate_call from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....tools import get_package @@ -40,6 +40,7 @@ def get_all_channels_info(cls, file_path: FilePath): _test_sonpy_installation() return cls.get_extractor().get_all_channels_info(file_path=file_path) + @validate_call def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Initialize reading of Spike2 file. diff --git a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py index a5ad98d52..4a63dc237 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py @@ -22,6 +22,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"].update(description="Path to SpikeGadgets (.rec) file.") return source_schema + # @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py index 9d40cde3d..6aeb36cec 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Optional -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from .spikeglxdatainterface import SpikeGLXRecordingInterface from .spikeglxnidqinterface import SpikeGLXNIDQInterface @@ -33,6 +33,7 @@ def get_streams(cls, folder_path: DirectoryPath) -> list[str]: return SpikeGLXRecordingExtractor.get_streams(folder_path=folder_path)[0] + @validate_call def __init__( self, folder_path: DirectoryPath, diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 7d97f7d25..d45a7f946 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -4,7 +4,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from .spikeglx_utils import ( add_recording_extractor_properties, @@ -42,6 +42,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX ap.bin or lf.bin file." return source_schema + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 3c0b886ec..9e903f3f1 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -25,6 +25,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX .nidq file." return source_schema + # @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py index 420d0f242..d01e63c5d 100644 --- a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py @@ -1,4 +1,4 @@ -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from ..baserecordingextractorinterface import BaseRecordingExtractorInterface @@ -10,6 +10,7 @@ class TdtRecordingInterface(BaseRecordingExtractorInterface): associated_suffixes = (".tbk", ".tbx", ".tev", ".tsq") info = "Interface for TDT recording data." + @validate_call def __init__( self, folder_path: DirectoryPath, diff --git a/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py b/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py index 5336b6116..535381466 100644 --- a/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py +++ b/src/neuroconv/datainterfaces/icephys/abf/abfdatainterface.py @@ -1,9 +1,10 @@ import json from datetime import datetime, timedelta from pathlib import Path +from typing import Optional from warnings import warn -from pydantic import FilePath +from pydantic import FilePath, validate_call from ..baseicephysinterface import BaseIcephysInterface @@ -50,8 +51,12 @@ def get_source_schema(cls) -> dict: ) return source_schema + @validate_call def __init__( - self, file_paths: list[FilePath], icephys_metadata: dict = None, icephys_metadata_file_path: FilePath = None + self, + file_paths: list[FilePath], + icephys_metadata: Optional[dict] = None, + icephys_metadata_file_path: Optional[FilePath] = None, ): """ ABF IcephysInterface based on Neo AxonIO. diff --git a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py index ff82ba391..76c9bf840 100644 --- a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py +++ b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py @@ -1,6 +1,7 @@ import importlib.util import numpy as np +from pydantic import FilePath, validate_call from pynwb import NWBFile from ...baseextractorinterface import BaseExtractorInterface @@ -24,7 +25,8 @@ def get_source_schema(cls) -> dict: source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=[]) return source_schema - def __init__(self, file_paths: list): + @validate_call + def __init__(self, file_paths: list[FilePath]): # Check if the ndx_dandi_icephys module is available dandi_icephys_spec = importlib.util.find_spec("ndx_dandi_icephys") if dandi_icephys_spec is not None: diff --git a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py index c980fcf41..2526f7fb3 100644 --- a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py +++ b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py @@ -13,6 +13,7 @@ class Hdf5ImagingInterface(BaseImagingExtractorInterface): associated_suffixes = (".h5", ".hdf5") info = "Interface for HDF5 imaging data." + # @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py index 4e758e742..17cbc95ed 100644 --- a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py @@ -1,5 +1,5 @@ from dateutil.parser import parse -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from ..baseimagingextractorinterface import BaseImagingExtractorInterface @@ -18,6 +18,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["folder_path"]["description"] = "The folder containing the OME-TIF image files." return source_schema + @validate_call def __init__(self, folder_path: DirectoryPath, verbose: bool = True): """ Data Interface for MicroManagerTiffImagingExtractor. diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py index 09aba9f0e..cfee8f027 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from pynwb import NWBFile from ... import MiniscopeBehaviorInterface, MiniscopeImagingInterface @@ -23,6 +23,7 @@ def get_source_schema(cls): source_schema["properties"]["folder_path"]["description"] = "The path to the main Miniscope folder." return source_schema + @validate_call def __init__(self, folder_path: DirectoryPath, verbose: bool = True): """ Initializes the data interfaces for the Miniscope recording and behavioral data stream. diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py index 594bf00dc..64a180c46 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py @@ -3,7 +3,7 @@ from typing import Literal, Optional import numpy as np -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from pynwb import NWBFile from ..baseimagingextractorinterface import BaseImagingExtractorInterface @@ -26,6 +26,7 @@ def get_source_schema(cls) -> dict: return source_schema + @validate_call def __init__(self, folder_path: DirectoryPath): """ Initialize reading the Miniscope imaging data. diff --git a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py index 2fa90a2bb..5b921f6f3 100644 --- a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import FilePath +from pydantic import FilePath, validate_call from ..baseimagingextractorinterface import BaseImagingExtractorInterface @@ -12,6 +12,7 @@ class SbxImagingInterface(BaseImagingExtractorInterface): associated_suffixes = (".sbx",) info = "Interface for Scanbox imaging data." + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py index 76a67ef9b..09e8f86d3 100644 --- a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py +++ b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py @@ -4,7 +4,7 @@ from typing import Optional from dateutil.parser import parse as dateparse -from pydantic import DirectoryPath, FilePath +from pydantic import DirectoryPath, FilePath, validate_call from ..baseimagingextractorinterface import BaseImagingExtractorInterface @@ -32,6 +32,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to Tiff file." return source_schema + @validate_call def __new__( cls, file_path: FilePath, @@ -90,6 +91,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to Tiff file." return source_schema + @validate_call def __init__( self, file_path: FilePath, @@ -168,6 +170,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["folder_path"]["description"] = "Path to the folder containing the TIFF files." return source_schema + @validate_call def __new__( cls, folder_path: DirectoryPath, @@ -226,6 +229,7 @@ class ScanImageMultiPlaneImagingInterface(BaseImagingExtractorInterface): ExtractorName = "ScanImageTiffMultiPlaneImagingExtractor" + @validate_call def __init__( self, file_path: FilePath, @@ -327,6 +331,7 @@ class ScanImageMultiPlaneMultiFileImagingInterface(BaseImagingExtractorInterface ExtractorName = "ScanImageTiffMultiPlaneMultiFileImagingExtractor" + @validate_call def __init__( self, folder_path: DirectoryPath, @@ -443,6 +448,7 @@ class ScanImageSinglePlaneImagingInterface(BaseImagingExtractorInterface): ExtractorName = "ScanImageTiffSinglePlaneImagingExtractor" + @validate_call def __init__( self, file_path: FilePath, @@ -560,6 +566,7 @@ class ScanImageSinglePlaneMultiFileImagingInterface(BaseImagingExtractorInterfac ExtractorName = "ScanImageTiffSinglePlaneMultiFileImagingExtractor" + @validate_call def __init__( self, folder_path: DirectoryPath, diff --git a/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py b/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py index 8bf34cc04..570e2cfe3 100644 --- a/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/sima/simadatainterface.py @@ -1,4 +1,4 @@ -from pydantic import FilePath +from pydantic import FilePath, validate_call from ..basesegmentationextractorinterface import BaseSegmentationExtractorInterface @@ -10,6 +10,7 @@ class SimaSegmentationInterface(BaseSegmentationExtractorInterface): associated_suffixes = (".sima",) info = "Interface for SIMA segmentation." + @validate_call def __init__(self, file_path: FilePath, sima_segmentation_label: str = "auto_ROIs"): """ diff --git a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py index 359dea25f..056616ce5 100644 --- a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py @@ -1,7 +1,7 @@ from copy import deepcopy from typing import Optional -from pydantic import DirectoryPath +from pydantic import DirectoryPath, validate_call from pynwb import NWBFile from ..basesegmentationextractorinterface import BaseSegmentationExtractorInterface @@ -72,6 +72,7 @@ def get_available_channels(cls, folder_path: DirectoryPath) -> dict: return Suite2pSegmentationExtractor.get_available_channels(folder_path=folder_path) + @validate_call def __init__( self, folder_path: DirectoryPath, diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py index 72c854634..aa58f6ae4 100644 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py @@ -6,7 +6,7 @@ import numpy as np import pytz -from pydantic import FilePath +from pydantic import DirectoryPath, validate_call from pynwb.file import NWBFile from neuroconv.basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -28,7 +28,8 @@ class TDTFiberPhotometryInterface(BaseTemporalAlignmentInterface): info = "Data Interface for converting fiber photometry data from TDT files." associated_suffixes = ("Tbk", "Tdx", "tev", "tin", "tsq") - def __init__(self, folder_path: FilePath, verbose: bool = True): + @validate_call + def __init__(self, folder_path: DirectoryPath, verbose: bool = True): """Initialize the TDTFiberPhotometryInterface. Parameters diff --git a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py index 017aa2e98..1eaa3b55e 100644 --- a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import FilePath +from pydantic import FilePath, validate_call from ..baseimagingextractorinterface import BaseImagingExtractorInterface @@ -18,6 +18,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to Tiff file." return source_schema + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py b/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py index cb4ae2330..92fbbbaa7 100644 --- a/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py @@ -1,7 +1,7 @@ from typing import Optional import pandas as pd -from pydantic import FilePath +from pydantic import FilePath, validate_call from ..timeintervalsinterface import TimeIntervalsInterface @@ -13,6 +13,7 @@ class ExcelTimeIntervalsInterface(TimeIntervalsInterface): associated_suffixes = (".xlsx", ".xls", ".xlsm") info = "Interface for writing a time intervals table from an excel file." + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py index b00779cb0..5f5b1107d 100644 --- a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py @@ -3,7 +3,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from ...basedatainterface import BaseDataInterface @@ -16,6 +16,7 @@ class TimeIntervalsInterface(BaseDataInterface): keywords = ("table", "trials", "epochs", "time intervals") + @validate_call def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index eabf6d772..1f3e7c9f8 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -7,7 +7,7 @@ from typing import Literal, Optional, Union from jsonschema import validate -from pydantic import FilePath +from pydantic import FilePath, validate_call from pynwb import NWBFile from .basedatainterface import BaseDataInterface @@ -73,6 +73,7 @@ def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = Tr if verbose: print("Source data is valid!") + @validate_call def __init__(self, source_data: dict[str, dict], verbose: bool = True): """Validate source_data against source_schema and initialize all data interfaces.""" self.verbose = verbose diff --git a/tests/test_behavior/test_audio_interface.py b/tests/test_behavior/test_audio_interface.py index 4fad4fe3d..50331d744 100644 --- a/tests/test_behavior/test_audio_interface.py +++ b/tests/test_behavior/test_audio_interface.py @@ -80,11 +80,6 @@ class AudioTestNWBConverter(NWBConverter): aligned_segment_starting_times=self.aligned_segment_starting_times ) - def test_unsupported_format(self): - exc_msg = "The currently supported file format for audio is WAV file. Some of the provided files does not match this format: ['.test']." - with pytest.raises(ValueError, match=re.escape(exc_msg)): - AudioInterface(file_paths=["test.test"]) - def test_get_metadata(self): audio_interface = AudioInterface(file_paths=self.file_paths) metadata = audio_interface.get_metadata() diff --git a/tests/test_on_data/ophys/test_fiber_photometry_interfaces.py b/tests/test_on_data/ophys/test_fiber_photometry_interfaces.py index b9121f08b..82cbb0e4a 100644 --- a/tests/test_on_data/ophys/test_fiber_photometry_interfaces.py +++ b/tests/test_on_data/ophys/test_fiber_photometry_interfaces.py @@ -400,8 +400,3 @@ def test_load_invalid_evtype(self): interface = self.data_interface_cls(**self.interface_kwargs) with self.assertRaises(AssertionError): interface.load(t2=1.0, evtype=["invalid"]) - - def test_load_invalid_folder_path(self): - interface = self.data_interface_cls(folder_path="invalid") - with self.assertRaises(AssertionError): - interface.load(t2=1.0) From 675ec48966eab421d25c15cd2ce6041ae6e8b34c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 4 Sep 2024 16:46:35 -0600 Subject: [PATCH 029/118] Make `config_file_path` optional on `DeepLabCutInterface` (#1031) --- CHANGELOG.md | 1 + .../behavior/deeplabcut/_dlc_utils.py | 104 +++++++++++------- .../deeplabcut/deeplabcutdatainterface.py | 21 ++-- .../behavior/test_behavior_interfaces.py | 26 +++++ 4 files changed, 105 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d61f7762..b75247d48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * The following classes and objects are now private `NWBMetaDataEncoder`, `NWBMetaDataEncoder`, `check_if_imaging_fits_into_memory`, `NoDatesSafeLoader` [PR #1050](https://github.com/catalystneuro/neuroconv/pull/1050) ### Features +* Make `config_file_path` optional in `DeepLabCutInterface`[PR #1031](https://github.com/catalystneuro/neuroconv/pull/1031) * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) ### Improvements diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 3704e859e..26c5b15fe 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -12,31 +12,31 @@ from ruamel.yaml import YAML -def _read_config(config_file_path): +def _read_config(config_file_path: FilePath) -> dict: """ Reads structured config file defining a project. """ + ruamelFile = YAML() path = Path(config_file_path) - if path.exists(): - try: - with open(path, "r") as f: - cfg = ruamelFile.load(f) - curr_dir = config_file_path.parent - if cfg["project_path"] != curr_dir: - cfg["project_path"] = curr_dir - except Exception as err: - if len(err.args) > 2: - if err.args[2] == "could not determine a constructor for the tag '!!python/tuple'": - with open(path, "r") as ymlfile: - cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) - else: - raise - else: - raise FileNotFoundError( - "Config file is not found. Please make sure that the file exists and/or that you passed the path of the config file correctly!" - ) + if not path.exists(): + raise FileNotFoundError(f"Config file {path} not found.") + + try: + with open(path, "r") as f: + cfg = ruamelFile.load(f) + curr_dir = config_file_path.parent + if cfg["project_path"] != curr_dir: + cfg["project_path"] = curr_dir + except Exception as err: + if len(err.args) > 2: + if err.args[2] == "could not determine a constructor for the tag '!!python/tuple'": + with open(path, "r") as ymlfile: + cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) + else: + raise + return cfg @@ -154,12 +154,30 @@ def _infer_nan_timestamps(timestamps): return timestamps -def _ensure_individuals_in_header(df, individual_name): +def _ensure_individuals_in_header(df, individual_name: str): + """ + Ensure that the 'individuals' column is present in the header of the given DataFrame. + + Parameters: + df (pandas.DataFrame): The DataFrame to modify. + individual_name (str): The name of the individual to add to the header. + + Returns: + pandas.DataFrame: The modified DataFrame with the 'individuals' column added to the header. + + Notes: + - If the 'individuals' column is already present in the header, no modifications are made. + - If the 'individuals' column is not present, a new DataFrame is created with the 'individual_name' + as the column name, and the 'individuals' column is added to the header of the DataFrame. + - The order of the columns in the header is preserved. + + """ if "individuals" not in df.columns.names: # Single animal project -> add individual row to # the header of single animal projects. temp = pd.concat({individual_name: df}, names=["individuals"], axis=1) df = temp.reorder_levels(["scorer", "individuals", "bodyparts", "coords"], axis=1) + return df @@ -220,7 +238,7 @@ def _get_video_info_from_config_file(config_file_path: Path, vidname: str): break if video is None: - warnings.warn(f"The corresponding video file could not be found...") + warnings.warn(f"The corresponding video file could not be found in the config file") video = None, "0, 0, 0, 0" # The video in the config_file looks like this: @@ -240,9 +258,6 @@ def _get_pes_args( ): h5file = Path(h5file) - if "DLC" not in h5file.name or not h5file.suffix == ".h5": - raise IOError("The file passed in is not a DeepLabCut h5 data file.") - _, scorer = h5file.stem.split("DLC") scorer = "DLC" + scorer @@ -256,7 +271,8 @@ def _write_pes_to_nwbfile( animal, df_animal, scorer, - video, # Expects this to be a tuple; first index is string path, second is the image shape as "0, width, 0, height" + video_file_path, + image_shape, paf_graph, timestamps, exclude_nans, @@ -295,12 +311,13 @@ def _write_pes_to_nwbfile( if is_deeplabcut_installed: deeplabcut_version = importlib.metadata.version(distribution_name="deeplabcut") + # TODO, taken from the original implementation, improve it if the video is passed + dimensions = [list(map(int, image_shape.split(",")))[1::2]] pose_estimation_default_kwargs = dict( pose_estimation_series=pose_estimation_series, description="2D keypoint coordinates estimated using DeepLabCut.", - original_videos=[video[0]], - # TODO check if this is a mandatory arg in ndx-pose (can skip if video is not found_ - dimensions=[list(map(int, video[1].split(",")))[1::2]], + original_videos=[video_file_path], + dimensions=dimensions, scorer=scorer, source_software="DeepLabCut", source_software_version=deeplabcut_version, @@ -326,7 +343,7 @@ def add_subject_to_nwbfile( nwbfile: NWBFile, h5file: FilePath, individual_name: str, - config_file: FilePath, + config_file: Optional[FilePath] = None, timestamps: Optional[Union[list, np.ndarray]] = None, pose_estimation_container_kwargs: Optional[dict] = None, ) -> NWBFile: @@ -342,7 +359,7 @@ def add_subject_to_nwbfile( individual_name : str Name of the subject (whose pose is predicted) for single-animal DLC project. For multi-animal projects, the names from the DLC project will be used directly. - config_file : str or path + config_file : str or path, optional Path to a project config.yaml file timestamps : list, np.ndarray or None, default: None Alternative timestamps vector. If None, then use the inferred timestamps from DLC2NWB @@ -356,18 +373,26 @@ def add_subject_to_nwbfile( """ h5file = Path(h5file) - scorer, df = _get_pes_args( - h5file=h5file, - individual_name=individual_name, - ) + if "DLC" not in h5file.name or not h5file.suffix == ".h5": + raise IOError("The file passed in is not a DeepLabCut h5 data file.") + + video_name, scorer = h5file.stem.split("DLC") + scorer = "DLC" + scorer + + df = _ensure_individuals_in_header(pd.read_hdf(h5file), individual_name) # Note the video here is a tuple of the video path and the image shape - vidname, scorer = h5file.stem.split("DLC") - video = _get_video_info_from_config_file(config_file_path=config_file, vidname=vidname) + if config_file is not None: + video_file_path, image_shape = _get_video_info_from_config_file( + config_file_path=config_file, + vidname=video_name, + ) + else: + video_file_path = None + image_shape = "0, 0, 0, 0" # find timestamps only if required:`` timestamps_available = timestamps is not None - video_file_path = video[0] if not timestamps_available: if video_file_path is None: timestamps = df.index.tolist() # setting timestamps to dummy @@ -375,7 +400,7 @@ def add_subject_to_nwbfile( timestamps = _get_movie_timestamps(video_file_path, infer_timestamps=True) # Fetch the corresponding metadata pickle file, we extract the edges graph from here - # TODO: This is the original implementation way to extract the file name but looks very brittle + # TODO: This is the original implementation way to extract the file name but looks very brittle. Improve it filename = str(h5file.parent / h5file.stem) for i, c in enumerate(filename[::-1]): if c.isnumeric(): @@ -393,7 +418,8 @@ def add_subject_to_nwbfile( individual_name, df_animal, scorer, - video, + video_file_path, + image_shape, paf_graph, timestamps, exclude_nans=False, diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index a0719470b..21b054e85 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -29,7 +29,7 @@ def get_source_schema(cls) -> dict: def __init__( self, file_path: FilePath, - config_file_path: FilePath, + config_file_path: Optional[FilePath] = None, subject_name: str = "ind1", verbose: bool = True, ): @@ -40,7 +40,7 @@ def __init__( ---------- file_path : FilePath path to the h5 file output by dlc. - config_file_path : FilePath + config_file_path : FilePath, optional path to .yml config file subject_name : str, default: "ind1" the name of the subject for which the :py:class:`~pynwb.file.NWBFile` is to be created. @@ -53,17 +53,22 @@ def __init__( if "DLC" not in file_path.stem or ".h5" not in file_path.suffixes: raise IOError("The file passed in is not a DeepLabCut h5 data file.") - self._config_file = _read_config(config_file_path=config_file_path) + self.config_dict = dict() + if config_file_path is not None: + self.config_dict = _read_config(config_file_path=config_file_path) self.subject_name = subject_name self.verbose = verbose super().__init__(file_path=file_path, config_file_path=config_file_path) def get_metadata(self): metadata = super().get_metadata() - metadata["NWBFile"].update( - session_description=self._config_file["Task"], - experimenter=[self._config_file["scorer"]], - ) + + if self.config_dict: + metadata["NWBFile"].update( + session_description=self.config_dict["Task"], + experimenter=[self.config_dict["scorer"]], + ) + return metadata def get_original_timestamps(self) -> np.ndarray: @@ -110,7 +115,7 @@ def add_to_nwbfile( nwbfile=nwbfile, h5file=str(self.source_data["file_path"]), individual_name=self.subject_name, - config_file=str(self.source_data["config_file_path"]), + config_file=self.source_data["config_file_path"], timestamps=self._timestamps, pose_estimation_container_kwargs=dict(name=container_name), ) diff --git a/tests/test_on_data/behavior/test_behavior_interfaces.py b/tests/test_on_data/behavior/test_behavior_interfaces.py index 2ec43f96e..348d37c49 100644 --- a/tests/test_on_data/behavior/test_behavior_interfaces.py +++ b/tests/test_on_data/behavior/test_behavior_interfaces.py @@ -365,6 +365,32 @@ def check_read_nwb(self, nwbfile_path: str): assert all(expected_pose_estimation_series_are_in_nwb_file) +class TestDeepLabCutInterfaceNoConfigFile(DataInterfaceTestMixin): + data_interface_cls = DeepLabCutInterface + interface_kwargs = dict( + file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5"), + config_file_path=None, + subject_name="ind1", + ) + save_directory = OUTPUT_PATH + + def check_read_nwb(self, nwbfile_path: str): + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces + assert "PoseEstimation" in processing_module_interfaces + + pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series + expected_pose_estimation_series = ["ind1_leftear", "ind1_rightear", "ind1_snout", "ind1_tailbase"] + + expected_pose_estimation_series_are_in_nwb_file = [ + pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series + ] + + assert all(expected_pose_estimation_series_are_in_nwb_file) + + class TestDeepLabCutInterfaceSetTimestamps(DeepLabCutInterfaceMixin): data_interface_cls = DeepLabCutInterface interface_kwargs = dict( From b01efebbe5adaf741f69f38a8b6d8ca3039db94f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:57:28 -0400 Subject: [PATCH 030/118] [Pydantic IVb] Pydantic validation on arrays (#1055) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 3 ++- .../ecephys/spikegadgets/spikegadgetsdatainterface.py | 4 ++-- .../datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py | 4 ++-- src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b75247d48..cd1580540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ ### Features * Make `config_file_path` optional in `DeepLabCutInterface`[PR #1031](https://github.com/catalystneuro/neuroconv/pull/1031) * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) +* Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) +* All remaining data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1055](https://github.com/catalystneuro/neuroconv/pull/1055) ### Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) @@ -41,7 +43,6 @@ * Added helper function `neuroconv.tools.data_transfers.submit_aws_batch_job` for basic automated submission of AWS batch jobs. [PR #384](https://github.com/catalystneuro/neuroconv/pull/384) * Data interfaces `run_conversion` method now performs metadata validation before running the conversion. [PR #949](https://github.com/catalystneuro/neuroconv/pull/949) * Introduced `null_values_for_properties` to `add_units_table` to give user control over null values behavior [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) -* Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) ### Bug fixes * Fixed the default naming of multiple electrical series in the `SpikeGLXConverterPipe`. [PR #957](https://github.com/catalystneuro/neuroconv/pull/957) diff --git a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py index 4a63dc237..b8b483dd0 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import FilePath +from pydantic import ConfigDict, FilePath, validate_call from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....utils import ArrayType, get_json_schema_from_method_signature @@ -22,7 +22,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"].update(description="Path to SpikeGadgets (.rec) file.") return source_schema - # @validate_call + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 9e903f3f1..42dad773d 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -1,7 +1,7 @@ from pathlib import Path import numpy as np -from pydantic import FilePath +from pydantic import ConfigDict, FilePath, validate_call from .spikeglx_utils import get_session_start_time from ..baserecordingextractorinterface import BaseRecordingExtractorInterface @@ -25,7 +25,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX .nidq file." return source_schema - # @validate_call + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, file_path: FilePath, diff --git a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py index 2526f7fb3..025e68b87 100644 --- a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py +++ b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py @@ -1,6 +1,6 @@ from typing import Literal -from pydantic import FilePath +from pydantic import ConfigDict, FilePath, validate_call from ..baseimagingextractorinterface import BaseImagingExtractorInterface from ....utils import ArrayType @@ -13,7 +13,7 @@ class Hdf5ImagingInterface(BaseImagingExtractorInterface): associated_suffixes = (".h5", ".hdf5") info = "Interface for HDF5 imaging data." - # @validate_call + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, file_path: FilePath, From 1498a50737c871a5deb826987b415ee3551191a7 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 6 Sep 2024 21:17:57 -0600 Subject: [PATCH 031/118] Fix Plexon2 in dev tests (#1058) --- .../ecephys/plexon/plexondatainterface.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py index 4a6a50fa5..fc3dbd3d0 100644 --- a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py @@ -82,7 +82,15 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele Allows verbosity. es_key : str, default: "ElectricalSeries" """ - stream_id = "3" + # TODO: when neo version 0.14.4 is out or higher change this to stream_name for clarify + import neo + from packaging.version import Version + + neo_version = Version(neo.__version__) + if neo_version <= Version("0.13.3"): + stream_id = "3" + else: + stream_id = "WB" assert Path(file_path).is_file(), f"Plexon file not found in: {file_path}" super().__init__( file_path=file_path, From 81a022def1eafcf1e9205a8dfe8b90bd62995a82 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 7 Sep 2024 15:37:12 -0600 Subject: [PATCH 032/118] Eliminate warning about setting timestamps in the ecephys stream (#1060) --- .../ecephys/baserecordingextractorinterface.py | 12 +++++++++--- .../ecephys/basesortingextractorinterface.py | 10 +++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 620b86224..23716c161 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -150,7 +150,7 @@ def set_aligned_timestamps(self, aligned_timestamps: np.ndarray): self._number_of_segments == 1 ), "This recording has multiple segments; please use 'align_segment_timestamps' instead." - self.recording_extractor.set_times(times=aligned_timestamps) + self.recording_extractor.set_times(times=aligned_timestamps, with_warning=False) def set_aligned_segment_timestamps(self, aligned_segment_timestamps: list[np.ndarray]): """ @@ -172,7 +172,9 @@ def set_aligned_segment_timestamps(self, aligned_segment_timestamps: list[np.nda for segment_index in range(self._number_of_segments): self.recording_extractor.set_times( - times=aligned_segment_timestamps[segment_index], segment_index=segment_index + times=aligned_segment_timestamps[segment_index], + segment_index=segment_index, + with_warning=False, ) def set_aligned_starting_time(self, aligned_starting_time: float): @@ -285,7 +287,11 @@ def subset_recording(self, stub_test: bool = False): for segment_index, end_frame in zip(range(number_of_segments), end_frame_list) ] for segment_index in range(number_of_segments): - recording_extractor_stubbed.set_times(times=times_stubbed[segment_index], segment_index=segment_index) + recording_extractor_stubbed.set_times( + times=times_stubbed[segment_index], + segment_index=segment_index, + with_warning=False, + ) return recording_extractor_stubbed diff --git a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py index b3cd25d24..cd8396154 100644 --- a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py @@ -124,7 +124,7 @@ def set_aligned_timestamps(self, aligned_timestamps: np.ndarray): ), "This recording has multiple segments; please use 'set_aligned_segment_timestamps' instead." if self._number_of_segments == 1: - self.sorting_extractor._recording.set_times(times=aligned_timestamps) + self.sorting_extractor._recording.set_times(times=aligned_timestamps, with_warning=False) else: assert isinstance( aligned_timestamps, list @@ -135,7 +135,9 @@ def set_aligned_timestamps(self, aligned_timestamps: np.ndarray): for segment_index in range(self._number_of_segments): self.sorting_extractor._recording.set_times( - times=aligned_timestamps[segment_index], segment_index=segment_index + times=aligned_timestamps[segment_index], + segment_index=segment_index, + with_warning=False, ) def set_aligned_segment_timestamps(self, aligned_segment_timestamps: list[np.ndarray]): @@ -164,7 +166,9 @@ def set_aligned_segment_timestamps(self, aligned_segment_timestamps: list[np.nda for segment_index in range(self._number_of_segments): self.sorting_extractor._recording.set_times( - times=aligned_segment_timestamps[segment_index], segment_index=segment_index + times=aligned_segment_timestamps[segment_index], + segment_index=segment_index, + with_warning=False, ) def set_aligned_starting_time(self, aligned_starting_time: float): From 100f7ab771d2c93fc3305adace1e3507e508c63c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Sep 2024 14:58:41 -0600 Subject: [PATCH 033/118] Fix a bug in `add_sorting_to_nwbfile` where `unit_electrode_indices` was only propagated if `waveform_means` was passed (#1057) --- CHANGELOG.md | 3 ++ .../tools/spikeinterface/spikeinterface.py | 5 +-- .../test_ecephys/test_tools_spikeinterface.py | 36 ++++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd1580540..320efc6b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Upcoming +### Bug fixes +* Fix a bug in `add_sorting_to_nwbfile` where `unit_electrode_indices` was only propagated if `waveform_means` was passed [PR #1057](https://github.com/catalystneuro/neuroconv/pull/1057) + ### Deprecations * The following classes and objects are now private `NWBMetaDataEncoder`, `NWBMetaDataEncoder`, `check_if_imaging_fits_into_memory`, `NoDatesSafeLoader` [PR #1050](https://github.com/catalystneuro/neuroconv/pull/1050) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 262e1eaa8..bce14699c 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -1542,8 +1542,9 @@ def add_units_table_to_nwbfile( unit_kwargs["waveform_mean"] = waveform_means[row] if waveform_sds is not None: unit_kwargs["waveform_sd"] = waveform_sds[row] - if unit_electrode_indices is not None: - unit_kwargs["electrodes"] = unit_electrode_indices[row] + if unit_electrode_indices is not None: + unit_kwargs["electrodes"] = unit_electrode_indices[row] + units_table.add_unit(spike_times=spike_times, **unit_kwargs, enforce_unique_id=True) # Add unit_name as a column and fill previously existing rows with unit_name equal to str(ids) diff --git a/tests/test_ecephys/test_tools_spikeinterface.py b/tests/test_ecephys/test_tools_spikeinterface.py index cf25bab0c..11c29b31f 100644 --- a/tests/test_ecephys/test_tools_spikeinterface.py +++ b/tests/test_ecephys/test_tools_spikeinterface.py @@ -22,6 +22,8 @@ from neuroconv.tools.spikeinterface import ( add_electrical_series_to_nwbfile, add_electrodes_to_nwbfile, + add_recording_to_nwbfile, + add_sorting_to_nwbfile, add_units_table_to_nwbfile, check_if_recording_traces_fit_into_memory, write_recording_to_nwbfile, @@ -921,7 +923,7 @@ def test_adding_doubled_ragged_arrays(self): [["s", "t", "u"], ["v", "w", "x"]], ] - # We add another properyt to recording 2 tat is not in recording 1 + # We add another property to recording 2 which is not in recording 1 self.recording_2.set_property(key="double_ragged_property2", values=second_doubled_nested_array) add_electrodes_to_nwbfile(recording=self.recording_2, nwbfile=self.nwbfile) @@ -1425,6 +1427,38 @@ def test_missing_bool_values(self): assert np.array_equal(extracted_complete_property, expected_complete_property) assert np.array_equal(extracted_incomplete_property, expected_incomplete_property) + def test_add_electrodes(self): + + sorting = generate_sorting(num_units=4) + sorting = sorting.rename_units(new_unit_ids=["a", "b", "c", "d"]) + + unit_electrode_indices = [[0], [1], [2], [0, 1, 2]] + + recording = generate_recording(num_channels=4, durations=[1.0]) + recording = recording.rename_channels(new_channel_ids=["A", "B", "C", "D"]) + + add_recording_to_nwbfile(recording=recording, nwbfile=self.nwbfile) + + assert self.nwbfile.electrodes is not None + + # add units table + add_sorting_to_nwbfile( + sorting=sorting, + nwbfile=self.nwbfile, + unit_electrode_indices=unit_electrode_indices, + ) + + units_table = self.nwbfile.units + assert "electrodes" in units_table.colnames + + electrode_table = self.nwbfile.electrodes + assert units_table["electrodes"].target.table == electrode_table + + assert units_table["electrodes"][0]["channel_name"].item() == "A" + assert units_table["electrodes"][1]["channel_name"].item() == "B" + assert units_table["electrodes"][2]["channel_name"].item() == "C" + assert units_table["electrodes"][3]["channel_name"].values.tolist() == ["A", "B", "C"] + from neuroconv.tools import get_package_version From d35b36420b82e242e1802bcb4d3529442722180f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Sep 2024 20:10:10 -0600 Subject: [PATCH 034/118] Fix bug in Intan interface where extra device is written (#1059) --- CHANGELOG.md | 5 +- .../ecephys/intan/intandatainterface.py | 46 ++++++------------- .../tools/spikeinterface/spikeinterface.py | 2 +- .../ecephys/test_raw_recordings.py | 18 -------- .../ecephys/test_recording_interfaces.py | 27 +++++++++-- 5 files changed, 41 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 320efc6b1..8a40fa620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Upcoming -### Bug fixes +### Bug Fixes +* Fixed a bug where `IntanRecordingInterface` added two devices [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) * Fix a bug in `add_sorting_to_nwbfile` where `unit_electrode_indices` was only propagated if `waveform_means` was passed [PR #1057](https://github.com/catalystneuro/neuroconv/pull/1057) ### Deprecations @@ -15,7 +16,7 @@ ### Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) * Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049) - +* Improved device metadata of `IntanRecordingInterface` by adding the type of controller used [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) ## v0.6.1 (August 30, 2024) diff --git a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py index d28214598..7bd0964ef 100644 --- a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py @@ -1,12 +1,9 @@ -import warnings -from typing import Optional +from pathlib import Path -from packaging.version import Version from pydantic import FilePath from pynwb.ecephys import ElectricalSeries from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....tools import get_package_version from ....utils import get_schema_from_hdmf_class @@ -31,7 +28,6 @@ def get_source_schema(cls) -> dict: def __init__( self, file_path: FilePath, - stream_id: Optional[str] = None, verbose: bool = True, es_key: str = "ElectricalSeries", ignore_integrity_checks: bool = False, @@ -43,8 +39,7 @@ def __init__( ---------- file_path : FilePathType Path to either a rhd or a rhs file - stream_id : str, optional - The stream of the data for spikeinterface, "0" by default. + verbose : bool, default: True Verbose es_key : str, default: "ElectricalSeries" @@ -53,35 +48,17 @@ def __init__( check performed is that timestamps are continuous. If False, an error will be raised if the check fails. """ - if stream_id is not None: - warnings.warn( - "Use of the 'stream_id' parameter is deprecated and it will be removed after September 2024.", - DeprecationWarning, - ) - self.stream_id = stream_id - else: - self.stream_id = "0" # These are the amplifier channels or to the stream_name 'RHD2000 amplifier channel' + self.file_path = Path(file_path) init_kwargs = dict( - file_path=file_path, + file_path=self.file_path, stream_id=self.stream_id, verbose=verbose, es_key=es_key, all_annotations=True, + ignore_integrity_checks=ignore_integrity_checks, ) - neo_version = get_package_version(name="neo") - spikeinterface_version = get_package_version(name="spikeinterface") - if neo_version < Version("0.13.1") or spikeinterface_version < Version("0.100.10"): - if ignore_integrity_checks: - warnings.warn( - "The 'ignore_integrity_checks' parameter is not supported for neo versions < 0.13.1. " - "or spikeinterface versions < 0.101.0.", - UserWarning, - ) - else: - init_kwargs["ignore_integrity_checks"] = ignore_integrity_checks - super().__init__(**init_kwargs) def get_metadata_schema(self) -> dict: @@ -96,14 +73,21 @@ def get_metadata(self) -> dict: ecephys_metadata = metadata["Ecephys"] # Add device - device = dict( + + system = self.file_path.suffix # .rhd or .rhs + device_description = {".rhd": "RHD Recording System", ".rhs": "RHS Stim/Recording System"}[system] + + intan_device = dict( name="Intan", - description="Intan recording", + description=device_description, manufacturer="Intan", ) - device_list = [device] + device_list = [intan_device] ecephys_metadata.update(Device=device_list) + electrode_group_metadata = ecephys_metadata["ElectrodeGroup"] + electrode_group_metadata[0]["device"] = intan_device["name"] + # Add electrodes and electrode groups ecephys_metadata.update( ElectricalSeriesRaw=dict(name="ElectricalSeriesRaw", description="Raw acquisition traces."), diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index bce14699c..b1f83ca26 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -177,7 +177,7 @@ def add_electrode_groups_to_nwbfile(recording: BaseRecording, nwbfile: pynwb.NWB device_name = group_metadata.get("device", defaults[0]["device"]) if device_name not in nwbfile.devices: new_device_metadata = dict(Ecephys=dict(Device=[dict(name=device_name)])) - add_devices(nwbfile=nwbfile, metadata=new_device_metadata) + add_devices_to_nwbfile(nwbfile=nwbfile, metadata=new_device_metadata) warnings.warn( f"Device '{device_name}' not detected in " "attempted link to electrode group! Automatically generating." diff --git a/tests/test_on_data/ecephys/test_raw_recordings.py b/tests/test_on_data/ecephys/test_raw_recordings.py index 97ca75976..f892d6457 100644 --- a/tests/test_on_data/ecephys/test_raw_recordings.py +++ b/tests/test_on_data/ecephys/test_raw_recordings.py @@ -10,7 +10,6 @@ from neuroconv import NWBConverter from neuroconv.datainterfaces import ( - IntanRecordingInterface, Plexon2RecordingInterface, ) @@ -49,23 +48,6 @@ class TestEcephysRawRecordingsNwbConversions(unittest.TestCase): ), ] - # Intan multiple files format - parameterized_recording_list.append( - param( - data_interface=IntanRecordingInterface, - interface_kwargs=dict(file_path=str(DATA_PATH / "intan" / "intan_fpc_test_231117_052630/info.rhd")), - case_name="one-file-per-channel", - ) - ) - - parameterized_recording_list.append( - param( - data_interface=IntanRecordingInterface, - interface_kwargs=dict(file_path=str(DATA_PATH / "intan" / "intan_fps_test_231117_052500/info.rhd")), - case_name="one-file-per-signal", - ) - ) - @parameterized.expand(input=parameterized_recording_list, name_func=custom_name_func) def test_recording_extractor_to_nwb(self, data_interface, interface_kwargs, case_name=""): nwbfile_path = str(self.savedir / f"{data_interface.__name__}_{case_name}.nwb") diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 66e6f2b7b..187e1bff8 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -209,17 +209,22 @@ def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal[ pass -class TestIntanRecordingInterface(RecordingExtractorInterfaceTestMixin): +class TestIntanRecordingInterfaceRHS(RecordingExtractorInterfaceTestMixin): + data_interface_cls = IntanRecordingInterface + interface_kwargs = dict(file_path=ECEPHY_DATA_PATH / "intan" / "intan_rhs_test_1.rhs") + + +class TestIntanRecordingInterfaceRHD(RecordingExtractorInterfaceTestMixin): data_interface_cls = IntanRecordingInterface - interface_kwargs = [] save_directory = OUTPUT_PATH @pytest.fixture( params=[ - dict(file_path=str(ECEPHY_DATA_PATH / "intan" / "intan_rhd_test_1.rhd")), - dict(file_path=str(ECEPHY_DATA_PATH / "intan" / "intan_rhs_test_1.rhs")), + dict(file_path=ECEPHY_DATA_PATH / "intan" / "intan_rhd_test_1.rhd"), + dict(file_path=ECEPHY_DATA_PATH / "intan" / "intan_fpc_test_231117_052630/info.rhd"), + dict(file_path=ECEPHY_DATA_PATH / "intan" / "intan_fps_test_231117_052500/info.rhd"), ], - ids=["rhd", "rhs"], + ids=["rhd", "one-file-per-channel", "one-file-per-signal"], ) def setup_interface(self, request): @@ -230,6 +235,18 @@ def setup_interface(self, request): return self.interface, self.test_name + def test_devices_written_correctly(self, setup_interface): + + from pynwb.testing.mock.file import mock_NWBFile + + nwbfile = mock_NWBFile() + self.interface.add_to_nwbfile(nwbfile=nwbfile) + + nwbfile.devices["Intan"].name == "Intan" + len(nwbfile.devices) == 1 + + nwbfile.devices["Intan"].description == "RHD Recording System" + @pytest.mark.skip(reason="This interface fails to load the necessary plugin sometimes.") class TestMaxOneRecordingInterface(RecordingExtractorInterfaceTestMixin): From ab37a4e252ba4fb4d0c6ce2aeab25f65f86dbd01 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Sep 2024 22:49:08 -0600 Subject: [PATCH 035/118] Using ruff rule to enforce the existence of docstrings in public functions (#1062) Co-authored-by: Paul Adkisson --- CHANGELOG.md | 1 + pyproject.toml | 32 ++++--- setup.py | 1 + .../blackrock/blackrockdatainterface.py | 6 +- .../ecephys/blackrock/header_tools.py | 56 +++++------ .../tools/roiextractors/roiextractors.py | 94 ++++++++++++++++++- .../tools/spikeinterface/spikeinterface.py | 5 +- src/neuroconv/tools/testing/mock_probes.py | 11 +++ src/neuroconv/utils/json_schema.py | 9 +- 9 files changed, 163 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a40fa620..e98f11b70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ ### Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) * Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049) +* Using ruff to enforce existence of public functions's docstrings [PR #1062](https://github.com/catalystneuro/neuroconv/pull/1062) * Improved device metadata of `IntanRecordingInterface` by adding the type of controller used [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) diff --git a/pyproject.toml b/pyproject.toml index 0eacf483b..dc5af94cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,15 +8,15 @@ version = "0.6.2" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ - {name = "Cody Baker"}, - {name = "Szonja Weigl"}, - {name = "Heberto Mayorquin"}, - {name = "Paul Adkisson"}, - {name = "Luiz Tauffer"}, - {name = "Ben Dichter", email = "ben.dichter@catalystneuro.com"} + { name = "Cody Baker" }, + { name = "Szonja Weigl" }, + { name = "Heberto Mayorquin" }, + { name = "Paul Adkisson" }, + { name = "Luiz Tauffer" }, + { name = "Ben Dichter", email = "ben.dichter@catalystneuro.com" }, ] urls = { "Homepage" = "https://github.com/catalystneuro/neuroconv" } -license = {file = "license.txt"} +license = { file = "license.txt" } keywords = ["nwb"] classifiers = [ "Intended Audience :: Science/Research", @@ -91,14 +91,10 @@ neuroconv = "neuroconv.tools.yaml_conversion_specification._yaml_conversion_spec [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra --doctest-glob='*.rst'" -testpaths = [ - "docs/conversion_examples_gallery/", - "tests" -] +testpaths = ["docs/conversion_examples_gallery/", "tests"] doctest_optionflags = "ELLIPSIS" - [tool.black] line-length = 120 target-version = ['py38', 'py39', 'py310'] @@ -121,17 +117,23 @@ extend-exclude = ''' ''' - [tool.ruff] [tool.ruff.lint] -select = ["F401", "I", "D101"] # TODO: eventually, expand to other 'F' linting +select = [ + "F401", # Unused import + "I", # All isort rules + "D101", # Missing docstring in public class + "D103", # Missing docstring in public function +] fixable = ["ALL"] [tool.ruff.lint.per-file-ignores] "**__init__.py" = ["F401", "I"] -"tests/**" = ["D"] # We are not enforcing docstrings in tests +"tests/**" = ["D"] # We are not enforcing docstrings in tests "src/neuroconv/tools/testing/data_interface_mixins.py" = ["D"] # We are not enforcing docstrings in the interface mixings +"docs/conf.py" = ["D"] # We are not enforcing docstrings in the conf.py file +"docs/conversion_examples_gallery/conftest.py" = ["D"] # We are not enforcing docstrings in the conversion examples [tool.ruff.lint.isort] relative-imports-order = "closest-to-furthest" diff --git a/setup.py b/setup.py index 3160c59fa..1f4b5b65a 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ def read_requirements(file): + """Read requirements from a file.""" with open(root / file) as f: return f.readlines() diff --git a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py index 19e34fdcd..d5122bf66 100644 --- a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py @@ -3,7 +3,7 @@ from pydantic import FilePath -from .header_tools import parse_nev_basic_header, parse_nsx_basic_header +from .header_tools import _parse_nev_basic_header, _parse_nsx_basic_header from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ..basesortingextractorinterface import BaseSortingExtractorInterface from ....utils import get_schema_from_method_signature @@ -60,7 +60,7 @@ def __init__( def get_metadata(self) -> dict: metadata = super().get_metadata() # Open file and extract headers - basic_header = parse_nsx_basic_header(self.source_data["file_path"]) + basic_header = _parse_nsx_basic_header(self.source_data["file_path"]) if "TimeOrigin" in basic_header: metadata["NWBFile"].update(session_start_time=basic_header["TimeOrigin"]) if "Comment" in basic_header: @@ -101,7 +101,7 @@ def __init__(self, file_path: FilePath, sampling_frequency: float = None, verbos def get_metadata(self) -> dict: metadata = super().get_metadata() # Open file and extract headers - basic_header = parse_nev_basic_header(self.source_data["file_path"]) + basic_header = _parse_nev_basic_header(self.source_data["file_path"]) if "TimeOrigin" in basic_header: session_start_time = basic_header["TimeOrigin"] metadata["NWBFile"].update(session_start_time=session_start_time.strftime("%Y-%m-%dT%H:%M:%S")) diff --git a/src/neuroconv/datainterfaces/ecephys/blackrock/header_tools.py b/src/neuroconv/datainterfaces/ecephys/blackrock/header_tools.py index 7b182a3f4..5ac8f94d5 100644 --- a/src/neuroconv/datainterfaces/ecephys/blackrock/header_tools.py +++ b/src/neuroconv/datainterfaces/ecephys/blackrock/header_tools.py @@ -5,7 +5,7 @@ from struct import calcsize, unpack -def processheaders(curr_file, packet_fields): +def _processheaders(curr_file, packet_fields): """ :param curr_file: {file} the current BR datafile to be processed :param packet_fields : {named tuple} the specific binary fields for the given header @@ -45,11 +45,11 @@ def processheaders(curr_file, packet_fields): return packet_formatted -def format_filespec(header_list): +def _format_filespec(header_list): return str(next(header_list)) + "." + str(next(header_list)) # eg 2.3 -def format_timeorigin(header_list): +def _format_timeorigin(header_list): year = next(header_list) month = next(header_list) _ = next(header_list) @@ -61,12 +61,12 @@ def format_timeorigin(header_list): return datetime(year, month, day, hour, minute, second, millisecond * 1000) -def format_stripstring(header_list): +def _format_stripstring(header_list): string = bytes.decode(next(header_list), "latin-1") return string.split(STRING_TERMINUS, 1)[0] -def format_none(header_list): +def _format_none(header_list): return next(header_list) @@ -74,38 +74,38 @@ def format_none(header_list): STRING_TERMINUS = "\x00" -def parse_nsx_basic_header(nsx_file): +def _parse_nsx_basic_header(nsx_file): nsx_basic_dict = [ - FieldDef("FileSpec", "2B", format_filespec), # 2 bytes - 2 unsigned char - FieldDef("BytesInHeader", "I", format_none), # 4 bytes - uint32 - FieldDef("Label", "16s", format_stripstring), # 16 bytes - 16 char array - FieldDef("Comment", "256s", format_stripstring), # 256 bytes - 256 char array - FieldDef("Period", "I", format_none), # 4 bytes - uint32 - FieldDef("TimeStampResolution", "I", format_none), # 4 bytes - uint32 - FieldDef("TimeOrigin", "8H", format_timeorigin), # 16 bytes - 8 uint16 - FieldDef("ChannelCount", "I", format_none), + FieldDef("FileSpec", "2B", _format_filespec), # 2 bytes - 2 unsigned char + FieldDef("BytesInHeader", "I", _format_none), # 4 bytes - uint32 + FieldDef("Label", "16s", _format_stripstring), # 16 bytes - 16 char array + FieldDef("Comment", "256s", _format_stripstring), # 256 bytes - 256 char array + FieldDef("Period", "I", _format_none), # 4 bytes - uint32 + FieldDef("TimeStampResolution", "I", _format_none), # 4 bytes - uint32 + FieldDef("TimeOrigin", "8H", _format_timeorigin), # 16 bytes - 8 uint16 + FieldDef("ChannelCount", "I", _format_none), ] # 4 bytes - uint32 datafile = open(nsx_file, "rb") filetype_id = bytes.decode(datafile.read(8), "latin-1") if filetype_id == "NEURALSG": # this won't contain fields that can be added to NWBFile metadata return dict() - return processheaders(datafile, nsx_basic_dict) + return _processheaders(datafile, nsx_basic_dict) -def parse_nev_basic_header(nev_file): +def _parse_nev_basic_header(nev_file): nev_basic_dict = [ - FieldDef("FileTypeID", "8s", format_stripstring), # 8 bytes - 8 char array - FieldDef("FileSpec", "2B", format_filespec), # 2 bytes - 2 unsigned char - FieldDef("AddFlags", "H", format_none), # 2 bytes - uint16 - FieldDef("BytesInHeader", "I", format_none), # 4 bytes - uint32 - FieldDef("BytesInDataPackets", "I", format_none), # 4 bytes - uint32 - FieldDef("TimeStampResolution", "I", format_none), # 4 bytes - uint32 - FieldDef("SampleTimeResolution", "I", format_none), # 4 bytes - uint32 - FieldDef("TimeOrigin", "8H", format_timeorigin), # 16 bytes - 8 x uint16 - FieldDef("CreatingApplication", "32s", format_stripstring), # 32 bytes - 32 char array - FieldDef("Comment", "256s", format_stripstring), # 256 bytes - 256 char array - FieldDef("NumExtendedHeaders", "I", format_none), + FieldDef("FileTypeID", "8s", _format_stripstring), # 8 bytes - 8 char array + FieldDef("FileSpec", "2B", _format_filespec), # 2 bytes - 2 unsigned char + FieldDef("AddFlags", "H", _format_none), # 2 bytes - uint16 + FieldDef("BytesInHeader", "I", _format_none), # 4 bytes - uint32 + FieldDef("BytesInDataPackets", "I", _format_none), # 4 bytes - uint32 + FieldDef("TimeStampResolution", "I", _format_none), # 4 bytes - uint32 + FieldDef("SampleTimeResolution", "I", _format_none), # 4 bytes - uint32 + FieldDef("TimeOrigin", "8H", _format_timeorigin), # 16 bytes - 8 x uint16 + FieldDef("CreatingApplication", "32s", _format_stripstring), # 32 bytes - 32 char array + FieldDef("Comment", "256s", _format_stripstring), # 256 bytes - 256 char array + FieldDef("NumExtendedHeaders", "I", _format_none), ] datafile = open(nev_file, "rb") - return processheaders(datafile, nev_basic_dict) + return _processheaders(datafile, nev_basic_dict) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 618d30b4a..f28631c77 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -682,9 +682,38 @@ def add_imaging_to_nwbfile( iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", -): +) -> NWBFile: + """ + Add imaging data from an ImagingExtractor object to an NWBFile. + + Parameters + ---------- + imaging : ImagingExtractor + The extractor object containing the imaging data. + nwbfile : NWBFile + The NWB file where the imaging data will be added. + metadata : dict, optional + Metadata for the NWBFile, by default None. + photon_series_type : {"TwoPhotonSeries", "OnePhotonSeries"}, optional + The type of photon series to be added, by default "TwoPhotonSeries". + photon_series_index : int, optional + The index of the photon series in the provided imaging data, by default 0. + iterator_type : str, optional + The type of iterator to use for adding the data. Commonly used to manage large datasets, by default "v2". + iterator_options : dict, optional + Additional options for controlling the iteration process, by default None. + parent_container : {"acquisition", "processing/ophys"}, optional + Specifies the parent container to which the photon series should be added, either as part of "acquisition" or + under the "processing/ophys" module, by default "acquisition". + + Returns + ------- + NWBFile + The NWB file with the imaging data added + + """ add_devices_to_nwbfile(nwbfile=nwbfile, metadata=metadata) - add_photon_series_to_nwbfile( + nwbfile = add_photon_series_to_nwbfile( imaging=imaging, nwbfile=nwbfile, metadata=metadata, @@ -695,6 +724,8 @@ def add_imaging_to_nwbfile( parent_container=parent_container, ) + return nwbfile + def write_imaging( imaging: ImagingExtractor, @@ -1158,8 +1189,31 @@ def add_background_plane_segmentation_to_nwbfile( iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: - # TODO needs docstring + """ + Add background plane segmentation data from a SegmentationExtractor object to an NWBFile. + Parameters + ---------- + segmentation_extractor : SegmentationExtractor + The extractor object containing background segmentation data. + nwbfile : NWBFile + The NWB file to which the background plane segmentation will be added. + metadata : dict, optional + Metadata for the NWBFile, by default None. + background_plane_segmentation_name : str, optional + The name of the background PlaneSegmentation object to be added, by default None. + mask_type : str, optional + Type of mask to use for segmentation; options are "image", "pixel", or "voxel", by default "image". + iterator_options : dict, optional + Options for iterating over the segmentation data, by default None. + compression_options : dict, optional + Deprecated: options for compression; will be removed after 2024-10-01, by default None. + + Returns + ------- + NWBFile + The NWBFile with the added background plane segmentation data. + """ # TODO: remove completely after 10/1/2024 if compression_options is not None: warnings.warn( @@ -1724,6 +1778,40 @@ def add_segmentation_to_nwbfile( iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: + """ + Add segmentation data from a SegmentationExtractor object to an NWBFile. + + Parameters + ---------- + segmentation_extractor : SegmentationExtractor + The extractor object containing segmentation data. + nwbfile : NWBFile + The NWB file where the segmentation data will be added. + metadata : dict, optional + Metadata for the NWBFile, by default None. + plane_segmentation_name : str, optional + The name of the PlaneSegmentation object to be added, by default None. + background_plane_segmentation_name : str, optional + The name of the background PlaneSegmentation, if any, by default None. + include_background_segmentation : bool, optional + If True, includes background plane segmentation, by default False. + include_roi_centroids : bool, optional + If True, includes the centroids of the regions of interest (ROIs), by default True. + include_roi_acceptance : bool, optional + If True, includes the acceptance status of ROIs, by default True. + mask_type : str, optional + Type of mask to use for segmentation; can be either "image" or "pixel", by default "image". + iterator_options : dict, optional + Options for iterating over the data, by default None. + compression_options : dict, optional + Deprecated: options for compression; will be removed after 2024-10-01, by default None. + + Returns + ------- + NWBFile + The NWBFile with the added segmentation data. + """ + # TODO: remove completely after 10/1/2024 if compression_options is not None: warnings.warn( diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index b1f83ca26..8b4db78be 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -1992,7 +1992,7 @@ def add_sorting_analyzer_to_nwbfile( sorting_copy.set_property(prop, tm[prop]) add_electrodes_info_to_nwbfile(recording, nwbfile=nwbfile, metadata=metadata) - electrode_group_indices = get_electrode_group_indices(recording, nwbfile=nwbfile) + electrode_group_indices = _get_electrode_group_indices(recording, nwbfile=nwbfile) unit_electrode_indices = [electrode_group_indices] * len(sorting.unit_ids) add_units_table_to_nwbfile( @@ -2214,7 +2214,8 @@ def add_waveforms( ) -def get_electrode_group_indices(recording, nwbfile): +def _get_electrode_group_indices(recording, nwbfile): + """ """ if "group_name" in recording.get_property_keys(): group_names = list(np.unique(recording.get_property("group_name"))) elif "group" in recording.get_property_keys(): diff --git a/src/neuroconv/tools/testing/mock_probes.py b/src/neuroconv/tools/testing/mock_probes.py index 8b41d0f9c..f70f0dfeb 100644 --- a/src/neuroconv/tools/testing/mock_probes.py +++ b/src/neuroconv/tools/testing/mock_probes.py @@ -2,6 +2,17 @@ def generate_mock_probe(num_channels: int, num_shanks: int = 3): + """ + Generate a mock probe with specified number of channels and shanks. + + Parameters: + num_channels (int): The number of channels in the probe. + num_shanks (int, optional): The number of shanks in the probe. Defaults to 3. + + Returns: + pi.Probe: The generated mock probe. + + """ import probeinterface as pi # The shank ids will be 0, 0, 0, ..., 1, 1, 1, ..., 2, 2, 2, ... diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index 6c1ba7245..182558b98 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -298,7 +298,14 @@ def get_schema_from_hdmf_class(hdmf_class): return schema -def get_metadata_schema_for_icephys(): +def get_metadata_schema_for_icephys() -> dict: + """ + Returns the metadata schema for icephys data. + + Returns: + dict: The metadata schema for icephys data. + + """ schema = get_base_schema(tag="Icephys") schema["required"] = ["Device", "Electrodes"] schema["properties"] = dict( From 0aa90874c8272f4589cbfe581f9366421eb6dff6 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Wed, 11 Sep 2024 02:12:24 +1000 Subject: [PATCH 036/118] unit table descriptions for phy and kilosort (#1053) --- CHANGELOG.md | 1 + .../ecephys/kilosort/kilosortdatainterface.py | 27 +++++ .../ecephys/phy/phydatainterface.py | 27 +++++ .../ecephys/test_sorting_interfaces.py | 101 ++++++++++++++++++ 4 files changed, 156 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e98f11b70..371c55d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ ### Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) * Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049) +* Added Unit Table descriptions for phy and kilosort: [PR #1053](https://github.com/catalystneuro/neuroconv/pull/1053) * Using ruff to enforce existence of public functions's docstrings [PR #1062](https://github.com/catalystneuro/neuroconv/pull/1062) * Improved device metadata of `IntanRecordingInterface` by adding the type of controller used [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) diff --git a/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py b/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py index fc6765823..aafde42f0 100644 --- a/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py @@ -36,3 +36,30 @@ def __init__( verbose: bool, default: True """ super().__init__(folder_path=folder_path, keep_good_only=keep_good_only, verbose=verbose) + + def get_metadata(self): + metadata = super().get_metadata() + # See Kilosort save_to_phy() docstring for more info on these fields: https://github.com/MouseLand/Kilosort/blob/main/kilosort/io.py + # Or see phy documentation: https://github.com/cortex-lab/phy/blob/master/phy/apps/base.py + metadata["Ecephys"]["UnitProperties"] = [ + dict(name="n_spikes", description="Number of spikes recorded from each unit."), + dict(name="fr", description="Average firing rate of each unit."), + dict(name="depth", description="Estimated depth of each unit in micrometers."), + dict(name="Amplitude", description="Per-template amplitudes, computed as the L2 norm of the template."), + dict( + name="ContamPct", + description="Contamination rate for each template, computed as fraction of refractory period violations relative to expectation based on a Poisson process.", + ), + dict( + name="KSLabel", + description="Label indicating whether each template is 'mua' (multi-unit activity) or 'good' (refractory).", + ), + dict(name="original_cluster_id", description="Original cluster ID assigned by Kilosort."), + dict( + name="amp", + description="For every template, the maximum amplitude of the template waveforms across all channels.", + ), + dict(name="ch", description="The channel label of the best channel, as defined by the user."), + dict(name="sh", description="The shank label of the best channel."), + ] + return metadata diff --git a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py index 07324b602..157fef2e5 100644 --- a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py @@ -43,3 +43,30 @@ def __init__( verbose : bool, default: True """ super().__init__(folder_path=folder_path, exclude_cluster_groups=exclude_cluster_groups, verbose=verbose) + + def get_metadata(self): + metadata = super().get_metadata() + # See Kilosort save_to_phy() docstring for more info on these fields: https://github.com/MouseLand/Kilosort/blob/main/kilosort/io.py + # Or see phy documentation: https://github.com/cortex-lab/phy/blob/master/phy/apps/base.py + metadata["Ecephys"]["UnitProperties"] = [ + dict(name="n_spikes", description="Number of spikes recorded from each unit."), + dict(name="fr", description="Average firing rate of each unit."), + dict(name="depth", description="Estimated depth of each unit in micrometers."), + dict(name="Amplitude", description="Per-template amplitudes, computed as the L2 norm of the template."), + dict( + name="ContamPct", + description="Contamination rate for each template, computed as fraction of refractory period violations relative to expectation based on a Poisson process.", + ), + dict( + name="KSLabel", + description="Label indicating whether each template is 'mua' (multi-unit activity) or 'good' (refractory).", + ), + dict(name="original_cluster_id", description="Original cluster ID assigned by Kilosort."), + dict( + name="amp", + description="For every template, the maximum amplitude of the template waveforms across all channels.", + ), + dict(name="ch", description="The channel label of the best channel, as defined by the user."), + dict(name="sh", description="The shank label of the best channel."), + ] + return metadata diff --git a/tests/test_on_data/ecephys/test_sorting_interfaces.py b/tests/test_on_data/ecephys/test_sorting_interfaces.py index 7c572c269..492c4ad9f 100644 --- a/tests/test_on_data/ecephys/test_sorting_interfaces.py +++ b/tests/test_on_data/ecephys/test_sorting_interfaces.py @@ -7,6 +7,7 @@ BlackrockRecordingInterface, BlackrockSortingInterface, CellExplorerSortingInterface, + KiloSortSortingInterface, NeuralynxSortingInterface, NeuroScopeSortingInterface, PhySortingInterface, @@ -193,6 +194,106 @@ class TestPhySortingInterface(SortingExtractorInterfaceTestMixin): interface_kwargs = dict(folder_path=str(DATA_PATH / "phy" / "phy_example_0")) save_directory = OUTPUT_PATH + def check_extracted_metadata(self, metadata: dict): + assert metadata["Ecephys"]["UnitProperties"] == [ + dict(name="n_spikes", description="Number of spikes recorded from each unit."), + dict(name="fr", description="Average firing rate of each unit."), + dict(name="depth", description="Estimated depth of each unit in micrometers."), + dict(name="Amplitude", description="Per-template amplitudes, computed as the L2 norm of the template."), + dict( + name="ContamPct", + description="Contamination rate for each template, computed as fraction of refractory period violations relative to expectation based on a Poisson process.", + ), + dict( + name="KSLabel", + description="Label indicating whether each template is 'mua' (multi-unit activity) or 'good' (refractory).", + ), + dict(name="original_cluster_id", description="Original cluster ID assigned by Kilosort."), + dict( + name="amp", + description="For every template, the maximum amplitude of the template waveforms across all channels.", + ), + dict(name="ch", description="The channel label of the best channel, as defined by the user."), + dict(name="sh", description="The shank label of the best channel."), + ] + + def check_units_table_propagation(self): + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + nwbfile = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) + + # example data does not contain n_spikes, fr, depth, amp, ch, and sh + assert ( + nwbfile.units["Amplitude"].description + == "Per-template amplitudes, computed as the L2 norm of the template." + ) + assert ( + nwbfile.units["ContamPct"].description + == "Contamination rate for each template, computed as fraction of refractory period violations relative to expectation based on a Poisson process." + ) + assert ( + nwbfile.units["KSLabel"].description + == "Label indicating whether each template is 'mua' (multi-unit activity) or 'good' (refractory)." + ) + assert nwbfile.units["original_cluster_id"].description == "Original cluster ID assigned by Kilosort." + + def run_custom_checks(self): + self.check_units_table_propagation() + + +class TestKilosortSortingInterface(SortingExtractorInterfaceTestMixin): + data_interface_cls = KiloSortSortingInterface + interface_kwargs = dict(folder_path=str(DATA_PATH / "phy" / "phy_example_0")) + save_directory = OUTPUT_PATH + + def check_extracted_metadata(self, metadata: dict): + assert metadata["Ecephys"]["UnitProperties"] == [ + dict(name="n_spikes", description="Number of spikes recorded from each unit."), + dict(name="fr", description="Average firing rate of each unit."), + dict(name="depth", description="Estimated depth of each unit in micrometers."), + dict(name="Amplitude", description="Per-template amplitudes, computed as the L2 norm of the template."), + dict( + name="ContamPct", + description="Contamination rate for each template, computed as fraction of refractory period violations relative to expectation based on a Poisson process.", + ), + dict( + name="KSLabel", + description="Label indicating whether each template is 'mua' (multi-unit activity) or 'good' (refractory).", + ), + dict(name="original_cluster_id", description="Original cluster ID assigned by Kilosort."), + dict( + name="amp", + description="For every template, the maximum amplitude of the template waveforms across all channels.", + ), + dict(name="ch", description="The channel label of the best channel, as defined by the user."), + dict(name="sh", description="The shank label of the best channel."), + ] + + def check_units_table_propagation(self): + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + nwbfile = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) + + # example data does not contain n_spikes, fr, depth, amp, ch, and sh + assert ( + nwbfile.units["Amplitude"].description + == "Per-template amplitudes, computed as the L2 norm of the template." + ) + assert ( + nwbfile.units["ContamPct"].description + == "Contamination rate for each template, computed as fraction of refractory period violations relative to expectation based on a Poisson process." + ) + assert ( + nwbfile.units["KSLabel"].description + == "Label indicating whether each template is 'mua' (multi-unit activity) or 'good' (refractory)." + ) + assert nwbfile.units["original_cluster_id"].description == "Original cluster ID assigned by Kilosort." + + def run_custom_checks(self): + self.check_units_table_propagation() + class TestPlexonSortingInterface(SortingExtractorInterfaceTestMixin): data_interface_cls = PlexonSortingInterface From 8e5d9ba581f3e3825a35494e543614a82402e626 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 10 Sep 2024 11:15:18 -0600 Subject: [PATCH 037/118] Release v0.6.2 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 371c55d5a..9c810454a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Upcoming +## v.0.6.2 + ### Bug Fixes * Fixed a bug where `IntanRecordingInterface` added two devices [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) * Fix a bug in `add_sorting_to_nwbfile` where `unit_electrode_indices` was only propagated if `waveform_means` was passed [PR #1057](https://github.com/catalystneuro/neuroconv/pull/1057) From 48f1fd6c9aeac527d10972add5c86a7a3de12c7b Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 10 Sep 2024 11:34:18 -0600 Subject: [PATCH 038/118] version bump --- .github/workflows/auto-publish.yml | 5 ++--- CHANGELOG.md | 13 ++++++++++++- pyproject.toml | 2 +- setup.py | 1 + 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/auto-publish.yml b/.github/workflows/auto-publish.yml index 8c1eb67bd..f61133dc9 100644 --- a/.github/workflows/auto-publish.yml +++ b/.github/workflows/auto-publish.yml @@ -15,14 +15,13 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip - pip install wheel - name: Build package run: | - python setup.py sdist bdist_wheel + python -m build - name: pypi-publish uses: pypa/gh-action-pypi-publish@v1.4.2 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c810454a..42d912bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ # Upcoming -## v.0.6.2 +## Bug Fixes + +## Deprecations + +## Features + +## Improvements + + + +## v0.6.2 (September 10, 2024) ### Bug Fixes * Fixed a bug where `IntanRecordingInterface` added two devices [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) @@ -15,6 +25,7 @@ * Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) * All remaining data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1055](https://github.com/catalystneuro/neuroconv/pull/1055) + ### Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) * Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049) diff --git a/pyproject.toml b/pyproject.toml index dc5af94cc..c4f049b08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.2" +version = "0.6.3" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ diff --git a/setup.py b/setup.py index 1f4b5b65a..75dce9ac1 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ def read_requirements(file): gin_config_file_base = root / "base_gin_test_config.json" gin_config_file_local = root / "tests/test_on_data/gin_test_config.json" if not gin_config_file_local.exists(): + gin_config_file_local.mkdir(parents=True, exist_ok=True) copy(src=gin_config_file_base, dst=gin_config_file_local) # Bug related to sonpy on M1 Mac being installed but not running properly From f8ea349388e4cbd954e2c6305923ef3ae27ddb49 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 10 Sep 2024 11:46:20 -0600 Subject: [PATCH 039/118] bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c4f049b08..743f4e57b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.3" +version = "0.6.4" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ From ba0bf5162a4d936ff3ff7ee1ac690b9905778c3a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 10 Sep 2024 13:20:49 -0600 Subject: [PATCH 040/118] fix action --- .github/workflows/auto-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/auto-publish.yml b/.github/workflows/auto-publish.yml index f61133dc9..40f5a05b2 100644 --- a/.github/workflows/auto-publish.yml +++ b/.github/workflows/auto-publish.yml @@ -16,9 +16,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install dependencies + - name: Install Building Dependencies run: | python -m pip install --upgrade pip + python -m pip install --upgrade build - name: Build package run: | python -m build From effbe0600c4d0fed9c3d36b1e74c2129aacbae1f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 11 Sep 2024 14:04:47 -0600 Subject: [PATCH 041/118] Fix paths (#1070) --- CHANGELOG.md | 1 + setup.py | 2 +- tests/test_on_data/setup_paths.py | 26 ++++++++++++++------------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42d912bff..09220c407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Upcoming ## Bug Fixes +Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) ## Deprecations diff --git a/setup.py b/setup.py index 75dce9ac1..5beebbf5f 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ def read_requirements(file): gin_config_file_base = root / "base_gin_test_config.json" gin_config_file_local = root / "tests/test_on_data/gin_test_config.json" if not gin_config_file_local.exists(): - gin_config_file_local.mkdir(parents=True, exist_ok=True) + gin_config_file_local.parent.mkdir(parents=True, exist_ok=True) copy(src=gin_config_file_base, dst=gin_config_file_local) # Bug related to sonpy on M1 Mac being installed but not running properly diff --git a/tests/test_on_data/setup_paths.py b/tests/test_on_data/setup_paths.py index 4d5944147..9554d27eb 100644 --- a/tests/test_on_data/setup_paths.py +++ b/tests/test_on_data/setup_paths.py @@ -4,29 +4,31 @@ from neuroconv.utils import load_dict_from_file +# Output by default to a temporary directory +OUTPUT_PATH = Path(tempfile.mkdtemp()) + + # Load the configuration for the data tests -file_path = Path(__file__).parent.parent.parent / "tests" / "test_on_data" / "gin_test_config.json" -test_config_dict = load_dict_from_file(file_path) -# GIN dataset: https://gin.g-node.org/CatalystNeuro/behavior_testing_data + if os.getenv("CI"): LOCAL_PATH = Path(".") # Must be set to "." for CI print("Running GIN tests on Github CI!") else: # Override LOCAL_PATH in the `gin_test_config.json` file to a point on your system that contains the dataset folder # Use DANDIHub at hub.dandiarchive.org for open, free use of data found in the /shared/catalystneuro/ directory + file_path = Path(__file__).parent / "gin_test_config.json" + assert file_path.exists(), f"File not found: {file_path}" + test_config_dict = load_dict_from_file(file_path) LOCAL_PATH = Path(test_config_dict["LOCAL_PATH"]) - print("Running GIN tests locally!") + + if test_config_dict["SAVE_OUTPUTS"]: + OUTPUT_PATH = LOCAL_PATH / "neuroconv_test_outputs" + OUTPUT_PATH.mkdir(exist_ok=True, parents=True) + BEHAVIOR_DATA_PATH = LOCAL_PATH / "behavior_testing_data" ECEPHY_DATA_PATH = LOCAL_PATH / "ephy_testing_data" OPHYS_DATA_PATH = LOCAL_PATH / "ophys_testing_data" -TEXT_DATA_PATH = file_path = Path(__file__).parent.parent.parent / "tests" / "test_text" - - -if test_config_dict["SAVE_OUTPUTS"]: - OUTPUT_PATH = LOCAL_PATH / "example_nwb_output" - OUTPUT_PATH.mkdir(exist_ok=True) -else: - OUTPUT_PATH = Path(tempfile.mkdtemp()) +TEXT_DATA_PATH = Path(__file__).parent.parent.parent / "tests" / "test_text" From 2611825d7a32d865e758501e9b4a2d0a280218c6 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 11 Sep 2024 18:11:48 -0600 Subject: [PATCH 042/118] Make `get_extractor` work with `MockImagingInterface` (#1076) --- CHANGELOG.md | 3 +- .../tools/testing/mock_interfaces.py | 32 +++++++++++++++++-- tests/test_ophys/test_ophys_interfaces.py | 25 ++++----------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09220c407..619648b20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ # Upcoming ## Bug Fixes -Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) +* Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) +* The method `get_extractor` now works for `MockImagingInterface` [PR #1076](https://github.com/catalystneuro/neuroconv/pull/1076) ## Deprecations diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index f05228b34..12f1dafd4 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -162,6 +162,9 @@ class MockImagingInterface(BaseImagingExtractorInterface): A mock imaging interface for testing purposes. """ + ExtractorModuleName = "roiextractors.testing" + ExtractorName = "generate_dummy_imaging_extractor" + def __init__( self, num_frames: int = 30, @@ -169,17 +172,40 @@ def __init__( num_columns: int = 10, sampling_frequency: float = 30, dtype: str = "uint16", - verbose: bool = True, + verbose: bool = False, + seed: int = 0, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", ): - from roiextractors.testing import generate_dummy_imaging_extractor + """ + Parameters + ---------- + num_frames : int, optional + The number of frames in the mock imaging data, by default 30. + num_rows : int, optional + The number of rows (height) in each frame of the mock imaging data, by default 10. + num_columns : int, optional + The number of columns (width) in each frame of the mock imaging data, by default 10. + sampling_frequency : float, optional + The sampling frequency of the mock imaging data in Hz, by default 30. + dtype : str, optional + The data type of the generated imaging data (e.g., 'uint16'), by default 'uint16'. + seed : int, optional + Random seed for reproducibility, by default 0. + photon_series_type : Literal["OnePhotonSeries", "TwoPhotonSeries"], optional + The type of photon series for the mock imaging data, either "OnePhotonSeries" or + "TwoPhotonSeries", by default "TwoPhotonSeries". + verbose : bool, default False + controls verbosity + """ - self.imaging_extractor = generate_dummy_imaging_extractor( + self.seed = seed + super().__init__( num_frames=num_frames, num_rows=num_rows, num_columns=num_columns, sampling_frequency=sampling_frequency, dtype=dtype, + verbose=verbose, ) self.verbose = verbose diff --git a/tests/test_ophys/test_ophys_interfaces.py b/tests/test_ophys/test_ophys_interfaces.py index 1a774f713..9ae151d5a 100644 --- a/tests/test_ophys/test_ophys_interfaces.py +++ b/tests/test_ophys/test_ophys_interfaces.py @@ -1,22 +1,9 @@ -import tempfile -import unittest -from pathlib import Path - -from pynwb.testing.mock.file import mock_NWBFile - +from neuroconv.tools.testing.data_interface_mixins import ( + ImagingExtractorInterfaceTestMixin, +) from neuroconv.tools.testing.mock_interfaces import MockImagingInterface -class TestMockImagingInterface(unittest.TestCase): - def setUp(self): - self.mock_imaging_interface = MockImagingInterface() - - def test_run_conversion(self): - - with tempfile.TemporaryDirectory() as tmpdir: - nwbfile_path = Path(tmpdir) / "test.nwb" - self.mock_imaging_interface.run_conversion(nwbfile_path=nwbfile_path) - - def test_add_to_nwbfile(self): - nwbfile = mock_NWBFile() - self.mock_imaging_interface.add_to_nwbfile(nwbfile) +class TestMockImagingInterface(ImagingExtractorInterfaceTestMixin): + data_interface_cls = MockImagingInterface + interface_kwargs = dict() From 78c1177a0b5cb78a36f96a59d2060f3949b28ba4 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Thu, 12 Sep 2024 06:48:54 -0400 Subject: [PATCH 043/118] [Cloud Deployment IVa] EFS creation and mounting (#1018) Co-authored-by: CodyCBakerPhD --- CHANGELOG.md | 4 + setup.py | 2 +- .../tools/aws/_submit_aws_batch_job.py | 121 ++++++++++++- tests/test_minimal/test_tools/aws_tools.py | 163 ++++++++++++++++-- 4 files changed, 273 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 619648b20..12ed1eaf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Upcoming +## v.0.6.3 + ## Bug Fixes * Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) * The method `get_extractor` now works for `MockImagingInterface` [PR #1076](https://github.com/catalystneuro/neuroconv/pull/1076) @@ -7,6 +9,7 @@ ## Deprecations ## Features +* Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) ## Improvements @@ -26,6 +29,7 @@ * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) * Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) * All remaining data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1055](https://github.com/catalystneuro/neuroconv/pull/1055) +* Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) ### Improvements diff --git a/setup.py b/setup.py index 5beebbf5f..53314e7e3 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def read_requirements(file): extras_require = defaultdict(list) -extras_require["full"] = ["dandi>=0.58.1", "hdf5plugin"] +extras_require["full"] = ["dandi>=0.58.1", "hdf5plugin", "boto3"] for modality in ["ophys", "ecephys", "icephys", "behavior", "text"]: modality_path = root / "src" / "neuroconv" / "datainterfaces" / modality diff --git a/src/neuroconv/tools/aws/_submit_aws_batch_job.py b/src/neuroconv/tools/aws/_submit_aws_batch_job.py index 0d36bee7f..9e3ba0488 100644 --- a/src/neuroconv/tools/aws/_submit_aws_batch_job.py +++ b/src/neuroconv/tools/aws/_submit_aws_batch_job.py @@ -14,6 +14,7 @@ def submit_aws_batch_job( docker_image: str, commands: Optional[list[str]] = None, environment_variables: Optional[dict[str, str]] = None, + efs_volume_name: Optional[str] = None, job_dependencies: Optional[list[dict[str, str]]] = None, status_tracker_table_name: str = "neuroconv_batch_status_tracker", iam_role_name: str = "neuroconv_batch_role", @@ -42,6 +43,9 @@ def submit_aws_batch_job( E.g., `commands=["echo", "'Hello, World!'"]`. environment_variables : dict, optional A dictionary of environment variables to pass to the Docker container. + efs_volume_name : str, optional + The name of an EFS volume to be created and attached to the job. + The path exposed to the container will always be `/mnt/efs`. job_dependencies : list of dict A list of job dependencies for this job to trigger. Structured as follows: [ @@ -88,6 +92,7 @@ def submit_aws_batch_job( import boto3 region = region or "us-east-2" + subregion = region + "a" # For anything that requires subregion, always default to "a" aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) @@ -116,6 +121,12 @@ def submit_aws_batch_job( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, ) + efs_client = boto3.client( + service_name="efs", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) # Get the tracking table and IAM role table = _create_or_get_status_tracker_table( @@ -131,10 +142,12 @@ def submit_aws_batch_job( job_queue_name=job_queue_name, compute_environment_name=compute_environment_name, batch_client=batch_client ) + efs_id = _create_or_get_efs_id(efs_volume_name=efs_volume_name, efs_client=efs_client, region=region) job_definition_name = job_definition_name or _generate_job_definition_name( docker_image=docker_image, minimum_worker_ram_in_gib=minimum_worker_ram_in_gib, minimum_worker_cpus=minimum_worker_cpus, + efs_id=efs_id, ) job_definition_arn = _ensure_job_definition_exists_and_get_arn( job_definition_name=job_definition_name, @@ -143,6 +156,7 @@ def submit_aws_batch_job( minimum_worker_cpus=minimum_worker_cpus, role_info=iam_role_info, batch_client=batch_client, + efs_id=efs_id, ) # Submit job and update status tracker @@ -160,6 +174,7 @@ def submit_aws_batch_job( container_overrides["environment"] = [{key: value} for key, value in environment_variables.items()] if commands is not None: container_overrides["command"] = commands + job_submission_info = batch_client.submit_job( jobName=job_name, dependsOn=job_dependencies, @@ -180,6 +195,7 @@ def submit_aws_batch_job( table.put_item(Item=table_submission_info) info = dict(job_submission_info=job_submission_info, table_submission_info=table_submission_info) + return info @@ -305,8 +321,8 @@ def _ensure_compute_environment_exists( "type": "EC2", "allocationStrategy": "BEST_FIT", # Note: not currently supporting spot due to interruptibility "instanceTypes": ["optimal"], - "minvCpus": 1, - "maxvCpus": 8, # Not: not currently exposing control over this since these are mostly I/O intensive + "minvCpus": 0, # Note: if not zero, will always keep an instance running in active state on standby + "maxvCpus": 8, # Note: not currently exposing control over this since these are mostly I/O intensive "instanceRole": "ecsInstanceRole", # Security groups and subnets last updated on 8/4/2024 "securityGroupIds": ["sg-001699e5b7496b226"], @@ -391,9 +407,20 @@ def _ensure_job_queue_exists( computeEnvironmentOrder=[ dict(order=1, computeEnvironment=compute_environment_name), ], + # Note: boto3 annotates the reason as a generic string + # But really it is Literal[ + # "MISCONFIGURATION:COMPUTE_ENVIRONMENT_MAX_RESOURCE", "MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT" + # ] + # And we should have limits on both jobStateTimeLimitActions=[ dict( - reason="Avoid zombie jobs.", + reason="MISCONFIGURATION:COMPUTE_ENVIRONMENT_MAX_RESOURCE", + state="RUNNABLE", + maxTimeSeconds=minimum_time_to_kill_in_seconds, + action="CANCEL", + ), + dict( + reason="MISCONFIGURATION:JOB_RESOURCE_REQUIREMENT", state="RUNNABLE", maxTimeSeconds=minimum_time_to_kill_in_seconds, action="CANCEL", @@ -418,11 +445,71 @@ def _ensure_job_queue_exists( return None +def _create_or_get_efs_id( + efs_volume_name: Optional[str], efs_client: "boto3.client.efs", region: str = "us-east-2" +) -> Optional[str]: # pragma: no cover + if efs_volume_name is None: + return None + + if region != "us-east-2": + raise NotImplementedError("EFS volumes are only supported in us-east-2 for now.") + + available_efs_volumes = efs_client.describe_file_systems() + matching_efs_volumes = [ + file_system + for file_system in available_efs_volumes["FileSystems"] + for tag in file_system["Tags"] + if tag["Key"] == "Name" and tag["Value"] == efs_volume_name + ] + + if len(matching_efs_volumes) > 1: + efs_volume = matching_efs_volumes[0] + efs_id = efs_volume["FileSystemId"] + + return efs_id + + # Existing volume not found - must create a fresh one and set mount targets on it + efs_volume = efs_client.create_file_system( + PerformanceMode="generalPurpose", # Only type supported in one-zone + Encrypted=False, + ThroughputMode="elastic", + # TODO: figure out how to make job spawn only on subregion for OneZone discount + # AvailabilityZoneName=subregion, + Backup=False, + Tags=[{"Key": "Name", "Value": efs_volume_name}], + ) + efs_id = efs_volume["FileSystemId"] + + # Takes a while to spin up - cannot assign mount targets until it is ready + # TODO: in a follow-up replace with more robust checking mechanism + time.sleep(60) + + # TODO: in follow-up, figure out how to fetch this automatically and from any region + # (might even resolve those previous OneZone issues) + region_to_subnet_id = { + "us-east-2a": "subnet-0890a93aedb42e73e", + "us-east-2b": "subnet-0e20bbcfb951b5387", + "us-east-2c": "subnet-0680e07980538b786", + } + for subnet_id in region_to_subnet_id.values(): + efs_client.create_mount_target( + FileSystemId=efs_id, + SubnetId=subnet_id, + SecurityGroups=[ + "sg-001699e5b7496b226", + ], + ) + time.sleep(60) # Also takes a while to create the mount targets so add some buffer time + + return efs_id + + def _generate_job_definition_name( *, docker_image: str, minimum_worker_ram_in_gib: int, minimum_worker_cpus: int, + efs_id: Optional[str] = None, ) -> str: # pragma: no cover """ Generate a job definition name for the AWS Batch job. @@ -449,6 +536,8 @@ def _generate_job_definition_name( job_definition_name += f"_{parsed_docker_image_name}-image" job_definition_name += f"_{minimum_worker_ram_in_gib}-GiB-RAM" job_definition_name += f"_{minimum_worker_cpus}-CPU" + if efs_id is not None: + job_definition_name += f"_{efs_id}" if docker_tag is None or docker_tag == "latest": date = datetime.now().strftime("%Y-%m-%d") job_definition_name += f"_created-on-{date}" @@ -464,6 +553,7 @@ def _ensure_job_definition_exists_and_get_arn( minimum_worker_cpus: int, role_info: dict, batch_client: "boto3.client.Batch", + efs_id: Optional[str] = None, max_retries: int = 12, ) -> str: # pragma: no cover """ @@ -494,6 +584,9 @@ def _ensure_job_definition_exists_and_get_arn( The IAM role information for the job. batch_client : boto3.client.Batch The AWS Batch client to use for the job. + efs_id : str, optional + The EFS volume information for the job. + The path exposed to the container will always be `/mnt/efs`. max_retries : int, default: 12 If the job definition does not already exist, then this is the maximum number of times to synchronously check for its successful creation before erroring. @@ -534,6 +627,20 @@ def _ensure_job_definition_exists_and_get_arn( minimum_time_to_kill_in_days = 1 # Note: eventually consider exposing this for very long jobs? minimum_time_to_kill_in_seconds = minimum_time_to_kill_in_days * 24 * 60 * 60 + volumes = [] + mountPoints = [] + if efs_id is not None: + volumes = [ + { + "name": "neuroconv_batch_efs_mounted", + "efsVolumeConfiguration": { + "fileSystemId": efs_id, + "transitEncryption": "DISABLED", + }, + }, + ] + mountPoints = [{"containerPath": "/mnt/efs/", "readOnly": False, "sourceVolume": "neuroconv_batch_efs_mounted"}] + # batch_client.register_job_definition() is not synchronous and so we need to wait a bit afterwards batch_client.register_job_definition( jobDefinitionName=job_definition_name, @@ -542,9 +649,13 @@ def _ensure_job_definition_exists_and_get_arn( containerProperties=dict( image=docker_image, resourceRequirements=resource_requirements, - jobRoleArn=role_info["Role"]["Arn"], - executionRoleArn=role_info["Role"]["Arn"], + # TODO: investigate if any IAM role is explicitly needed in conjunction with the credentials + # jobRoleArn=role_info["Role"]["Arn"], + # executionRoleArn=role_info["Role"]["Arn"], + volumes=volumes, + mountPoints=mountPoints, ), + platformCapabilities=["EC2"], ) job_definition_request = batch_client.describe_job_definitions(jobDefinitions=[job_definition_with_revision]) diff --git a/tests/test_minimal/test_tools/aws_tools.py b/tests/test_minimal/test_tools/aws_tools.py index 69de5d31a..2e7598178 100644 --- a/tests/test_minimal/test_tools/aws_tools.py +++ b/tests/test_minimal/test_tools/aws_tools.py @@ -1,3 +1,4 @@ +import datetime import os import time @@ -5,6 +6,8 @@ from neuroconv.tools.aws import submit_aws_batch_job +_RETRY_STATES = ["RUNNABLE", "PENDING", "STARTING", "RUNNING"] + def test_submit_aws_batch_job(): region = "us-east-2" @@ -35,14 +38,24 @@ def test_submit_aws_batch_job(): time.sleep(60) job_id = info["job_submission_info"]["jobId"] + job = None + max_retries = 10 + retry = 0 + while retry < max_retries: + job_description_response = batch_client.describe_jobs(jobs=[job_id]) + assert job_description_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + jobs = job_description_response["jobs"] + assert len(jobs) == 1 - all_jobs_response = batch_client.describe_jobs(jobs=[job_id]) - assert all_jobs_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + job = jobs[0] - jobs = all_jobs_response["jobs"] - assert len(jobs) == 1 + if job["status"] in _RETRY_STATES: + retry += 1 + time.sleep(60) + else: + break - job = jobs[0] assert job["jobName"] == job_name assert "neuroconv_batch_queue" in job["jobQueue"] assert "neuroconv_batch_ubuntu-latest-image_4-GiB-RAM_4-CPU" in job["jobDefinition"] @@ -106,18 +119,29 @@ def test_submit_aws_batch_job_with_dependencies(): ) # Wait for AWS to process the jobs - time.sleep(120) + time.sleep(60) job_id_1 = job_info_1["job_submission_info"]["jobId"] job_id_2 = job_info_2["job_submission_info"]["jobId"] + job_1 = None + max_retries = 10 + retry = 0 + while retry < max_retries: + all_job_descriptions_response = batch_client.describe_jobs(jobs=[job_id_1, job_id_2]) + assert all_job_descriptions_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + jobs_by_id = {job["jobId"]: job for job in all_job_descriptions_response["jobs"]} + assert len(jobs_by_id) == 2 - all_jobs_response = batch_client.describe_jobs(jobs=[job_id_1, job_id_2]) - assert all_jobs_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + job_1 = jobs_by_id[job_id_1] + job_2 = jobs_by_id[job_id_2] - jobs_by_id = {job["jobId"]: job for job in all_jobs_response["jobs"]} - assert len(jobs_by_id) == 2 + if job_1["status"] in _RETRY_STATES or job_2["status"] in _RETRY_STATES: + retry += 1 + time.sleep(60) + else: + break - job_1 = jobs_by_id[job_id_1] assert job_1["jobName"] == job_name_1 assert "neuroconv_batch_queue" in job_1["jobQueue"] assert "neuroconv_batch_ubuntu-latest-image_4-GiB-RAM_4-CPU" in job_1["jobDefinition"] @@ -156,3 +180,120 @@ def test_submit_aws_batch_job_with_dependencies(): table.update_item( Key={"id": table_submission_id_2}, AttributeUpdates={"status": {"Action": "PUT", "Value": "Test passed."}} ) + + +def test_submit_aws_batch_job_with_efs_mount(): + """ + It was confirmed manually that a job using this definition will fail if the /mnt/efs/ directory does not exist. + + It is, however, prohibitively difficult to automatically check if the file exists on the EFS volume. + + If desired, you can manually check the EFS volume by following these instructions: + https://repost.aws/knowledge-center/efs-mount-automount-unmount-steps + """ + region = "us-east-2" + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) + aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) + + dynamodb_resource = boto3.resource( + service_name="dynamodb", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + batch_client = boto3.client( + service_name="batch", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + efs_client = boto3.client( + service_name="efs", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + + job_name = "test_submit_aws_batch_job_with_efs" + docker_image = "ubuntu:latest" + date = datetime.datetime.now().date().strftime("%y%m%d") + commands = ["touch", f"/mnt/efs/test_{date}.txt"] + + # TODO: to reduce costs even more, find a good combinations of memory/CPU to minimize size of instance + efs_volume_name = f"test_neuroconv_batch_with_efs_{date}" + info = submit_aws_batch_job( + job_name=job_name, + docker_image=docker_image, + commands=commands, + efs_volume_name=efs_volume_name, + ) + + # Wait for AWS to process the job + time.sleep(60) + + job_id = info["job_submission_info"]["jobId"] + job = None + max_retries = 10 + retry = 0 + while retry < max_retries: + job_description_response = batch_client.describe_jobs(jobs=[job_id]) + assert job_description_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + jobs = job_description_response["jobs"] + assert len(jobs) == 1 + + job = jobs[0] + + if job["status"] in _RETRY_STATES: + retry += 1 + time.sleep(60) + else: + break + + # Check EFS specific details + efs_volumes = efs_client.describe_file_systems() + matching_efs_volumes = [ + file_system + for file_system in efs_volumes["FileSystems"] + for tag in file_system["Tags"] + if tag["Key"] == "Name" and tag["Value"] == efs_volume_name + ] + assert len(matching_efs_volumes) == 1 + efs_volume = matching_efs_volumes[0] + efs_id = efs_volume["FileSystemId"] + + # Check normal job completion + assert job["jobName"] == job_name + assert "neuroconv_batch_queue" in job["jobQueue"] + assert "fs-" in job["jobDefinition"] + assert job["status"] == "SUCCEEDED" + + status_tracker_table_name = "neuroconv_batch_status_tracker" + table = dynamodb_resource.Table(name=status_tracker_table_name) + table_submission_id = info["table_submission_info"]["id"] + + table_item_response = table.get_item(Key={"id": table_submission_id}) + assert table_item_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + table_item = table_item_response["Item"] + assert table_item["job_name"] == job_name + assert table_item["job_id"] == job_id + assert table_item["status"] == "Job submitted..." + + table.update_item( + Key={"id": table_submission_id}, + AttributeUpdates={"status": {"Action": "PUT", "Value": "Test passed - cleaning up..."}}, + ) + + # Cleanup EFS after testing is complete - must clear mount targets first, then wait before deleting the volume + # TODO: cleanup job definitions? (since built daily) + mount_targets = efs_client.describe_mount_targets(FileSystemId=efs_id) + for mount_target in mount_targets["MountTargets"]: + efs_client.delete_mount_target(MountTargetId=mount_target["MountTargetId"]) + + time.sleep(60) + efs_client.delete_file_system(FileSystemId=efs_id) + + table.update_item( + Key={"id": table_submission_id}, AttributeUpdates={"status": {"Action": "PUT", "Value": "Test passed."}} + ) From 679418c151cc46eecaf9ca016f4a7721d3e7be38 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Fri, 13 Sep 2024 04:28:01 +1000 Subject: [PATCH 044/118] Added chunking/compression for string-only objects (#1042) Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Co-authored-by: Heberto Mayorquin --- CHANGELOG.md | 4 +- .../_configuration_models/_base_dataset_io.py | 28 +++++++++++-- .../test_dataset_io_configuration_model.py | 42 +++++++++++++++++++ 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12ed1eaf9..cf4f6bfbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Upcoming -## v.0.6.3 +## v0.6.4 ## Bug Fixes * Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) @@ -9,10 +9,12 @@ ## Deprecations ## Features +* Added chunking/compression for string-only compound objects: [PR #1042](https://github.com/catalystneuro/neuroconv/pull/1042) * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) ## Improvements +## v0.6.3 ## v0.6.2 (September 10, 2024) diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py index 8b40e9a9e..9cdb97405 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py @@ -277,10 +277,30 @@ def from_neurodata_object(cls, neurodata_object: Container, dataset_name: Litera ) compression_method = "gzip" elif dtype == np.dtype("object"): # Unclear what default chunking/compression should be for compound objects - raise NotImplementedError( - f"Unable to create a `DatasetIOConfiguration` for the dataset at '{location_in_file}'" - f"for neurodata object '{neurodata_object}' of type '{type(neurodata_object)}'!" - ) + # pandas reads in strings as objects by default: https://pandas.pydata.org/docs/user_guide/text.html + all_elements_are_strings = all([isinstance(element, str) for element in candidate_dataset[:].flat]) + if all_elements_are_strings: + dtype = np.array([element for element in candidate_dataset[:].flat]).dtype + chunk_shape = SliceableDataChunkIterator.estimate_default_chunk_shape( + chunk_mb=10.0, maxshape=full_shape, dtype=dtype + ) + buffer_shape = SliceableDataChunkIterator.estimate_default_buffer_shape( + buffer_gb=0.5, chunk_shape=chunk_shape, maxshape=full_shape, dtype=dtype + ) + compression_method = "gzip" + else: + raise NotImplementedError( + f"Unable to create a `DatasetIOConfiguration` for the dataset at '{location_in_file}'" + f"for neurodata object '{neurodata_object}' of type '{type(neurodata_object)}'!" + ) + # TODO: Add support for compound objects with non-string elements + # chunk_shape = full_shape # validate_all_shapes fails if chunk_shape or buffer_shape is None + # buffer_shape = full_shape + # compression_method = None + # warnings.warn( + # f"Default chunking and compression options for compound objects are not optimized. " + # f"Consider manually specifying DatasetIOConfiguration for dataset at '{location_in_file}'." + # ) return cls( object_id=neurodata_object.object_id, diff --git a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_models/test_dataset_io_configuration_model.py b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_models/test_dataset_io_configuration_model.py index 901a82d63..616c6e9d4 100644 --- a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_models/test_dataset_io_configuration_model.py +++ b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_models/test_dataset_io_configuration_model.py @@ -2,6 +2,7 @@ import numpy as np import pytest +from pynwb.testing.mock.file import mock_NWBFile from neuroconv.tools.nwb_helpers import DatasetIOConfiguration @@ -53,3 +54,44 @@ def test_model_json_schema_generator_assertion(): DatasetIOConfiguration.model_json_schema(schema_generator="anything") assert "The 'schema_generator' of this method cannot be changed." == str(error_info.value) + + +# TODO: Add support for compound objects with non-string elements +# def test_from_neurodata_object_dtype_object(): +# class TestDatasetIOConfiguration(DatasetIOConfiguration): +# def get_data_io_kwargs(self): +# super().get_data_io_kwargs() + +# nwbfile = mock_NWBFile() +# nwbfile.add_trial(start_time=0.0, stop_time=1.0) +# nwbfile.add_trial(start_time=1.0, stop_time=2.0) +# nwbfile.add_trial(start_time=2.0, stop_time=3.0) +# data = np.array(["test", 5, False], dtype=object) +# nwbfile.add_trial_column(name="test", description="test column with object dtype", data=data) +# neurodata_object = nwbfile.trials.columns[2] + +# dataset_io_configuration = TestDatasetIOConfiguration.from_neurodata_object(neurodata_object, dataset_name="data") + +# assert dataset_io_configuration.chunk_shape == (3,) +# assert dataset_io_configuration.buffer_shape == (3,) +# assert dataset_io_configuration.compression_method is None + + +def test_from_neurodata_object_dtype_object_all_strings(): + class TestDatasetIOConfiguration(DatasetIOConfiguration): + def get_data_io_kwargs(self): + super().get_data_io_kwargs() + + nwbfile = mock_NWBFile() + nwbfile.add_trial(start_time=0.0, stop_time=1.0) + nwbfile.add_trial(start_time=1.0, stop_time=2.0) + nwbfile.add_trial(start_time=2.0, stop_time=3.0) + data = np.array(["test", "string", "abc"], dtype=object) + nwbfile.add_trial_column(name="test", description="test column with object dtype but all strings", data=data) + neurodata_object = nwbfile.trials.columns[2] + + dataset_io_configuration = TestDatasetIOConfiguration.from_neurodata_object(neurodata_object, dataset_name="data") + + assert dataset_io_configuration.chunk_shape == (3,) + assert dataset_io_configuration.buffer_shape == (3,) + assert dataset_io_configuration.compression_method == "gzip" From f7e9c4e73e013952be8650c190ca7e3a3eaa674f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 12 Sep 2024 18:33:35 -0600 Subject: [PATCH 045/118] Github CI only run doctests when tests are not run (#1077) Co-authored-by: Paul Adkisson --- .github/workflows/assess-file-changes.yml | 2 +- .github/workflows/deploy-tests.yml | 2 +- CHANGELOG.md | 1 + docs/conversion_examples_gallery/conftest.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/assess-file-changes.yml b/.github/workflows/assess-file-changes.yml index 2a234659c..6c7887f63 100644 --- a/.github/workflows/assess-file-changes.yml +++ b/.github/workflows/assess-file-changes.yml @@ -41,7 +41,7 @@ jobs: echo "CHANGELOG_UPDATED=false" >> $GITHUB_OUTPUT for file in ${{ steps.changed-files.outputs.all_changed_files }}; do echo $file - if [[ $file == "src/"* || $file == "tests/"* || $file == "requirements-minimal.txt" || $file == "requirements-testing.txt" || $file == "setup.py" || $file == ".github/"* ]] + if [[ $file == "src/"* || $file == "tests/"* || $file == "pyproject.toml" || $file == "setup.py" || $file == ".github/"* ]] then echo "Source changed" echo "SOURCE_CHANGED=true" >> $GITHUB_OUTPUT diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index a1a6ae790..49af30be0 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -57,7 +57,7 @@ jobs: run-doctests-only: needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' || needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} + if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml check-final-status: diff --git a/CHANGELOG.md b/CHANGELOG.md index cf4f6bfbb..05d393e9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) ## Improvements +* Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) ## v0.6.3 diff --git a/docs/conversion_examples_gallery/conftest.py b/docs/conversion_examples_gallery/conftest.py index 134b198b4..21c392bf0 100644 --- a/docs/conversion_examples_gallery/conftest.py +++ b/docs/conversion_examples_gallery/conftest.py @@ -17,5 +17,5 @@ def add_data_space(doctest_namespace, tmp_path): doctest_namespace["OPHYS_DATA_PATH"] = OPHYS_DATA_PATH doctest_namespace["TEXT_DATA_PATH"] = TEXT_DATA_PATH - doctest_namespace["path_to_save_nwbfile"] = Path(tmp_path) / "file.nwb" + doctest_namespace["path_to_save_nwbfile"] = Path(tmp_path) / "doctest_file.nwb" doctest_namespace["output_folder"] = Path(tmp_path) From 96c8ed4d76bd734e335acd999c015770f9cfd92a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 13 Sep 2024 01:00:26 -0600 Subject: [PATCH 046/118] Enable zarr backend testing in data tests [1] (#1056) Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- CHANGELOG.md | 1 + .../behavior/video/videodatainterface.py | 2 +- .../tools/testing/data_interface_mixins.py | 93 ++++++++++++++----- tests/test_behavior/test_video_interface.py | 6 +- .../ecephys/test_recording_interfaces.py | 35 ++++--- 5 files changed, 91 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05d393e9e..0aec170e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) ## Improvements +* Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) ## v0.6.3 diff --git a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py index 7a28c2d2f..a544f9c27 100644 --- a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py @@ -97,7 +97,7 @@ def get_metadata(self): metadata = super().get_metadata() behavior_metadata = { self.metadata_key_name: [ - dict(name=f"Video: {Path(file_path).stem}", description="Video recorded by camera.", unit="Frames") + dict(name=f"Video {Path(file_path).stem}", description="Video recorded by camera.", unit="Frames") for file_path in self.source_data["file_paths"] ] } diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 24042feee..2b8252154 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -67,9 +67,8 @@ class DataInterfaceTestMixin: @pytest.fixture def setup_interface(self, request): - + """Add this as a fixture when you want freshly created interface in the test.""" self.test_name: str = "" - self.conversion_options = self.conversion_options or dict() self.interface = self.data_interface_cls(**self.interface_kwargs) return self.interface, self.test_name @@ -88,31 +87,26 @@ def check_conversion_options_schema_valid(self): schema = self.interface.get_conversion_options_schema() Draft7Validator.check_schema(schema=schema) - def check_metadata_schema_valid(self): + def test_metadata_schema_valid(self, setup_interface): schema = self.interface.get_metadata_schema() Draft7Validator.check_schema(schema=schema) def check_metadata(self): - schema = self.interface.get_metadata_schema() + # Validate metadata now happens on the class itself metadata = self.interface.get_metadata() - if "session_start_time" not in metadata["NWBFile"]: - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - # handle json encoding of datetimes and other tricky types - metadata_for_validation = json.loads(json.dumps(metadata, cls=_NWBMetaDataEncoder)) - validate(metadata_for_validation, schema) self.check_extracted_metadata(metadata) - def check_no_metadata_mutation(self): - """Ensure the metadata object was not altered by `add_to_nwbfile` method.""" - metadata = self.interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - metadata_in = deepcopy(metadata) + def test_no_metadata_mutation(self, setup_interface): + """Ensure the metadata object is not altered by `add_to_nwbfile` method.""" nwbfile = mock_NWBFile() + + metadata = self.interface.get_metadata() + metadata_before_add_method = deepcopy(metadata) + self.interface.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **self.conversion_options) - assert metadata == metadata_in + assert metadata == metadata_before_add_method def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"): metadata = self.interface.get_metadata() @@ -223,6 +217,26 @@ def run_custom_checks(self): """Override this in child classes to inject additional custom checks.""" pass + @pytest.mark.parametrize("backend", ["hdf5", "zarr"]) + def test_run_conversion_with_backend(self, setup_interface, tmp_path, backend): + + nwbfile_path = str(tmp_path / f"conversion_with_backend{backend}-{self.test_name}.nwb") + + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + self.interface.run_conversion( + nwbfile_path=nwbfile_path, + overwrite=True, + metadata=metadata, + backend=backend, + **self.conversion_options, + ) + + if backend == "zarr": + self.check_basic_zarr_read(nwbfile_path) + def test_all_conversion_checks(self, setup_interface, tmp_path): interface, test_name = setup_interface @@ -231,16 +245,13 @@ def test_all_conversion_checks(self, setup_interface, tmp_path): self.nwbfile_path = nwbfile_path # Now run the checks using the setup objects - self.check_metadata_schema_valid() self.check_conversion_options_schema_valid() self.check_metadata() - self.check_no_metadata_mutation() self.check_configure_backend_for_equivalent_nwbfiles() self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - self.check_run_conversion_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") self.check_run_conversion_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") self.check_read_nwb(nwbfile_path=nwbfile_path) @@ -733,16 +744,13 @@ def test_all_conversion_checks(self, setup_interface, tmp_path): nwbfile_path = str(tmp_path / f"{self.__class__.__name__}_{self.test_name}.nwb") # Now run the checks using the setup objects - self.check_metadata_schema_valid() self.check_conversion_options_schema_valid() self.check_metadata() - self.check_no_metadata_mutation() self.check_configure_backend_for_equivalent_nwbfiles() self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - self.check_run_conversion_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") self.check_run_conversion_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") self.check_read_nwb(nwbfile_path=nwbfile_path) @@ -813,7 +821,7 @@ def check_read_nwb(self, nwbfile_path: str): with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() video_type = Path(self.interface_kwargs["file_paths"][0]).suffix[1:] - assert f"Video: video_{video_type}" in nwbfile.acquisition + assert f"Video video_{video_type}" in nwbfile.acquisition def check_interface_set_aligned_timestamps(self): all_unaligned_timestamps = self.interface.get_original_timestamps() @@ -883,6 +891,29 @@ class MedPCInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): A mixin for testing MedPC interfaces. """ + def test_metadata_schema_valid(self): + pass + + def test_run_conversion_with_backend(self): + pass + + def test_no_metadata_mutation(self): + pass + + def check_metadata_schema_valid(self): + schema = self.interface.get_metadata_schema() + Draft7Validator.check_schema(schema=schema) + + def check_metadata(self): + schema = self.interface.get_metadata_schema() + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + # handle json encoding of datetimes and other tricky types + metadata_for_validation = json.loads(json.dumps(metadata, cls=_NWBMetaDataEncoder)) + validate(metadata_for_validation, schema) + self.check_extracted_metadata(metadata) + def check_no_metadata_mutation(self, metadata: dict): """Ensure the metadata object was not altered by `add_to_nwbfile` method.""" @@ -1220,6 +1251,22 @@ def check_read_nwb(self, nwbfile_path: str): class TDTFiberPhotometryInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): """Mixin for testing TDT Fiber Photometry interfaces.""" + def test_metadata_schema_valid(self): + pass + + def test_no_metadata_mutation(self): + pass + + def test_run_conversion_with_backend(self): + pass + + def test_no_metadata_mutation(self): + pass + + def check_metadata_schema_valid(self): + schema = self.interface.get_metadata_schema() + Draft7Validator.check_schema(schema=schema) + def check_no_metadata_mutation(self, metadata: dict): """Ensure the metadata object was not altered by `add_to_nwbfile` method.""" diff --git a/tests/test_behavior/test_video_interface.py b/tests/test_behavior/test_video_interface.py index b1ce7f1f4..b367d406d 100644 --- a/tests/test_behavior/test_video_interface.py +++ b/tests/test_behavior/test_video_interface.py @@ -137,8 +137,8 @@ def test_video_external_mode(self): nwbfile = io.read() module = nwbfile.acquisition metadata = self.nwb_converter.get_metadata() - self.assertListEqual(list1=list(module["Video: test1"].external_file[:]), list2=self.video_files[0:2]) - self.assertListEqual(list1=list(module["Video: test3"].external_file[:]), list2=[self.video_files[2]]) + self.assertListEqual(list1=list(module["Video test1"].external_file[:]), list2=self.video_files[0:2]) + self.assertListEqual(list1=list(module["Video test3"].external_file[:]), list2=[self.video_files[2]]) def test_video_irregular_timestamps(self): aligned_timestamps = [np.array([1.0, 2.0, 4.0]), np.array([5.0, 6.0, 7.0])] @@ -157,7 +157,7 @@ def test_video_irregular_timestamps(self): expected_timestamps = timestamps = np.array([1.0, 2.0, 4.0, 55.0, 56.0, 57.0]) with NWBHDF5IO(path=self.nwbfile_path, mode="r") as io: nwbfile = io.read() - np.testing.assert_array_equal(expected_timestamps, nwbfile.acquisition["Video: test1"].timestamps[:]) + np.testing.assert_array_equal(expected_timestamps, nwbfile.acquisition["Video test1"].timestamps[:]) def test_starting_frames_type_error(self): timestamps = [np.array([2.2, 2.4, 2.6]), np.array([3.2, 3.4, 3.6])] diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 187e1bff8..cc83625dc 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -1,7 +1,6 @@ from datetime import datetime from platform import python_version from sys import platform -from typing import Literal import numpy as np import pytest @@ -180,32 +179,30 @@ class TestEDFRecordingInterface(RecordingExtractorInterfaceTestMixin): def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 3, 2, 10, 42, 19) - def test_interface_alignment(self): - interface_kwargs = self.interface_kwargs + def test_all_conversion_checks(self, setup_interface, tmp_path): + # Create a unique test name and file path + nwbfile_path = str(tmp_path / f"{self.__class__.__name__}.nwb") + self.nwbfile_path = nwbfile_path + + # Now run the checks using the setup objects + self.check_conversion_options_schema_valid() + self.check_metadata() - # TODO - debug hanging I/O from pyedflib - # self.check_interface_get_original_timestamps() - # self.check_interface_get_timestamps() - # self.check_align_starting_time_internal() - # self.check_align_starting_time_external() - # self.check_interface_align_timestamps() - # self.check_shift_timestamps_by_start_time() - # self.check_interface_original_timestamps_inmutability() + self.check_run_conversion_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") - self.check_nwbfile_temporal_alignment() + self.check_read_nwb(nwbfile_path=nwbfile_path) # EDF has simultaneous access issues; can't have multiple interfaces open on the same file at once... - def check_run_conversion_in_nwbconverter_with_backend( - self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5" - ): + def test_metadata_schema_valid(self): pass - def check_run_conversion_in_nwbconverter_with_backend_configuration( - self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5" - ): + def test_no_metadata_mutation(self): pass - def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"): + def test_run_conversion_with_backend(self): + pass + + def test_interface_alignment(self): pass From 113527f5650e10c7d0a61b10f9f362837f12b117 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 13 Sep 2024 11:21:16 -0600 Subject: [PATCH 047/118] Only run changelog request when status is not draft (#1078) Co-authored-by: Paul Adkisson --- .github/workflows/deploy-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 49af30be0..9cbf6ba94 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -2,6 +2,7 @@ name: Deploy tests on: pull_request: + types: [synchronize, opened, reopened, ready_for_review] # defaults + ready_for_review merge_group: workflow_dispatch: @@ -16,7 +17,7 @@ jobs: detect-changelog-updates: needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} + if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' && github.event.pull_request.draft == false }} name: Auto-detecting CHANGELOG.md updates runs-on: ubuntu-latest steps: From ad1e2a1a54068e4b1711f8fbc5e224c983131c16 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 13 Sep 2024 14:49:14 -0600 Subject: [PATCH 048/118] Add `MockSegmentationInterface` (#1067) Co-authored-by: Szonja Weigl Co-authored-by: Paul Adkisson --- CHANGELOG.md | 3 + .../basesegmentationextractorinterface.py | 3 +- .../tools/testing/mock_interfaces.py | 74 +++++++++++++++++++ tests/test_ophys/test_ophys_interfaces.py | 12 ++- 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aec170e4..dfa9612aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) * Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) * All remaining data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1055](https://github.com/catalystneuro/neuroconv/pull/1055) +* Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) @@ -44,6 +45,8 @@ * Improved device metadata of `IntanRecordingInterface` by adding the type of controller used [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) + + ## v0.6.1 (August 30, 2024) ### Bug fixes diff --git a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py index 6b55b5afb..0f2e41bb9 100644 --- a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py @@ -18,8 +18,9 @@ class BaseSegmentationExtractorInterface(BaseExtractorInterface): ExtractorModuleName = "roiextractors" - def __init__(self, **source_data): + def __init__(self, verbose: bool = False, **source_data): super().__init__(**source_data) + self.verbose = verbose self.segmentation_extractor = self.get_extractor()(**source_data) def get_metadata_schema(self) -> dict: diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 12f1dafd4..87f6dcf8e 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -14,6 +14,9 @@ from ...datainterfaces.ophys.baseimagingextractorinterface import ( BaseImagingExtractorInterface, ) +from ...datainterfaces.ophys.basesegmentationextractorinterface import ( + BaseSegmentationExtractorInterface, +) from ...utils import ArrayType, get_schema_from_method_signature @@ -216,3 +219,74 @@ def get_metadata(self, photon_series_type: Optional[Literal["OnePhotonSeries", " metadata = super().get_metadata(photon_series_type=photon_series_type) metadata["NWBFile"]["session_start_time"] = session_start_time return metadata + + +class MockSegmentationInterface(BaseSegmentationExtractorInterface): + """A mock segmentation interface for testing purposes.""" + + ExtractorModuleName = "roiextractors.testing" + ExtractorName = "generate_dummy_segmentation_extractor" + + def __init__( + self, + num_rois: int = 10, + num_frames: int = 30, + num_rows: int = 25, + num_columns: int = 25, + sampling_frequency: float = 30.0, + has_summary_images: bool = True, + has_raw_signal: bool = True, + has_dff_signal: bool = True, + has_deconvolved_signal: bool = True, + has_neuropil_signal: bool = True, + seed: int = 0, + verbose: bool = False, + ): + """ + Parameters + ---------- + num_rois : int, optional + number of regions of interest, by default 10. + num_frames : int, optional + description, by default 30. + num_rows : int, optional + number of rows in the hypothetical video from which the data was extracted, by default 25. + num_columns : int, optional + number of columns in the hypothetical video from which the data was extracted, by default 25. + sampling_frequency : float, optional + sampling frequency of the hypothetical video from which the data was extracted, by default 30.0. + has_summary_images : bool, optional + whether the dummy segmentation extractor has summary images or not (mean and correlation). + has_raw_signal : bool, optional + whether a raw fluorescence signal is desired in the object, by default True. + has_dff_signal : bool, optional + whether a relative (df/f) fluorescence signal is desired in the object, by default True. + has_deconvolved_signal : bool, optional + whether a deconvolved signal is desired in the object, by default True. + has_neuropil_signal : bool, optional + whether a neuropil signal is desired in the object, by default True. + seed: int, default 0 + seed for the random number generator, by default 0 + verbose : bool, optional + controls verbosity, by default False. + """ + + super().__init__( + num_rois=num_rois, + num_frames=num_frames, + num_rows=num_rows, + num_columns=num_columns, + sampling_frequency=sampling_frequency, + has_summary_images=has_summary_images, + has_raw_signal=has_raw_signal, + has_dff_signal=has_dff_signal, + has_deconvolved_signal=has_deconvolved_signal, + has_neuropil_signal=has_neuropil_signal, + verbose=verbose, + ) + + def get_metadata(self) -> dict: + session_start_time = datetime.now().astimezone() + metadata = super().get_metadata() + metadata["NWBFile"]["session_start_time"] = session_start_time + return metadata diff --git a/tests/test_ophys/test_ophys_interfaces.py b/tests/test_ophys/test_ophys_interfaces.py index 9ae151d5a..c2c9b4a0c 100644 --- a/tests/test_ophys/test_ophys_interfaces.py +++ b/tests/test_ophys/test_ophys_interfaces.py @@ -1,9 +1,19 @@ from neuroconv.tools.testing.data_interface_mixins import ( ImagingExtractorInterfaceTestMixin, + SegmentationExtractorInterfaceTestMixin, +) +from neuroconv.tools.testing.mock_interfaces import ( + MockImagingInterface, + MockSegmentationInterface, ) -from neuroconv.tools.testing.mock_interfaces import MockImagingInterface class TestMockImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = MockImagingInterface interface_kwargs = dict() + + +class TestMockSegmentationInterface(SegmentationExtractorInterfaceTestMixin): + + data_interface_cls = MockSegmentationInterface + interface_kwargs = dict() From ad9f25d55ad6cb5ea092d5abb19401ecbef1d773 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 16 Sep 2024 10:05:33 -0600 Subject: [PATCH 049/118] Add `MockSortingInterface` (#1065) --- CHANGELOG.md | 4 +- src/neuroconv/tools/testing/__init__.py | 8 ++- .../tools/testing/mock_interfaces.py | 51 +++++++++++++++++++ tests/test_ecephys/test_ecephys_interfaces.py | 33 +++++++++++- 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfa9612aa..7570045b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ ## Features * Added chunking/compression for string-only compound objects: [PR #1042](https://github.com/catalystneuro/neuroconv/pull/1042) * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) +* Added a `MockSortingInterface` for testing purposes. [PR #1065](https://github.com/catalystneuro/neuroconv/pull/1065) + ## Improvements * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) @@ -33,8 +35,6 @@ * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) * Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) * All remaining data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1055](https://github.com/catalystneuro/neuroconv/pull/1055) -* Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) -* Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) ### Improvements diff --git a/src/neuroconv/tools/testing/__init__.py b/src/neuroconv/tools/testing/__init__.py index 7179a7544..79b54d3f9 100644 --- a/src/neuroconv/tools/testing/__init__.py +++ b/src/neuroconv/tools/testing/__init__.py @@ -5,5 +5,11 @@ mock_ZarrDatasetIOConfiguration, ) from .mock_files import generate_path_expander_demo_ibl -from .mock_interfaces import MockBehaviorEventInterface, MockSpikeGLXNIDQInterface +from .mock_interfaces import ( + MockBehaviorEventInterface, + MockSpikeGLXNIDQInterface, + MockRecordingInterface, + MockImagingInterface, + MockSortingInterface, +) from .mock_ttl_signals import generate_mock_ttl_signal, regenerate_test_cases diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 87f6dcf8e..43f8c2dd2 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -11,6 +11,9 @@ from ...datainterfaces.ecephys.baserecordingextractorinterface import ( BaseRecordingExtractorInterface, ) +from ...datainterfaces.ecephys.basesortingextractorinterface import ( + BaseSortingExtractorInterface, +) from ...datainterfaces.ophys.baseimagingextractorinterface import ( BaseImagingExtractorInterface, ) @@ -160,6 +163,54 @@ def get_metadata(self) -> dict: return metadata +class MockSortingInterface(BaseSortingExtractorInterface): + """A mock sorting extractor interface for generating synthetic sorting data.""" + + # TODO: Implement this class with the lazy generator once is merged + # https://github.com/SpikeInterface/spikeinterface/pull/2227 + + ExtractorModuleName = "spikeinterface.core.generate" + ExtractorName = "generate_sorting" + + def __init__( + self, + num_units: int = 4, + sampling_frequency: float = 30_000.0, + durations: tuple[float] = (1.0,), + seed: int = 0, + verbose: bool = True, + ): + """ + Parameters + ---------- + num_units : int, optional + Number of units to generate, by default 4. + sampling_frequency : float, optional + Sampling frequency of the generated data in Hz, by default 30,000.0 Hz. + durations : tuple of float, optional + Durations of the segments in seconds, by default (1.0,). + seed : int, optional + Seed for the random number generator, by default 0. + verbose : bool, optional + Control whether to display verbose messages during writing, by default True. + + """ + + super().__init__( + num_units=num_units, + sampling_frequency=sampling_frequency, + durations=durations, + seed=seed, + verbose=verbose, + ) + + def get_metadata(self) -> dict: # noqa D102 + metadata = super().get_metadata() + session_start_time = datetime.now().astimezone() + metadata["NWBFile"]["session_start_time"] = session_start_time + return metadata + + class MockImagingInterface(BaseImagingExtractorInterface): """ A mock imaging interface for testing purposes. diff --git a/tests/test_ecephys/test_ecephys_interfaces.py b/tests/test_ecephys/test_ecephys_interfaces.py index 6372b3535..5591bb6fb 100644 --- a/tests/test_ecephys/test_ecephys_interfaces.py +++ b/tests/test_ecephys/test_ecephys_interfaces.py @@ -20,7 +20,10 @@ BaseSortingExtractorInterface, ) from neuroconv.tools.nwb_helpers import get_module -from neuroconv.tools.testing.mock_interfaces import MockRecordingInterface +from neuroconv.tools.testing.mock_interfaces import ( + MockRecordingInterface, + MockSortingInterface, +) python_version = Version(get_python_version()) @@ -67,7 +70,33 @@ def test_spike2_import_assertions_3_11(self): Spike2RecordingInterface.get_all_channels_info(file_path="does_not_matter.smrx") -class TestSortingInterface(unittest.TestCase): +class TestSortingInterface: + + def test_run_conversion(self, tmp_path): + + nwbfile_path = Path(tmp_path) / "test_sorting.nwb" + num_units = 4 + interface = MockSortingInterface(num_units=num_units, durations=(1.0,)) + interface.sorting_extractor = interface.sorting_extractor.rename_units(new_unit_ids=["a", "b", "c", "d"]) + + interface.run_conversion(nwbfile_path=nwbfile_path) + with NWBHDF5IO(nwbfile_path, "r") as io: + nwbfile = io.read() + + units = nwbfile.units + assert len(units) == num_units + units_df = units.to_dataframe() + # Get index in units table + for unit_id in interface.sorting_extractor.unit_ids: + # In pynwb we write unit name as unit_id + row = units_df.query(f"unit_name == '{unit_id}'") + spike_times = interface.sorting_extractor.get_unit_spike_train(unit_id=unit_id, return_times=True) + written_spike_times = row["spike_times"].iloc[0] + + np.testing.assert_array_equal(spike_times, written_spike_times) + + +class TestSortingInterfaceOld(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.test_dir = Path(mkdtemp()) From 4c7e02724369b1656e6eb0200c0dbcac8124a60b Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 16 Sep 2024 12:10:02 -0600 Subject: [PATCH 050/118] Enable testing arm64 Mac architecture in the CI. (#1061) --- .github/workflows/doctests.yml | 2 +- .github/workflows/live-service-testing.yml | 2 +- .github/workflows/testing.yml | 2 +- CHANGELOG.md | 1 + docs/conversion_examples_gallery/conftest.py | 16 +++++++++++++++ .../behavior/deeplabcut/_dlc_utils.py | 5 ++++- .../behavior/deeplabcut/requirements.txt | 5 ++--- .../behavior/test_behavior_interfaces.py | 20 +++++++++++++++++++ 8 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index c3e467fe0..42af37800 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, macos-13, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index 28be2d94a..c35cf8a06 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, macos-13, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1d0fb9429..2e6528e4a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, macos-13, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/CHANGELOG.md b/CHANGELOG.md index 7570045b5..ae45a6856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ## Improvements +* Testing on mac sillicon [PR #1061](https://github.com/catalystneuro/neuroconv/pull/1061) * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) diff --git a/docs/conversion_examples_gallery/conftest.py b/docs/conversion_examples_gallery/conftest.py index 21c392bf0..6618d6d52 100644 --- a/docs/conversion_examples_gallery/conftest.py +++ b/docs/conversion_examples_gallery/conftest.py @@ -1,6 +1,8 @@ +import platform from pathlib import Path import pytest +from packaging import version from tests.test_on_data.setup_paths import ( BEHAVIOR_DATA_PATH, @@ -19,3 +21,17 @@ def add_data_space(doctest_namespace, tmp_path): doctest_namespace["path_to_save_nwbfile"] = Path(tmp_path) / "doctest_file.nwb" doctest_namespace["output_folder"] = Path(tmp_path) + + + +python_version = platform.python_version() +os = platform.system() +# Hook to conditionally skip doctests in deeplabcut.rst for Python 3.9 on macOS (Darwin) +def pytest_runtest_setup(item): + if isinstance(item, pytest.DoctestItem): + # Check if we are running the doctest from deeplabcut.rst + test_file = Path(item.fspath) + if test_file.name == "deeplabcut.rst": + # Check if Python version is 3.9 and platform is Darwin (macOS) + if version.parse(python_version) < version.parse("3.10") and os == "Darwin": + pytest.skip("Skipping doctests for deeplabcut.rst on Python 3.9 and macOS") diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 26c5b15fe..9e368fb39 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -379,7 +379,10 @@ def add_subject_to_nwbfile( video_name, scorer = h5file.stem.split("DLC") scorer = "DLC" + scorer - df = _ensure_individuals_in_header(pd.read_hdf(h5file), individual_name) + # TODO probably could be read directly with h5py + # This requires pytables + data_frame_from_hdf5 = pd.read_hdf(h5file) + df = _ensure_individuals_in_header(data_frame_from_hdf5, individual_name) # Note the video here is a tuple of the video path and the image shape if config_file is not None: diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt b/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt index 03e0ee0b0..3a5f2fd19 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt @@ -1,4 +1,3 @@ -tables<3.9.2;sys_platform=="darwin" -tables;sys_platform=="linux" or sys_platform=="win32" +tables>=3.10.1; platform_system == 'Darwin' and python_version >= '3.10' +tables; platform_system != 'Darwin' ndx-pose==0.1.1 -neuroconv[video] diff --git a/tests/test_on_data/behavior/test_behavior_interfaces.py b/tests/test_on_data/behavior/test_behavior_interfaces.py index 348d37c49..8e3e01d61 100644 --- a/tests/test_on_data/behavior/test_behavior_interfaces.py +++ b/tests/test_on_data/behavior/test_behavior_interfaces.py @@ -320,6 +320,18 @@ class TestFicTracDataInterfaceTiming(TemporalAlignmentMixin): save_directory = OUTPUT_PATH +from platform import python_version + +from packaging import version + +python_version = version.parse(python_version()) +from sys import platform + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10"), + reason="interface not supported on macOS with Python < 3.10", +) class TestDeepLabCutInterface(DeepLabCutInterfaceMixin): data_interface_cls = DeepLabCutInterface interface_kwargs = dict( @@ -365,6 +377,10 @@ def check_read_nwb(self, nwbfile_path: str): assert all(expected_pose_estimation_series_are_in_nwb_file) +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10"), + reason="interface not supported on macOS with Python < 3.10", +) class TestDeepLabCutInterfaceNoConfigFile(DataInterfaceTestMixin): data_interface_cls = DeepLabCutInterface interface_kwargs = dict( @@ -391,6 +407,10 @@ def check_read_nwb(self, nwbfile_path: str): assert all(expected_pose_estimation_series_are_in_nwb_file) +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10"), + reason="interface not supported on macOS with Python < 3.10", +) class TestDeepLabCutInterfaceSetTimestamps(DeepLabCutInterfaceMixin): data_interface_cls = DeepLabCutInterface interface_kwargs = dict( From f950792abe6135dfe7a26f80c3a8977fe7d2f463 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Tue, 17 Sep 2024 08:09:29 +1000 Subject: [PATCH 051/118] Consolidate daily workflows into one workflow and add email notifs (#1081) --- .../build_and_upload_docker_image_dev.yml | 8 +- .github/workflows/dailies.yml | 115 +++++++++++++++++- .github/workflows/dev-testing.yml | 3 +- .github/workflows/live-service-testing.yml | 3 +- .../workflows/neuroconv_docker_testing.yml | 10 +- .github/workflows/rclone_docker_testing.yml | 10 +- .github/workflows/update-s3-testing-data.yml | 10 +- CHANGELOG.md | 1 + ...ocker_yaml_conversion_specification_cli.py | 52 ++++++-- 9 files changed, 187 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build_and_upload_docker_image_dev.yml b/.github/workflows/build_and_upload_docker_image_dev.yml index d7c72b052..74d60e283 100644 --- a/.github/workflows/build_and_upload_docker_image_dev.yml +++ b/.github/workflows/build_and_upload_docker_image_dev.yml @@ -1,9 +1,13 @@ name: Build and Upload Docker Image of Current Dev Branch to GHCR on: - schedule: - - cron: "0 13 * * *" # Daily at 9 EST workflow_dispatch: + workflow_call: + secrets: + DOCKER_UPLOADER_USERNAME: + required: true + DOCKER_UPLOADER_PASSWORD: + required: true concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 54f8d0235..459246091 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -3,9 +3,15 @@ name: Daily workflows on: workflow_dispatch: schedule: - - cron: "0 16 * * *" # Daily at noon EST + - cron: "0 4 * * *" # Daily at 8PM PST, 11PM EST, 5AM CET to avoid working hours jobs: + build-and-upload-docker-image-dev: + uses: ./.github/workflows/build_and_upload_docker_image_dev.yml + secrets: + DOCKER_UPLOADER_USERNAME: ${{ secrets.DOCKER_UPLOADER_USERNAME }} + DOCKER_UPLOADER_PASSWORD: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} + run-daily-tests: uses: ./.github/workflows/testing.yml secrets: @@ -14,9 +20,52 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + run-daily-dev-tests: + uses: ./.github/workflows/dev-testing.yml + secrets: + DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + + run-daily-live-service-testing: + uses: ./.github/workflows/live-service-testing.yml + secrets: + DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + + run-daily-neuroconv-docker-testing: + uses: ./.github/workflows/neuroconv_docker_testing.yml + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + + run-daily-rclone-docker-testing: + uses: ./.github/workflows/rclone_docker_testing.yml + secrets: + RCLONE_DRIVE_ACCESS_TOKEN: ${{ secrets.RCLONE_DRIVE_ACCESS_TOKEN }} + RCLONE_DRIVE_REFRESH_TOKEN: ${{ secrets.RCLONE_DRIVE_REFRESH_TOKEN }} + RCLONE_EXPIRY_TOKEN: ${{ secrets.RCLONE_EXPIRY_TOKEN }} + run-daily-doc-link-checks: uses: ./.github/workflows/test-external-links.yml + notify-build-and-upload-docker-image-dev: + runs-on: ubuntu-latest + needs: [build-and-upload-docker-image-dev] + if: ${{ always() && needs.build-and-upload-docker-image-dev.result == 'failure' }} + steps: + - uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: NeuroConv Daily Docker Image Build and Upload Failure + to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} + from: NeuroConv + body: "The daily build and upload of the Docker image failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml" + notify-test-failure: runs-on: ubuntu-latest needs: [run-daily-tests] @@ -33,6 +82,70 @@ jobs: from: NeuroConv body: "The daily test workflow failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml" + notify-dev-test-failure: + runs-on: ubuntu-latest + needs: [run-daily-dev-tests] + if: ${{ always() && needs.run-daily-dev-tests.result == 'failure' }} + steps: + - uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: NeuroConv Daily Dev Test Failure + to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} + from: NeuroConv + body: "The daily dev test workflow failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml" + + notify-live-service-test-failure: + runs-on: ubuntu-latest + needs: [run-daily-live-service-testing] + if: ${{ always() && needs.run-daily-live-service-testing.result == 'failure' }} + steps: + - uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: NeuroConv Daily Live Service Test Failure + to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} + from: NeuroConv + body: "The daily live service test workflow failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml" + + notify-neuroconv-docker-test-failure: + runs-on: ubuntu-latest + needs: [run-daily-neuroconv-docker-testing] + if: ${{ always() && needs.run-daily-neuroconv-docker-testing.result == 'failure' }} + steps: + - uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: NeuroConv Daily NeuroConv Docker Test Failure + to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} + from: NeuroConv + body: "The daily neuroconv docker test workflow failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml" + + notify-rclone-docker-test-failure: + runs-on: ubuntu-latest + needs: [run-daily-rclone-docker-testing] + if: ${{ always() && needs.run-daily-rclone-docker-testing.result == 'failure' }} + steps: + - uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: NeuroConv Daily Rclone Docker Test Failure + to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} + from: NeuroConv + body: "The daily rclone docker test workflow failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/dailies.yml" + notify-link-check-failure: runs-on: ubuntu-latest needs: [run-daily-doc-link-checks] diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index 65be5bffd..463ac4403 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -1,7 +1,6 @@ name: Dev Branch Testing on: - schedule: - - cron: "0 16 * * *" # Daily at noon EST + workflow_dispatch: workflow_call: secrets: DANDI_API_KEY: diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index c35cf8a06..28c9fff26 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -1,7 +1,6 @@ name: Live service testing on: - schedule: - - cron: "0 16 * * *" # Daily at noon EST + workflow_dispatch: workflow_call: secrets: DANDI_API_KEY: diff --git a/.github/workflows/neuroconv_docker_testing.yml b/.github/workflows/neuroconv_docker_testing.yml index 282da7937..dd151d607 100644 --- a/.github/workflows/neuroconv_docker_testing.yml +++ b/.github/workflows/neuroconv_docker_testing.yml @@ -1,8 +1,14 @@ name: NeuroConv Docker CLI tests on: - schedule: - - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: + workflow_call: + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + S3_GIN_BUCKET: + required: true jobs: run: diff --git a/.github/workflows/rclone_docker_testing.yml b/.github/workflows/rclone_docker_testing.yml index 2e8ea9e17..de1c333d6 100644 --- a/.github/workflows/rclone_docker_testing.yml +++ b/.github/workflows/rclone_docker_testing.yml @@ -1,8 +1,14 @@ name: Rclone Docker Tests on: - schedule: - - cron: "0 16 * * *" # Daily at noon EST workflow_dispatch: + workflow_call: + secrets: + RCLONE_DRIVE_ACCESS_TOKEN: + required: true + RCLONE_DRIVE_REFRESH_TOKEN: + required: true + RCLONE_EXPIRY_TOKEN: + required: true jobs: run: diff --git a/.github/workflows/update-s3-testing-data.yml b/.github/workflows/update-s3-testing-data.yml index 902770864..563f05ac5 100644 --- a/.github/workflows/update-s3-testing-data.yml +++ b/.github/workflows/update-s3-testing-data.yml @@ -1,8 +1,14 @@ name: Update S3 Testing Data on: - schedule: - - cron: "0 0 * * *" workflow_dispatch: + workflow_call: + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + S3_GIN_BUCKET: + required: true jobs: run: diff --git a/CHANGELOG.md b/CHANGELOG.md index ae45a6856..79d4c3218 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * Testing on mac sillicon [PR #1061](https://github.com/catalystneuro/neuroconv/pull/1061) * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) +* Consolidated daily workflows into one workflow and added email notifications [PR #1081](https://github.com/catalystneuro/neuroconv/pull/1081) ## v0.6.3 diff --git a/tests/docker_yaml_conversion_specification_cli.py b/tests/docker_yaml_conversion_specification_cli.py index c15d2e780..586b487df 100644 --- a/tests/docker_yaml_conversion_specification_cli.py +++ b/tests/docker_yaml_conversion_specification_cli.py @@ -21,20 +21,33 @@ class TestLatestDockerYAMLConversionSpecification(TestCase): test_folder = OUTPUT_PATH tag = os.getenv("NEUROCONV_DOCKER_TESTS_TAG", "latest") source_volume = os.getenv("NEUROCONV_DOCKER_TESTS_SOURCE_VOLUME", "/home/runner/work/neuroconv/neuroconv") + # If running locally, export NEUROCONV_DOCKER_TESTS_SOURCE_VOLUME=/path/to/neuroconv def test_run_conversion_from_yaml_cli(self): - path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" + path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "test_yaml" / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" - - output = deploy_process( - command=( + if self.source_volume == "/home/runner/work/neuroconv/neuroconv": # in CI + command = ( "docker run -t " f"--volume {self.source_volume}:{self.source_volume} " f"--volume {self.test_folder}:{self.test_folder} " f"ghcr.io/catalystneuro/neuroconv:{self.tag} " f"neuroconv {yaml_file_path} " f"--data-folder-path {self.source_volume}/{DATA_PATH} --output-folder-path {self.test_folder} --overwrite" - ), + ) + else: # running locally + command = ( + "docker run -t " + f"--volume {self.source_volume}:{self.source_volume} " + f"--volume {self.test_folder}:{self.test_folder} " + f"--volume {DATA_PATH}:{DATA_PATH} " + f"ghcr.io/catalystneuro/neuroconv:{self.tag} " + f"neuroconv {yaml_file_path} " + f"--data-folder-path {DATA_PATH} --output-folder-path {self.test_folder} --overwrite" + ) + + output = deploy_process( + command=command, catch_output=True, ) print(output) @@ -64,7 +77,7 @@ def test_run_conversion_from_yaml_cli(self): assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() - assert nwbfile.session_description == "Auto-generated by neuroconv" + assert nwbfile.session_description == "" assert nwbfile.lab == "My Lab" assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") @@ -72,7 +85,7 @@ def test_run_conversion_from_yaml_cli(self): assert "spike_times" in nwbfile.units def test_run_conversion_from_yaml_variable(self): - path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "conversion_specifications" + path_to_test_yml_files = Path(__file__).parent / "test_on_data" / "test_yaml" / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification.yml" with open(file=yaml_file_path, mode="r") as io: @@ -80,11 +93,11 @@ def test_run_conversion_from_yaml_variable(self): yaml_string = "".join(yaml_lines) os.environ["NEUROCONV_YAML"] = yaml_string - os.environ["NEUROCONV_DATA_PATH"] = self.source_volume + str(DATA_PATH) os.environ["NEUROCONV_OUTPUT_PATH"] = str(self.test_folder) - output = deploy_process( - command=( + if self.source_volume == "/home/runner/work/neuroconv/neuroconv": # in CI + os.environ["NEUROCONV_DATA_PATH"] = self.source_volume + str(DATA_PATH) + command = ( "docker run -t " f"--volume {self.source_volume}:{self.source_volume} " f"--volume {self.test_folder}:{self.test_folder} " @@ -92,7 +105,22 @@ def test_run_conversion_from_yaml_variable(self): '-e NEUROCONV_DATA_PATH="$NEUROCONV_DATA_PATH" ' '-e NEUROCONV_OUTPUT_PATH="$NEUROCONV_OUTPUT_PATH" ' "ghcr.io/catalystneuro/neuroconv:yaml_variable" - ), + ) + else: # running locally + os.environ["NEUROCONV_DATA_PATH"] = str(DATA_PATH) + command = ( + "docker run -t " + f"--volume {self.source_volume}:{self.source_volume} " + f"--volume {self.test_folder}:{self.test_folder} " + f"--volume {DATA_PATH}:{DATA_PATH} " + '-e NEUROCONV_YAML="$NEUROCONV_YAML" ' + '-e NEUROCONV_DATA_PATH="$NEUROCONV_DATA_PATH" ' + '-e NEUROCONV_OUTPUT_PATH="$NEUROCONV_OUTPUT_PATH" ' + "ghcr.io/catalystneuro/neuroconv:yaml_variable" + ) + + output = deploy_process( + command=command, catch_output=True, ) print(output) @@ -122,7 +150,7 @@ def test_run_conversion_from_yaml_variable(self): assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() - assert nwbfile.session_description == "Auto-generated by neuroconv" + assert nwbfile.session_description == "" assert nwbfile.lab == "My Lab" assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-11T21:19:09+00:00") From 36464df599214140a2626591d23f38415dc689f1 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 16 Sep 2024 19:53:26 -0600 Subject: [PATCH 052/118] Fix stream in Plexon 1 interface (#1013) --- .github/workflows/dev-testing.yml | 2 +- .github/workflows/doctests.yml | 2 +- .../formatwise-installation-testing.yml | 2 +- .github/workflows/neuroconv_docker_testing.yml | 2 +- .github/workflows/testing.yml | 2 +- CHANGELOG.md | 1 + .../recording/plexon.rst | 2 +- .../ecephys/plexon/plexondatainterface.py | 17 +++++++++++++++-- .../datainterfaces/ecephys/requirements.txt | 2 +- .../ecephys/test_recording_interfaces.py | 5 ++--- 10 files changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index 463ac4403..c85d80fe0 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -74,7 +74,7 @@ jobs: id: cache-ephys-datasets with: path: ./ephy_testing_data - key: ephys-datasets-2024-06-21-ubuntu-latest-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + key: ephys-datasets-2024-08-30-ubuntu-latest-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index 42af37800..ed4bd7cd6 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -42,7 +42,7 @@ jobs: id: cache-ephys-datasets with: path: ./ephy_testing_data - key: ephys-datasets-2024-08-07-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + key: ephys-datasets-2024-08-30-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" diff --git a/.github/workflows/formatwise-installation-testing.yml b/.github/workflows/formatwise-installation-testing.yml index ea6af41c9..0dbcd2483 100644 --- a/.github/workflows/formatwise-installation-testing.yml +++ b/.github/workflows/formatwise-installation-testing.yml @@ -44,7 +44,7 @@ jobs: id: cache-ephys-datasets with: path: ./ephy_testing_data - key: ephys-datasets-2024-08-27-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + key: ephys-datasets-2024-08-30-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" diff --git a/.github/workflows/neuroconv_docker_testing.yml b/.github/workflows/neuroconv_docker_testing.yml index dd151d607..c86a83d80 100644 --- a/.github/workflows/neuroconv_docker_testing.yml +++ b/.github/workflows/neuroconv_docker_testing.yml @@ -43,7 +43,7 @@ jobs: id: cache-ephys-datasets with: path: ./ephy_testing_data - key: ephys-datasets-2023-06-26-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + key: ephys-datasets-2024-08-30-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2e6528e4a..5ccdb6c33 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -94,7 +94,7 @@ jobs: id: cache-ephys-datasets with: path: ./ephy_testing_data - key: ephys-datasets-2024-08-07-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + key: ephys-datasets-2024-08-30-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" diff --git a/CHANGELOG.md b/CHANGELOG.md index 79d4c3218..693b6d184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bug Fixes * Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) * The method `get_extractor` now works for `MockImagingInterface` [PR #1076](https://github.com/catalystneuro/neuroconv/pull/1076) +* Solved a bug of `PlexonRecordingInterface` where data with multiple streams could not be opened [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) ## Deprecations diff --git a/docs/conversion_examples_gallery/recording/plexon.rst b/docs/conversion_examples_gallery/recording/plexon.rst index 33146ceb1..0d11a9c6e 100644 --- a/docs/conversion_examples_gallery/recording/plexon.rst +++ b/docs/conversion_examples_gallery/recording/plexon.rst @@ -16,7 +16,7 @@ Convert Plexon recording data to NWB using :py:class:`~neuroconv.datainterfaces. >>> from pathlib import Path >>> from neuroconv.datainterfaces import PlexonRecordingInterface >>> - >>> file_path = f"{ECEPHY_DATA_PATH}/plexon/File_plexon_3.plx" + >>> file_path = f"{ECEPHY_DATA_PATH}/plexon/4chDemoPLX.plx" >>> # Change the file_path to the location in your system >>> interface = PlexonRecordingInterface(file_path=file_path, verbose=False) >>> diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py index fc3dbd3d0..5653b1586 100644 --- a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py @@ -25,7 +25,13 @@ def get_source_schema(cls) -> dict: return source_schema @validate_call - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__( + self, + file_path: FilePath, + verbose: bool = True, + es_key: str = "ElectricalSeries", + stream_name: str = "WB-Wideband", + ): """ Load and prepare data for Plexon. @@ -36,8 +42,15 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele verbose : bool, default: True Allows verbosity. es_key : str, default: "ElectricalSeries" + stream_name: str, optional + Only pass a stream if you modified the channel prefixes in the Plexon file and you know the prefix of + the wideband data. """ - super().__init__(file_path=file_path, verbose=verbose, es_key=es_key) + + invalid_stream_names = ["FPl-Low Pass Filtered", "SPKC-High Pass Filtered", "AI-Auxiliary Input"] + assert stream_name not in invalid_stream_names, f"Invalid stream name: {stream_name}" + + super().__init__(file_path=file_path, verbose=verbose, es_key=es_key, stream_name=stream_name) def get_metadata(self) -> DeepDict: metadata = super().get_metadata() diff --git a/src/neuroconv/datainterfaces/ecephys/requirements.txt b/src/neuroconv/datainterfaces/ecephys/requirements.txt index 938056d7a..bcd36de3f 100644 --- a/src/neuroconv/datainterfaces/ecephys/requirements.txt +++ b/src/neuroconv/datainterfaces/ecephys/requirements.txt @@ -1,2 +1,2 @@ spikeinterface>=0.101.0 -neo>=0.13.2 +neo>=0.13.3 diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index cc83625dc..7eb94c7e1 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -693,10 +693,9 @@ def check_read_nwb(self, nwbfile_path: str): class TestPlexonRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = PlexonRecordingInterface interface_kwargs = dict( - # Only File_plexon_3.plx has an ecephys recording stream - file_path=str(ECEPHY_DATA_PATH / "plexon" / "File_plexon_3.plx"), + file_path=str(ECEPHY_DATA_PATH / "plexon" / "4chDemoPLX.plx"), ) save_directory = OUTPUT_PATH def check_extracted_metadata(self, metadata: dict): - assert metadata["NWBFile"]["session_start_time"] == datetime(2010, 2, 22, 20, 0, 57) + assert metadata["NWBFile"]["session_start_time"] == datetime(2013, 11, 19, 13, 48, 13) From fa6990035a8e6e4729760f6271b583d8e45580b7 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Wed, 18 Sep 2024 00:19:41 +1000 Subject: [PATCH 053/118] updated opencv version for security (#1087) --- CHANGELOG.md | 1 + src/neuroconv/datainterfaces/behavior/video/requirements.txt | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 693b6d184..f5bebee40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bug Fixes * Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) * The method `get_extractor` now works for `MockImagingInterface` [PR #1076](https://github.com/catalystneuro/neuroconv/pull/1076) +* Updated opencv version for security [PR #1087](https://github.com/catalystneuro/neuroconv/pull/1087) * Solved a bug of `PlexonRecordingInterface` where data with multiple streams could not be opened [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) ## Deprecations diff --git a/src/neuroconv/datainterfaces/behavior/video/requirements.txt b/src/neuroconv/datainterfaces/behavior/video/requirements.txt index 3df644bd1..650f0426f 100644 --- a/src/neuroconv/datainterfaces/behavior/video/requirements.txt +++ b/src/neuroconv/datainterfaces/behavior/video/requirements.txt @@ -1,3 +1 @@ -opencv-python-headless>=4.5.1.48 -opencv-python-headless>=4.5.1.48,<4.7.0.72; sys_platform == 'darwin' -opencv-python-headless>=4.5.1.48,<4.7; sys_platform == 'darwin' and python_version>='3.11' +opencv-python-headless>=4.8.1.78 From 4dad0ef4929dbe8e50d5540bab0f6cd9a22c8a2d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 15:40:12 -0600 Subject: [PATCH 054/118] Enable zarr backend testing in data tests [2] (#1083) --- CHANGELOG.md | 4 ++ src/neuroconv/basedatainterface.py | 11 ++-- .../tools/testing/data_interface_mixins.py | 53 +++++++------------ .../ecephys/test_recording_interfaces.py | 16 ++++++ 4 files changed, 44 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bebee40..abe20834c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ## Features * Added chunking/compression for string-only compound objects: [PR #1042](https://github.com/catalystneuro/neuroconv/pull/1042) * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) +* Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) * Added a `MockSortingInterface` for testing purposes. [PR #1065](https://github.com/catalystneuro/neuroconv/pull/1065) @@ -21,6 +22,9 @@ * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) * Consolidated daily workflows into one workflow and added email notifications [PR #1081](https://github.com/catalystneuro/neuroconv/pull/1081) +* Added zarr tests for the test on data with checking equivalent backends [PR #1083](https://github.com/catalystneuro/neuroconv/pull/1083) + + ## v0.6.3 diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 59415b043..682ed2dba 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -127,11 +127,8 @@ def run_conversion( nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - # TODO: when all H5DataIO prewraps are gone, introduce Zarr safely - # backend: Union[Literal["hdf5", "zarr"]], - # backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None, - backend: Optional[Literal["hdf5"]] = None, - backend_configuration: Optional[HDF5BackendConfiguration] = None, + backend: Optional[Literal["hdf5", "zarr"]] = None, + backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None, **conversion_options, ): """ @@ -148,11 +145,11 @@ def run_conversion( overwrite : bool, default: False Whether to overwrite the NWBFile if one exists at the nwbfile_path. The default is False (append mode). - backend : "hdf5", optional + backend : {"hdf5", "zarr"}, optional The type of backend to use when writing the file. If a `backend_configuration` is not specified, the default type will be "hdf5". If a `backend_configuration` is specified, then the type will be auto-detected. - backend_configuration : HDF5BackendConfiguration, optional + backend_configuration : HDF5BackendConfiguration or ZarrBackendConfiguration, optional The configuration model to use when configuring the datasets for this backend. To customize, call the `.get_default_backend_configuration(...)` method, modify the returned BackendConfiguration object, and pass that instead. diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 2b8252154..d3107650b 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -105,33 +105,8 @@ def test_no_metadata_mutation(self, setup_interface): metadata_before_add_method = deepcopy(metadata) self.interface.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **self.conversion_options) - assert metadata == metadata_before_add_method - def check_run_conversion_with_backend(self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5"): - metadata = self.interface.get_metadata() - if "session_start_time" not in metadata["NWBFile"]: - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - self.interface.run_conversion( - nwbfile_path=nwbfile_path, - overwrite=True, - metadata=metadata, - backend=backend, - **self.conversion_options, - ) - - def check_configure_backend_for_equivalent_nwbfiles(self, backend: Literal["hdf5", "zarr"] = "hdf5"): - metadata = self.interface.get_metadata() - if "session_start_time" not in metadata["NWBFile"]: - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - nwbfile_1 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) - nwbfile_2 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) - - backend_configuration = get_default_backend_configuration(nwbfile=nwbfile_1, backend=backend) - configure_backend(nwbfile=nwbfile_2, backend_configuration=backend_configuration) - def check_run_conversion_with_backend_configuration( self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5" ): @@ -204,11 +179,6 @@ def check_read_nwb(self, nwbfile_path: str): """Read the produced NWB file and compare it to the interface.""" pass - def check_basic_zarr_read(self, nwbfile_path: str): - """Ensure NWBZarrIO can read the file.""" - with NWBZarrIO(path=nwbfile_path, mode="r") as io: - io.read() - def check_extracted_metadata(self, metadata: dict): """Override this method to make assertions about specific extracted metadata values.""" pass @@ -235,7 +205,20 @@ def test_run_conversion_with_backend(self, setup_interface, tmp_path, backend): ) if backend == "zarr": - self.check_basic_zarr_read(nwbfile_path) + with NWBZarrIO(path=nwbfile_path, mode="r") as io: + io.read() + + @pytest.mark.parametrize("backend", ["hdf5", "zarr"]) + def test_configure_backend_for_equivalent_nwbfiles(self, setup_interface, tmp_path, backend): + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + nwbfile_1 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) + nwbfile_2 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) + + backend_configuration = get_default_backend_configuration(nwbfile=nwbfile_1, backend=backend) + configure_backend(nwbfile=nwbfile_2, backend_configuration=backend_configuration) def test_all_conversion_checks(self, setup_interface, tmp_path): interface, test_name = setup_interface @@ -247,7 +230,6 @@ def test_all_conversion_checks(self, setup_interface, tmp_path): # Now run the checks using the setup objects self.check_conversion_options_schema_valid() self.check_metadata() - self.check_configure_backend_for_equivalent_nwbfiles() self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") @@ -746,7 +728,6 @@ def test_all_conversion_checks(self, setup_interface, tmp_path): # Now run the checks using the setup objects self.check_conversion_options_schema_valid() self.check_metadata() - self.check_configure_backend_for_equivalent_nwbfiles() self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") @@ -900,6 +881,9 @@ def test_run_conversion_with_backend(self): def test_no_metadata_mutation(self): pass + def test_configure_backend_for_equivalent_nwbfiles(self): + pass + def check_metadata_schema_valid(self): schema = self.interface.get_metadata_schema() Draft7Validator.check_schema(schema=schema) @@ -1263,6 +1247,9 @@ def test_run_conversion_with_backend(self): def test_no_metadata_mutation(self): pass + def test_configure_backend_for_equivalent_nwbfiles(self): + pass + def check_metadata_schema_valid(self): schema = self.interface.get_metadata_schema() Draft7Validator.check_schema(schema=schema) diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 7eb94c7e1..00630afbf 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -179,6 +179,19 @@ class TestEDFRecordingInterface(RecordingExtractorInterfaceTestMixin): def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 3, 2, 10, 42, 19) + def check_run_conversion_with_backend(self, nwbfile_path: str, backend="hdf5"): + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + self.interface.run_conversion( + nwbfile_path=nwbfile_path, + overwrite=True, + metadata=metadata, + backend=backend, + **self.conversion_options, + ) + def test_all_conversion_checks(self, setup_interface, tmp_path): # Create a unique test name and file path nwbfile_path = str(tmp_path / f"{self.__class__.__name__}.nwb") @@ -205,6 +218,9 @@ def test_run_conversion_with_backend(self): def test_interface_alignment(self): pass + def test_configure_backend_for_equivalent_nwbfiles(self): + pass + class TestIntanRecordingInterfaceRHS(RecordingExtractorInterfaceTestMixin): data_interface_cls = IntanRecordingInterface From 5d154770cfd3474373913b115567b31d1358184a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 16:56:25 -0600 Subject: [PATCH 055/118] Release v0.6.4 --- CHANGELOG.md | 10 +++++----- pyproject.toml | 11 +++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abe20834c..86ea9028a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Upcoming -## v0.6.4 +# v0.6.4 (September 17, 2024) ## Bug Fixes * Fixed a setup bug introduced in `v0.6.2` where installation process created a directory instead of a file for test configuration file [PR #1070](https://github.com/catalystneuro/neuroconv/pull/1070) @@ -26,10 +26,10 @@ -## v0.6.3 +# v0.6.3 -## v0.6.2 (September 10, 2024) +# v0.6.2 (September 10, 2024) ### Bug Fixes * Fixed a bug where `IntanRecordingInterface` added two devices [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) @@ -55,14 +55,14 @@ -## v0.6.1 (August 30, 2024) +# v0.6.1 (August 30, 2024) ### Bug fixes * Fixed the JSON schema inference warning on excluded fields; also improved error message reporting of which method triggered the error. [PR #1037](https://github.com/catalystneuro/neuroconv/pull/1037) -## v0.6.0 (August 27, 2024) +# v0.6.0 (August 27, 2024) ### Deprecations * Deprecated `WaveformExtractor` usage. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821) diff --git a/pyproject.toml b/pyproject.toml index 743f4e57b..9baf07049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,9 +15,10 @@ authors = [ { name = "Luiz Tauffer" }, { name = "Ben Dichter", email = "ben.dichter@catalystneuro.com" }, ] -urls = { "Homepage" = "https://github.com/catalystneuro/neuroconv" } + + license = { file = "license.txt" } -keywords = ["nwb"] +keywords = ["nwb", "NeurodataWithoutBorders"] classifiers = [ "Intended Audience :: Science/Research", "Programming Language :: Python :: 3.9", @@ -53,6 +54,12 @@ dependencies = [ ] +[project.urls] +"Homepage" = "https://github.com/catalystneuro/neuroconv" +"Documentation" = "https://neuroconv.readthedocs.io/en/latest/" +"Changelog" = "https://github.com/catalystneuro/neuroconv/blob/main/CHANGELOG.md" + + [project.optional-dependencies] test = [ "pytest", From 9b00afd9598dab695e3ffa55b301e5bc8802b6d6 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 17:18:11 -0600 Subject: [PATCH 056/118] bump version --- CHANGELOG.md | 26 +++++++++++++++++--------- pyproject.toml | 4 ++-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86ea9028a..38fdcd5a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Upcoming +## Bug Fixes + +## Deprecations + +## Features + +## Improvements + # v0.6.4 (September 17, 2024) ## Bug Fixes @@ -31,21 +39,21 @@ # v0.6.2 (September 10, 2024) -### Bug Fixes +## Bug Fixes * Fixed a bug where `IntanRecordingInterface` added two devices [PR #1059](https://github.com/catalystneuro/neuroconv/pull/1059) * Fix a bug in `add_sorting_to_nwbfile` where `unit_electrode_indices` was only propagated if `waveform_means` was passed [PR #1057](https://github.com/catalystneuro/neuroconv/pull/1057) -### Deprecations +## Deprecations * The following classes and objects are now private `NWBMetaDataEncoder`, `NWBMetaDataEncoder`, `check_if_imaging_fits_into_memory`, `NoDatesSafeLoader` [PR #1050](https://github.com/catalystneuro/neuroconv/pull/1050) -### Features +## Features * Make `config_file_path` optional in `DeepLabCutInterface`[PR #1031](https://github.com/catalystneuro/neuroconv/pull/1031) * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039) * Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022) * All remaining data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1055](https://github.com/catalystneuro/neuroconv/pull/1055) -### Improvements +## Improvements * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034) * Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049) * Added Unit Table descriptions for phy and kilosort: [PR #1053](https://github.com/catalystneuro/neuroconv/pull/1053) @@ -57,14 +65,14 @@ # v0.6.1 (August 30, 2024) -### Bug fixes +## Bug fixes * Fixed the JSON schema inference warning on excluded fields; also improved error message reporting of which method triggered the error. [PR #1037](https://github.com/catalystneuro/neuroconv/pull/1037) # v0.6.0 (August 27, 2024) -### Deprecations +## Deprecations * Deprecated `WaveformExtractor` usage. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821) * Changed the `tools.spikeinterface` functions (e.g. `add_recording`, `add_sorting`) to have `_to_nwbfile` as suffix [PR #1015](https://github.com/catalystneuro/neuroconv/pull/1015) * Deprecated use of `compression` and `compression_options` in `VideoInterface` [PR #1005](https://github.com/catalystneuro/neuroconv/pull/1005) @@ -73,7 +81,7 @@ * Changed the `tools.roiextractors` function (e.g. `add_imaging` and `add_segmentation`) to have the `_to_nwbfile` suffix [PR #1017](https://github.com/catalystneuro/neuroconv/pull/1027) -### Features +## Features * Added `MedPCInterface` for operant behavioral output files. [PR #883](https://github.com/catalystneuro/neuroconv/pull/883) * Support `SortingAnalyzer` in the `SpikeGLXConverterPipe`. [PR #821](https://github.com/catalystneuro/neuroconv/pull/821) * Added `TDTFiberPhotometryInterface` data interface, for converting fiber photometry data from TDT file formats. [PR #920](https://github.com/catalystneuro/neuroconv/pull/920) @@ -84,12 +92,12 @@ * Data interfaces `run_conversion` method now performs metadata validation before running the conversion. [PR #949](https://github.com/catalystneuro/neuroconv/pull/949) * Introduced `null_values_for_properties` to `add_units_table` to give user control over null values behavior [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) -### Bug fixes +## Bug fixes * Fixed the default naming of multiple electrical series in the `SpikeGLXConverterPipe`. [PR #957](https://github.com/catalystneuro/neuroconv/pull/957) * Write new properties to the electrode table use the global identifier channel_name, group [PR #984](https://github.com/catalystneuro/neuroconv/pull/984) * Removed a bug where int64 was casted lossy to float [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) -### Improvements +## Improvements * The `OpenEphysBinaryRecordingInterface` now uses `lxml` for extracting the session start time from the settings.xml file and does not depend on `pyopenephys` anymore. [PR #971](https://github.com/catalystneuro/neuroconv/pull/971) * Swap the majority of package setup and build steps to `pyproject.toml` instead of `setup.py`. [PR #955](https://github.com/catalystneuro/neuroconv/pull/955) * The `DeeplabcutInterface` now skips inferring timestamps from movie when timestamps are specified, running faster. [PR #967](https://github.com/catalystneuro/neuroconv/pull/967) diff --git a/pyproject.toml b/pyproject.toml index 9baf07049..d7cf25813 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.4" +version = "0.6.5" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ @@ -56,7 +56,7 @@ dependencies = [ [project.urls] "Homepage" = "https://github.com/catalystneuro/neuroconv" -"Documentation" = "https://neuroconv.readthedocs.io/en/latest/" +"Documentation" = "https://neuroconv.readthedocs.io/" "Changelog" = "https://github.com/catalystneuro/neuroconv/blob/main/CHANGELOG.md" From 9dc05c6eb4dc792c33bf6232903915c69e756bc6 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 17:20:50 -0600 Subject: [PATCH 057/118] Using in-house GenericDataChunkIterator (#1068) --- CHANGELOG.md | 4 +++- src/neuroconv/datainterfaces/behavior/video/video_utils.py | 3 ++- .../nwb_helpers/_configuration_models/_base_dataset_io.py | 3 +-- .../tools/roiextractors/imagingextractordatachunkiterator.py | 3 ++- .../spikeinterfacerecordingdatachunkiterator.py | 3 ++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38fdcd5a1..22eae5727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,12 @@ # Upcoming +## Deprecations + ## Bug Fixes -## Deprecations ## Features +* Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) ## Improvements diff --git a/src/neuroconv/datainterfaces/behavior/video/video_utils.py b/src/neuroconv/datainterfaces/behavior/video/video_utils.py index a8f2412aa..fe817a3b2 100644 --- a/src/neuroconv/datainterfaces/behavior/video/video_utils.py +++ b/src/neuroconv/datainterfaces/behavior/video/video_utils.py @@ -2,10 +2,11 @@ from typing import Optional, Tuple import numpy as np -from hdmf.data_utils import GenericDataChunkIterator from pydantic import FilePath from tqdm import tqdm +from neuroconv.tools.hdmf import GenericDataChunkIterator + from ....tools import get_package diff --git a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py index 9cdb97405..8b4e98436 100644 --- a/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py +++ b/src/neuroconv/tools/nwb_helpers/_configuration_models/_base_dataset_io.py @@ -9,7 +9,6 @@ import numpy as np import zarr from hdmf import Container -from hdmf.data_utils import GenericDataChunkIterator from hdmf.utils import get_data_shape from pydantic import ( BaseModel, @@ -25,7 +24,7 @@ from neuroconv.utils.str_utils import human_readable_size from ._pydantic_pure_json_schema_generator import PureJSONSchemaGenerator -from ...hdmf import SliceableDataChunkIterator +from ...hdmf import GenericDataChunkIterator, SliceableDataChunkIterator def _recursively_find_location_in_memory_nwbfile(current_location: str, neurodata_object: Container) -> str: diff --git a/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py b/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py index 506967be0..43b077730 100644 --- a/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py +++ b/src/neuroconv/tools/roiextractors/imagingextractordatachunkiterator.py @@ -4,10 +4,11 @@ from typing import Optional import numpy as np -from hdmf.data_utils import GenericDataChunkIterator from roiextractors import ImagingExtractor from tqdm import tqdm +from neuroconv.tools.hdmf import GenericDataChunkIterator + class ImagingExtractorDataChunkIterator(GenericDataChunkIterator): """DataChunkIterator for ImagingExtractor objects primarily used when writing imaging data to an NWB file.""" diff --git a/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py b/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py index 5d6407f19..4218641cd 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterfacerecordingdatachunkiterator.py @@ -1,9 +1,10 @@ from typing import Iterable, Optional -from hdmf.data_utils import GenericDataChunkIterator from spikeinterface import BaseRecording from tqdm import tqdm +from neuroconv.tools.hdmf import GenericDataChunkIterator + class SpikeInterfaceRecordingDataChunkIterator(GenericDataChunkIterator): """DataChunkIterator specifically for use on RecordingExtractor objects.""" From 88a4c2f4085ae5db912b9e5b26328d26c3ebc52d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 17:21:24 -0600 Subject: [PATCH 058/118] Provide a fast CI option for draft status (#1082) --- .github/workflows/deploy-tests.yml | 39 +++++++++++++--------- .github/workflows/dev-testing.yml | 8 ++++- .github/workflows/doctests.yml | 16 +++++++-- .github/workflows/live-service-testing.yml | 16 +++++++-- .github/workflows/testing.yml | 22 ++++++++---- CHANGELOG.md | 1 + 6 files changed, 76 insertions(+), 26 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 9cbf6ba94..3afd5e956 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -2,11 +2,13 @@ name: Deploy tests on: pull_request: - types: [synchronize, opened, reopened, ready_for_review] # defaults + ready_for_review + types: [synchronize, opened, reopened, ready_for_review] + # Synchronize, open and reopened are the default types for pull request + # We add ready_for_review to trigger the check for changelog and full tests when ready for review is clicked merge_group: workflow_dispatch: -concurrency: # Cancel previous workflows on the same pull request +concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -21,9 +23,9 @@ jobs: name: Auto-detecting CHANGELOG.md updates runs-on: ubuntu-latest steps: - - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'true' }} + - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'true' }} run: echo "CHANGELOG.md has been updated." - - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'false' }} + - if: ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'false' }} run: | echo "CHANGELOG.md has not been updated." 0 @@ -37,7 +39,9 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} run-live-service-tests: needs: assess-file-changes @@ -45,6 +49,9 @@ jobs: uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} run-dev-tests: needs: assess-file-changes @@ -55,25 +62,27 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} run-doctests-only: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml + with: # Ternary operator: condition && value_if_true || value_if_false + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} check-final-status: name: All tests passing if: always() - needs: - - run-tests - - run-doctests-only - + - run-tests + - run-doctests-only runs-on: ubuntu-latest - steps: - - name: Decide whether the all jobs succeeded or at least one failed - uses: re-actors/alls-green@release/v1 - with: - allowed-skips: run-tests, run-doctests-only # Each has the option to skip depending on whether src changed - jobs: ${{ toJSON(needs) }} + - name: Decide whether all jobs succeeded or at least one failed + uses: re-actors/alls-green@release/v1 + with: + allowed-skips: run-tests, run-doctests-only + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index c85d80fe0..acd7a3d74 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -2,6 +2,12 @@ name: Dev Branch Testing on: workflow_dispatch: workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' secrets: DANDI_API_KEY: required: true @@ -22,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ${{ fromJson(inputs.python-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index ed4bd7cd6..d816dbd02 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -1,6 +1,18 @@ name: Run doctests on: workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: + description: 'List of OS versions to use in matrix, as JSON string' + required: true + type: string + default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + jobs: run: @@ -9,8 +21,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ${{ fromJson(inputs.python-versions) }} + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index 28c9fff26..b9a425a8d 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -2,6 +2,18 @@ name: Live service testing on: workflow_dispatch: workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: + description: 'List of OS versions to use in matrix, as JSON string' + required: true + type: string + default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + secrets: DANDI_API_KEY: required: true @@ -16,8 +28,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ${{ fromJson(inputs.python-versions) }} + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5ccdb6c33..736da6030 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,6 +1,20 @@ name: Minimal and Full Tests + on: workflow_call: + inputs: + python-versions: + description: 'List of Python versions to use in matrix, as JSON string' + required: true + type: string + default: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: + description: 'List of OS versions to use in matrix, as JSON string' + required: true + type: string + default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + + secrets: AWS_ACCESS_KEY_ID: required: true @@ -19,8 +33,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ${{ fromJson(inputs.python-versions) }} + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags @@ -79,13 +93,9 @@ jobs: #- name: Run icephys tests # There are no icephys specific tests without data # run: pytest tests/test_icephys -rsx -n auto --dist loadscope - - - name: Install full requirements run: pip install .[full] - - - name: Get ephy_testing_data current head hash id: ephys run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" diff --git a/CHANGELOG.md b/CHANGELOG.md index 22eae5727..495184eea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) * Consolidated daily workflows into one workflow and added email notifications [PR #1081](https://github.com/catalystneuro/neuroconv/pull/1081) +* Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) * Added zarr tests for the test on data with checking equivalent backends [PR #1083](https://github.com/catalystneuro/neuroconv/pull/1083) From 55c94a579b6b1ed99b12ac4be3ba31f3c8b1174a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:21:56 -0600 Subject: [PATCH 059/118] [pre-commit.ci] pre-commit autoupdate (#1066) Co-authored-by: Heberto Mayorquin --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8ee7373a..810976eb5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.3 + rev: v0.6.5 hooks: - id: ruff args: [ --fix ] From fddd4d28cb7839920479d8154f49417f78a391af Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 17 Sep 2024 17:26:07 -0600 Subject: [PATCH 060/118] update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 495184eea..5bd9c2160 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,11 @@ ## Bug Fixes - ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) ## Improvements +* Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) # v0.6.4 (September 17, 2024) @@ -32,7 +32,6 @@ * Add writing to zarr test for to the test on data [PR #1056](https://github.com/catalystneuro/neuroconv/pull/1056) * Modified the CI to avoid running doctests twice [PR #1077](https://github.com/catalystneuro/neuroconv/pull/#1077) * Consolidated daily workflows into one workflow and added email notifications [PR #1081](https://github.com/catalystneuro/neuroconv/pull/1081) -* Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) * Added zarr tests for the test on data with checking equivalent backends [PR #1083](https://github.com/catalystneuro/neuroconv/pull/1083) From 4c3edc968c304579ce476152a4e1bc5611e58a0c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 18 Sep 2024 11:03:04 -0600 Subject: [PATCH 061/118] Avoid doing link check on draft (#1093) --- .github/workflows/test-external-links.yml | 8 ++++++++ CHANGELOG.md | 2 ++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/test-external-links.yml b/.github/workflows/test-external-links.yml index fa782a476..039ca25d1 100644 --- a/.github/workflows/test-external-links.yml +++ b/.github/workflows/test-external-links.yml @@ -1,11 +1,19 @@ name: Testing External Links on: pull_request: + types: [synchronize, opened, reopened, ready_for_review] + # Synchronize, open and reopened are the default types for pull request + # We add ready_for_review to trigger the check for changelog and full tests when ready for review is clicked merge_group: workflow_call: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build-and-test: + if: ${{ github.event.pull_request.draft == false }} name: Testing External Links runs-on: ubuntu-latest strategy: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bd9c2160..dc8d4112d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ ## Improvements * Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) +* Avoid running link test when the PR is on draft [PR #1093](https://github.com/catalystneuro/neuroconv/pull/1093) + # v0.6.4 (September 17, 2024) From 60cc8c73818e5bfd9c92bac4cb1b32f4952b2be4 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Thu, 19 Sep 2024 11:41:29 +1000 Subject: [PATCH 062/118] reorganized weeklies into dedicated workflow like dailies (#1088) Co-authored-by: Heberto Mayorquin --- ...upload_docker_image_rclone_with_config.yml | 8 +++-- .github/workflows/weeklies.yml | 29 +++++++++++++++++++ CHANGELOG.md | 1 + 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/weeklies.yml diff --git a/.github/workflows/build_and_upload_docker_image_rclone_with_config.yml b/.github/workflows/build_and_upload_docker_image_rclone_with_config.yml index 7ff197bdc..ee1c0032d 100644 --- a/.github/workflows/build_and_upload_docker_image_rclone_with_config.yml +++ b/.github/workflows/build_and_upload_docker_image_rclone_with_config.yml @@ -1,9 +1,13 @@ name: Build and Upload Docker Image of Rclone With Config to GHCR on: - schedule: - - cron: "0 16 * * 1" # Weekly at noon EST on Monday workflow_dispatch: + workflow_call: + secrets: + DOCKER_UPLOADER_USERNAME: + required: true + DOCKER_UPLOADER_PASSWORD: + required: true concurrency: # Cancel previous workflows on the same pull request group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/weeklies.yml b/.github/workflows/weeklies.yml new file mode 100644 index 000000000..8397fcbd7 --- /dev/null +++ b/.github/workflows/weeklies.yml @@ -0,0 +1,29 @@ +name: Weekly workflows + +on: + workflow_dispatch: + schedule: + - cron: "0 2 * * 0" # Weekly at 6PM PST, 9PM EST, 3AM CET on Sunday to avoid working hours + +jobs: + build-and-upload-docker-image-rclone-with-config: + uses: ./.github/workflows/build_and_upload_docker_image_rclone_with_config.yml + secrets: + DOCKER_UPLOADER_USERNAME: ${{ secrets.DOCKER_UPLOADER_USERNAME }} + DOCKER_UPLOADER_PASSWORD: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} + + notify-build-and-upload-docker-image-rclone-with-config: + runs-on: ubuntu-latest + needs: [build-and-upload-docker-image-rclone-with-config] + if: ${{ always() && needs.build-and-upload-docker-image-rclone-with-config.result == 'failure' }} + steps: + - uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + subject: NeuroConv Weekly Docker Image Build and Upload Failure + to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }} + from: NeuroConv + body: "The weekly build and upload of the Docker image failed, please check status at https://github.com/catalystneuro/neuroconv/actions/workflows/weeklies.yml" diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8d4112d..78fed6276 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ## Improvements * Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) +* Consolidated weekly workflows into one workflow and added email notifications [PR #1088](https://github.com/catalystneuro/neuroconv/pull/1088) * Avoid running link test when the PR is on draft [PR #1093](https://github.com/catalystneuro/neuroconv/pull/1093) From f51c6faa7d933803782d1c85ed2ee0eaa2fd665e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 19 Sep 2024 06:38:29 -0600 Subject: [PATCH 063/118] Add `always_write_timestamps` method to base recording interfaces (#1091) --- CHANGELOG.md | 1 + .../baserecordingextractorinterface.py | 14 ++++- .../tools/spikeinterface/spikeinterface.py | 52 +++++++++++++------ tests/test_ecephys/test_ecephys_interfaces.py | 14 +++++ 4 files changed, 63 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78fed6276..2e39c45aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) * Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) * Added a `MockSortingInterface` for testing purposes. [PR #1065](https://github.com/catalystneuro/neuroconv/pull/1065) +* BaseRecordingInterfaces have a new conversion options `always_write_timestamps` that ca be used to force writing timestamps even if neuroconv heuristic indicates regular sampling rate [PR #1091](https://github.com/catalystneuro/neuroconv/pull/1091) ## Improvements diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 23716c161..642ec5b78 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -307,6 +307,7 @@ def add_to_nwbfile( compression_opts: Optional[int] = None, iterator_type: Optional[str] = "v2", iterator_opts: Optional[dict] = None, + always_write_timestamps: bool = False, ): """ Primary function for converting raw (unprocessed) RecordingExtractor data to the NWB standard. @@ -325,7 +326,12 @@ def add_to_nwbfile( Sets the starting time of the ElectricalSeries to a manually set value. stub_test : bool, default: False If True, will truncate the data to run the conversion faster and take up less memory. - write_as : {'raw', 'lfp', 'processed'} + write_as : {'raw', 'processed', 'lfp'}, default='raw' + Specifies how to save the trace data in the NWB file. Options are: + - 'raw': Save the data in the acquisition group. + - 'processed': Save the data as FilteredEphys in a processing module. + - 'lfp': Save the data as LFP in a processing module. + write_electrical_series : bool, default: True Electrical series are written in acquisition. If False, only device, electrode_groups, and electrodes are written to NWB. @@ -353,6 +359,11 @@ def add_to_nwbfile( * progress_bar_options : dict, optional Dictionary of keyword arguments to be passed directly to tqdm. See https://github.com/tqdm/tqdm#parameters for options. + always_write_timestamps : bool, default: False + Set to True to always write timestamps. + By default (False), the function checks if the timestamps are uniformly sampled, and if so, stores the data + using a regular sampling rate instead of explicit timestamps. If set to True, timestamps will be written + explicitly, regardless of whether the sampling rate is uniform. """ from ...tools.spikeinterface import add_recording_to_nwbfile @@ -376,4 +387,5 @@ def add_to_nwbfile( compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, + always_write_timestamps=always_write_timestamps, ) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 8b4db78be..9128078a6 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -791,6 +791,7 @@ def add_electrical_series_to_nwbfile( compression_opts: Optional[int] = None, iterator_type: Optional[str] = "v2", iterator_opts: Optional[dict] = None, + always_write_timestamps: bool = False, ): """ Adds traces from recording object as ElectricalSeries to an NWBFile object. @@ -833,6 +834,11 @@ def add_electrical_series_to_nwbfile( Dictionary of options for the iterator. See https://hdmf.readthedocs.io/en/stable/hdmf.data_utils.html#hdmf.data_utils.GenericDataChunkIterator for the full list of options. + always_write_timestamps : bool, default: False + Set to True to always write timestamps. + By default (False), the function checks if the timestamps are uniformly sampled, and if so, stores the data + using a regular sampling rate instead of explicit timestamps. If set to True, timestamps will be written + explicitly, regardless of whether the sampling rate is uniform. Notes ----- @@ -928,25 +934,31 @@ def add_electrical_series_to_nwbfile( ) eseries_kwargs.update(data=ephys_data_iterator) - # Now we decide whether to store the timestamps as a regular series or as an irregular series. - if recording.has_time_vector(segment_index=segment_index): - # First we check if the recording has a time vector to avoid creating artificial timestamps - timestamps = recording.get_times(segment_index=segment_index) - rate = calculate_regular_series_rate(series=timestamps) # Returns None if it is not regular - recording_t_start = timestamps[0] - else: - rate = recording.get_sampling_frequency() - recording_t_start = recording._recording_segments[segment_index].t_start or 0 - starting_time = starting_time if starting_time is not None else 0 - if rate: - starting_time = float(starting_time + recording_t_start) - # Note that we call the sampling frequency again because the estimated rate might be different from the - # sampling frequency of the recording extractor by some epsilon. - eseries_kwargs.update(starting_time=starting_time, rate=recording.get_sampling_frequency()) - else: + if always_write_timestamps: + timestamps = recording.get_times(segment_index=segment_index) shifted_timestamps = starting_time + timestamps eseries_kwargs.update(timestamps=shifted_timestamps) + else: + # By default we write the rate if the timestamps are regular + recording_has_timestamps = recording.has_time_vector(segment_index=segment_index) + if recording_has_timestamps: + timestamps = recording.get_times(segment_index=segment_index) + rate = calculate_regular_series_rate(series=timestamps) # Returns None if it is not regular + recording_t_start = timestamps[0] + else: + rate = recording.get_sampling_frequency() + recording_t_start = recording._recording_segments[segment_index].t_start or 0 + + # Shift timestamps if starting_time is set + if rate: + starting_time = float(starting_time + recording_t_start) + # Note that we call the sampling frequency again because the estimated rate might be different from the + # sampling frequency of the recording extractor by some epsilon. + eseries_kwargs.update(starting_time=starting_time, rate=recording.get_sampling_frequency()) + else: + shifted_timestamps = starting_time + timestamps + eseries_kwargs.update(timestamps=shifted_timestamps) # Create ElectricalSeries object and add it to nwbfile es = pynwb.ecephys.ElectricalSeries(**eseries_kwargs) @@ -1048,6 +1060,7 @@ def add_recording_to_nwbfile( compression_opts: Optional[int] = None, iterator_type: str = "v2", iterator_opts: Optional[dict] = None, + always_write_timestamps: bool = False, ): """ Add traces from a recording object as an ElectricalSeries to an NWBFile object. @@ -1074,7 +1087,6 @@ def add_recording_to_nwbfile( the starting time is taken from the recording extractor. write_as : {'raw', 'processed', 'lfp'}, default='raw' Specifies how to save the trace data in the NWB file. Options are: - - 'raw': Save the data in the acquisition group. - 'processed': Save the data as FilteredEphys in a processing module. - 'lfp': Save the data as LFP in a processing module. @@ -1094,6 +1106,11 @@ def add_recording_to_nwbfile( Dictionary of options for the iterator. Refer to the documentation at https://hdmf.readthedocs.io/en/stable/hdmf.data_utils.html#hdmf.data_utils.GenericDataChunkIterator for a full list of available options. + always_write_timestamps : bool, default: False + Set to True to always write timestamps. + By default (False), the function checks if the timestamps are uniformly sampled, and if so, stores the data + using a regular sampling rate instead of explicit timestamps. If set to True, timestamps will be written + explicitly, regardless of whether the sampling rate is uniform. Notes ----- @@ -1125,6 +1142,7 @@ def add_recording_to_nwbfile( compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, + always_write_timestamps=always_write_timestamps, ) diff --git a/tests/test_ecephys/test_ecephys_interfaces.py b/tests/test_ecephys/test_ecephys_interfaces.py index 5591bb6fb..4d4232bf2 100644 --- a/tests/test_ecephys/test_ecephys_interfaces.py +++ b/tests/test_ecephys/test_ecephys_interfaces.py @@ -52,6 +52,20 @@ def test_no_slash_in_name(self): interface.validate_metadata(metadata) +class TestAlwaysWriteTimestamps: + + def test_always_write_timestamps(self): + # By default the MockRecordingInterface has a uniform sampling rate + interface = MockRecordingInterface(durations=[1.0], sampling_frequency=30_000.0) + + nwbfile = interface.create_nwbfile(always_write_timestamps=True) + electrical_series = nwbfile.acquisition["ElectricalSeries"] + + expected_timestamps = interface.recording_extractor.get_times() + + np.testing.assert_array_equal(electrical_series.timestamps[:], expected_timestamps) + + class TestAssertions(TestCase): @pytest.mark.skipif(python_version.minor != 10, reason="Only testing with Python 3.10!") def test_spike2_import_assertions_3_10(self): From 1ccdb2abed757f4a133985dd1f73ffe54b9e26cf Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 19 Sep 2024 12:16:36 -0600 Subject: [PATCH 064/118] Remove dev tests from the PR tests (#1092) --- .github/workflows/deploy-tests.yml | 25 ++++++++----------------- CHANGELOG.md | 1 + 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 3afd5e956..4f67d15de 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -43,36 +43,27 @@ jobs: python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} - run-live-service-tests: + # If the conversion gallery is the only thing that changed, run doctests only + run-doctests-only: needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} - uses: ./.github/workflows/live-service-testing.yml - secrets: - DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} + uses: ./.github/workflows/doctests.yml with: # Ternary operator: condition && value_if_true || value_if_false python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} - run-dev-tests: + + run-live-service-tests: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} - uses: ./.github/workflows/dev-testing.yml + uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} - with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - - run-doctests-only: - needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} - uses: ./.github/workflows/doctests.yml with: # Ternary operator: condition && value_if_true || value_if_false python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + check-final-status: name: All tests passing if: always() diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e39c45aa..d73940856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) ## Improvements +* Remove dev test from PR [PR #1092](https://github.com/catalystneuro/neuroconv/pull/1092) * Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) * Consolidated weekly workflows into one workflow and added email notifications [PR #1088](https://github.com/catalystneuro/neuroconv/pull/1088) * Avoid running link test when the PR is on draft [PR #1093](https://github.com/catalystneuro/neuroconv/pull/1093) From 9f67ec60611c0e4f3d6f52df2b50f47fcbd2acb0 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 19 Sep 2024 18:36:55 -0600 Subject: [PATCH 065/118] Add json schema validation for source data at the interface level (#1090) --- CHANGELOG.md | 4 ++- .../behavior/lightningpose.rst | 2 +- .../fiberphotometry/tdt_fp.rst | 1 + src/neuroconv/basedatainterface.py | 20 ++++++++++++ src/neuroconv/baseextractorinterface.py | 10 ++++++ .../alphaomega/alphaomegadatainterface.py | 8 ++++- .../ecephys/axona/axonadatainterface.py | 23 +++++++++++-- .../baserecordingextractorinterface.py | 9 ++++-- .../blackrock/blackrockdatainterface.py | 11 +++++-- .../cellexplorer/cellexplorerdatainterface.py | 9 +++++- .../ecephys/intan/intandatainterface.py | 9 ++++-- .../neuralynx/neuralynxdatainterface.py | 13 ++++++-- .../ecephys/plexon/plexondatainterface.py | 13 +++++--- .../ecephys/spikeglx/spikeglxdatainterface.py | 23 +++++++++---- .../ecephys/spikeglx/spikeglxnidqinterface.py | 15 +++++++-- .../ecephys/tdt/tdtdatainterface.py | 8 +++++ .../icephys/baseicephysinterface.py | 3 +- .../ophys/baseimagingextractorinterface.py | 4 +-- .../ophys/hdf5/hdf5datainterface.py | 10 +++--- .../ophys/sbx/sbxdatainterface.py | 4 +-- .../scanimage/scanimageimaginginterfaces.py | 32 +++++++++++++++++-- .../tools/testing/mock_interfaces.py | 4 +-- 22 files changed, 191 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d73940856..da1f33811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) +* Data interfaces now perform source (argument inputs) validation with the json schema [PR #1020](https://github.com/catalystneuro/neuroconv/pull/1020) ## Improvements * Remove dev test from PR [PR #1092](https://github.com/catalystneuro/neuroconv/pull/1092) @@ -27,7 +28,7 @@ ## Features * Added chunking/compression for string-only compound objects: [PR #1042](https://github.com/catalystneuro/neuroconv/pull/1042) * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) -* Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) +* Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) * Added a `MockSortingInterface` for testing purposes. [PR #1065](https://github.com/catalystneuro/neuroconv/pull/1065) * BaseRecordingInterfaces have a new conversion options `always_write_timestamps` that ca be used to force writing timestamps even if neuroconv heuristic indicates regular sampling rate [PR #1091](https://github.com/catalystneuro/neuroconv/pull/1091) @@ -99,6 +100,7 @@ * Data interfaces `run_conversion` method now performs metadata validation before running the conversion. [PR #949](https://github.com/catalystneuro/neuroconv/pull/949) * Introduced `null_values_for_properties` to `add_units_table` to give user control over null values behavior [PR #989](https://github.com/catalystneuro/neuroconv/pull/989) + ## Bug fixes * Fixed the default naming of multiple electrical series in the `SpikeGLXConverterPipe`. [PR #957](https://github.com/catalystneuro/neuroconv/pull/957) * Write new properties to the electrode table use the global identifier channel_name, group [PR #984](https://github.com/catalystneuro/neuroconv/pull/984) diff --git a/docs/conversion_examples_gallery/behavior/lightningpose.rst b/docs/conversion_examples_gallery/behavior/lightningpose.rst index 80f38e60a..a024a6799 100644 --- a/docs/conversion_examples_gallery/behavior/lightningpose.rst +++ b/docs/conversion_examples_gallery/behavior/lightningpose.rst @@ -23,7 +23,7 @@ Convert LightningPose pose estimation data to NWB using :py:class:`~neuroconv.da >>> labeled_video_file_path = str(folder_path / "labeled_videos/test_vid_labeled.mp4") >>> converter = LightningPoseConverter(file_path=file_path, original_video_file_path=original_video_file_path, labeled_video_file_path=labeled_video_file_path, verbose=False) - + Source data is valid! >>> metadata = converter.get_metadata() >>> # For data provenance we add the time zone information to the conversion >>> session_start_time = metadata["NWBFile"]["session_start_time"] diff --git a/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst b/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst index b82e4a801..52ceb866c 100644 --- a/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst +++ b/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst @@ -208,6 +208,7 @@ Convert TDT Fiber Photometry data to NWB using >>> editable_metadata_path = LOCAL_PATH / "tests" / "test_on_data" / "ophys" / "fiber_photometry_metadata.yaml" >>> interface = TDTFiberPhotometryInterface(folder_path=folder_path, verbose=True) + Source data is valid! >>> metadata = interface.get_metadata() >>> metadata["NWBFile"]["session_start_time"] = datetime.now(tz=ZoneInfo("US/Pacific")) >>> editable_metadata = load_dict_from_file(editable_metadata_path) diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 682ed2dba..adcec89b5 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -24,6 +24,7 @@ load_dict_from_file, ) from .utils.dict import DeepDict +from .utils.json_schema import _NWBSourceDataEncoder class BaseDataInterface(ABC): @@ -39,11 +40,30 @@ def get_source_schema(cls) -> dict: """Infer the JSON schema for the source_data from the method signature (annotation typing).""" return get_json_schema_from_method_signature(cls, exclude=["source_data"]) + @classmethod + def validate_source(cls, source_data: dict, verbose: bool = False): + """Validate source_data against Converter source_schema.""" + cls._validate_source_data(source_data=source_data, verbose=verbose) + + def _validate_source_data(self, source_data: dict, verbose: bool = False): + + encoder = _NWBSourceDataEncoder() + # The encoder produces a serialized object, so we deserialized it for comparison + + serialized_source_data = encoder.encode(source_data) + decoded_source_data = json.loads(serialized_source_data) + source_schema = self.get_source_schema() + validate(instance=decoded_source_data, schema=source_schema) + if verbose: + print("Source data is valid!") + @validate_call def __init__(self, verbose: bool = False, **source_data): self.verbose = verbose self.source_data = source_data + self._validate_source_data(source_data=source_data, verbose=verbose) + def get_metadata_schema(self) -> dict: """Retrieve JSON schema for metadata.""" metadata_schema = load_dict_from_file(Path(__file__).parent / "schemas" / "base_metadata_schema.json") diff --git a/src/neuroconv/baseextractorinterface.py b/src/neuroconv/baseextractorinterface.py index 5dc0b25a1..a75fbe1f0 100644 --- a/src/neuroconv/baseextractorinterface.py +++ b/src/neuroconv/baseextractorinterface.py @@ -29,3 +29,13 @@ def get_extractor(cls): ) cls.Extractor = extractor return extractor + + def __init__(self, **source_data): + super().__init__(**source_data) + self.extractor = self.get_extractor() + self.extractor_kwargs = self._source_data_to_extractor_kwargs(source_data) + self._extractor_instance = self.extractor(**self.extractor_kwargs) + + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + """This functions maps the source_data to kwargs required to initialize the Extractor.""" + return source_data diff --git a/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py b/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py index 308f6bc56..97f89abca 100644 --- a/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py @@ -13,6 +13,7 @@ class AlphaOmegaRecordingInterface(BaseRecordingExtractorInterface): display_name = "AlphaOmega Recording" associated_suffixes = (".mpx",) info = "Interface class for converting AlphaOmega recording data." + stream_id = "RAW" @classmethod def get_source_schema(cls) -> dict: @@ -20,6 +21,11 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["folder_path"]["description"] = "Path to the folder of .mpx files." return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs["stream_id"] = self.stream_id + return extractor_kwargs + def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ Load and prepare data for AlphaOmega. @@ -33,7 +39,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str Default is True. es_key: str, default: "ElectricalSeries" """ - super().__init__(folder_path=folder_path, stream_id="RAW", verbose=verbose, es_key=es_key) + super().__init__(folder_path=folder_path, verbose=verbose, es_key=es_key) def get_metadata(self) -> dict: metadata = super().get_metadata() diff --git a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py index 731aec168..ba6adf4a1 100644 --- a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py @@ -30,6 +30,12 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to .bin file." return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs["all_annotations"] = True + + return extractor_kwargs + def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ @@ -41,7 +47,7 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele es_key: str, default: "ElectricalSeries" """ - super().__init__(file_path=file_path, all_annotations=True, verbose=verbose, es_key=es_key) + super().__init__(file_path=file_path, verbose=verbose, es_key=es_key) self.source_data = dict(file_path=file_path, verbose=verbose) self.metadata_in_set_file = self.recording_extractor.neo_reader.file_parameters["set"]["file_header"] @@ -134,6 +140,7 @@ def __init__(self, file_path: FilePath, noise_std: float = 3.5): class AxonaLFPDataInterface(BaseLFPExtractorInterface): """ Primary data interface class for converting Axona LFP data. + Note that this interface is not lazy and will load all data into memory. """ display_name = "Axona LFP" @@ -151,10 +158,20 @@ def get_source_schema(cls) -> dict: additionalProperties=False, ) + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("file_path") + extractor_kwargs["traces_list"] = self.traces_list + extractor_kwargs["sampling_frequency"] = self.sampling_frequency + + return extractor_kwargs + def __init__(self, file_path: FilePath): data = read_all_eeg_file_lfp_data(file_path).T - sampling_frequency = get_eeg_sampling_frequency(file_path) - super().__init__(traces_list=[data], sampling_frequency=sampling_frequency) + self.traces_list = [data] + self.sampling_frequency = get_eeg_sampling_frequency(file_path) + super().__init__(file_path=file_path) self.source_data = dict(file_path=file_path) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 642ec5b78..e2c747378 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -32,8 +32,9 @@ def __init__(self, verbose: bool = True, es_key: str = "ElectricalSeries", **sou The key-value pairs of extractor-specific arguments. """ + super().__init__(**source_data) - self.recording_extractor = self.get_extractor()(**source_data) + self.recording_extractor = self._extractor_instance property_names = self.recording_extractor.get_property_keys() # TODO remove this and go and change all the uses of channel_name once spikeinterface > 0.101.0 is released if "channel_name" not in property_names and "channel_names" in property_names: @@ -118,7 +119,11 @@ def get_original_timestamps(self) -> Union[np.ndarray, list[np.ndarray]]: The timestamps for the data stream; if the recording has multiple segments, then a list of timestamps is returned. """ new_recording = self.get_extractor()( - **{keyword: value for keyword, value in self.source_data.items() if keyword not in ["verbose", "es_key"]} + **{ + keyword: value + for keyword, value in self.extractor_kwargs.items() + if keyword not in ["verbose", "es_key"] + } ) if self._number_of_segments == 1: return new_recording.get_times() diff --git a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py index d5122bf66..e84719431 100644 --- a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py @@ -25,6 +25,12 @@ def get_source_schema(cls): ] = "Path to the Blackrock file with suffix being .ns1, .ns2, .ns3, .ns4m .ns4, or .ns6." return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs["stream_id"] = self.stream_id + + return extractor_kwargs + def __init__( self, file_path: FilePath, @@ -55,7 +61,8 @@ def __init__( nsx_to_load = int(file_path.suffix[-1]) self.file_path = file_path - super().__init__(file_path=file_path, stream_id=str(nsx_to_load), verbose=verbose, es_key=es_key) + self.stream_id = str(nsx_to_load) + super().__init__(file_path=file_path, verbose=verbose, es_key=es_key) def get_metadata(self) -> dict: metadata = super().get_metadata() @@ -83,7 +90,7 @@ def get_source_schema(cls) -> dict: metadata_schema["properties"]["file_path"].update(description="Path to Blackrock .nev file.") return metadata_schema - def __init__(self, file_path: FilePath, sampling_frequency: float = None, verbose: bool = True): + def __init__(self, file_path: FilePath, sampling_frequency: Optional[float] = None, verbose: bool = True): """ Parameters ---------- diff --git a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py index e388e5e44..46e825fb5 100644 --- a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py @@ -419,6 +419,12 @@ class CellExplorerSortingInterface(BaseSortingExtractorInterface): associated_suffixes = (".mat", ".sessionInfo", ".spikes", ".cellinfo") info = "Interface for CellExplorer sorting data." + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs["sampling_frequency"] = self.sampling_frequency + + return extractor_kwargs + def __init__(self, file_path: FilePath, verbose: bool = True): """ Initialize read of Cell Explorer file. @@ -454,7 +460,8 @@ def __init__(self, file_path: FilePath, verbose: bool = True): if "extracellular" in session_data.keys(): sampling_frequency = session_data["extracellular"].get("sr", None) - super().__init__(file_path=file_path, sampling_frequency=sampling_frequency, verbose=verbose) + self.sampling_frequency = sampling_frequency + super().__init__(file_path=file_path, verbose=verbose) self.source_data = dict(file_path=file_path) spikes_matfile_path = Path(file_path) diff --git a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py index 7bd0964ef..2d7c849f2 100644 --- a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py @@ -25,6 +25,13 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to either a .rhd or a .rhs file" return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs["all_annotations"] = True + extractor_kwargs["stream_id"] = self.stream_id + + return extractor_kwargs + def __init__( self, file_path: FilePath, @@ -52,10 +59,8 @@ def __init__( init_kwargs = dict( file_path=self.file_path, - stream_id=self.stream_id, verbose=verbose, es_key=es_key, - all_annotations=True, ignore_integrity_checks=ignore_integrity_checks, ) diff --git a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py index d4ceb6b38..446c06302 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py @@ -32,6 +32,12 @@ def get_source_schema(cls) -> dict: ] = 'Path to Neuralynx directory containing ".ncs", ".nse", ".ntt", ".nse", or ".nev" files.' return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs["all_annotations"] = True + + return extractor_kwargs + def __init__( self, folder_path: DirectoryPath, @@ -53,7 +59,10 @@ def __init__( es_key : str, default: "ElectricalSeries" """ super().__init__( - folder_path=folder_path, stream_name=stream_name, verbose=verbose, all_annotations=True, es_key=es_key + folder_path=folder_path, + stream_name=stream_name, + verbose=verbose, + es_key=es_key, ) # convert properties of object dtype (e.g. datetime) and bool as these are not supported by nwb @@ -103,7 +112,7 @@ class NeuralynxSortingInterface(BaseSortingExtractorInterface): associated_suffixes = (".nse", ".ntt", ".nse", ".nev") info = "Interface for Neuralynx sorting data." - def __init__(self, folder_path: DirectoryPath, sampling_frequency: float = None, verbose: bool = True): + def __init__(self, folder_path: DirectoryPath, sampling_frequency: Optional[float] = None, verbose: bool = True): """_summary_ Parameters diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py index 5653b1586..d1dcc4d45 100644 --- a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py @@ -82,6 +82,13 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .pl2 file." return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs["all_annotations"] = True + extractor_kwargs["stream_id"] = self.stream_id + + return extractor_kwargs + @validate_call def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): """ @@ -101,16 +108,14 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele neo_version = Version(neo.__version__) if neo_version <= Version("0.13.3"): - stream_id = "3" + self.stream_id = "3" else: - stream_id = "WB" + self.stream_id = "WB" assert Path(file_path).is_file(), f"Plexon file not found in: {file_path}" super().__init__( file_path=file_path, verbose=verbose, es_key=es_key, - stream_id=stream_id, - all_annotations=True, ) def get_metadata(self) -> DeepDict: diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index d45a7f946..c15516431 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -13,7 +13,7 @@ get_session_start_time, ) from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class SpikeGLXRecordingInterface(BaseRecordingExtractorInterface): @@ -38,10 +38,19 @@ class SpikeGLXRecordingInterface(BaseRecordingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["x_pitch", "y_pitch"]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["x_pitch", "y_pitch"]) source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX ap.bin or lf.bin file." return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("file_path") + extractor_kwargs["folder_path"] = self.folder_path + extractor_kwargs["all_annotations"] = True + extractor_kwargs["stream_id"] = self.stream_id + return extractor_kwargs + @validate_call def __init__( self, @@ -68,13 +77,12 @@ def __init__( else: raise ValueError("Cannot automatically determine es_key from path") file_path = Path(file_path) - folder_path = file_path.parent + self.folder_path = file_path.parent + super().__init__( - folder_path=folder_path, - stream_id=self.stream_id, + file_path=file_path, verbose=verbose, es_key=es_key, - all_annotations=True, ) self.source_data["file_path"] = str(file_path) self.meta = self.recording_extractor.neo_reader.signals_info_dict[(0, self.stream_id)]["meta"] @@ -124,7 +132,8 @@ def get_metadata(self) -> dict: def get_original_timestamps(self) -> np.ndarray: new_recording = self.get_extractor()( - folder_path=self.source_data["folder_path"], stream_id=self.source_data["stream_id"] + folder_path=self.folder_path, + stream_id=self.stream_id, ) # TODO: add generic method for aliasing from NeuroConv signature to SI init if self._number_of_segments == 1: return new_recording.get_times() diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 42dad773d..fab9e5b5f 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -18,6 +18,7 @@ class SpikeGLXNIDQInterface(BaseRecordingExtractorInterface): info = "Interface for NIDQ board recording data." ExtractorName = "SpikeGLXRecordingExtractor" + stream_id = "nidq" @classmethod def get_source_schema(cls) -> dict: @@ -25,6 +26,14 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX .nidq file." return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("file_path") + extractor_kwargs["folder_path"] = self.folder_path + extractor_kwargs["stream_id"] = self.stream_id + return extractor_kwargs + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, @@ -50,10 +59,10 @@ def __init__( es_key : str, default: "ElectricalSeriesNIDQ" """ - folder_path = Path(file_path).parent + self.file_path = Path(file_path) + self.folder_path = self.file_path.parent super().__init__( - folder_path=folder_path, - stream_id="nidq", + file_path=self.file_path, verbose=verbose, load_sync_channel=load_sync_channel, es_key=es_key, diff --git a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py index d01e63c5d..a46f3e0f8 100644 --- a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py @@ -10,6 +10,13 @@ class TdtRecordingInterface(BaseRecordingExtractorInterface): associated_suffixes = (".tbk", ".tbx", ".tev", ".tsq") info = "Interface for TDT recording data." + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("gain") + + return extractor_kwargs + @validate_call def __init__( self, @@ -44,6 +51,7 @@ def __init__( stream_id=stream_id, verbose=verbose, es_key=es_key, + gain=gain, ) # Fix channel name format diff --git a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py index 76c9bf840..f8bad53d6 100644 --- a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py +++ b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py @@ -41,7 +41,8 @@ def __init__(self, file_paths: list[FilePath]): from ...tools.neo import get_number_of_electrodes, get_number_of_segments - super().__init__(file_paths=file_paths) + self.source_data = dict() + self.source_data["file_paths"] = file_paths self.readers_list = list() for f in file_paths: diff --git a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py index 548b57d0c..5125af3cc 100644 --- a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py @@ -41,7 +41,7 @@ def __init__( **source_data, ): super().__init__(**source_data) - self.imaging_extractor = self.get_extractor()(**source_data) + self.imaging_extractor = self._extractor_instance self.verbose = verbose self.photon_series_type = photon_series_type @@ -128,7 +128,7 @@ def get_metadata( return metadata def get_original_timestamps(self) -> np.ndarray: - reinitialized_extractor = self.get_extractor()(**self.source_data) + reinitialized_extractor = self.get_extractor()(**self.extractor_kwargs) return reinitialized_extractor.frame_to_time(frames=np.arange(stop=reinitialized_extractor.get_num_frames())) def get_timestamps(self) -> np.ndarray: diff --git a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py index 025e68b87..4bb13f86b 100644 --- a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py +++ b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Literal, Optional from pydantic import ConfigDict, FilePath, validate_call @@ -18,10 +18,10 @@ def __init__( self, file_path: FilePath, mov_field: str = "mov", - sampling_frequency: float = None, - start_time: float = None, - metadata: dict = None, - channel_names: ArrayType = None, + sampling_frequency: Optional[float] = None, + start_time: Optional[float] = None, + metadata: Optional[dict] = None, + channel_names: Optional[ArrayType] = None, verbose: bool = True, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", ): diff --git a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py index 5b921f6f3..554cc5aba 100644 --- a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Literal, Optional from pydantic import FilePath, validate_call @@ -16,7 +16,7 @@ class SbxImagingInterface(BaseImagingExtractorInterface): def __init__( self, file_path: FilePath, - sampling_frequency: float = None, + sampling_frequency: Optional[float] = None, verbose: bool = True, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", ): diff --git a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py index 09e8f86d3..c74161e55 100644 --- a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py +++ b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py @@ -91,6 +91,13 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to Tiff file." return source_schema + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("fallback_sampling_frequency", None) + extractor_kwargs["sampling_frequency"] = self.sampling_frequency + + return extractor_kwargs + @validate_call def __init__( self, @@ -128,7 +135,8 @@ def __init__( assert fallback_sampling_frequency is not None, assert_msg sampling_frequency = fallback_sampling_frequency - super().__init__(file_path=file_path, sampling_frequency=sampling_frequency, verbose=verbose) + self.sampling_frequency = sampling_frequency + super().__init__(file_path=file_path, fallback_sampling_frequency=fallback_sampling_frequency, verbose=verbose) def get_metadata(self) -> dict: device_number = 0 # Imaging plane metadata is a list with metadata for each plane @@ -229,6 +237,13 @@ class ScanImageMultiPlaneImagingInterface(BaseImagingExtractorInterface): ExtractorName = "ScanImageTiffMultiPlaneImagingExtractor" + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("image_metadata") + extractor_kwargs["metadata"] = self.image_metadata + + return extractor_kwargs + @validate_call def __init__( self, @@ -278,10 +293,12 @@ def __init__( two_photon_series_name_suffix = f"{channel_name.replace(' ', '')}" self.two_photon_series_name_suffix = two_photon_series_name_suffix + self.metadata = image_metadata + self.parsed_metadata = parsed_metadata super().__init__( file_path=file_path, channel_name=channel_name, - metadata=image_metadata, + image_metadata=image_metadata, parsed_metadata=parsed_metadata, verbose=verbose, ) @@ -448,6 +465,13 @@ class ScanImageSinglePlaneImagingInterface(BaseImagingExtractorInterface): ExtractorName = "ScanImageTiffSinglePlaneImagingExtractor" + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("image_metadata") + extractor_kwargs["metadata"] = self.image_metadata + + return extractor_kwargs + @validate_call def __init__( self, @@ -512,11 +536,13 @@ def __init__( two_photon_series_name_suffix = f"{two_photon_series_name_suffix}Plane{plane_name}" self.two_photon_series_name_suffix = two_photon_series_name_suffix + self.metadata = image_metadata + self.parsed_metadata = parsed_metadata super().__init__( file_path=file_path, channel_name=channel_name, plane_name=plane_name, - metadata=image_metadata, + image_metadata=image_metadata, parsed_metadata=parsed_metadata, verbose=verbose, ) diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 43f8c2dd2..04dc57250 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -142,7 +142,7 @@ def __init__( self, num_channels: int = 4, sampling_frequency: float = 30_000.0, - durations: tuple[float] = (1.0,), + durations: tuple[float, ...] = (1.0,), seed: int = 0, verbose: bool = True, es_key: str = "ElectricalSeries", @@ -176,7 +176,7 @@ def __init__( self, num_units: int = 4, sampling_frequency: float = 30_000.0, - durations: tuple[float] = (1.0,), + durations: tuple[float, ...] = (1.0,), seed: int = 0, verbose: bool = True, ): From 7ea96d84c0bbdcfd4318048430daa101c9d84ee7 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 23 Sep 2024 10:53:58 -0600 Subject: [PATCH 066/118] Enable zarr backend testing in data tests [3] (#1094) --- CHANGELOG.md | 1 + .../tools/testing/data_interface_mixins.py | 196 +++++++++--------- .../ecephys/test_recording_interfaces.py | 22 +- tests/test_ophys/test_ophys_interfaces.py | 4 + 4 files changed, 120 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1f33811..751d7cc55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ## Improvements * Remove dev test from PR [PR #1092](https://github.com/catalystneuro/neuroconv/pull/1092) * Run only the most basic testing while a PR is on draft [PR #1082](https://github.com/catalystneuro/neuroconv/pull/1082) +* Test that zarr backend_configuration works in gin data tests [PR #1094](https://github.com/catalystneuro/neuroconv/pull/1094) * Consolidated weekly workflows into one workflow and added email notifications [PR #1088](https://github.com/catalystneuro/neuroconv/pull/1088) * Avoid running link test when the PR is on draft [PR #1093](https://github.com/catalystneuro/neuroconv/pull/1093) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index d3107650b..946b3fd6c 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -83,7 +83,7 @@ def test_source_schema_valid(self): schema = self.data_interface_cls.get_source_schema() Draft7Validator.check_schema(schema=schema) - def check_conversion_options_schema_valid(self): + def test_conversion_options_schema_valid(self, setup_interface): schema = self.interface.get_conversion_options_schema() Draft7Validator.check_schema(schema=schema) @@ -91,11 +91,15 @@ def test_metadata_schema_valid(self, setup_interface): schema = self.interface.get_metadata_schema() Draft7Validator.check_schema(schema=schema) - def check_metadata(self): + def test_metadata(self, setup_interface): # Validate metadata now happens on the class itself metadata = self.interface.get_metadata() self.check_extracted_metadata(metadata) + def check_extracted_metadata(self, metadata: dict): + """Override this method to make assertions about specific extracted metadata values.""" + pass + def test_no_metadata_mutation(self, setup_interface): """Ensure the metadata object is not altered by `add_to_nwbfile` method.""" @@ -107,13 +111,35 @@ def test_no_metadata_mutation(self, setup_interface): self.interface.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **self.conversion_options) assert metadata == metadata_before_add_method - def check_run_conversion_with_backend_configuration( - self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5" - ): + @pytest.mark.parametrize("backend", ["hdf5", "zarr"]) + def test_run_conversion_with_backend(self, setup_interface, tmp_path, backend): + + nwbfile_path = str(tmp_path / f"conversion_with_backend{backend}-{self.test_name}.nwb") + + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + self.interface.run_conversion( + nwbfile_path=nwbfile_path, + overwrite=True, + metadata=metadata, + backend=backend, + **self.conversion_options, + ) + + if backend == "zarr": + with NWBZarrIO(path=nwbfile_path, mode="r") as io: + io.read() + + @pytest.mark.parametrize("backend", ["hdf5", "zarr"]) + def test_run_conversion_with_backend_configuration(self, setup_interface, tmp_path, backend): metadata = self.interface.get_metadata() if "session_start_time" not in metadata["NWBFile"]: metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + nwbfile_path = str(tmp_path / f"conversion_with_backend_configuration{backend}-{self.test_name}.nwb") + nwbfile = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) backend_configuration = self.interface.get_default_backend_configuration(nwbfile=nwbfile, backend=backend) self.interface.run_conversion( @@ -125,6 +151,42 @@ def check_run_conversion_with_backend_configuration( **self.conversion_options, ) + @pytest.mark.parametrize("backend", ["hdf5", "zarr"]) + def test_configure_backend_for_equivalent_nwbfiles(self, setup_interface, tmp_path, backend): + metadata = self.interface.get_metadata() + if "session_start_time" not in metadata["NWBFile"]: + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + nwbfile_1 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) + nwbfile_2 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) + + backend_configuration = get_default_backend_configuration(nwbfile=nwbfile_1, backend=backend) + configure_backend(nwbfile=nwbfile_2, backend_configuration=backend_configuration) + + def test_all_conversion_checks(self, setup_interface, tmp_path): + interface, test_name = setup_interface + + # Create a unique test name and file path + nwbfile_path = str(tmp_path / f"{self.__class__.__name__}_{self.test_name}.nwb") + self.nwbfile_path = nwbfile_path + + self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") + self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") + + self.check_read_nwb(nwbfile_path=nwbfile_path) + + # Any extra custom checks to run + self.run_custom_checks() + + @abstractmethod + def check_read_nwb(self, nwbfile_path: str): + """Read the produced NWB file and compare it to the interface.""" + pass + + def run_custom_checks(self): + """Override this in child classes to inject additional custom checks.""" + pass + def check_run_conversion_in_nwbconverter_with_backend( self, nwbfile_path: str, backend: Literal["hdf5", "zarr"] = "hdf5" ): @@ -174,73 +236,6 @@ class TestNWBConverter(NWBConverter): conversion_options=conversion_options, ) - @abstractmethod - def check_read_nwb(self, nwbfile_path: str): - """Read the produced NWB file and compare it to the interface.""" - pass - - def check_extracted_metadata(self, metadata: dict): - """Override this method to make assertions about specific extracted metadata values.""" - pass - - def run_custom_checks(self): - """Override this in child classes to inject additional custom checks.""" - pass - - @pytest.mark.parametrize("backend", ["hdf5", "zarr"]) - def test_run_conversion_with_backend(self, setup_interface, tmp_path, backend): - - nwbfile_path = str(tmp_path / f"conversion_with_backend{backend}-{self.test_name}.nwb") - - metadata = self.interface.get_metadata() - if "session_start_time" not in metadata["NWBFile"]: - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - self.interface.run_conversion( - nwbfile_path=nwbfile_path, - overwrite=True, - metadata=metadata, - backend=backend, - **self.conversion_options, - ) - - if backend == "zarr": - with NWBZarrIO(path=nwbfile_path, mode="r") as io: - io.read() - - @pytest.mark.parametrize("backend", ["hdf5", "zarr"]) - def test_configure_backend_for_equivalent_nwbfiles(self, setup_interface, tmp_path, backend): - metadata = self.interface.get_metadata() - if "session_start_time" not in metadata["NWBFile"]: - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - nwbfile_1 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) - nwbfile_2 = self.interface.create_nwbfile(metadata=metadata, **self.conversion_options) - - backend_configuration = get_default_backend_configuration(nwbfile=nwbfile_1, backend=backend) - configure_backend(nwbfile=nwbfile_2, backend_configuration=backend_configuration) - - def test_all_conversion_checks(self, setup_interface, tmp_path): - interface, test_name = setup_interface - - # Create a unique test name and file path - nwbfile_path = str(tmp_path / f"{self.__class__.__name__}_{self.test_name}.nwb") - self.nwbfile_path = nwbfile_path - - # Now run the checks using the setup objects - self.check_conversion_options_schema_valid() - self.check_metadata() - - self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") - self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - - self.check_run_conversion_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - - self.check_read_nwb(nwbfile_path=nwbfile_path) - - # Any extra custom checks to run - self.run_custom_checks() - class TemporalAlignmentMixin: """ @@ -718,27 +713,6 @@ def check_shift_segment_timestamps_by_starting_times(self): ): assert_array_equal(x=retrieved_aligned_timestamps, y=expected_aligned_timestamps) - def test_all_conversion_checks(self, setup_interface, tmp_path): - # The fixture `setup_interface` sets up the necessary objects - interface, test_name = setup_interface - - # Create a unique test name and file path - nwbfile_path = str(tmp_path / f"{self.__class__.__name__}_{self.test_name}.nwb") - - # Now run the checks using the setup objects - self.check_conversion_options_schema_valid() - self.check_metadata() - - self.check_run_conversion_in_nwbconverter_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") - self.check_run_conversion_in_nwbconverter_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - - self.check_run_conversion_with_backend_configuration(nwbfile_path=nwbfile_path, backend="hdf5") - - self.check_read_nwb(nwbfile_path=nwbfile_path) - - # Any extra custom checks to run - self.run_custom_checks() - def test_interface_alignment(self, setup_interface): # TODO sorting can have times without associated recordings, test this later @@ -872,12 +846,21 @@ class MedPCInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): A mixin for testing MedPC interfaces. """ + def test_metadata(self): + pass + + def test_conversion_options_schema_valid(self): + pass + def test_metadata_schema_valid(self): pass def test_run_conversion_with_backend(self): pass + def test_run_conversion_with_backend_configuration(self): + pass + def test_no_metadata_mutation(self): pass @@ -888,6 +871,10 @@ def check_metadata_schema_valid(self): schema = self.interface.get_metadata_schema() Draft7Validator.check_schema(schema=schema) + def check_conversion_options_schema_valid(self): + schema = self.interface.get_conversion_options_schema() + Draft7Validator.check_schema(schema=schema) + def check_metadata(self): schema = self.interface.get_metadata_schema() metadata = self.interface.get_metadata() @@ -1158,9 +1145,8 @@ def check_read_nwb(self, nwbfile_path: str): assert one_photon_series.starting_frame is None assert one_photon_series.timestamps.shape == (15,) - imaging_extractor = self.interface.imaging_extractor - times_from_extractor = imaging_extractor._times - assert_array_equal(one_photon_series.timestamps, times_from_extractor) + interface_times = self.interface.get_original_timestamps() + assert_array_equal(one_photon_series.timestamps, interface_times) class ScanImageSinglePlaneImagingInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): @@ -1235,25 +1221,43 @@ def check_read_nwb(self, nwbfile_path: str): class TDTFiberPhotometryInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): """Mixin for testing TDT Fiber Photometry interfaces.""" + def test_metadata(self): + pass + def test_metadata_schema_valid(self): pass def test_no_metadata_mutation(self): pass + def test_conversion_options_schema_valid(self): + pass + def test_run_conversion_with_backend(self): pass + def test_run_conversion_with_backend_configuration(self): + pass + def test_no_metadata_mutation(self): pass def test_configure_backend_for_equivalent_nwbfiles(self): pass + def check_metadata(self): + # Validate metadata now happens on the class itself + metadata = self.interface.get_metadata() + self.check_extracted_metadata(metadata) + def check_metadata_schema_valid(self): schema = self.interface.get_metadata_schema() Draft7Validator.check_schema(schema=schema) + def check_conversion_options_schema_valid(self): + schema = self.interface.get_conversion_options_schema() + Draft7Validator.check_schema(schema=schema) + def check_no_metadata_mutation(self, metadata: dict): """Ensure the metadata object was not altered by `add_to_nwbfile` method.""" diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 00630afbf..7677ded22 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -158,7 +158,7 @@ def test_add_channel_metadata_to_nwb(self, setup_interface): else: assert expected_value == extracted_value - # Test addition to electrodes table + # Test addition to electrodes table!~ with NWBHDF5IO(self.nwbfile_path, "r") as io: nwbfile = io.read() electrode_table = nwbfile.electrodes.to_dataframe() @@ -176,9 +176,6 @@ class TestEDFRecordingInterface(RecordingExtractorInterfaceTestMixin): interface_kwargs = dict(file_path=str(ECEPHY_DATA_PATH / "edf" / "edf+C.edf")) save_directory = OUTPUT_PATH - def check_extracted_metadata(self, metadata: dict): - assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 3, 2, 10, 42, 19) - def check_run_conversion_with_backend(self, nwbfile_path: str, backend="hdf5"): metadata = self.interface.get_metadata() if "session_start_time" not in metadata["NWBFile"]: @@ -198,11 +195,10 @@ def test_all_conversion_checks(self, setup_interface, tmp_path): self.nwbfile_path = nwbfile_path # Now run the checks using the setup objects - self.check_conversion_options_schema_valid() - self.check_metadata() + metadata = self.interface.get_metadata() + assert metadata["NWBFile"]["session_start_time"] == datetime(2022, 3, 2, 10, 42, 19) self.check_run_conversion_with_backend(nwbfile_path=nwbfile_path, backend="hdf5") - self.check_read_nwb(nwbfile_path=nwbfile_path) # EDF has simultaneous access issues; can't have multiple interfaces open on the same file at once... @@ -215,12 +211,24 @@ def test_no_metadata_mutation(self): def test_run_conversion_with_backend(self): pass + def test_run_conversion_with_backend_configuration(self): + pass + def test_interface_alignment(self): pass def test_configure_backend_for_equivalent_nwbfiles(self): pass + def test_conversion_options_schema_valid(self): + pass + + def test_metadata(self): + pass + + def test_conversion_options_schema_valid(self): + pass + class TestIntanRecordingInterfaceRHS(RecordingExtractorInterfaceTestMixin): data_interface_cls = IntanRecordingInterface diff --git a/tests/test_ophys/test_ophys_interfaces.py b/tests/test_ophys/test_ophys_interfaces.py index c2c9b4a0c..4381faf8b 100644 --- a/tests/test_ophys/test_ophys_interfaces.py +++ b/tests/test_ophys/test_ophys_interfaces.py @@ -12,6 +12,10 @@ class TestMockImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = MockImagingInterface interface_kwargs = dict() + # TODO: fix this by setting a seed on the dummy imaging extractor + def test_all_conversion_checks(self): + pass + class TestMockSegmentationInterface(SegmentationExtractorInterfaceTestMixin): From f73bbee23b8f2258b1a72037f91e04449d64fb7e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 23 Sep 2024 13:35:20 -0600 Subject: [PATCH 067/118] Use a github action to centralize gin data preparation (#1095) --- .github/actions/load-data/action.yml | 97 ++++++++++++++++++++++++++++ .github/workflows/dev-testing.yml | 51 ++------------- .github/workflows/doctests.yml | 33 ++-------- .github/workflows/testing.yml | 54 +++------------- CHANGELOG.md | 2 +- 5 files changed, 118 insertions(+), 119 deletions(-) create mode 100644 .github/actions/load-data/action.yml diff --git a/.github/actions/load-data/action.yml b/.github/actions/load-data/action.yml new file mode 100644 index 000000000..77dd3ce93 --- /dev/null +++ b/.github/actions/load-data/action.yml @@ -0,0 +1,97 @@ +name: 'Prepare Datasets' +description: 'Restores data from caches or downloads it from S3.' +inputs: + aws-access-key-id: + description: 'AWS Access Key ID' + required: true + aws-secret-access-key: + description: 'AWS Secret Access Key' + required: true + s3-gin-bucket: + description: 'S3 GIN Bucket URL' + required: true + os: + description: 'Operating system' + required: true +runs: + using: 'composite' + steps: + - name: Get ephy_testing_data current head hash + id: ephys + shell: bash + run: | + HASH=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1) + echo "HASH_EPHY_DATASET=$HASH" >> $GITHUB_OUTPUT + + - name: Cache ephys dataset + uses: actions/cache@v4 + id: cache-ephys-datasets + with: + path: ./ephy_testing_data + key: ephys-datasets-${{ inputs.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + + - name: Get ophys_testing_data current head hash + id: ophys + shell: bash + run: | + HASH=$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1) + echo "HASH_OPHYS_DATASET=$HASH" >> $GITHUB_OUTPUT + + - name: Cache ophys dataset + uses: actions/cache@v4 + id: cache-ophys-datasets + with: + path: ./ophys_testing_data + key: ophys-datasets-${{ inputs.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + + - name: Get behavior_testing_data current head hash + id: behavior + shell: bash + run: | + HASH=$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1) + echo "HASH_BEHAVIOR_DATASET=$HASH" >> $GITHUB_OUTPUT + + - name: Cache behavior dataset + uses: actions/cache@v4 + id: cache-behavior-datasets + with: + path: ./behavior_testing_data + key: behavior-datasets-${{ inputs.os }}-${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} + + - name: Determine if downloads are required + id: download-check + shell: bash # Added shell property + run: | + if [[ "${{ steps.cache-ephys-datasets.outputs.cache-hit }}" != 'true' || \ + "${{ steps.cache-ophys-datasets.outputs.cache-hit }}" != 'true' || \ + "${{ steps.cache-behavior-datasets.outputs.cache-hit }}" != 'true' ]]; then + echo "DOWNLOAD_REQUIRED=true" >> $GITHUB_OUTPUT + else + echo "DOWNLOAD_REQUIRED=false" >> $GITHUB_OUTPUT + fi + + - if: ${{ steps.download-check.outputs.DOWNLOAD_REQUIRED == 'true' }} + name: Install and configure AWS CLI + shell: bash + run: | + pip install awscli + aws configure set aws_access_key_id "${{ inputs.aws-access-key-id }}" + aws configure set aws_secret_access_key "${{ inputs.aws-secret-access-key }}" + + - if: ${{ steps.cache-ephys-datasets.outputs.cache-hit != 'true' }} + name: Download ephys dataset from S3 + shell: bash + run: | + aws s3 cp --recursive "${{ inputs.s3-gin-bucket }}/ephy_testing_data" ./ephy_testing_data + + - if: ${{ steps.cache-ophys-datasets.outputs.cache-hit != 'true' }} + name: Download ophys dataset from S3 + shell: bash + run: | + aws s3 cp --recursive "${{ inputs.s3-gin-bucket }}/ophys_testing_data" ./ophys_testing_data + + - if: ${{ steps.cache-behavior-datasets.outputs.cache-hit != 'true' }} + name: Download behavior dataset from S3 + shell: bash + run: | + aws s3 cp --recursive "${{ inputs.s3-gin-bucket }}/behavior_testing_data" ./behavior_testing_data diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index acd7a3d74..31b5329a8 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -72,52 +72,13 @@ jobs: run: | pip list - - name: Get ephy_testing_data current head hash - id: ephys - run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" - - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - uses: actions/cache@v4 - id: cache-ephys-datasets + - name: Prepare data for tests + uses: ./.github/actions/load-data with: - path: ./ephy_testing_data - key: ephys-datasets-2024-08-30-ubuntu-latest-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - - name: Get ophys_testing_data current head hash - id: ophys - run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - uses: actions/cache@v4 - id: cache-ophys-datasets - with: - path: ./ophys_testing_data - key: ophys-datasets-2022-08-18-ubuntu-latest-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - name: Get behavior_testing_data current head hash - id: behavior - run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" - - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} - uses: actions/cache@v4 - id: cache-behavior-datasets - with: - path: ./behavior_testing_data - key: behavior-datasets-2023-07-26-ubuntu-latest-${{ steps.behavior.outputs.HASH_behavior_DATASET }} - - - - - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' || steps.cache-ophys-datasets.outputs.cache-hit != 'true' || steps.cache-behavior-datasets.outputs.cache-hit != 'true' - name: Install and configure AWS CLI - run: | - pip install awscli - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' - name: Download ephys dataset from S3 - run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ephy_testing_data ./ephy_testing_data - - if: steps.cache-ophys-datasets.outputs.cache-hit != 'true' - name: Download ophys dataset from S3 - run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ophys_testing_data ./ophys_testing_data - - if: steps.cache-behavior-datasets.outputs.cache-hit != 'true' - name: Download behavior dataset from S3 - run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data - + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + s3-gin-bucket: ${{ secrets.S3_GIN_BUCKET }} + os: ${{ matrix.os }} - name: Run full pytest diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index d816dbd02..e492eda0c 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -46,34 +46,13 @@ jobs: with: os: ${{ runner.os }} - - name: Get ephy_testing_data current head hash - id: ephys - run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" - - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - uses: actions/cache@v4 - id: cache-ephys-datasets + - name: Prepare data for tests + uses: ./.github/actions/load-data with: - path: ./ephy_testing_data - key: ephys-datasets-2024-08-30-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - - name: Get ophys_testing_data current head hash - id: ophys - run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - uses: actions/cache@v4 - id: cache-ophys-datasets - with: - path: ./ophys_testing_data - key: ophys-datasets-2022-08-18-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - name: Get behavior_testing_data current head hash - id: behavior - run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" - - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} - uses: actions/cache@v4 - id: cache-behavior-datasets - with: - path: ./behavior_testing_data - key: behavior-datasets-2023-07-26-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_behavior_DATASET }} - + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + s3-gin-bucket: ${{ secrets.S3_GIN_BUCKET }} + os: ${{ matrix.os }} - name: Run doctests diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 736da6030..06de82c4c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -93,54 +93,16 @@ jobs: #- name: Run icephys tests # There are no icephys specific tests without data # run: pytest tests/test_icephys -rsx -n auto --dist loadscope - - name: Install full requirements - run: pip install .[full] - - - name: Get ephy_testing_data current head hash - id: ephys - run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" - - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - uses: actions/cache@v4 - id: cache-ephys-datasets - with: - path: ./ephy_testing_data - key: ephys-datasets-2024-08-30-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} - - name: Get ophys_testing_data current head hash - id: ophys - run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" - - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - uses: actions/cache@v4 - id: cache-ophys-datasets - with: - path: ./ophys_testing_data - key: ophys-datasets-2022-08-18-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - name: Get behavior_testing_data current head hash - id: behavior - run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" - - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} - uses: actions/cache@v4 - id: cache-behavior-datasets + - name: Prepare data for tests + uses: ./.github/actions/load-data with: - path: ./behavior_testing_data - key: behavior-datasets-2023-07-26-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_behavior_DATASET }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + s3-gin-bucket: ${{ secrets.S3_GIN_BUCKET }} + os: ${{ matrix.os }} - - - - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' || steps.cache-ophys-datasets.outputs.cache-hit != 'true' || steps.cache-behavior-datasets.outputs.cache-hit != 'true' - name: Install and configure AWS CLI - run: | - pip install awscli - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' - name: Download ephys dataset from S3 - run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ephy_testing_data ./ephy_testing_data - - if: steps.cache-ophys-datasets.outputs.cache-hit != 'true' - name: Download ophys dataset from S3 - run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/ophys_testing_data ./ophys_testing_data - - if: steps.cache-behavior-datasets.outputs.cache-hit != 'true' - name: Download behavior dataset from S3 - run: aws s3 cp --recursive ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data + - name: Install full requirements + run: pip install .[full] - name: Run full pytest with coverage run: pytest -vv -rsx -n auto --dist loadscope --cov=neuroconv --cov-report xml:./codecov.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 751d7cc55..9c7bc13be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ * Test that zarr backend_configuration works in gin data tests [PR #1094](https://github.com/catalystneuro/neuroconv/pull/1094) * Consolidated weekly workflows into one workflow and added email notifications [PR #1088](https://github.com/catalystneuro/neuroconv/pull/1088) * Avoid running link test when the PR is on draft [PR #1093](https://github.com/catalystneuro/neuroconv/pull/1093) - +* Centralize gin data preparation in a github action [PR #1095](https://github.com/catalystneuro/neuroconv/pull/1095) # v0.6.4 (September 17, 2024) From 44c2475c3d18d4f7db53b952a184f713f9932ed1 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 24 Sep 2024 16:07:53 -0400 Subject: [PATCH 068/118] Update docker_demo.rst to format yml (#1100) --- docs/user_guide/docker_demo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst index e089b5748..a943cfa58 100644 --- a/docs/user_guide/docker_demo.rst +++ b/docs/user_guide/docker_demo.rst @@ -30,7 +30,7 @@ It relies on some of the GIN data from the main testing suite, see :ref:`example 3. Create a file in this folder named ``demo_neuroconv_docker_yaml.yml`` with the following content... -.. code:: +.. code-block:: yaml metadata: NWBFile: From 80325265ba51ed0cb802c03c32a476cb7f69f284 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sat, 28 Sep 2024 17:44:54 -0400 Subject: [PATCH 069/118] adjust docker tag on main to rebuild --- .../workflows/build_and_upload_docker_image_yaml_variable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml index 3c2a06d93..d5c6da058 100644 --- a/.github/workflows/build_and_upload_docker_image_yaml_variable.yml +++ b/.github/workflows/build_and_upload_docker_image_yaml_variable.yml @@ -31,7 +31,7 @@ jobs: uses: docker/build-push-action@v5 with: push: true # Push is a shorthand for --output=type=registry - tags: ghcr.io/catalystneuro/neuroconv:yaml_variable + tags: ghcr.io/catalystneuro/neuroconv_yaml_variable:latest context: . file: dockerfiles/neuroconv_latest_yaml_variable provenance: false From 6735e097a09326772a52f70a667f71770d9a95ea Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sat, 28 Sep 2024 18:14:16 -0400 Subject: [PATCH 070/118] adjust dockerfile on main to rebuild --- dockerfiles/neuroconv_latest_yaml_variable | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_latest_yaml_variable b/dockerfiles/neuroconv_latest_yaml_variable index ea411ee44..5500f14f0 100644 --- a/dockerfiles/neuroconv_latest_yaml_variable +++ b/dockerfiles/neuroconv_latest_yaml_variable @@ -1,4 +1,4 @@ FROM ghcr.io/catalystneuro/neuroconv:latest LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv LABEL org.opencontainers.image.description="A docker image for the most recent official release of the NeuroConv package. Modified to take in environment variables for the YAML conversion specification and other command line arguments." -CMD echo "$NEUROCONV_YAML" > run.yml && python -m neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" --overwrite +CMD printf "$NEUROCONV_YAML" > ./run.yml && neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" --overwrite From 0cc66de36d91b43f718507c97447ca56382dcb74 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sat, 28 Sep 2024 20:57:45 -0400 Subject: [PATCH 071/118] hotfix base image --- dockerfiles/neuroconv_latest_yaml_variable | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfiles/neuroconv_latest_yaml_variable b/dockerfiles/neuroconv_latest_yaml_variable index 5500f14f0..04d2f0847 100644 --- a/dockerfiles/neuroconv_latest_yaml_variable +++ b/dockerfiles/neuroconv_latest_yaml_variable @@ -1,4 +1,5 @@ -FROM ghcr.io/catalystneuro/neuroconv:latest +# TODO: make this neuroconv:latest once optional installations are working again +FROM ghcr.io/catalystneuro/neuroconv:dev LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv LABEL org.opencontainers.image.description="A docker image for the most recent official release of the NeuroConv package. Modified to take in environment variables for the YAML conversion specification and other command line arguments." CMD printf "$NEUROCONV_YAML" > ./run.yml && neuroconv run.yml --data-folder-path "$NEUROCONV_DATA_PATH" --output-folder-path "$NEUROCONV_OUTPUT_PATH" --overwrite From ccc4a1e7bee62a5bd07d9da5c17072f26ab2bea8 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:09:07 -0400 Subject: [PATCH 072/118] Slimmer code blocks in Docker usage docs (#1102) --- docs/user_guide/docker_demo.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst index a943cfa58..ddf255070 100644 --- a/docs/user_guide/docker_demo.rst +++ b/docs/user_guide/docker_demo.rst @@ -102,7 +102,11 @@ It relies on some of the GIN data from the main testing suite, see :ref:`example .. code:: - docker run -t --volume /home/user/demo_neuroconv_docker:/demo_neuroconv_docker ghcr.io/catalystneuro/neuroconv:latest neuroconv /demo_neuroconv_docker/demo_neuroconv_docker_yaml.yml --output-folder-path /demo_neuroconv_docker/demo_output + docker run -t \ + --volume /home/user/demo_neuroconv_docker:/demo_neuroconv_docker \ + ghcr.io/catalystneuro/neuroconv:latest \ + neuroconv /demo_neuroconv_docker/demo_neuroconv_docker_yaml.yml \ + --output-folder-path /demo_neuroconv_docker/demo_output Voilà! If everything occurred successfully, you should see... @@ -142,6 +146,10 @@ Then, you can use the following command to run the Rclone Docker image: .. code:: - docker run -t --volume destination_folder:destination_folder -e RCLONE_CONFIG="$RCLONE_CONFIG" -e RCLONE_COMMAND="$RCLONE_COMMAND" ghcr.io/catalystneuro/rclone_with_config:latest + docker run -t \ + --volume destination_folder:destination_folder \ + -e RCLONE_CONFIG="$RCLONE_CONFIG" \ + -e RCLONE_COMMAND="$RCLONE_COMMAND" \ + ghcr.io/catalystneuro/rclone_with_config:latest This image is particularly designed for convenience with AWS Batch (EC2) tools that rely heavily on atomic Docker operations. Alternative AWS approaches would have relied on transferring the Rclone configuration file to the EC2 instances using separate transfer protocols or dependent steps, both of which add complexity to the workflow. From bfbbe4bff8245760622e5b873cd846d346c6ae06 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:17:53 -0400 Subject: [PATCH 073/118] [pre-commit.ci] pre-commit autoupdate (#1098) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 810976eb5..b2c1d900c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.5 + rev: v0.6.9 hooks: - id: ruff args: [ --fix ] From 52cd6aa0ea46c0d143739767da4951ade17c3b74 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 11 Oct 2024 08:54:53 -0600 Subject: [PATCH 074/118] Add skip channels to EDF interface (#1110) --- CHANGELOG.md | 1 + .../ecephys/edf/edfdatainterface.py | 32 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c7bc13be..d748d79e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) * Data interfaces now perform source (argument inputs) validation with the json schema [PR #1020](https://github.com/catalystneuro/neuroconv/pull/1020) +* Added `channels_to_skip` to `EDFRecordingInterface` so the user can skip non-neural channels [PR #1110](https://github.com/catalystneuro/neuroconv/pull/1110) ## Improvements * Remove dev test from PR [PR #1092](https://github.com/catalystneuro/neuroconv/pull/1092) diff --git a/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py b/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py index 119e9f8d2..ef169f66f 100644 --- a/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py @@ -1,3 +1,5 @@ +from typing import Optional + from pydantic import FilePath from ..baserecordingextractorinterface import BaseRecordingExtractorInterface @@ -23,7 +25,22 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .edf file." return source_schema - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: + + extractor_kwargs = source_data.copy() + extractor_kwargs.pop("channels_to_skip") + extractor_kwargs["all_annotations"] = True + extractor_kwargs["use_names_as_ids"] = True + + return extractor_kwargs + + def __init__( + self, + file_path: FilePath, + verbose: bool = True, + es_key: str = "ElectricalSeries", + channels_to_skip: Optional[list] = None, + ): """ Load and prepare data for EDF. Currently, only continuous EDF+ files (EDF+C) and original EDF files (EDF) are supported @@ -36,15 +53,24 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele verbose : bool, default: True Allows verbose. es_key : str, default: "ElectricalSeries" + Key for the ElectricalSeries metadata + channels_to_skip : list, default: None + Channels to skip when adding the data to the nwbfile. These parameter can be used to skip non-neural + channels that are present in the EDF file. + """ get_package( package_name="pyedflib", - excluded_platforms_and_python_versions=dict(darwin=dict(arm=["3.8", "3.9"])), + excluded_platforms_and_python_versions=dict(darwin=dict(arm=["3.9"])), ) - super().__init__(file_path=file_path, verbose=verbose, es_key=es_key) + super().__init__(file_path=file_path, verbose=verbose, es_key=es_key, channels_to_skip=channels_to_skip) self.edf_header = self.recording_extractor.neo_reader.edf_header + # We remove the channels that are not neural + if channels_to_skip: + self.recording_extractor = self.recording_extractor.remove_channels(remove_channel_ids=channels_to_skip) + def extract_nwb_file_metadata(self) -> dict: nwbfile_metadata = dict( session_start_time=self.edf_header["startdate"], From 165cb31068a3dac62eee65921b4ab09ba1111d85 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 22 Oct 2024 08:12:01 -0600 Subject: [PATCH 075/118] Add more friendly error when writing recording with multiple offsets (#1111) --- CHANGELOG.md | 1 + .../tools/spikeinterface/spikeinterface.py | 33 ++++++++++++++++--- .../test_ecephys/test_tools_spikeinterface.py | 33 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d748d79e6..931383066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) * Data interfaces now perform source (argument inputs) validation with the json schema [PR #1020](https://github.com/catalystneuro/neuroconv/pull/1020) +* Improve the error message when writing a recording extractor with multiple offsets [PR #1111](https://github.com/catalystneuro/neuroconv/pull/1111) * Added `channels_to_skip` to `EDFRecordingInterface` so the user can skip non-neural channels [PR #1110](https://github.com/catalystneuro/neuroconv/pull/1110) ## Improvements diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 9128078a6..1be86862a 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -99,7 +99,8 @@ def add_devices_to_nwbfile(nwbfile: pynwb.NWBFile, metadata: Optional[DeepDict] metadata["Ecephys"]["Device"] = [defaults] for device_metadata in metadata["Ecephys"]["Device"]: if device_metadata.get("name", defaults["name"]) not in nwbfile.devices: - nwbfile.create_device(**dict(defaults, **device_metadata)) + device_kwargs = dict(defaults, **device_metadata) + nwbfile.create_device(**device_kwargs) def add_electrode_groups(recording: BaseRecording, nwbfile: pynwb.NWBFile, metadata: dict = None): @@ -778,6 +779,28 @@ def add_electrical_series( ) +def _report_variable_offset(channel_offsets, channel_ids): + """ + Helper function to report variable offsets per channel IDs. + Groups the different available offsets per channel IDs and raises a ValueError. + """ + # Group the different offsets per channel IDs + offset_to_channel_ids = {} + for offset, channel_id in zip(channel_offsets, channel_ids): + if offset not in offset_to_channel_ids: + offset_to_channel_ids[offset] = [] + offset_to_channel_ids[offset].append(channel_id) + + # Create a user-friendly message + message_lines = ["Recording extractors with heterogeneous offsets are not supported."] + message_lines.append("Multiple offsets were found per channel IDs:") + for offset, ids in offset_to_channel_ids.items(): + message_lines.append(f" Offset {offset}: Channel IDs {ids}") + message = "\n".join(message_lines) + + raise ValueError(message) + + def add_electrical_series_to_nwbfile( recording: BaseRecording, nwbfile: pynwb.NWBFile, @@ -905,14 +928,16 @@ def add_electrical_series_to_nwbfile( # Spikeinterface guarantees data in micro volts when return_scaled=True. This multiplies by gain and adds offsets # In nwb to get traces in Volts we take data*channel_conversion*conversion + offset channel_conversion = recording.get_channel_gains() - channel_offset = recording.get_channel_offsets() + channel_offsets = recording.get_channel_offsets() unique_channel_conversion = np.unique(channel_conversion) unique_channel_conversion = unique_channel_conversion[0] if len(unique_channel_conversion) == 1 else None - unique_offset = np.unique(channel_offset) + unique_offset = np.unique(channel_offsets) if unique_offset.size > 1: - raise ValueError("Recording extractors with heterogeneous offsets are not supported") + channel_ids = recording.get_channel_ids() + # This prints a user friendly error where the user is provided with a map from offset to channels + _report_variable_offset(channel_offsets, channel_ids) unique_offset = unique_offset[0] if unique_offset[0] is not None else 0 micro_to_volts_conversion_factor = 1e-6 diff --git a/tests/test_ecephys/test_tools_spikeinterface.py b/tests/test_ecephys/test_tools_spikeinterface.py index 11c29b31f..3436a2e70 100644 --- a/tests/test_ecephys/test_tools_spikeinterface.py +++ b/tests/test_ecephys/test_tools_spikeinterface.py @@ -1,3 +1,4 @@ +import re import unittest from datetime import datetime from pathlib import Path @@ -8,9 +9,11 @@ import numpy as np import psutil import pynwb.ecephys +import pytest from hdmf.data_utils import DataChunkIterator from hdmf.testing import TestCase from pynwb import NWBFile +from pynwb.testing.mock.file import mock_NWBFile from spikeinterface.core.generate import ( generate_ground_truth_recording, generate_recording, @@ -394,6 +397,36 @@ def test_variable_offsets_assertion(self): ) +def test_error_with_multiple_offset(): + # Generate a mock recording with 5 channels and 1 second duration + recording = generate_recording(num_channels=5, durations=[1.0]) + # Rename channels to specific identifiers for clarity in error messages + recording = recording.rename_channels(new_channel_ids=["a", "b", "c", "d", "e"]) + # Set different offsets for the channels + recording.set_channel_offsets(offsets=[0, 0, 1, 1, 2]) + + # Create a mock NWBFile object + nwbfile = mock_NWBFile() + + # Expected error message + expected_message_lines = [ + "Recording extractors with heterogeneous offsets are not supported.", + "Multiple offsets were found per channel IDs:", + " Offset 0: Channel IDs ['a', 'b']", + " Offset 1: Channel IDs ['c', 'd']", + " Offset 2: Channel IDs ['e']", + ] + expected_message = "\n".join(expected_message_lines) + + # Use re.escape to escape any special regex characters in the expected message + expected_message_regex = re.escape(expected_message) + + # Attempt to add electrical series to the NWB file + # Expecting a ValueError due to multiple offsets, matching the expected message + with pytest.raises(ValueError, match=expected_message_regex): + add_electrical_series_to_nwbfile(recording=recording, nwbfile=nwbfile) + + class TestAddElectricalSeriesChunking(unittest.TestCase): @classmethod def setUpClass(cls): From 60c5e2a9a1736c237ce46f1545abbe0a318070e6 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Thu, 24 Oct 2024 11:25:13 +1100 Subject: [PATCH 076/118] Fix Failing Dailies (#1113) --- .github/workflows/all_os_versions.txt | 1 + .github/workflows/all_python_versions.txt | 1 + .github/workflows/dailies.yml | 23 +++++++++++++++++ .github/workflows/deploy-tests.yml | 29 +++++++++++++++------- .github/workflows/dev-testing.yml | 1 - .github/workflows/doctests.yml | 2 -- .github/workflows/live-service-testing.yml | 2 -- .github/workflows/testing.yml | 3 --- CHANGELOG.md | 1 + 9 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/all_os_versions.txt create mode 100644 .github/workflows/all_python_versions.txt diff --git a/.github/workflows/all_os_versions.txt b/.github/workflows/all_os_versions.txt new file mode 100644 index 000000000..93d2aa0a7 --- /dev/null +++ b/.github/workflows/all_os_versions.txt @@ -0,0 +1 @@ +["ubuntu-latest", "macos-latest", "windows-latest", "macos-13"] diff --git a/.github/workflows/all_python_versions.txt b/.github/workflows/all_python_versions.txt new file mode 100644 index 000000000..7d1db34cd --- /dev/null +++ b/.github/workflows/all_python_versions.txt @@ -0,0 +1 @@ +["3.9", "3.10", "3.11", "3.12"] diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 459246091..136590652 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -6,6 +6,18 @@ on: - cron: "0 4 * * *" # Daily at 8PM PST, 11PM EST, 5AM CET to avoid working hours jobs: + load_python_and_os_versions: + runs-on: ubuntu-latest + outputs: + ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} + ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} + steps: + - uses: actions/checkout@v4 + - id: load_python_versions + run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" + - id: load_os_versions + run: echo "os_versions=$(cat ./.github/workflows/all_os_versions.txt)" >> "$GITHUB_OUTPUT" + build-and-upload-docker-image-dev: uses: ./.github/workflows/build_and_upload_docker_image_dev.yml secrets: @@ -13,25 +25,36 @@ jobs: DOCKER_UPLOADER_PASSWORD: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} run-daily-tests: + needs: load_python_and_os_versions uses: ./.github/workflows/testing.yml secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-daily-dev-tests: + needs: load_python_and_os_versions uses: ./.github/workflows/dev-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} run-daily-live-service-testing: + needs: load_python_and_os_versions uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-daily-neuroconv-docker-testing: uses: ./.github/workflows/neuroconv_docker_testing.yml diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 4f67d15de..a18fe8310 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -13,6 +13,17 @@ concurrency: cancel-in-progress: true jobs: + load_python_and_os_versions: + runs-on: ubuntu-latest + outputs: + ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} + ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} + steps: + - uses: actions/checkout@v4 + - id: load_python_versions + run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" + - id: load_os_versions + run: echo "os_versions=$(cat ./.github/workflows/all_os_versions.txt)" >> "$GITHUB_OUTPUT" assess-file-changes: uses: ./.github/workflows/assess-file-changes.yml @@ -31,7 +42,7 @@ jobs: 0 run-tests: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} uses: ./.github/workflows/testing.yml secrets: @@ -40,28 +51,28 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} # If the conversion gallery is the only thing that changed, run doctests only run-doctests-only: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-live-service-tests: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} check-final-status: diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index 31b5329a8..65c011653 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -7,7 +7,6 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' secrets: DANDI_API_KEY: required: true diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index e492eda0c..5d96bf4a3 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -6,12 +6,10 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' os-versions: description: 'List of OS versions to use in matrix, as JSON string' required: true type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' jobs: diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index b9a425a8d..24eda7bc3 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -7,12 +7,10 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' os-versions: description: 'List of OS versions to use in matrix, as JSON string' required: true type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' secrets: DANDI_API_KEY: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 06de82c4c..d8c5bb9fd 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -7,13 +7,10 @@ on: description: 'List of Python versions to use in matrix, as JSON string' required: true type: string - default: '["3.9", "3.10", "3.11", "3.12"]' os-versions: description: 'List of OS versions to use in matrix, as JSON string' required: true type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' - secrets: AWS_ACCESS_KEY_ID: diff --git a/CHANGELOG.md b/CHANGELOG.md index 931383066..268a773e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Deprecations ## Bug Fixes +* Fixed dailies [PR #1113](https://github.com/catalystneuro/neuroconv/pull/1113) ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) From 1897b0047d1945b99ed23476532ae5d9d5f162d0 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 1 Nov 2024 08:38:21 -0600 Subject: [PATCH 077/118] Fix installation extras (#1118) --- CHANGELOG.md | 1 + pyproject.toml | 259 +++++++++++++++++- setup.py | 56 ---- tests/test_on_data/icephys/__init__.py | 0 .../test_on_data/icephys/test_gin_icephys.py | 31 +-- tests/test_on_data/setup_paths.py | 15 +- 6 files changed, 272 insertions(+), 90 deletions(-) delete mode 100644 setup.py create mode 100644 tests/test_on_data/icephys/__init__.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 268a773e9..3fef5faff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Deprecations ## Bug Fixes +* Fixed formatwise installation from pipy [PR #1118](https://github.com/catalystneuro/neuroconv/pull/1118) * Fixed dailies [PR #1113](https://github.com/catalystneuro/neuroconv/pull/1113) ## Features diff --git a/pyproject.toml b/pyproject.toml index d7cf25813..2ee338c8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,9 @@ dependencies = [ "Changelog" = "https://github.com/catalystneuro/neuroconv/blob/main/CHANGELOG.md" +[project.scripts] +neuroconv = "neuroconv.tools.yaml_conversion_specification._yaml_conversion_specification:run_conversion_from_yaml_cli" + [project.optional-dependencies] test = [ "pytest", @@ -83,17 +86,265 @@ docs = [ "spikeinterface>=0.101.0", # Needed for the API documentation "pydata_sphinx_theme==0.12.0" ] + dandi = ["dandi>=0.58.1"] compressors = ["hdf5plugin"] aws = ["boto3"] -[tool.setuptools.packages.find] -where = ["src"] +########################## +# Modality-specific Extras +########################## +## Text +csv = [ +] +excel = [ + "openpyxl", + "xlrd", +] +text = [ + "neuroconv[csv]", + "neuroconv[excel]", +] -[project.scripts] -neuroconv = "neuroconv.tools.yaml_conversion_specification._yaml_conversion_specification:run_conversion_from_yaml_cli" +## Behavior +audio = [ + "ndx-sound>=0.2.0", +] +sleap = [ + "av>=10.0.0", + "sleap-io>=0.0.2,<0.0.12; python_version<'3.9'", + "sleap-io>=0.0.2; python_version>='3.9'", +] +deeplabcut = [ + "ndx-pose==0.1.1", + "tables; platform_system != 'Darwin'", + "tables>=3.10.1; platform_system == 'Darwin' and python_version >= '3.10'", +] +fictrac = [ +] +video = [ + "opencv-python-headless>=4.8.1.78", +] +lightningpose = [ + "ndx-pose==0.1.1", + "neuroconv[video]", +] +medpc = [ + "ndx-events==0.2.0", +] +behavior = [ + "neuroconv[sleap]", + "neuroconv[audio]", + "neuroconv[deeplabcut]", + "neuroconv[fictrac]", + "neuroconv[video]", + "neuroconv[lightningpose]", + "neuroconv[medpc]", + "ndx-miniscope>=0.5.1", # This is for the miniscope behavior data interface, not sure is needed +] + + +## Ecephys +alphaomega = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +axona = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +biocam = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +blackrock = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +cellexplorer = [ + "hdf5storage>=0.1.18", + "neo>=0.13.3", + "pymatreader>=0.0.32", + "spikeinterface>=0.101.0", +] +edf = [ + "neo>=0.13.3", + "pyedflib>=0.1.36", + "spikeinterface>=0.101.0", +] +intan = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +kilosort = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] + +maxwell = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +mcsraw = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +mearec = [ + "MEArec>=1.8.0", + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +neuralynx = [ + "natsort>=7.1.1", + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +neuroscope = [ + "lxml>=4.6.5", + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +openephys = [ + "lxml>=4.9.4", + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +phy = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +plexon = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", + "zugbruecke >= 0.2.1; platform_system != 'Windows'", +] +spike2 = [ + "neo>=0.13.3", + "sonpy>=1.7.1; python_version=='3.9' and platform_system != 'Darwin'", + "spikeinterface>=0.101.0", +] +spikegadgets = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +spikeglx = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +tdt = [ + "neo>=0.13.3", + "spikeinterface>=0.101.0", +] +ecephys = [ + "neuroconv[alphaomega]", + "neuroconv[axona]", + "neuroconv[biocam]", + "neuroconv[blackrock]", + "neuroconv[cellexplorer]", + "neuroconv[edf]", + "neuroconv[intan]", + "neuroconv[kilosort]", + "neuroconv[maxwell]", + "neuroconv[mcsraw]", + "neuroconv[mearec]", + "neuroconv[neuralynx]", + "neuroconv[neuroscope]", + "neuroconv[openephys]", + "neuroconv[phy]", + "neuroconv[plexon]", + "neuroconv[spike2]", + "neuroconv[spikegadgets]", + "neuroconv[spikeglx]", + "neuroconv[tdt]", +] + +## Icephys +abf = [ + "ndx-dandi-icephys>=0.4.0", + "neo>=0.13.2", +] +icephys = [ + "neuroconv[abf]", +] + +## Ophys +brukertiff = [ + "roiextractors>=0.5.7", + "tifffile>=2023.3.21", +] +caiman = [ + "roiextractors>=0.5.7", +] +cnmfe = [ + "roiextractors>=0.5.7", +] +extract = [ + "roiextractors>=0.5.7", +] +hdf5 = [ + "roiextractors>=0.5.7", +] +micromanagertiff = [ + "roiextractors>=0.5.7", + "tifffile>=2023.3.21", +] +miniscope = [ + "natsort>=8.3.1", + "ndx-miniscope>=0.5.1", + "roiextractors>=0.5.7", +] +sbx = [ + "roiextractors>=0.5.7", +] +scanimage = [ + "roiextractors>=0.5.7", + "scanimage-tiff-reader>=1.4.1", +] +sima = [ + "roiextractors>=0.5.7", +] +suite2p = [ + "roiextractors>=0.5.7", +] +tdt_fp = [ + "ndx-fiber-photometry", + "roiextractors>=0.5.7", + "tdt", +] +tiff = [ + "roiextractors>=0.5.7", + "tiffile>=2018.10.18", +] +ophys = [ + "neuroconv[brukertiff]", + "neuroconv[caiman]", + "neuroconv[cnmfe]", + "neuroconv[extract]", + "neuroconv[hdf5]", + "neuroconv[micromanagertiff]", + "neuroconv[miniscope]", + "neuroconv[sbx]", + "neuroconv[scanimage]", + "neuroconv[sima]", + "neuroconv[suite2p]", + "neuroconv[tdt_fp]", + "neuroconv[tiff]", +] +# Note these are references to the package in pipy (not local) +full = [ + "neuroconv[aws]", + "neuroconv[compressors]", + "neuroconv[dandi]", + "neuroconv[behavior]", + "neuroconv[ecephys]", + "neuroconv[icephys]", + "neuroconv[ophys]", + "neuroconv[text]", +] +[tool.setuptools.packages.find] +where = ["src"] [tool.pytest.ini_options] minversion = "6.0" diff --git a/setup.py b/setup.py deleted file mode 100644 index 53314e7e3..000000000 --- a/setup.py +++ /dev/null @@ -1,56 +0,0 @@ -import platform -import sys -from collections import defaultdict -from pathlib import Path -from shutil import copy - -from setuptools import setup - -root = Path(__file__).parent - - -def read_requirements(file): - """Read requirements from a file.""" - with open(root / file) as f: - return f.readlines() - - -extras_require = defaultdict(list) -extras_require["full"] = ["dandi>=0.58.1", "hdf5plugin", "boto3"] - -for modality in ["ophys", "ecephys", "icephys", "behavior", "text"]: - modality_path = root / "src" / "neuroconv" / "datainterfaces" / modality - modality_requirement_file = modality_path / "requirements.txt" - if modality_requirement_file.exists(): - modality_requirements = read_requirements(modality_requirement_file) - extras_require["full"].extend(modality_requirements) - extras_require[modality] = modality_requirements - else: - modality_requirements = [] - - format_subpaths = [path for path in modality_path.iterdir() if path.is_dir() and path.name != "__pycache__"] - for format_subpath in format_subpaths: - format_requirement_file = format_subpath / "requirements.txt" - extras_require[format_subpath.name] = modality_requirements.copy() - if format_requirement_file.exists(): - format_requirements = read_requirements(format_requirement_file) - extras_require["full"].extend(format_requirements) - extras_require[modality].extend(format_requirements) - extras_require[format_subpath.name].extend(format_requirements) - -# Create a local copy for the gin test configuration file based on the master file `base_gin_test_config.json` -gin_config_file_base = root / "base_gin_test_config.json" -gin_config_file_local = root / "tests/test_on_data/gin_test_config.json" -if not gin_config_file_local.exists(): - gin_config_file_local.parent.mkdir(parents=True, exist_ok=True) - copy(src=gin_config_file_base, dst=gin_config_file_local) - -# Bug related to sonpy on M1 Mac being installed but not running properly -if sys.platform == "darwin" and platform.processor() == "arm": - extras_require.pop("spike2", None) - extras_require["ecephys"] = [req for req in extras_require["ecephys"] if "sonpy" not in req] - extras_require["full"] = [req for req in extras_require["full"] if "sonpy" not in req] - -setup( - extras_require=extras_require, -) diff --git a/tests/test_on_data/icephys/__init__.py b/tests/test_on_data/icephys/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_on_data/icephys/test_gin_icephys.py b/tests/test_on_data/icephys/test_gin_icephys.py index 2d71e0636..2c8f4e22a 100644 --- a/tests/test_on_data/icephys/test_gin_icephys.py +++ b/tests/test_on_data/icephys/test_gin_icephys.py @@ -1,8 +1,5 @@ -import os -import tempfile import unittest from datetime import datetime -from pathlib import Path import numpy.testing as npt import pytest @@ -11,7 +8,8 @@ from neuroconv import NWBConverter from neuroconv.datainterfaces import AbfInterface from neuroconv.tools.neo import get_number_of_electrodes, get_number_of_segments -from neuroconv.utils import load_dict_from_file + +from ..setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH try: from parameterized import param, parameterized @@ -19,30 +17,9 @@ HAVE_PARAMETERIZED = True except ImportError: HAVE_PARAMETERIZED = False -# Load the configuration for the data tests -test_config_dict = load_dict_from_file(Path(__file__).parent.parent / "gin_test_config.json") - -# GIN dataset: https://gin.g-node.org/NeuralEnsemble/ephy_testing_data -if os.getenv("CI"): - LOCAL_PATH = Path(".") # Must be set to "." for CI - print("Running GIN tests on Github CI!") -else: - # Override LOCAL_PATH in the `gin_test_config.json` file to a point on your system that contains the dataset folder - # Use DANDIHub at hub.dandiarchive.org for open, free use of data found in the /shared/catalystneuro/ directory - LOCAL_PATH = Path(test_config_dict["LOCAL_PATH"]) - print("Running GIN tests locally!") -DATA_PATH = LOCAL_PATH / "ephy_testing_data" -HAVE_DATA = DATA_PATH.exists() - -if test_config_dict["SAVE_OUTPUTS"]: - OUTPUT_PATH = LOCAL_PATH / "example_nwb_output" - OUTPUT_PATH.mkdir(exist_ok=True) -else: - OUTPUT_PATH = Path(tempfile.mkdtemp()) + if not HAVE_PARAMETERIZED: pytest.fail("parameterized module is not installed! Please install (`pip install parameterized`).") -if not HAVE_DATA: - pytest.fail(f"No ephy_testing_data folder found in location: {DATA_PATH}!") def custom_name_func(testcase_func, param_num, param): @@ -59,7 +36,7 @@ class TestIcephysNwbConversions(unittest.TestCase): param( data_interface=AbfInterface, interface_kwargs=dict( - file_paths=[str(DATA_PATH / "axon" / "File_axon_1.abf")], + file_paths=[str(ECEPHY_DATA_PATH / "axon" / "File_axon_1.abf")], icephys_metadata={ "recording_sessions": [ {"abf_file_name": "File_axon_1.abf", "icephys_experiment_type": "voltage_clamp"} diff --git a/tests/test_on_data/setup_paths.py b/tests/test_on_data/setup_paths.py index 9554d27eb..3f7bf4123 100644 --- a/tests/test_on_data/setup_paths.py +++ b/tests/test_on_data/setup_paths.py @@ -1,6 +1,7 @@ import os import tempfile from pathlib import Path +from shutil import copy from neuroconv.utils import load_dict_from_file @@ -17,9 +18,17 @@ else: # Override LOCAL_PATH in the `gin_test_config.json` file to a point on your system that contains the dataset folder # Use DANDIHub at hub.dandiarchive.org for open, free use of data found in the /shared/catalystneuro/ directory - file_path = Path(__file__).parent / "gin_test_config.json" - assert file_path.exists(), f"File not found: {file_path}" - test_config_dict = load_dict_from_file(file_path) + test_config_path = Path(__file__).parent / "gin_test_config.json" + config_file_exists = test_config_path.exists() + if not config_file_exists: + + root = test_config_path.parent.parent + base_test_config_path = root / "base_gin_test_config.json" + + test_config_path.parent.mkdir(parents=True, exist_ok=True) + copy(src=base_test_config_path, dst=test_config_path) + + test_config_dict = load_dict_from_file(test_config_path) LOCAL_PATH = Path(test_config_dict["LOCAL_PATH"]) if test_config_dict["SAVE_OUTPUTS"]: From 2fc153adc566a7b53ee1e855407cc512c6d0d204 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 08:58:51 -0600 Subject: [PATCH 078/118] [pre-commit.ci] pre-commit autoupdate (#1120) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Heberto Mayorquin --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b2c1d900c..6a6977633 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 + rev: v0.7.1 hooks: - id: ruff args: [ --fix ] From f50a6f55b213316e0845f5a956e41109812921b9 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 1 Nov 2024 15:15:15 -0600 Subject: [PATCH 079/118] Release 0.6.5 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fef5faff..bc816e350 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Upcoming +# v0.6.5 (November 1, 2024) + ## Deprecations ## Bug Fixes From 419ab11104ff9e67d63f6400aacb3e12081c90d1 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 1 Nov 2024 15:19:18 -0600 Subject: [PATCH 080/118] bump version --- CHANGELOG.md | 8 ++++++++ pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc816e350..fa679434a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Upcoming +## Deprecations + +## Bug Fixes + +## Features + +## Improvements + # v0.6.5 (November 1, 2024) ## Deprecations diff --git a/pyproject.toml b/pyproject.toml index 2ee338c8c..a83380467 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.5" +version = "0.6.6" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ From cbf68d4c6d7a3a5c942586e228a90063fe161b22 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 7 Nov 2024 09:59:53 -0600 Subject: [PATCH 081/118] Change deprecated `get_schema_from_method_signature` to `get_json_schema_from_method_signature` (#1130) --- .../behavior/lightningpose/lightningposeconverter.py | 6 +++--- .../datainterfaces/ecephys/axona/axonadatainterface.py | 4 ++-- .../ecephys/blackrock/blackrockdatainterface.py | 6 +++--- .../ecephys/openephys/openephysbinarydatainterface.py | 4 ++-- .../ecephys/openephys/openephyssortingdatainterface.py | 4 ++-- .../datainterfaces/ecephys/spike2/spike2datainterface.py | 4 ++-- .../datainterfaces/ecephys/spikeglx/spikeglxconverter.py | 4 ++-- .../ecephys/spikeglx/spikeglxnidqinterface.py | 4 ++-- .../datainterfaces/icephys/baseicephysinterface.py | 4 ++-- .../datainterfaces/ophys/brukertiff/brukertiffconverter.py | 6 +++--- .../datainterfaces/ophys/miniscope/miniscopeconverter.py | 4 ++-- src/neuroconv/tools/testing/mock_interfaces.py | 6 +++--- src/neuroconv/utils/__init__.py | 2 +- .../test_get_json_schema_from_method_signature.py | 6 +++--- 14 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py index dee848f19..505aa144d 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py @@ -10,7 +10,7 @@ from neuroconv.utils import ( DeepDict, dict_deep_update, - get_schema_from_method_signature, + get_json_schema_from_method_signature, ) @@ -24,7 +24,7 @@ class LightningPoseConverter(NWBConverter): @classmethod def get_source_schema(cls): - return get_schema_from_method_signature(cls) + return get_json_schema_from_method_signature(cls) @validate_call def __init__( @@ -71,7 +71,7 @@ def __init__( self.data_interface_objects.update(dict(LabeledVideo=VideoInterface(file_paths=[labeled_video_file_path]))) def get_conversion_options_schema(self) -> dict: - conversion_options_schema = get_schema_from_method_signature( + conversion_options_schema = get_json_schema_from_method_signature( method=self.add_to_nwbfile, exclude=["nwbfile", "metadata"] ) diff --git a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py index ba6adf4a1..3c8a1067c 100644 --- a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py @@ -12,7 +12,7 @@ from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....basedatainterface import BaseDataInterface from ....tools.nwb_helpers import get_module -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class AxonaRecordingInterface(BaseRecordingExtractorInterface): @@ -186,7 +186,7 @@ class AxonaPositionDataInterface(BaseDataInterface): @classmethod def get_source_schema(cls) -> dict: - return get_schema_from_method_signature(cls.__init__) + return get_json_schema_from_method_signature(cls.__init__) def __init__(self, file_path: str): """ diff --git a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py index e84719431..7e6e499d1 100644 --- a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py @@ -6,7 +6,7 @@ from .header_tools import _parse_nev_basic_header, _parse_nsx_basic_header from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ..basesortingextractorinterface import BaseSortingExtractorInterface -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class BlackrockRecordingInterface(BaseRecordingExtractorInterface): @@ -19,7 +19,7 @@ class BlackrockRecordingInterface(BaseRecordingExtractorInterface): @classmethod def get_source_schema(cls): - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["block_index", "seg_index"]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["block_index", "seg_index"]) source_schema["properties"]["file_path"][ "description" ] = "Path to the Blackrock file with suffix being .ns1, .ns2, .ns3, .ns4m .ns4, or .ns6." @@ -85,7 +85,7 @@ class BlackrockSortingInterface(BaseSortingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: - metadata_schema = get_schema_from_method_signature(method=cls.__init__) + metadata_schema = get_json_schema_from_method_signature(method=cls.__init__) metadata_schema["additionalProperties"] = True metadata_schema["properties"]["file_path"].update(description="Path to Blackrock .nev file.") return metadata_schema diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py index 371b96f94..ef67fde40 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py @@ -3,7 +3,7 @@ from pydantic import DirectoryPath from ..baserecordingextractorinterface import BaseRecordingExtractorInterface -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class OpenEphysBinaryRecordingInterface(BaseRecordingExtractorInterface): @@ -29,7 +29,7 @@ def get_stream_names(cls, folder_path: DirectoryPath) -> list[str]: @classmethod def get_source_schema(cls) -> dict: """Compile input schema for the RecordingExtractor.""" - source_schema = get_schema_from_method_signature( + source_schema = get_json_schema_from_method_signature( method=cls.__init__, exclude=["recording_id", "experiment_id", "stub_test"] ) source_schema["properties"]["folder_path"][ diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py index 2d53e6331..ecf2067f1 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py @@ -1,7 +1,7 @@ from pydantic import DirectoryPath, validate_call from ..basesortingextractorinterface import BaseSortingExtractorInterface -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class OpenEphysSortingInterface(BaseSortingExtractorInterface): @@ -14,7 +14,7 @@ class OpenEphysSortingInterface(BaseSortingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: """Compile input schema for the SortingExtractor.""" - metadata_schema = get_schema_from_method_signature( + metadata_schema = get_json_schema_from_method_signature( method=cls.__init__, exclude=["recording_id", "experiment_id"] ) metadata_schema["properties"]["folder_path"].update( diff --git a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py index ccd98a369..bf0ddc860 100644 --- a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py @@ -4,7 +4,7 @@ from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....tools import get_package -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature def _test_sonpy_installation() -> None: @@ -29,7 +29,7 @@ class Spike2RecordingInterface(BaseRecordingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["smrx_channel_ids"]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["smrx_channel_ids"]) source_schema.update(additionalProperties=True) source_schema["properties"]["file_path"].update(description="Path to .smrx file.") return source_schema diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py index 6aeb36cec..007c3177c 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py @@ -6,7 +6,7 @@ from .spikeglxdatainterface import SpikeGLXRecordingInterface from .spikeglxnidqinterface import SpikeGLXNIDQInterface from ....nwbconverter import ConverterPipe -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class SpikeGLXConverterPipe(ConverterPipe): @@ -23,7 +23,7 @@ class SpikeGLXConverterPipe(ConverterPipe): @classmethod def get_source_schema(cls): - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["streams"]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["streams"]) source_schema["properties"]["folder_path"]["description"] = "Path to the folder containing SpikeGLX streams." return source_schema diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index fab9e5b5f..3cf50080a 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -6,7 +6,7 @@ from .spikeglx_utils import get_session_start_time from ..baserecordingextractorinterface import BaseRecordingExtractorInterface from ....tools.signal_processing import get_rising_frames_from_ttl -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class SpikeGLXNIDQInterface(BaseRecordingExtractorInterface): @@ -22,7 +22,7 @@ class SpikeGLXNIDQInterface(BaseRecordingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["x_pitch", "y_pitch"]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["x_pitch", "y_pitch"]) source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX .nidq file." return source_schema diff --git a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py index f8bad53d6..092ec1e36 100644 --- a/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py +++ b/src/neuroconv/datainterfaces/icephys/baseicephysinterface.py @@ -7,9 +7,9 @@ from ...baseextractorinterface import BaseExtractorInterface from ...tools.nwb_helpers import make_nwbfile_from_metadata from ...utils import ( + get_json_schema_from_method_signature, get_metadata_schema_for_icephys, get_schema_from_hdmf_class, - get_schema_from_method_signature, ) @@ -22,7 +22,7 @@ class BaseIcephysInterface(BaseExtractorInterface): @classmethod def get_source_schema(cls) -> dict: - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=[]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=[]) return source_schema @validate_call diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py index 86e8edc1f..2a67da720 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py @@ -9,7 +9,7 @@ ) from ....nwbconverter import NWBConverter from ....tools.nwb_helpers import make_or_load_nwbfile -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class BrukerTiffMultiPlaneConverter(NWBConverter): @@ -24,7 +24,7 @@ class BrukerTiffMultiPlaneConverter(NWBConverter): @classmethod def get_source_schema(cls): - source_schema = get_schema_from_method_signature(cls) + source_schema = get_json_schema_from_method_signature(cls) source_schema["properties"]["folder_path"][ "description" ] = "The folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env)." @@ -138,7 +138,7 @@ class BrukerTiffSinglePlaneConverter(NWBConverter): @classmethod def get_source_schema(cls): - return get_schema_from_method_signature(cls) + return get_json_schema_from_method_signature(cls) def get_conversion_options_schema(self): interface_name = list(self.data_interface_objects.keys())[0] diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py index cfee8f027..d1a0fb701 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py @@ -6,7 +6,7 @@ from ... import MiniscopeBehaviorInterface, MiniscopeImagingInterface from ....nwbconverter import NWBConverter from ....tools.nwb_helpers import make_or_load_nwbfile -from ....utils import get_schema_from_method_signature +from ....utils import get_json_schema_from_method_signature class MiniscopeConverter(NWBConverter): @@ -19,7 +19,7 @@ class MiniscopeConverter(NWBConverter): @classmethod def get_source_schema(cls): - source_schema = get_schema_from_method_signature(cls) + source_schema = get_json_schema_from_method_signature(cls) source_schema["properties"]["folder_path"]["description"] = "The path to the main Miniscope folder." return source_schema diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 04dc57250..4ba7bb639 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -20,7 +20,7 @@ from ...datainterfaces.ophys.basesegmentationextractorinterface import ( BaseSegmentationExtractorInterface, ) -from ...utils import ArrayType, get_schema_from_method_signature +from ...utils import ArrayType, get_json_schema_from_method_signature class MockBehaviorEventInterface(BaseTemporalAlignmentInterface): @@ -30,7 +30,7 @@ class MockBehaviorEventInterface(BaseTemporalAlignmentInterface): @classmethod def get_source_schema(cls) -> dict: - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["event_times"]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["event_times"]) source_schema["additionalProperties"] = True return source_schema @@ -74,7 +74,7 @@ class MockSpikeGLXNIDQInterface(SpikeGLXNIDQInterface): @classmethod def get_source_schema(cls) -> dict: - source_schema = get_schema_from_method_signature(method=cls.__init__, exclude=["ttl_times"]) + source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["ttl_times"]) source_schema["additionalProperties"] = True return source_schema diff --git a/src/neuroconv/utils/__init__.py b/src/neuroconv/utils/__init__.py index f59cf59c5..c0061a983 100644 --- a/src/neuroconv/utils/__init__.py +++ b/src/neuroconv/utils/__init__.py @@ -12,7 +12,7 @@ get_base_schema, get_metadata_schema_for_icephys, get_schema_from_hdmf_class, - get_schema_from_method_signature, + get_json_schema_from_method_signature, unroot_schema, get_json_schema_from_method_signature, ) diff --git a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py index 10139f7e1..e6fe27e16 100644 --- a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py +++ b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py @@ -110,7 +110,7 @@ def basic_method( assert test_json_schema == expected_json_schema -def test_get_schema_from_method_signature_init(): +def test_get_json_schema_from_method_signature_init(): """Test that 'self' is automatically skipped.""" class TestClass: @@ -141,7 +141,7 @@ def __init__( assert test_json_schema == expected_json_schema -def test_get_schema_from_method_signature_class_static(): +def test_get_json_schema_from_method_signature_class_static(): """Ensuring that signature assembly prior to passing to Pydantic is not affected by bound or static methods.""" class TestClass: @@ -165,7 +165,7 @@ def test_static_method(integer: int, string: str, boolean: bool): assert test_json_schema == expected_json_schema -def test_get_schema_from_method_signature_class_method(): +def test_get_json_schema_from_method_signature_class_method(): """Test that 'cls' is automatically skipped.""" class TestClass: From 9448f95f1b92b4035b792a7d87571dedcd2fb044 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 11 Nov 2024 10:02:38 -0600 Subject: [PATCH 082/118] Remove deprecations and add docstring to `BaseImagingExtractorInterface` (#1126) --- CHANGELOG.md | 1 + .../behavior/fictrac/fictracdatainterface.py | 14 ------ .../ecephys/baselfpextractorinterface.py | 4 -- .../baserecordingextractorinterface.py | 4 -- .../ophys/baseimagingextractorinterface.py | 45 ++++++++++--------- src/neuroconv/tools/neo/neo.py | 36 --------------- .../tools/spikeinterface/spikeinterface.py | 38 ---------------- .../tools/testing/mock_interfaces.py | 4 +- .../test_baseimagingextractorinterface.py | 15 ------- 9 files changed, 28 insertions(+), 133 deletions(-) delete mode 100644 tests/test_ophys/test_baseimagingextractorinterface.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fa679434a..75c6ea917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Upcoming ## Deprecations +* Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) ## Bug Fixes diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index 1b9686fd1..1d822f919 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -1,6 +1,5 @@ import json import re -import warnings from datetime import datetime, timezone from pathlib import Path from typing import Optional, Union @@ -210,8 +209,6 @@ def add_to_nwbfile( self, nwbfile: NWBFile, metadata: Optional[dict] = None, - compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 - compression_opts: Optional[int] = None, # TODO: remove completely after 10/1/2024 ): """ Parameters @@ -223,17 +220,6 @@ def add_to_nwbfile( """ import pandas as pd - # TODO: remove completely after 10/1/2024 - if compression is not None or compression_opts is not None: - warnings.warn( - message=( - "Specifying compression methods and their options at the level of tool functions has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) - fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, names=self.columns_in_dat_file) # Get the timestamps diff --git a/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py index 7ce6bb9e4..af16601bb 100644 --- a/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py @@ -26,8 +26,6 @@ def add_to_nwbfile( starting_time: Optional[float] = None, write_as: Literal["raw", "lfp", "processed"] = "lfp", write_electrical_series: bool = True, - compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 - compression_opts: Optional[int] = None, iterator_type: str = "v2", iterator_opts: Optional[dict] = None, ): @@ -38,8 +36,6 @@ def add_to_nwbfile( starting_time=starting_time, write_as=write_as, write_electrical_series=write_electrical_series, - compression=compression, - compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, ) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index e2c747378..6d0df14c1 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -308,8 +308,6 @@ def add_to_nwbfile( starting_time: Optional[float] = None, write_as: Literal["raw", "lfp", "processed"] = "raw", write_electrical_series: bool = True, - compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 - compression_opts: Optional[int] = None, iterator_type: Optional[str] = "v2", iterator_opts: Optional[dict] = None, always_write_timestamps: bool = False, @@ -388,8 +386,6 @@ def add_to_nwbfile( write_as=write_as, write_electrical_series=write_electrical_series, es_key=self.es_key, - compression=compression, - compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, always_write_timestamps=always_write_timestamps, diff --git a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py index 5125af3cc..9f88b861f 100644 --- a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py @@ -1,6 +1,5 @@ """Author: Ben Dichter.""" -import warnings from typing import Literal, Optional import numpy as np @@ -46,17 +45,9 @@ def __init__( self.photon_series_type = photon_series_type def get_metadata_schema( - self, photon_series_type: Optional[Literal["OnePhotonSeries", "TwoPhotonSeries"]] = None + self, ) -> dict: - if photon_series_type is not None: - warnings.warn( - "The 'photon_series_type' argument is deprecated and will be removed in a future version. " - "Please set 'photon_series_type' during the initialization of the BaseImagingExtractorInterface instance.", - DeprecationWarning, - stacklevel=2, - ) - self.photon_series_type = photon_series_type metadata_schema = super().get_metadata_schema() metadata_schema["required"] = ["Ophys"] @@ -100,18 +91,9 @@ def get_metadata_schema( return metadata_schema def get_metadata( - self, photon_series_type: Optional[Literal["OnePhotonSeries", "TwoPhotonSeries"]] = None + self, ) -> DeepDict: - if photon_series_type is not None: - warnings.warn( - "The 'photon_series_type' argument is deprecated and will be removed in a future version. " - "Please set 'photon_series_type' during the initialization of the BaseImagingExtractorInterface instance.", - DeprecationWarning, - stacklevel=2, - ) - self.photon_series_type = photon_series_type - from ...tools.roiextractors import get_nwb_imaging_metadata metadata = super().get_metadata() @@ -147,6 +129,29 @@ def add_to_nwbfile( stub_test: bool = False, stub_frames: int = 100, ): + """ + Add imaging data to the NWB file + + Parameters + ---------- + nwbfile : NWBFile + The NWB file where the imaging data will be added. + metadata : dict, optional + Metadata for the NWBFile, by default None. + photon_series_type : {"TwoPhotonSeries", "OnePhotonSeries"}, optional + The type of photon series to be added, by default "TwoPhotonSeries". + photon_series_index : int, optional + The index of the photon series in the provided imaging data, by default 0. + parent_container : {"acquisition", "processing/ophys"}, optional + Specifies the parent container to which the photon series should be added, either as part of "acquisition" or + under the "processing/ophys" module, by default "acquisition". + stub_test : bool, optional + If True, only writes a small subset of frames for testing purposes, by default False. + stub_frames : int, optional + The number of frames to write when stub_test is True. Will use min(stub_frames, total_frames) to avoid + exceeding available frames, by default 100. + """ + from ...tools.roiextractors import add_imaging_to_nwbfile if stub_test: diff --git a/src/neuroconv/tools/neo/neo.py b/src/neuroconv/tools/neo/neo.py index 220c64de0..ccef706e5 100644 --- a/src/neuroconv/tools/neo/neo.py +++ b/src/neuroconv/tools/neo/neo.py @@ -214,7 +214,6 @@ def add_icephys_recordings( icephys_experiment_type: str = "voltage_clamp", stimulus_type: str = "not described", skip_electrodes: tuple[int] = (), - compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 ): """ Add icephys recordings (stimulus/response pairs) to nwbfile object. @@ -230,16 +229,6 @@ def add_icephys_recordings( skip_electrodes : tuple, default: () Electrode IDs to skip. """ - # TODO: remove completely after 10/1/2024 - if compression is not None: - warn( - message=( - "Specifying compression methods and their options at the level of tool functions has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) n_segments = get_number_of_segments(neo_reader, block=0) @@ -380,7 +369,6 @@ def add_neo_to_nwb( neo_reader, nwbfile: pynwb.NWBFile, metadata: dict = None, - compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 icephys_experiment_type: str = "voltage_clamp", stimulus_type: Optional[str] = None, skip_electrodes: tuple[int] = (), @@ -409,15 +397,6 @@ def add_neo_to_nwb( assert isinstance(nwbfile, pynwb.NWBFile), "'nwbfile' should be of type pynwb.NWBFile" # TODO: remove completely after 10/1/2024 - if compression is not None: - warn( - message=( - "Specifying compression methods and their options at the level of tool functions has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) add_device_from_metadata(nwbfile=nwbfile, modality="Icephys", metadata=metadata) @@ -443,7 +422,6 @@ def write_neo_to_nwb( overwrite: bool = False, nwbfile=None, metadata: dict = None, - compression: Optional[str] = None, # TODO: remove completely after 10/1/2024 icephys_experiment_type: Optional[str] = None, stimulus_type: Optional[str] = None, skip_electrodes: Optional[tuple] = (), @@ -499,9 +477,6 @@ def write_neo_to_nwb( Note that data intended to be added to the electrodes table of the NWBFile should be set as channel properties in the RecordingExtractor object. - compression: str (optional, defaults to "gzip") - Type of compression to use. Valid types are "gzip" and "lzf". - Set to None to disable all compression. icephys_experiment_type: str (optional) Type of Icephys experiment. Allowed types are: 'voltage_clamp', 'current_clamp' and 'izero'. If no value is passed, 'voltage_clamp' is used as default. @@ -518,17 +493,6 @@ def write_neo_to_nwb( assert save_path is None or nwbfile is None, "Either pass a save_path location, or nwbfile object, but not both!" - # TODO: remove completely after 10/1/2024 - if compression is not None: - warn( - message=( - "Specifying compression methods and their options at the level of tool functions has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) - if metadata is None: metadata = get_nwb_metadata(neo_reader=neo_reader) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 1be86862a..5aa3c8925 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -749,8 +749,6 @@ def add_electrical_series( write_as: Literal["raw", "processed", "lfp"] = "raw", es_key: str = None, write_scaled: bool = False, - compression: Optional[str] = None, - compression_opts: Optional[int] = None, iterator_type: Optional[str] = "v2", iterator_opts: Optional[dict] = None, ): @@ -772,8 +770,6 @@ def add_electrical_series( write_as=write_as, es_key=es_key, write_scaled=write_scaled, - compression=compression, - compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, ) @@ -810,8 +806,6 @@ def add_electrical_series_to_nwbfile( write_as: Literal["raw", "processed", "lfp"] = "raw", es_key: str = None, write_scaled: bool = False, - compression: Optional[str] = None, - compression_opts: Optional[int] = None, iterator_type: Optional[str] = "v2", iterator_opts: Optional[dict] = None, always_write_timestamps: bool = False, @@ -847,7 +841,6 @@ def add_electrical_series_to_nwbfile( write_scaled : bool, default: False If True, writes the traces in uV with the right conversion. If False , the data is stored as it is and the right conversions factors are added to the nwbfile. - Only applies to compression="gzip". Controls the level of the GZIP. iterator_type: {"v2", None}, default: 'v2' The type of DataChunkIterator to use. 'v1' is the original DataChunkIterator of the hdmf data_utils. @@ -868,16 +861,6 @@ def add_electrical_series_to_nwbfile( Missing keys in an element of metadata['Ecephys']['ElectrodeGroup'] will be auto-populated with defaults whenever possible. """ - # TODO: remove completely after 10/1/2024 - if compression is not None or compression_opts is not None: - warnings.warn( - message=( - "Specifying compression methods and their options at the level of tool functions has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) assert write_as in [ "raw", @@ -1042,8 +1025,6 @@ def add_recording( es_key: Optional[str] = None, write_electrical_series: bool = True, write_scaled: bool = False, - compression: Optional[str] = "gzip", - compression_opts: Optional[int] = None, iterator_type: str = "v2", iterator_opts: Optional[dict] = None, ): @@ -1065,8 +1046,6 @@ def add_recording( es_key=es_key, write_electrical_series=write_electrical_series, write_scaled=write_scaled, - compression=compression, - compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, ) @@ -1081,8 +1060,6 @@ def add_recording_to_nwbfile( es_key: Optional[str] = None, write_electrical_series: bool = True, write_scaled: bool = False, - compression: Optional[str] = "gzip", - compression_opts: Optional[int] = None, iterator_type: str = "v2", iterator_opts: Optional[dict] = None, always_write_timestamps: bool = False, @@ -1163,8 +1140,6 @@ def add_recording_to_nwbfile( write_as=write_as, es_key=es_key, write_scaled=write_scaled, - compression=compression, - compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, always_write_timestamps=always_write_timestamps, @@ -1183,8 +1158,6 @@ def write_recording( es_key: Optional[str] = None, write_electrical_series: bool = True, write_scaled: bool = False, - compression: Optional[str] = "gzip", - compression_opts: Optional[int] = None, iterator_type: Optional[str] = "v2", iterator_opts: Optional[dict] = None, ): @@ -1209,8 +1182,6 @@ def write_recording( es_key=es_key, write_electrical_series=write_electrical_series, write_scaled=write_scaled, - compression=compression, - compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, ) @@ -1228,8 +1199,6 @@ def write_recording_to_nwbfile( es_key: Optional[str] = None, write_electrical_series: bool = True, write_scaled: bool = False, - compression: Optional[str] = "gzip", - compression_opts: Optional[int] = None, iterator_type: Optional[str] = "v2", iterator_opts: Optional[dict] = None, ) -> pynwb.NWBFile: @@ -1303,11 +1272,6 @@ def write_recording_to_nwbfile( and electrodes are written to NWB. write_scaled: bool, default: True If True, writes the scaled traces (return_scaled=True) - compression: {None, 'gzip', 'lzp'}, default: 'gzip' - Type of compression to use. Set to None to disable all compression. - To use the `configure_backend` function, you should set this to None. - compression_opts: int, optional, default: 4 - Only applies to compression="gzip". Controls the level of the GZIP. iterator_type: {"v2", "v1", None} The type of DataChunkIterator to use. 'v1' is the original DataChunkIterator of the hdmf data_utils. @@ -1348,8 +1312,6 @@ def write_recording_to_nwbfile( es_key=es_key, write_electrical_series=write_electrical_series, write_scaled=write_scaled, - compression=compression, - compression_opts=compression_opts, iterator_type=iterator_type, iterator_opts=iterator_opts, ) diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 4ba7bb639..44d1adf61 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -265,9 +265,9 @@ def __init__( self.verbose = verbose self.photon_series_type = photon_series_type - def get_metadata(self, photon_series_type: Optional[Literal["OnePhotonSeries", "TwoPhotonSeries"]] = None) -> dict: + def get_metadata(self) -> dict: session_start_time = datetime.now().astimezone() - metadata = super().get_metadata(photon_series_type=photon_series_type) + metadata = super().get_metadata() metadata["NWBFile"]["session_start_time"] = session_start_time return metadata diff --git a/tests/test_ophys/test_baseimagingextractorinterface.py b/tests/test_ophys/test_baseimagingextractorinterface.py deleted file mode 100644 index 863a978d2..000000000 --- a/tests/test_ophys/test_baseimagingextractorinterface.py +++ /dev/null @@ -1,15 +0,0 @@ -from hdmf.testing import TestCase - -from neuroconv.tools.testing.mock_interfaces import MockImagingInterface - - -class TestBaseImagingExtractorInterface(TestCase): - def setUp(self): - self.mock_imaging_interface = MockImagingInterface() - - def test_photon_series_type_warning_triggered_in_get_metadata(self): - with self.assertWarnsWith( - warn_type=DeprecationWarning, - exc_msg="The 'photon_series_type' argument is deprecated and will be removed in a future version. Please set 'photon_series_type' during the initialization of the BaseImagingExtractorInterface instance.", - ): - self.mock_imaging_interface.get_metadata(photon_series_type="TwoPhotonSeries") From 6960872de0b11f9f04d4f0efecbec4aa9c010c12 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 11 Nov 2024 14:17:24 -0600 Subject: [PATCH 083/118] Add `always_write_timestamps` conversion option to imaging interfaces (#1125) --- CHANGELOG.md | 3 +- pyproject.toml | 26 +++++++------- .../behavior/video/videodatainterface.py | 13 ------- .../ophys/baseimagingextractorinterface.py | 2 ++ .../tools/roiextractors/roiextractors.py | 34 +++++++++++++++---- .../tools/testing/mock_interfaces.py | 2 ++ tests/test_ophys/test_ophys_interfaces.py | 14 +++++++- 7 files changed, 59 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75c6ea917..a06ddf300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ## Bug Fixes ## Features +* Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) ## Improvements @@ -46,7 +47,7 @@ * Added automated EFS volume creation and mounting to the `submit_aws_job` helper function. [PR #1018](https://github.com/catalystneuro/neuroconv/pull/1018) * Added a mock for segmentation extractors interfaces in ophys: `MockSegmentationInterface` [PR #1067](https://github.com/catalystneuro/neuroconv/pull/1067) * Added a `MockSortingInterface` for testing purposes. [PR #1065](https://github.com/catalystneuro/neuroconv/pull/1065) -* BaseRecordingInterfaces have a new conversion options `always_write_timestamps` that ca be used to force writing timestamps even if neuroconv heuristic indicates regular sampling rate [PR #1091](https://github.com/catalystneuro/neuroconv/pull/1091) +* BaseRecordingInterfaces have a new conversion options `always_write_timestamps` that can be used to force writing timestamps even if neuroconv heuristic indicates regular sampling rate [PR #1091](https://github.com/catalystneuro/neuroconv/pull/1091) ## Improvements diff --git a/pyproject.toml b/pyproject.toml index a83380467..5efd432f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -270,50 +270,50 @@ icephys = [ ## Ophys brukertiff = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", "tifffile>=2023.3.21", ] caiman = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] cnmfe = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] extract = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] hdf5 = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] micromanagertiff = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", "tifffile>=2023.3.21", ] miniscope = [ "natsort>=8.3.1", "ndx-miniscope>=0.5.1", - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] sbx = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] scanimage = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", "scanimage-tiff-reader>=1.4.1", ] sima = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] suite2p = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", ] tdt_fp = [ "ndx-fiber-photometry", - "roiextractors>=0.5.7", + "roiextractors>=0.5.10", "tdt", ] tiff = [ - "roiextractors>=0.5.7", + "roiextractors>=0.5.9", "tiffile>=2018.10.18", ] ophys = [ diff --git a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py index a544f9c27..aaa875f3e 100644 --- a/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/video/videodatainterface.py @@ -269,8 +269,6 @@ def add_to_nwbfile( chunk_data: bool = True, module_name: Optional[str] = None, module_description: Optional[str] = None, - compression: Optional[str] = "gzip", - compression_options: Optional[int] = None, ): """ Convert the video data files to :py:class:`~pynwb.image.ImageSeries` and write them in the @@ -431,17 +429,6 @@ def add_to_nwbfile( pbar.update(1) iterable = video - # TODO: remove completely after 03/1/2024 - if compression is not None or compression_options is not None: - warnings.warn( - message=( - "Specifying compression methods and their options for this interface has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) - image_series_kwargs.update(data=iterable) if timing_type == "starting_time and rate": diff --git a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py index 9f88b861f..0019b8bd7 100644 --- a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py @@ -128,6 +128,7 @@ def add_to_nwbfile( parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", stub_test: bool = False, stub_frames: int = 100, + always_write_timestamps: bool = False, ): """ Add imaging data to the NWB file @@ -167,4 +168,5 @@ def add_to_nwbfile( photon_series_type=photon_series_type, photon_series_index=photon_series_index, parent_container=parent_container, + always_write_timestamps=always_write_timestamps, ) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index f28631c77..27d3b5f9c 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -445,6 +445,7 @@ def add_photon_series_to_nwbfile( parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, + always_write_timestamps: bool = False, ) -> NWBFile: """ Auxiliary static method for nwbextractor. @@ -472,6 +473,11 @@ def add_photon_series_to_nwbfile( iterator_type: str, default: 'v2' The type of iterator to use when adding the photon series to the NWB file. iterator_options: dict, optional + always_write_timestamps : bool, default: False + Set to True to always write timestamps. + By default (False), the function checks if the timestamps are uniformly sampled, and if so, stores the data + using a regular sampling rate instead of explicit timestamps. If set to True, timestamps will be written + explicitly, regardless of whether the sampling rate is uniform. Returns ------- @@ -530,16 +536,23 @@ def add_photon_series_to_nwbfile( photon_series_kwargs.update(dimension=imaging.get_image_size()) # Add timestamps or rate - if imaging.has_time_vector(): + if always_write_timestamps: timestamps = imaging.frame_to_time(np.arange(imaging.get_num_frames())) - estimated_rate = calculate_regular_series_rate(series=timestamps) + photon_series_kwargs.update(timestamps=timestamps) + else: + imaging_has_timestamps = imaging.has_time_vector() + if imaging_has_timestamps: + timestamps = imaging.frame_to_time(np.arange(imaging.get_num_frames())) + estimated_rate = calculate_regular_series_rate(series=timestamps) + starting_time = timestamps[0] + else: + estimated_rate = float(imaging.get_sampling_frequency()) + starting_time = 0.0 + if estimated_rate: - photon_series_kwargs.update(starting_time=timestamps[0], rate=estimated_rate) + photon_series_kwargs.update(rate=estimated_rate, starting_time=starting_time) else: - photon_series_kwargs.update(timestamps=timestamps, rate=None) - else: - rate = float(imaging.get_sampling_frequency()) - photon_series_kwargs.update(rate=rate) + photon_series_kwargs.update(timestamps=timestamps) # Add the photon series to the nwbfile (either as OnePhotonSeries or TwoPhotonSeries) photon_series = dict( @@ -682,6 +695,7 @@ def add_imaging_to_nwbfile( iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, parent_container: Literal["acquisition", "processing/ophys"] = "acquisition", + always_write_timestamps: bool = False, ) -> NWBFile: """ Add imaging data from an ImagingExtractor object to an NWBFile. @@ -705,6 +719,11 @@ def add_imaging_to_nwbfile( parent_container : {"acquisition", "processing/ophys"}, optional Specifies the parent container to which the photon series should be added, either as part of "acquisition" or under the "processing/ophys" module, by default "acquisition". + always_write_timestamps : bool, default: False + Set to True to always write timestamps. + By default (False), the function checks if the timestamps are uniformly sampled, and if so, stores the data + using a regular sampling rate instead of explicit timestamps. If set to True, timestamps will be written + explicitly, regardless of whether the sampling rate is uniform. Returns ------- @@ -722,6 +741,7 @@ def add_imaging_to_nwbfile( iterator_type=iterator_type, iterator_options=iterator_options, parent_container=parent_container, + always_write_timestamps=always_write_timestamps, ) return nwbfile diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 44d1adf61..dd3ec12c2 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -260,6 +260,7 @@ def __init__( sampling_frequency=sampling_frequency, dtype=dtype, verbose=verbose, + seed=seed, ) self.verbose = verbose @@ -334,6 +335,7 @@ def __init__( has_deconvolved_signal=has_deconvolved_signal, has_neuropil_signal=has_neuropil_signal, verbose=verbose, + seed=seed, ) def get_metadata(self) -> dict: diff --git a/tests/test_ophys/test_ophys_interfaces.py b/tests/test_ophys/test_ophys_interfaces.py index 4381faf8b..3ea329e5f 100644 --- a/tests/test_ophys/test_ophys_interfaces.py +++ b/tests/test_ophys/test_ophys_interfaces.py @@ -1,3 +1,5 @@ +import numpy as np + from neuroconv.tools.testing.data_interface_mixins import ( ImagingExtractorInterfaceTestMixin, SegmentationExtractorInterfaceTestMixin, @@ -12,7 +14,17 @@ class TestMockImagingInterface(ImagingExtractorInterfaceTestMixin): data_interface_cls = MockImagingInterface interface_kwargs = dict() - # TODO: fix this by setting a seed on the dummy imaging extractor + def test_always_write_timestamps(self, setup_interface): + # By default the MockImagingInterface has a uniform sampling rate + + nwbfile = self.interface.create_nwbfile(always_write_timestamps=True) + two_photon_series = nwbfile.acquisition["TwoPhotonSeries"] + imaging = self.interface.imaging_extractor + expected_timestamps = imaging.frame_to_time(np.arange(imaging.get_num_frames())) + + np.testing.assert_array_equal(two_photon_series.timestamps[:], expected_timestamps) + + # Remove this after roiextractors 0.5.10 is released def test_all_conversion_checks(self): pass From e3cde1f38d9de4f970ffdb1ffd2f73078bb04e0b Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 14 Nov 2024 07:47:34 -0600 Subject: [PATCH 084/118] Support datetime in conversion options (#1139) --- CHANGELOG.md | 1 + src/neuroconv/basedatainterface.py | 4 ++- src/neuroconv/nwbconverter.py | 29 ++++++++++++------ .../tools/testing/mock_interfaces.py | 21 +++++++++++++ src/neuroconv/utils/json_schema.py | 17 +++++++++++ .../test_minimal/test_interface_validation.py | 30 +++++++++++++++++++ 6 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 tests/test_minimal/test_interface_validation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a06ddf300..cdc70223f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) ## Bug Fixes +* datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) ## Features * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index adcec89b5..272abbd0c 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -126,7 +126,7 @@ def create_nwbfile(self, metadata: Optional[dict] = None, **conversion_options) return nwbfile @abstractmethod - def add_to_nwbfile(self, nwbfile: NWBFile, **conversion_options) -> None: + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict], **conversion_options) -> None: """ Define a protocol for mapping the data from this interface to NWB neurodata objects. @@ -136,6 +136,8 @@ def add_to_nwbfile(self, nwbfile: NWBFile, **conversion_options) -> None: ---------- nwbfile : pynwb.NWBFile The in-memory object to add the data to. + metadata : dict + Metadata dictionary with information used to create the NWBFile. **conversion_options Additional keyword arguments to pass to the `.add_to_nwbfile` method. """ diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index 1f3e7c9f8..fe1b09915 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -29,7 +29,11 @@ unroot_schema, ) from .utils.dict import DeepDict -from .utils.json_schema import _NWBMetaDataEncoder, _NWBSourceDataEncoder +from .utils.json_schema import ( + _NWBConversionOptionsEncoder, + _NWBMetaDataEncoder, + _NWBSourceDataEncoder, +) class NWBConverter: @@ -63,11 +67,10 @@ def validate_source(cls, source_data: dict[str, dict], verbose: bool = True): def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = True): + # We do this to ensure that python objects are in string format for the JSON schema encoder = _NWBSourceDataEncoder() - # The encoder produces a serialized object, so we deserialized it for comparison - - serialized_source_data = encoder.encode(source_data) - decoded_source_data = json.loads(serialized_source_data) + encoded_source_data = encoder.encode(source_data) + decoded_source_data = json.loads(encoded_source_data) validate(instance=decoded_source_data, schema=self.get_source_schema()) if verbose: @@ -106,9 +109,10 @@ def get_metadata(self) -> DeepDict: def validate_metadata(self, metadata: dict[str, dict], append_mode: bool = False): """Validate metadata against Converter metadata_schema.""" encoder = _NWBMetaDataEncoder() - # The encoder produces a serialized object, so we deserialized it for comparison - serialized_metadata = encoder.encode(metadata) - decoded_metadata = json.loads(serialized_metadata) + + # We do this to ensure that python objects are in string format for the JSON schema + encoded_metadta = encoder.encode(metadata) + decoded_metadata = json.loads(encoded_metadta) metadata_schema = self.get_metadata_schema() if append_mode: @@ -138,7 +142,14 @@ def get_conversion_options_schema(self) -> dict: def validate_conversion_options(self, conversion_options: dict[str, dict]): """Validate conversion_options against Converter conversion_options_schema.""" - validate(instance=conversion_options or {}, schema=self.get_conversion_options_schema()) + + conversion_options = conversion_options or dict() + + # We do this to ensure that python objects are in string format for the JSON schema + encoded_conversion_options = _NWBConversionOptionsEncoder().encode(conversion_options) + decoded_conversion_options = json.loads(encoded_conversion_options) + + validate(instance=decoded_conversion_options, schema=self.get_conversion_options_schema()) if self.verbose: print("conversion_options is valid!") diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index dd3ec12c2..0652284e7 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -6,6 +6,7 @@ from pynwb.base import DynamicTable from .mock_ttl_signals import generate_mock_ttl_signal +from ...basedatainterface import BaseDataInterface from ...basetemporalalignmentinterface import BaseTemporalAlignmentInterface from ...datainterfaces import SpikeGLXNIDQInterface from ...datainterfaces.ecephys.baserecordingextractorinterface import ( @@ -23,6 +24,26 @@ from ...utils import ArrayType, get_json_schema_from_method_signature +class MockInterface(BaseDataInterface): + """ + A mock interface for testing basic command passing without side effects. + """ + + def __init__(self, verbose: bool = False, **source_data): + + super().__init__(verbose=verbose, **source_data) + + def get_metadata(self) -> dict: + metadata = super().get_metadata() + session_start_time = datetime.now().astimezone() + metadata["NWBFile"]["session_start_time"] = session_start_time + return metadata + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict], **conversion_options): + + return None + + class MockBehaviorEventInterface(BaseTemporalAlignmentInterface): """ A mock behavior event interface for testing purposes. diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index 182558b98..07dc3321f 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -60,6 +60,23 @@ def default(self, obj): return super().default(obj) +class _NWBConversionOptionsEncoder(_NWBMetaDataEncoder): + """ + Custom JSON encoder for conversion options of the data interfaces and converters (i.e. kwargs). + + This encoder extends the default JSONEncoder class and provides custom serialization + for certain data types commonly used in interface source data. + """ + + def default(self, obj): + + # Over-write behaviors for Paths + if isinstance(obj, Path): + return str(obj) + + return super().default(obj) + + def get_base_schema( tag: Optional[str] = None, root: bool = False, diff --git a/tests/test_minimal/test_interface_validation.py b/tests/test_minimal/test_interface_validation.py new file mode 100644 index 000000000..1bc409b06 --- /dev/null +++ b/tests/test_minimal/test_interface_validation.py @@ -0,0 +1,30 @@ +from datetime import datetime +from typing import Optional + +from pynwb import NWBFile + +from neuroconv import ConverterPipe +from neuroconv.tools.testing.mock_interfaces import ( + MockInterface, +) + + +def test_conversion_options_validation(tmp_path): + + class InterfaceWithDateTimeConversionOptions(MockInterface): + "class for testing how a file with datetime object is validated" + + def add_to_nwbfile(self, nwbfile: NWBFile, metadata: Optional[dict], datetime_option: datetime): + pass + + interface = InterfaceWithDateTimeConversionOptions() + + nwbfile_path = tmp_path / "interface_test.nwb" + interface.run_conversion(nwbfile_path=nwbfile_path, datetime_option=datetime.now(), overwrite=True) + + data_interfaces = {"InterfaceWithDateTimeConversionOptions": interface} + conversion_options = {"InterfaceWithDateTimeConversionOptions": {"datetime_option": datetime.now()}} + converter = ConverterPipe(data_interfaces=data_interfaces) + + nwbfile_path = tmp_path / "converter_test.nwb" + converter.run_conversion(nwbfile_path=nwbfile_path, overwrite=True, conversion_options=conversion_options) From 56673dddd246f806ec2d7ee1911d86fcd21414ae Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Fri, 15 Nov 2024 06:52:44 +1100 Subject: [PATCH 085/118] Added CSV support to DeepLabCutInterface (#1140) --- CHANGELOG.md | 1 + .../behavior/deeplabcut.rst | 5 +- .../behavior/deeplabcut/_dlc_utils.py | 41 ++++-------- .../deeplabcut/deeplabcutdatainterface.py | 30 +++++---- .../tools/testing/data_interface_mixins.py | 24 ------- .../behavior/test_behavior_interfaces.py | 65 ++++++++++++++++--- 6 files changed, 91 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdc70223f..0545001d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ## Features * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) +* Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) ## Improvements diff --git a/docs/conversion_examples_gallery/behavior/deeplabcut.rst b/docs/conversion_examples_gallery/behavior/deeplabcut.rst index c20dd057d..64201ea72 100644 --- a/docs/conversion_examples_gallery/behavior/deeplabcut.rst +++ b/docs/conversion_examples_gallery/behavior/deeplabcut.rst @@ -8,6 +8,7 @@ Install NeuroConv with the additional dependencies necessary for reading DeepLab pip install "neuroconv[deeplabcut]" Convert DeepLabCut pose estimation data to NWB using :py:class:`~neuroconv.datainterfaces.behavior.deeplabcut.deeplabcutdatainterface.DeepLabCutInterface`. +This interface supports both .h5 and .csv output files from DeepLabCut. .. code-block:: python @@ -16,8 +17,8 @@ Convert DeepLabCut pose estimation data to NWB using :py:class:`~neuroconv.datai >>> from pathlib import Path >>> from neuroconv.datainterfaces import DeepLabCutInterface - >>> file_path = BEHAVIOR_DATA_PATH / "DLC" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" - >>> config_file_path = BEHAVIOR_DATA_PATH / "DLC" / "config.yaml" + >>> file_path = BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + >>> config_file_path = BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml" >>> interface = DeepLabCutInterface(file_path=file_path, config_file_path=config_file_path, subject_name="ind1", verbose=False) diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 9e368fb39..5d1224e85 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -251,21 +251,6 @@ def _get_video_info_from_config_file(config_file_path: Path, vidname: str): return video_file_path, image_shape -def _get_pes_args( - *, - h5file: Path, - individual_name: str, -): - h5file = Path(h5file) - - _, scorer = h5file.stem.split("DLC") - scorer = "DLC" + scorer - - df = _ensure_individuals_in_header(pd.read_hdf(h5file), individual_name) - - return scorer, df - - def _write_pes_to_nwbfile( nwbfile, animal, @@ -339,23 +324,23 @@ def _write_pes_to_nwbfile( return nwbfile -def add_subject_to_nwbfile( +def _add_subject_to_nwbfile( nwbfile: NWBFile, - h5file: FilePath, + file_path: FilePath, individual_name: str, config_file: Optional[FilePath] = None, timestamps: Optional[Union[list, np.ndarray]] = None, pose_estimation_container_kwargs: Optional[dict] = None, ) -> NWBFile: """ - Given the subject name, add the DLC .h5 file to an in-memory NWBFile object. + Given the subject name, add the DLC output file (.h5 or .csv) to an in-memory NWBFile object. Parameters ---------- nwbfile : pynwb.NWBFile The in-memory nwbfile object to which the subject specific pose estimation series will be added. - h5file : str or path - Path to the DeepLabCut .h5 output file. + file_path : str or path + Path to the DeepLabCut .h5 or .csv output file. individual_name : str Name of the subject (whose pose is predicted) for single-animal DLC project. For multi-animal projects, the names from the DLC project will be used directly. @@ -371,18 +356,18 @@ def add_subject_to_nwbfile( nwbfile : pynwb.NWBFile nwbfile with pes written in the behavior module """ - h5file = Path(h5file) - - if "DLC" not in h5file.name or not h5file.suffix == ".h5": - raise IOError("The file passed in is not a DeepLabCut h5 data file.") + file_path = Path(file_path) - video_name, scorer = h5file.stem.split("DLC") + video_name, scorer = file_path.stem.split("DLC") scorer = "DLC" + scorer # TODO probably could be read directly with h5py # This requires pytables - data_frame_from_hdf5 = pd.read_hdf(h5file) - df = _ensure_individuals_in_header(data_frame_from_hdf5, individual_name) + if ".h5" in file_path.suffixes: + df = pd.read_hdf(file_path) + elif ".csv" in file_path.suffixes: + df = pd.read_csv(file_path, header=[0, 1, 2], index_col=0) + df = _ensure_individuals_in_header(df, individual_name) # Note the video here is a tuple of the video path and the image shape if config_file is not None: @@ -404,7 +389,7 @@ def add_subject_to_nwbfile( # Fetch the corresponding metadata pickle file, we extract the edges graph from here # TODO: This is the original implementation way to extract the file name but looks very brittle. Improve it - filename = str(h5file.parent / h5file.stem) + filename = str(file_path.parent / file_path.stem) for i, c in enumerate(filename[::-1]): if c.isnumeric(): break diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index 21b054e85..f45913061 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -5,6 +5,7 @@ from pydantic import FilePath, validate_call from pynwb.file import NWBFile +# import ndx_pose from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -13,7 +14,7 @@ class DeepLabCutInterface(BaseTemporalAlignmentInterface): display_name = "DeepLabCut" keywords = ("DLC",) - associated_suffixes = (".h5",) + associated_suffixes = (".h5", ".csv") info = "Interface for handling data from DeepLabCut." _timestamps = None @@ -21,8 +22,8 @@ class DeepLabCutInterface(BaseTemporalAlignmentInterface): @classmethod def get_source_schema(cls) -> dict: source_schema = super().get_source_schema() - source_schema["properties"]["file_path"]["description"] = "Path to the .h5 file output by dlc." - source_schema["properties"]["config_file_path"]["description"] = "Path to .yml config file" + source_schema["properties"]["file_path"]["description"] = "Path to the file output by dlc (.h5 or .csv)." + source_schema["properties"]["config_file_path"]["description"] = "Path to .yml config file." return source_schema @validate_call @@ -34,24 +35,25 @@ def __init__( verbose: bool = True, ): """ - Interface for writing DLC's h5 files to nwb using dlc2nwb. + Interface for writing DLC's output files to nwb using dlc2nwb. Parameters ---------- file_path : FilePath - path to the h5 file output by dlc. + Path to the file output by dlc (.h5 or .csv). config_file_path : FilePath, optional - path to .yml config file + Path to .yml config file subject_name : str, default: "ind1" - the name of the subject for which the :py:class:`~pynwb.file.NWBFile` is to be created. + The name of the subject for which the :py:class:`~pynwb.file.NWBFile` is to be created. verbose: bool, default: True - controls verbosity. + Controls verbosity. """ from ._dlc_utils import _read_config file_path = Path(file_path) - if "DLC" not in file_path.stem or ".h5" not in file_path.suffixes: - raise IOError("The file passed in is not a DeepLabCut h5 data file.") + suffix_is_valid = ".h5" in file_path.suffixes or ".csv" in file_path.suffixes + if not "DLC" in file_path.stem or not suffix_is_valid: + raise IOError("The file passed in is not a valid DeepLabCut output data file.") self.config_dict = dict() if config_file_path is not None: @@ -108,12 +110,14 @@ def add_to_nwbfile( nwb file to which the recording information is to be added metadata: dict metadata info for constructing the nwb file (optional). + container_name: str, default: "PoseEstimation" + Name of the container to store the pose estimation. """ - from ._dlc_utils import add_subject_to_nwbfile + from ._dlc_utils import _add_subject_to_nwbfile - add_subject_to_nwbfile( + _add_subject_to_nwbfile( nwbfile=nwbfile, - h5file=str(self.source_data["file_path"]), + file_path=str(self.source_data["file_path"]), individual_name=self.subject_name, config_file=self.source_data["config_file_path"], timestamps=self._timestamps, diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 946b3fd6c..5187ff2e4 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -743,30 +743,6 @@ def test_interface_alignment(self): pass -class DeepLabCutInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): - """ - A mixin for testing DeepLabCut interfaces. - """ - - def check_interface_get_original_timestamps(self): - pass # TODO in separate PR - - def check_interface_get_timestamps(self): - pass # TODO in separate PR - - def check_interface_set_aligned_timestamps(self): - pass # TODO in separate PR - - def check_shift_timestamps_by_start_time(self): - pass # TODO in separate PR - - def check_interface_original_timestamps_inmutability(self): - pass # TODO in separate PR - - def check_nwbfile_temporal_alignment(self): - pass # TODO in separate PR - - class VideoInterfaceMixin(DataInterfaceTestMixin, TemporalAlignmentMixin): """ A mixin for testing Video interfaces. diff --git a/tests/test_on_data/behavior/test_behavior_interfaces.py b/tests/test_on_data/behavior/test_behavior_interfaces.py index 8e3e01d61..b43e65206 100644 --- a/tests/test_on_data/behavior/test_behavior_interfaces.py +++ b/tests/test_on_data/behavior/test_behavior_interfaces.py @@ -29,7 +29,6 @@ ) from neuroconv.tools.testing.data_interface_mixins import ( DataInterfaceTestMixin, - DeepLabCutInterfaceMixin, MedPCInterfaceMixin, TemporalAlignmentMixin, VideoInterfaceMixin, @@ -332,11 +331,16 @@ class TestFicTracDataInterfaceTiming(TemporalAlignmentMixin): platform == "darwin" and python_version < version.parse("3.10"), reason="interface not supported on macOS with Python < 3.10", ) -class TestDeepLabCutInterface(DeepLabCutInterfaceMixin): +class TestDeepLabCutInterface(DataInterfaceTestMixin): data_interface_cls = DeepLabCutInterface interface_kwargs = dict( - file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5"), - config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "config.yaml"), + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), subject_name="ind1", ) save_directory = OUTPUT_PATH @@ -384,7 +388,12 @@ def check_read_nwb(self, nwbfile_path: str): class TestDeepLabCutInterfaceNoConfigFile(DataInterfaceTestMixin): data_interface_cls = DeepLabCutInterface interface_kwargs = dict( - file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5"), + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), config_file_path=None, subject_name="ind1", ) @@ -411,11 +420,16 @@ def check_read_nwb(self, nwbfile_path: str): platform == "darwin" and python_version < version.parse("3.10"), reason="interface not supported on macOS with Python < 3.10", ) -class TestDeepLabCutInterfaceSetTimestamps(DeepLabCutInterfaceMixin): +class TestDeepLabCutInterfaceSetTimestamps(DataInterfaceTestMixin): data_interface_cls = DeepLabCutInterface interface_kwargs = dict( - file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5"), - config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "config.yaml"), + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), subject_name="ind1", ) @@ -454,6 +468,41 @@ def check_read_nwb(self, nwbfile_path: str): pass +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10"), + reason="interface not supported on macOS with Python < 3.10", +) +class TestDeepLabCutInterfaceFromCSV(DataInterfaceTestMixin): + data_interface_cls = DeepLabCutInterface + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "SL18_csv" + / "SL18_D19_S01_F01_BOX_SLP_20230503_112642.1DLC_resnet50_SubLearnSleepBoxRedLightJun26shuffle1_100000_stubbed.csv" + ), + config_file_path=None, + subject_name="SL18", + ) + save_directory = OUTPUT_PATH + + def check_read_nwb(self, nwbfile_path: str): + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces + assert "PoseEstimation" in processing_module_interfaces + + pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series + expected_pose_estimation_series = ["SL18_redled", "SL18_shoulder", "SL18_haunch", "SL18_baseoftail"] + + expected_pose_estimation_series_are_in_nwb_file = [ + pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series + ] + + assert all(expected_pose_estimation_series_are_in_nwb_file) + + class TestSLEAPInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): data_interface_cls = SLEAPInterface interface_kwargs = dict( From 64fb9e01a5f4070fd3b01ebab50d8fc19a2fe953 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 14 Nov 2024 21:33:52 -0600 Subject: [PATCH 086/118] Use mixing tests for mocks (#1136) --- CHANGELOG.md | 1 + .../tools/testing/data_interface_mixins.py | 1 - tests/test_ecephys/test_ecephys_interfaces.py | 114 +++++++----------- .../test_mock_recording_interface.py | 9 -- 4 files changed, 43 insertions(+), 82 deletions(-) delete mode 100644 tests/test_ecephys/test_mock_recording_interface.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0545001d1..92f4e6b5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) ## Improvements +* Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) # v0.6.5 (November 1, 2024) diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index 5187ff2e4..fab049165 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -92,7 +92,6 @@ def test_metadata_schema_valid(self, setup_interface): Draft7Validator.check_schema(schema=schema) def test_metadata(self, setup_interface): - # Validate metadata now happens on the class itself metadata = self.interface.get_metadata() self.check_extracted_metadata(metadata) diff --git a/tests/test_ecephys/test_ecephys_interfaces.py b/tests/test_ecephys/test_ecephys_interfaces.py index 4d4232bf2..24393923f 100644 --- a/tests/test_ecephys/test_ecephys_interfaces.py +++ b/tests/test_ecephys/test_ecephys_interfaces.py @@ -27,42 +27,61 @@ python_version = Version(get_python_version()) +from neuroconv.tools.testing.data_interface_mixins import ( + RecordingExtractorInterfaceTestMixin, + SortingExtractorInterfaceTestMixin, +) -class TestRecordingInterface(TestCase): - @classmethod - def setUpClass(cls): - cls.single_segment_recording_interface = MockRecordingInterface(durations=[0.100]) - cls.multi_segment_recording_interface = MockRecordingInterface(durations=[0.100, 0.100]) - def test_stub_single_segment(self): - interface = self.single_segment_recording_interface +class TestSortingInterface(SortingExtractorInterfaceTestMixin): + + data_interface_cls = MockSortingInterface + interface_kwargs = dict(num_units=4, durations=[0.100]) + + def test_propagate_conversion_options(self, setup_interface): + interface = self.interface metadata = interface.get_metadata() - interface.create_nwbfile(stub_test=True, metadata=metadata) + nwbfile = interface.create_nwbfile( + stub_test=True, + metadata=metadata, + write_as="processing", + units_name="processed_units", + units_description="The processed units.", + ) - def test_stub_multi_segment(self): - interface = self.multi_segment_recording_interface + ecephys = get_module(nwbfile, "ecephys") + + assert nwbfile.units is None + assert "processed_units" in ecephys.data_interfaces + + +class TestRecordingInterface(RecordingExtractorInterfaceTestMixin): + data_interface_cls = MockRecordingInterface + interface_kwargs = dict(durations=[0.100]) + + def test_stub(self, setup_interface): + interface = self.interface metadata = interface.get_metadata() interface.create_nwbfile(stub_test=True, metadata=metadata) - def test_no_slash_in_name(self): - interface = self.single_segment_recording_interface + def test_no_slash_in_name(self, setup_interface): + interface = self.interface metadata = interface.get_metadata() metadata["Ecephys"]["ElectricalSeries"]["name"] = "test/slash" - with self.assertRaises(jsonschema.exceptions.ValidationError): + with pytest.raises(jsonschema.exceptions.ValidationError): interface.validate_metadata(metadata) + def test_stub_multi_segment(self): -class TestAlwaysWriteTimestamps: + interface = MockRecordingInterface(durations=[0.100, 0.100]) + metadata = interface.get_metadata() + interface.create_nwbfile(stub_test=True, metadata=metadata) - def test_always_write_timestamps(self): - # By default the MockRecordingInterface has a uniform sampling rate - interface = MockRecordingInterface(durations=[1.0], sampling_frequency=30_000.0) + def test_always_write_timestamps(self, setup_interface): - nwbfile = interface.create_nwbfile(always_write_timestamps=True) + nwbfile = self.interface.create_nwbfile(always_write_timestamps=True) electrical_series = nwbfile.acquisition["ElectricalSeries"] - - expected_timestamps = interface.recording_extractor.get_times() - + expected_timestamps = self.interface.recording_extractor.get_times() np.testing.assert_array_equal(electrical_series.timestamps[:], expected_timestamps) @@ -84,33 +103,9 @@ def test_spike2_import_assertions_3_11(self): Spike2RecordingInterface.get_all_channels_info(file_path="does_not_matter.smrx") -class TestSortingInterface: - - def test_run_conversion(self, tmp_path): - - nwbfile_path = Path(tmp_path) / "test_sorting.nwb" - num_units = 4 - interface = MockSortingInterface(num_units=num_units, durations=(1.0,)) - interface.sorting_extractor = interface.sorting_extractor.rename_units(new_unit_ids=["a", "b", "c", "d"]) - - interface.run_conversion(nwbfile_path=nwbfile_path) - with NWBHDF5IO(nwbfile_path, "r") as io: - nwbfile = io.read() - - units = nwbfile.units - assert len(units) == num_units - units_df = units.to_dataframe() - # Get index in units table - for unit_id in interface.sorting_extractor.unit_ids: - # In pynwb we write unit name as unit_id - row = units_df.query(f"unit_name == '{unit_id}'") - spike_times = interface.sorting_extractor.get_unit_spike_train(unit_id=unit_id, return_times=True) - written_spike_times = row["spike_times"].iloc[0] - - np.testing.assert_array_equal(spike_times, written_spike_times) - - class TestSortingInterfaceOld(unittest.TestCase): + """Old-style tests for the SortingInterface. Remove once we we are sure all the behaviors are covered by the mock.""" + @classmethod def setUpClass(cls) -> None: cls.test_dir = Path(mkdtemp()) @@ -194,28 +189,3 @@ def test_sorting_full(self): nwbfile = io.read() for i, start_times in enumerate(self.sorting_start_frames): assert len(nwbfile.units["spike_times"][i]) == self.num_frames - start_times - - def test_sorting_propagate_conversion_options(self): - minimal_nwbfile = self.test_dir / "temp2.nwb" - metadata = self.test_sorting_interface.get_metadata() - metadata["NWBFile"]["session_start_time"] = datetime.now().astimezone() - units_description = "The processed units." - conversion_options = dict( - TestSortingInterface=dict( - write_as="processing", - units_name="processed_units", - units_description=units_description, - ) - ) - self.test_sorting_interface.run_conversion( - nwbfile_path=minimal_nwbfile, - metadata=metadata, - conversion_options=conversion_options, - ) - - with NWBHDF5IO(minimal_nwbfile, "r") as io: - nwbfile = io.read() - ecephys = get_module(nwbfile, "ecephys") - self.assertIsNone(nwbfile.units) - self.assertIn("processed_units", ecephys.data_interfaces) - self.assertEqual(ecephys["processed_units"].description, units_description) diff --git a/tests/test_ecephys/test_mock_recording_interface.py b/tests/test_ecephys/test_mock_recording_interface.py deleted file mode 100644 index a33f3acd1..000000000 --- a/tests/test_ecephys/test_mock_recording_interface.py +++ /dev/null @@ -1,9 +0,0 @@ -from neuroconv.tools.testing.data_interface_mixins import ( - RecordingExtractorInterfaceTestMixin, -) -from neuroconv.tools.testing.mock_interfaces import MockRecordingInterface - - -class TestMockRecordingInterface(RecordingExtractorInterfaceTestMixin): - data_interface_cls = MockRecordingInterface - interface_kwargs = dict(durations=[0.100]) From a608e904ad6fd75f7f983c1555bea1832f850f8e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 15 Nov 2024 12:24:14 -0600 Subject: [PATCH 087/118] Propagate `unit_electrode_indices` to `SortingInterface` (#1124) --- CHANGELOG.md | 1 + .../ecephys/basesortingextractorinterface.py | 8 +++++ .../tools/spikeinterface/spikeinterface.py | 19 +++++++--- tests/test_ecephys/test_ecephys_interfaces.py | 35 +++++++++++++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92f4e6b5f..002aed660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) ## Features +* Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) diff --git a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py index cd8396154..dca2dea5f 100644 --- a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py @@ -288,6 +288,7 @@ def add_to_nwbfile( write_as: Literal["units", "processing"] = "units", units_name: str = "units", units_description: str = "Autogenerated by neuroconv.", + unit_electrode_indices: Optional[list[list[int]]] = None, ): """ Primary function for converting the data in a SortingExtractor to NWB format. @@ -312,9 +313,15 @@ def add_to_nwbfile( units_name : str, default: 'units' The name of the units table. If write_as=='units', then units_name must also be 'units'. units_description : str, default: 'Autogenerated by neuroconv.' + unit_electrode_indices : list of lists of int, optional + A list of lists of integers indicating the indices of the electrodes that each unit is associated with. + The length of the list must match the number of units in the sorting extractor. """ from ...tools.spikeinterface import add_sorting_to_nwbfile + if metadata is None: + metadata = self.get_metadata() + metadata_copy = deepcopy(metadata) if write_ecephys_metadata: self.add_channel_metadata_to_nwb(nwbfile=nwbfile, metadata=metadata_copy) @@ -346,4 +353,5 @@ def add_to_nwbfile( write_as=write_as, units_name=units_name, units_description=units_description, + unit_electrode_indices=unit_electrode_indices, ) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 5aa3c8925..fa00d58ed 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -1368,7 +1368,7 @@ def add_units_table_to_nwbfile( write_in_processing_module: bool = False, waveform_means: Optional[np.ndarray] = None, waveform_sds: Optional[np.ndarray] = None, - unit_electrode_indices=None, + unit_electrode_indices: Optional[list[list[int]]] = None, null_values_for_properties: Optional[dict] = None, ): """ @@ -1405,8 +1405,9 @@ def add_units_table_to_nwbfile( Waveform standard deviation for each unit. Shape: (num_units, num_samples, num_channels). unit_electrode_indices : list of lists of int, optional For each unit, a list of electrode indices corresponding to waveform data. - null_values_for_properties: dict, optional - A dictionary mapping properties to null values to use when the property is not present + unit_electrode_indices : list of lists of int, optional + A list of lists of integers indicating the indices of the electrodes that each unit is associated with. + The length of the list must match the number of units in the sorting extractor. """ unit_table_description = unit_table_description or "Autogenerated by neuroconv." @@ -1414,6 +1415,13 @@ def add_units_table_to_nwbfile( nwbfile, pynwb.NWBFile ), f"'nwbfile' should be of type pynwb.NWBFile but is of type {type(nwbfile)}" + if unit_electrode_indices is not None: + electrodes_table = nwbfile.electrodes + if electrodes_table is None: + raise ValueError( + "Electrodes table is required to map units to electrodes. Add an electrode table to the NWBFile first." + ) + null_values_for_properties = dict() if null_values_for_properties is None else null_values_for_properties if not write_in_processing_module and units_table_name != "units": @@ -1668,7 +1676,7 @@ def add_sorting_to_nwbfile( units_description: str = "Autogenerated by neuroconv.", waveform_means: Optional[np.ndarray] = None, waveform_sds: Optional[np.ndarray] = None, - unit_electrode_indices=None, + unit_electrode_indices: Optional[list[list[int]]] = None, ): """Add sorting data (units and their properties) to an NWBFile. @@ -1703,7 +1711,8 @@ def add_sorting_to_nwbfile( waveform_sds : np.ndarray, optional Waveform standard deviation for each unit. Shape: (num_units, num_samples, num_channels). unit_electrode_indices : list of lists of int, optional - For each unit, a list of electrode indices corresponding to waveform data. + A list of lists of integers indicating the indices of the electrodes that each unit is associated with. + The length of the list must match the number of units in the sorting extractor. """ if skip_features is not None: diff --git a/tests/test_ecephys/test_ecephys_interfaces.py b/tests/test_ecephys/test_ecephys_interfaces.py index 24393923f..e036ccb81 100644 --- a/tests/test_ecephys/test_ecephys_interfaces.py +++ b/tests/test_ecephys/test_ecephys_interfaces.py @@ -54,6 +54,41 @@ def test_propagate_conversion_options(self, setup_interface): assert nwbfile.units is None assert "processed_units" in ecephys.data_interfaces + def test_electrode_indices(self, setup_interface): + + recording_interface = MockRecordingInterface(num_channels=4, durations=[0.100]) + recording_extractor = recording_interface.recording_extractor + recording_extractor = recording_extractor.rename_channels(new_channel_ids=["a", "b", "c", "d"]) + recording_extractor.set_property(key="property", values=["A", "B", "C", "D"]) + recording_interface.recording_extractor = recording_extractor + + nwbfile = recording_interface.create_nwbfile() + + unit_electrode_indices = [[3], [0, 1], [1], [2]] + expected_properties_matching = [["D"], ["A", "B"], ["B"], ["C"]] + self.interface.add_to_nwbfile(nwbfile=nwbfile, unit_electrode_indices=unit_electrode_indices) + + unit_table = nwbfile.units + + for unit_row, electrode_indices, property in zip( + unit_table.to_dataframe().itertuples(index=False), + unit_electrode_indices, + expected_properties_matching, + ): + electrode_table_region = unit_row.electrodes + electrode_table_region_indices = electrode_table_region.index.to_list() + assert electrode_table_region_indices == electrode_indices + + electrode_table_region_properties = electrode_table_region["property"].to_list() + assert electrode_table_region_properties == property + + def test_electrode_indices_assertion_error_when_missing_table(self, setup_interface): + with pytest.raises( + ValueError, + match="Electrodes table is required to map units to electrodes. Add an electrode table to the NWBFile first.", + ): + self.interface.create_nwbfile(unit_electrode_indices=[[0], [1], [2], [3]]) + class TestRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MockRecordingInterface From dcd248b3faec7f9b6f003784dbd899945b565df8 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 22 Nov 2024 12:55:53 -0600 Subject: [PATCH 088/118] Move imports of extensions to the init of the interfaces (#1144) Co-authored-by: Paul Adkisson --- CHANGELOG.md | 3 +- .../behavior/audio/audiointerface.py | 6 ++- .../behavior/deeplabcut/_dlc_utils.py | 4 +- .../deeplabcut/deeplabcutdatainterface.py | 4 +- .../lightningposedatainterface.py | 4 ++ .../behavior/medpc/medpcdatainterface.py | 4 ++ .../tdt_fp/tdtfiberphotometrydatainterface.py | 1 + src/neuroconv/tools/audio/audio.py | 12 ----- .../behavior/test_behavior_interfaces.py | 47 ++++++++++++++++++- .../behavior/test_lightningpose_converter.py | 3 +- 10 files changed, 69 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 002aed660..c0c3bd729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # Upcoming ## Deprecations -* Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) +* Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) +* Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) diff --git a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py index fc3f08fb8..6561096b5 100644 --- a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py +++ b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py @@ -46,6 +46,10 @@ def __init__(self, file_paths: list[FilePath], verbose: bool = False): verbose : bool, default: False """ + # This import is to assure that ndx_sound is in the global namespace when an pynwb.io object is created. + # For more detail, see https://github.com/rly/ndx-pose/issues/36 + import ndx_sound # noqa: F401 + suffixes = [suffix for file_path in file_paths for suffix in Path(file_path).suffixes] format_is_not_supported = [ suffix for suffix in suffixes if suffix not in [".wav"] @@ -166,7 +170,6 @@ def add_to_nwbfile( stub_frames: int = 1000, write_as: Literal["stimulus", "acquisition"] = "stimulus", iterator_options: Optional[dict] = None, - compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 overwrite: bool = False, verbose: bool = True, ): @@ -224,7 +227,6 @@ def add_to_nwbfile( write_as=write_as, starting_time=starting_times[file_index], iterator_options=iterator_options, - compression_options=compression_options, # TODO: remove completely after 10/1/2024; still passing for deprecation warning ) return nwbfile diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 5d1224e85..14866510d 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -279,13 +279,14 @@ def _write_pes_to_nwbfile( else: timestamps_cleaned = timestamps + timestamps = np.asarray(timestamps_cleaned).astype("float64", copy=False) pes = PoseEstimationSeries( name=f"{animal}_{keypoint}" if animal else keypoint, description=f"Keypoint {keypoint} from individual {animal}.", data=data[:, :2], unit="pixels", reference_frame="(0,0) corresponds to the bottom left corner of the video.", - timestamps=timestamps_cleaned, + timestamps=timestamps, confidence=data[:, 2], confidence_definition="Softmax output of the deep neural network.", ) @@ -298,6 +299,7 @@ def _write_pes_to_nwbfile( # TODO, taken from the original implementation, improve it if the video is passed dimensions = [list(map(int, image_shape.split(",")))[1::2]] + dimensions = np.array(dimensions, dtype="uint32") pose_estimation_default_kwargs = dict( pose_estimation_series=pose_estimation_series, description="2D keypoint coordinates estimated using DeepLabCut.", diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index f45913061..147fcf6ea 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -5,7 +5,6 @@ from pydantic import FilePath, validate_call from pynwb.file import NWBFile -# import ndx_pose from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface @@ -48,6 +47,9 @@ def __init__( verbose: bool, default: True Controls verbosity. """ + # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created + from ndx_pose import PoseEstimation, PoseEstimationSeries # noqa: F401 + from ._dlc_utils import _read_config file_path = Path(file_path) diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py index f103b7c9a..dbd425b5b 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py @@ -80,6 +80,10 @@ def __init__( verbose : bool, default: True controls verbosity. ``True`` by default. """ + # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created + # For more detail, see https://github.com/rly/ndx-pose/issues/36 + import ndx_pose # noqa: F401 + from neuroconv.datainterfaces.behavior.video.video_utils import ( VideoCaptureContext, ) diff --git a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py index 6a4127663..09f9111d7 100644 --- a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py @@ -73,6 +73,10 @@ def __init__( verbose : bool, optional Whether to print verbose output, by default True """ + # This import is to assure that the ndx_events is in the global namespace when an pynwb.io object is created + # For more detail, see https://github.com/rly/ndx-pose/issues/36 + import ndx_events # noqa: F401 + if aligned_timestamp_names is None: aligned_timestamp_names = [] super().__init__( diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py index aa58f6ae4..8b092464e 100644 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py @@ -43,6 +43,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True): folder_path=folder_path, verbose=verbose, ) + # This module should be here so ndx_fiber_photometry is in the global namespace when an pynwb.io object is created import ndx_fiber_photometry # noqa: F401 def get_metadata(self) -> DeepDict: diff --git a/src/neuroconv/tools/audio/audio.py b/src/neuroconv/tools/audio/audio.py index 44d10de63..064f36155 100644 --- a/src/neuroconv/tools/audio/audio.py +++ b/src/neuroconv/tools/audio/audio.py @@ -15,7 +15,6 @@ def add_acoustic_waveform_series( starting_time: float = 0.0, write_as: Literal["stimulus", "acquisition"] = "stimulus", iterator_options: Optional[dict] = None, - compression_options: Optional[dict] = None, # TODO: remove completely after 10/1/2024 ) -> NWBFile: """ @@ -53,17 +52,6 @@ def add_acoustic_waveform_series( "acquisition", ], "Acoustic series can be written either as 'stimulus' or 'acquisition'." - # TODO: remove completely after 10/1/2024 - if compression_options is not None: - warn( - message=( - "Specifying compression methods and their options at the level of tool functions has been deprecated. " - "Please use the `configure_backend` tool function for this purpose." - ), - category=DeprecationWarning, - stacklevel=2, - ) - iterator_options = iterator_options or dict() container = nwbfile.acquisition if write_as == "acquisition" else nwbfile.stimulus diff --git a/tests/test_on_data/behavior/test_behavior_interfaces.py b/tests/test_on_data/behavior/test_behavior_interfaces.py index b43e65206..0b5c63376 100644 --- a/tests/test_on_data/behavior/test_behavior_interfaces.py +++ b/tests/test_on_data/behavior/test_behavior_interfaces.py @@ -1,3 +1,4 @@ +import sys import unittest from datetime import datetime, timezone from pathlib import Path @@ -10,7 +11,6 @@ from natsort import natsorted from ndx_miniscope import Miniscope from ndx_miniscope.utils import get_timestamps -from ndx_pose import PoseEstimation, PoseEstimationSeries from numpy.testing import assert_array_equal from parameterized import param, parameterized from pynwb import NWBHDF5IO @@ -105,6 +105,8 @@ def check_extracted_metadata(self, metadata: dict): assert metadata["Behavior"][self.pose_estimation_name] == self.expected_metadata[self.pose_estimation_name] def check_read_nwb(self, nwbfile_path: str): + from ndx_pose import PoseEstimation, PoseEstimationSeries + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() @@ -381,6 +383,49 @@ def check_read_nwb(self, nwbfile_path: str): assert all(expected_pose_estimation_series_are_in_nwb_file) +@pytest.fixture +def clean_pose_extension_import(): + modules_to_remove = [m for m in sys.modules if m.startswith("ndx_pose")] + for module in modules_to_remove: + del sys.modules[module] + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10"), + reason="interface not supported on macOS with Python < 3.10", +) +def test_deep_lab_cut_import_pose_extension_bug(clean_pose_extension_import, tmp_path): + """ + Test that the DeepLabCutInterface writes correctly without importing the ndx-pose extension. + See issues: + https://github.com/catalystneuro/neuroconv/issues/1114 + https://github.com/rly/ndx-pose/issues/36 + + """ + + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), + ) + + interface = DeepLabCutInterface(**interface_kwargs) + metadata = interface.get_metadata() + metadata["NWBFile"]["session_start_time"] = datetime(2023, 7, 24, 9, 30, 55, 440600, tzinfo=timezone.utc) + + nwbfile_path = tmp_path / "test.nwb" + interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + read_nwbfile = io.read() + pose_estimation_container = read_nwbfile.processing["behavior"]["PoseEstimation"] + + assert len(pose_estimation_container.fields) > 0 + + @pytest.mark.skipif( platform == "darwin" and python_version < version.parse("3.10"), reason="interface not supported on macOS with Python < 3.10", diff --git a/tests/test_on_data/behavior/test_lightningpose_converter.py b/tests/test_on_data/behavior/test_lightningpose_converter.py index 4d0f8ab89..dd93632a4 100644 --- a/tests/test_on_data/behavior/test_lightningpose_converter.py +++ b/tests/test_on_data/behavior/test_lightningpose_converter.py @@ -5,7 +5,6 @@ from warnings import warn from hdmf.testing import TestCase -from ndx_pose import PoseEstimation from pynwb import NWBHDF5IO from pynwb.image import ImageSeries @@ -134,6 +133,8 @@ def test_run_conversion_add_conversion_options(self): self.assertNWBFileStructure(nwbfile_path=nwbfile_path, **self.conversion_options) def assertNWBFileStructure(self, nwbfile_path: str, stub_test: bool = False): + from ndx_pose import PoseEstimation + with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() From 006184ae120f4c4dd73349b354b4717345f7507d Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 26 Nov 2024 18:44:53 -0500 Subject: [PATCH 089/118] [Cloud Deployment IVd] AWS usage docs (#1104) Co-authored-by: Heberto Mayorquin --- docs/api/tools.aws.rst | 5 ++ docs/api/tools.rst | 1 + docs/user_guide/aws_demo.rst | 136 +++++++++++++++++++++++++++++++++++ docs/user_guide/index.rst | 1 + 4 files changed, 143 insertions(+) create mode 100644 docs/api/tools.aws.rst create mode 100644 docs/user_guide/aws_demo.rst diff --git a/docs/api/tools.aws.rst b/docs/api/tools.aws.rst new file mode 100644 index 000000000..6f3ed0f86 --- /dev/null +++ b/docs/api/tools.aws.rst @@ -0,0 +1,5 @@ +.. _api_docs_aws_tools: + +AWS Tools +--------- +.. automodule:: neuroconv.tools.aws diff --git a/docs/api/tools.rst b/docs/api/tools.rst index 41515facd..793e0dbeb 100644 --- a/docs/api/tools.rst +++ b/docs/api/tools.rst @@ -13,3 +13,4 @@ Tools tools.signal_processing tools.data_transfers tools.nwb_helpers + tools.aws diff --git a/docs/user_guide/aws_demo.rst b/docs/user_guide/aws_demo.rst new file mode 100644 index 000000000..7002b7057 --- /dev/null +++ b/docs/user_guide/aws_demo.rst @@ -0,0 +1,136 @@ +NeuroConv AWS Demo +------------------ + +The :ref:`neuroconv.tools.aws ` submodule provides a number of tools for deploying NWB conversions +within AWS cloud services. These tools are primarily for facilitating source data transfers from cloud storage +sources to AWS, where the NWB conversion takes place, following by immediate direct upload to the `Dandi Archive `_. + +The following is an explicit demonstration of how to use these to create a pipeline to run a remote data conversion. + +This tutorial relies on setting up several cloud-based aspects ahead of time: + +a. Download some of the GIN data from the main testing suite, see :ref:`example_data` for more +details. Specifically, you will need the ``spikeglx`` and ``phy`` folders. + +b. Have access to a `Google Drive `_ folder to mimic a typical remote storage +location. The example data from (a) only takes up about 20 MB of space, so ensure you have that available. In +practice, any `cloud storage provider that can be accessed via Rclone `_ can be used. + +c. Install `Rclone `_, run ``rclone config``, and follow all instructions while giving your +remote the name ``test_google_drive_remote``. This step is necessary to provide the necessary credentials to access +the Google Drive folder from other locations by creating a file called ``rclone.conf``. You can find the path to +file, which you will need for a later step, by running ``rclone config file``. + +d. Have access to an `AWS account `_. Then, from +the `AWS console `_, sign in and navigate to the "IAM" page. Here, you will +generate some credentials by creating a new user with programmatic access. Save your access key and secret key +somewhere safe (such as installing the `AWS CLI `_ and running ``aws configure`` +to store the values on your local device). + +e. Have access to an account on both the `staging/testing server `_ (you +will probably want one on the main archive as well, but please do not upload demonstration data to the primary +server). This request can take a few days for the admin team to process. Once you have access, you will need +to create a new Dandiset on the staging server and record the six-digit Dandiset ID. + +.. warning:: + + *Cloud costs*. While the operations deployed on your behalf by NeuroConv are optimized to the best extent we can, cloud services can still become expensive. Please be aware of the costs associated with running these services and ensure you have the necessary permissions and budget to run these operations. While NeuroConv makes every effort to ensure there are no stalled resources, it is ultimately your responsibility to monitor and manage these resources. We recommend checking the AWS dashboards regularly while running these operations, manually removing any spurious resources, and setting up billing alerts to ensure you do not exceed your budget. + +Then, to setup the remaining steps of the tutorial: + +1. In your Google Drive, make a new folder for this demo conversion named ``demo_neuroconv_aws`` at the outermost +level (not nested in any other folders). + +2. Create a file on your local device named ``demo_neuroconv_aws.yml`` with the following content: + +.. code-block:: yaml + + metadata: + NWBFile: + lab: My Lab + institution: My Institution + + data_interfaces: + ap: SpikeGLXRecordingInterface + phy: PhySortingInterface + + upload_to_dandiset: "< enter your six-digit Dandiset ID here >" + + experiments: + my_experiment: + metadata: + NWBFile: + session_description: My session. + + sessions: + - source_data: + ap: + file_path: spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "1" + sex: F + age: P35D + species: Mus musculus + - metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + Subject: + subject_id: "002" + sex: F + age: P35D + species: Mus musculus + source_data: + phy: + folder_path: phy/phy_example_0/ + + +3. Copy and paste the ``Noise4Sam_g0`` and ``phy_example_0`` folders from the :ref:`example_data` into this demo +folder so that you have the following structure... + +.. code:: + + demo_neuroconv_aws/ + ¦ demo_output/ + ¦ spikeglx/ + ¦ +-- Noise4Sam_g0/ + ¦ +-- ... # .nidq streams + ¦ ¦ +-- Noise4Sam_g0_imec0/ + ¦ ¦ +-- Noise4Sam_g0_t0.imec0.ap.bin + ¦ ¦ +-- Noise4Sam_g0_t0.imec0.ap.meta + ¦ ¦ +-- ... # .lf streams + ¦ phy/ + ¦ +-- phy_example_0/ + ¦ ¦ +-- ... # The various file contents from the example Phy folder + +4. Now run the following Python code to deploy the AWS Batch job: + +.. code:: python + + from neuroconv.tools.aws import deploy_neuroconv_batch_job + + rclone_command = ( + "rclone copy test_google_drive_remote:demo_neuroconv_aws /mnt/efs/source " + "--verbose --progress --config ./rclone.conf" + ) + + # Remember - you can find this via `rclone config file` + rclone_config_file_path = "/path/to/rclone.conf" + + yaml_specification_file_path = "/path/to/demo_neuroconv_aws.yml" + + job_name = "demo_deploy_neuroconv_batch_job" + efs_volume_name = "demo_deploy_neuroconv_batch_job" + deploy_neuroconv_batch_job( + rclone_command=rclone_command, + yaml_specification_file_path=yaml_specification_file_path, + job_name=job_name, + efs_volume_name=efs_volume_name, + rclone_config_file_path=rclone_config_file_path, + ) + +Voilà! If everything occurred successfully, you should eventually (~2-10 minutes) see the files uploaded to your +Dandiset on the staging server. You should also be able to monitor the resources running in the AWS Batch dashboard +as well as on the DynamoDB table. diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 4077f49be..bf9aaf253 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -27,3 +27,4 @@ and synchronize data across multiple sources. backend_configuration yaml docker_demo + aws_demo From c4afad37a8600aed3c21fa73ce45fbb24ba82ea4 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 27 Nov 2024 20:18:35 -0500 Subject: [PATCH 090/118] [Cloud Deployment IVb] Rclone in AWS on EFS (#1085) Co-authored-by: Heberto Mayorquin --- .../{aws_tests.yml => generic_aws_tests.yml} | 7 +- .github/workflows/rclone_aws_tests.yml | 46 +++++ CHANGELOG.md | 9 + src/neuroconv/tools/aws/__init__.py | 3 +- .../tools/aws/_rclone_transfer_batch_job.py | 113 +++++++++++ .../tools/aws/_submit_aws_batch_job.py | 15 +- tests/docker_rclone_with_config_cli.py | 3 +- .../{aws_tools.py => aws_tools_tests.py} | 0 .../test_yaml/yaml_aws_tools_tests.py | 176 ++++++++++++++++++ 9 files changed, 361 insertions(+), 11 deletions(-) rename .github/workflows/{aws_tests.yml => generic_aws_tests.yml} (88%) create mode 100644 .github/workflows/rclone_aws_tests.yml create mode 100644 src/neuroconv/tools/aws/_rclone_transfer_batch_job.py rename tests/test_minimal/test_tools/{aws_tools.py => aws_tools_tests.py} (100%) create mode 100644 tests/test_on_data/test_yaml/yaml_aws_tools_tests.py diff --git a/.github/workflows/aws_tests.yml b/.github/workflows/generic_aws_tests.yml similarity index 88% rename from .github/workflows/aws_tests.yml rename to .github/workflows/generic_aws_tests.yml index 0ecbb4d7b..20886a178 100644 --- a/.github/workflows/aws_tests.yml +++ b/.github/workflows/generic_aws_tests.yml @@ -11,7 +11,6 @@ concurrency: # Cancel previous workflows on the same pull request env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} jobs: run: @@ -36,8 +35,8 @@ jobs: git config --global user.email "CI@example.com" git config --global user.name "CI Almighty" - - name: Install full requirements + - name: Install AWS requirements run: pip install .[aws,test] - - name: Run subset of tests that use S3 live services - run: pytest -rsx -n auto tests/test_minimal/test_tools/aws_tools.py + - name: Run generic AWS tests + run: pytest -rsx -n auto tests/test_minimal/test_tools/aws_tools_tests.py diff --git a/.github/workflows/rclone_aws_tests.yml b/.github/workflows/rclone_aws_tests.yml new file mode 100644 index 000000000..bcfbeb5c7 --- /dev/null +++ b/.github/workflows/rclone_aws_tests.yml @@ -0,0 +1,46 @@ +name: Rclone AWS Tests +on: + schedule: + - cron: "0 16 * * 2" # Weekly at noon on Tuesday + workflow_dispatch: + +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + RCLONE_DRIVE_ACCESS_TOKEN: ${{ secrets.RCLONE_DRIVE_ACCESS_TOKEN }} + RCLONE_DRIVE_REFRESH_TOKEN: ${{ secrets.RCLONE_DRIVE_REFRESH_TOKEN }} + RCLONE_EXPIRY_TOKEN: ${{ secrets.RCLONE_EXPIRY_TOKEN }} + DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + +jobs: + run: + name: ${{ matrix.os }} Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - run: git fetch --prune --unshallow --tags + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Global Setup + run: | + python -m pip install -U pip # Official recommended way + git config --global user.email "CI@example.com" + git config --global user.name "CI Almighty" + + - name: Install AWS requirements + run: pip install .[aws,test] + + - name: Run RClone on AWS tests + run: pytest -rsx -n auto tests/test_on_data/test_yaml/yaml_aws_tools_tests.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c0c3bd729..a37093e39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Upcoming +## Features +* Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) + + + +## v0.6.4 + ## Deprecations * Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) @@ -37,6 +44,8 @@ * Avoid running link test when the PR is on draft [PR #1093](https://github.com/catalystneuro/neuroconv/pull/1093) * Centralize gin data preparation in a github action [PR #1095](https://github.com/catalystneuro/neuroconv/pull/1095) + + # v0.6.4 (September 17, 2024) ## Bug Fixes diff --git a/src/neuroconv/tools/aws/__init__.py b/src/neuroconv/tools/aws/__init__.py index d40ddb2dd..88144fb01 100644 --- a/src/neuroconv/tools/aws/__init__.py +++ b/src/neuroconv/tools/aws/__init__.py @@ -1,3 +1,4 @@ from ._submit_aws_batch_job import submit_aws_batch_job +from ._rclone_transfer_batch_job import rclone_transfer_batch_job -__all__ = ["submit_aws_batch_job"] +__all__ = ["submit_aws_batch_job", "rclone_transfer_batch_job"] diff --git a/src/neuroconv/tools/aws/_rclone_transfer_batch_job.py b/src/neuroconv/tools/aws/_rclone_transfer_batch_job.py new file mode 100644 index 000000000..65bef7824 --- /dev/null +++ b/src/neuroconv/tools/aws/_rclone_transfer_batch_job.py @@ -0,0 +1,113 @@ +"""Collection of helper functions for assessing and performing automated data transfers related to AWS.""" + +import warnings +from typing import Optional + +from pydantic import FilePath, validate_call + +from ._submit_aws_batch_job import submit_aws_batch_job + + +@validate_call +def rclone_transfer_batch_job( + *, + rclone_command: str, + job_name: str, + efs_volume_name: str, + rclone_config_file_path: Optional[FilePath] = None, + status_tracker_table_name: str = "neuroconv_batch_status_tracker", + compute_environment_name: str = "neuroconv_batch_environment", + job_queue_name: str = "neuroconv_batch_queue", + job_definition_name: Optional[str] = None, + minimum_worker_ram_in_gib: int = 4, + minimum_worker_cpus: int = 4, + submission_id: Optional[str] = None, + region: Optional[str] = None, +) -> dict[str, str]: + """ + Submit a job to AWS Batch for processing. + + Requires AWS credentials saved to files in the `~/.aws/` folder or set as environment variables. + + Parameters + ---------- + rclone_command : str + The command to pass directly to Rclone running on the EC2 instance. + E.g.: "rclone copy my_drive:testing_rclone /mnt/efs" + Must move data from or to '/mnt/efs'. + job_name : str + The name of the job to submit. + efs_volume_name : str + The name of an EFS volume to be created and attached to the job. + The path exposed to the container will always be `/mnt/efs`. + rclone_config_file_path : FilePath, optional + The path to the Rclone configuration file to use for the job. + If unspecified, method will attempt to find the file in `~/.rclone` and will raise an error if it cannot. + status_tracker_table_name : str, default: "neuroconv_batch_status_tracker" + The name of the DynamoDB table to use for tracking job status. + compute_environment_name : str, default: "neuroconv_batch_environment" + The name of the compute environment to use for the job. + job_queue_name : str, default: "neuroconv_batch_queue" + The name of the job queue to use for the job. + job_definition_name : str, optional + The name of the job definition to use for the job. + If unspecified, a name starting with 'neuroconv_batch_' will be generated. + minimum_worker_ram_in_gib : int, default: 4 + The minimum amount of base worker memory required to run this job. + Determines the EC2 instance type selected by the automatic 'best fit' selector. + Recommended to be several GiB to allow comfortable buffer space for data chunk iterators. + minimum_worker_cpus : int, default: 4 + The minimum number of CPUs required to run this job. + A minimum of 4 is required, even if only one will be used in the actual process. + submission_id : str, optional + The unique ID to pair with this job submission when tracking the status via DynamoDB. + Defaults to a random UUID4. + region : str, optional + The AWS region to use for the job. + If not provided, we will attempt to load the region from your local AWS configuration. + If that file is not found on your system, we will default to "us-east-2", the location of the DANDI Archive. + + Returns + ------- + info : dict + A dictionary containing information about this AWS Batch job. + + info["job_submission_info"] is the return value of `boto3.client.submit_job` which contains the job ID. + info["table_submission_info"] is the initial row data inserted into the DynamoDB status tracking table. + """ + docker_image = "ghcr.io/catalystneuro/rclone_with_config:latest" + + if "/mnt/efs" not in rclone_command: + message = ( + f"The Rclone command '{rclone_command}' does not contain a reference to '/mnt/efs'. " + "Without utilizing the EFS mount, the instance is unlikely to have enough local disk space." + ) + warnings.warn(message=message, stacklevel=2) + + rclone_config_file_path = rclone_config_file_path or pathlib.Path.home() / ".rclone" / "rclone.conf" + if not rclone_config_file_path.exists(): + raise FileNotFoundError( + f"Rclone configuration file not found at: {rclone_config_file_path}! " + "Please check that `rclone config` successfully created the file." + ) + with open(file=rclone_config_file_path, mode="r") as io: + rclone_config_file_stream = io.read() + + region = region or "us-east-2" + + info = submit_aws_batch_job( + job_name=job_name, + docker_image=docker_image, + environment_variables={"RCLONE_CONFIG": rclone_config_file_stream, "RCLONE_COMMAND": rclone_command}, + efs_volume_name=efs_volume_name, + status_tracker_table_name=status_tracker_table_name, + compute_environment_name=compute_environment_name, + job_queue_name=job_queue_name, + job_definition_name=job_definition_name, + minimum_worker_ram_in_gib=minimum_worker_ram_in_gib, + minimum_worker_cpus=minimum_worker_cpus, + submission_id=submission_id, + region=region, + ) + + return info diff --git a/src/neuroconv/tools/aws/_submit_aws_batch_job.py b/src/neuroconv/tools/aws/_submit_aws_batch_job.py index 9e3ba0488..748f25399 100644 --- a/src/neuroconv/tools/aws/_submit_aws_batch_job.py +++ b/src/neuroconv/tools/aws/_submit_aws_batch_job.py @@ -171,7 +171,9 @@ def submit_aws_batch_job( job_dependencies = job_dependencies or [] container_overrides = dict() if environment_variables is not None: - container_overrides["environment"] = [{key: value} for key, value in environment_variables.items()] + container_overrides["environment"] = [ + {"name": key, "value": value} for key, value in environment_variables.items() + ] if commands is not None: container_overrides["command"] = commands @@ -294,7 +296,7 @@ def _ensure_compute_environment_exists( The AWS Batch client to use for the job. max_retries : int, default: 12 If the compute environment does not already exist, then this is the maximum number of times to synchronously - check for its successful creation before erroring. + check for its successful creation before raising an error. This is essential for a clean setup of the entire pipeline, or else later steps might error because they tried to launch before the compute environment was ready. """ @@ -530,7 +532,11 @@ def _generate_job_definition_name( """ docker_tags = docker_image.split(":")[1:] docker_tag = docker_tags[0] if len(docker_tags) > 1 else None - parsed_docker_image_name = docker_image.replace(":", "-") # AWS Batch does not allow colons in job definition names + + # AWS Batch does not allow colons, slashes, or periods in job definition names + parsed_docker_image_name = str(docker_image) + for disallowed_character in [":", r"/", "."]: + parsed_docker_image_name = parsed_docker_image_name.replace(disallowed_character, "-") job_definition_name = f"neuroconv_batch" job_definition_name += f"_{parsed_docker_image_name}-image" @@ -540,7 +546,6 @@ def _generate_job_definition_name( job_definition_name += f"_{efs_id}" if docker_tag is None or docker_tag == "latest": date = datetime.now().strftime("%Y-%m-%d") - job_definition_name += f"_created-on-{date}" return job_definition_name @@ -641,7 +646,7 @@ def _ensure_job_definition_exists_and_get_arn( ] mountPoints = [{"containerPath": "/mnt/efs/", "readOnly": False, "sourceVolume": "neuroconv_batch_efs_mounted"}] - # batch_client.register_job_definition() is not synchronous and so we need to wait a bit afterwards + # batch_client.register_job_definition is not synchronous and so we need to wait a bit afterwards batch_client.register_job_definition( jobDefinitionName=job_definition_name, type="container", diff --git a/tests/docker_rclone_with_config_cli.py b/tests/docker_rclone_with_config_cli.py index ed472bdf2..9b1e265dd 100644 --- a/tests/docker_rclone_with_config_cli.py +++ b/tests/docker_rclone_with_config_cli.py @@ -61,7 +61,8 @@ def test_direct_usage_of_rclone_with_config(self): os.environ["RCLONE_CONFIG"] = rclone_config_file_stream os.environ["RCLONE_COMMAND"] = ( - f"rclone copy test_google_drive_remote:testing_rclone_with_config {self.test_folder} --verbose --progress --config ./rclone.conf" + f"rclone copy test_google_drive_remote:testing_rclone_with_config {self.test_folder} " + "--verbose --progress --config ./rclone.conf" ) command = ( diff --git a/tests/test_minimal/test_tools/aws_tools.py b/tests/test_minimal/test_tools/aws_tools_tests.py similarity index 100% rename from tests/test_minimal/test_tools/aws_tools.py rename to tests/test_minimal/test_tools/aws_tools_tests.py diff --git a/tests/test_on_data/test_yaml/yaml_aws_tools_tests.py b/tests/test_on_data/test_yaml/yaml_aws_tools_tests.py new file mode 100644 index 000000000..7ea49e644 --- /dev/null +++ b/tests/test_on_data/test_yaml/yaml_aws_tools_tests.py @@ -0,0 +1,176 @@ +import datetime +import os +import time +import unittest + +import boto3 + +from neuroconv.tools.aws import rclone_transfer_batch_job + +from ..setup_paths import OUTPUT_PATH + +_RETRY_STATES = ["RUNNABLE", "PENDING", "STARTING", "RUNNING"] + + +class TestRcloneTransferBatchJob(unittest.TestCase): + """ + To allow this test to work, the developer must create a folder on the outer level of their personal Google Drive + called 'testing_rclone_spikegl_and_phy' with the following structure: + + testing_rclone_spikeglx_and_phy + ├── ci_tests + ├──── spikeglx + ├────── Noise4Sam_g0 + ├──── phy + ├────── phy_example_0 + + Where 'Noise4Sam' is from the 'spikeglx/Noise4Sam_g0' GIN ephys dataset and 'phy_example_0' is likewise from the + 'phy' folder of the same dataset. + + Then the developer must install Rclone and call `rclone config` to generate tokens in their own `rclone.conf` file. + The developer can easily find the location of the config file on their system using `rclone config file`. + """ + + test_folder = OUTPUT_PATH / "aws_rclone_tests" + test_config_file_path = test_folder / "rclone.conf" + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) + aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) + region = "us-east-2" + + def setUp(self): + self.test_folder.mkdir(exist_ok=True) + + # Pretend as if .conf file already exists on the system (created via interactive `rclone config` command) + token_dictionary = dict( + access_token=os.environ["RCLONE_DRIVE_ACCESS_TOKEN"], + token_type="Bearer", + refresh_token=os.environ["RCLONE_DRIVE_REFRESH_TOKEN"], + expiry=os.environ["RCLONE_EXPIRY_TOKEN"], + ) + token_string = str(token_dictionary).replace("'", '"').replace(" ", "") + rclone_config_contents = [ + "[test_google_drive_remote]\n", + "type = drive\n", + "scope = drive\n", + f"token = {token_string}\n", + "team_drive = \n", + "\n", + ] + with open(file=self.test_config_file_path, mode="w") as io: + io.writelines(rclone_config_contents) + + self.efs_client = boto3.client( + service_name="efs", + region_name=self.region, + aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + ) + + def tearDown(self): + efs_client = self.efs_client + + # Cleanup EFS after testing is complete - must clear mount targets first, then wait before deleting the volume + # TODO: cleanup job definitions? (since built daily) + mount_targets = efs_client.describe_mount_targets(FileSystemId=self.efs_id) + for mount_target in mount_targets["MountTargets"]: + efs_client.delete_mount_target(MountTargetId=mount_target["MountTargetId"]) + + time.sleep(60) + efs_client.delete_file_system(FileSystemId=self.efs_id) + + def test_rclone_transfer_batch_job(self): + region = self.region + aws_access_key_id = self.aws_access_key_id + aws_secret_access_key = self.aws_secret_access_key + + dynamodb_resource = boto3.resource( + service_name="dynamodb", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + batch_client = boto3.client( + service_name="batch", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + efs_client = self.efs_client + + rclone_command = ( + "rclone copy test_google_drive_remote:testing_rclone_spikeglx_and_phy /mnt/efs " + "--verbose --progress --config ./rclone.conf" # TODO: should just include this in helper function? + ) + rclone_config_file_path = self.test_config_file_path + + today = datetime.datetime.now().date().isoformat() + job_name = f"test_rclone_transfer_batch_job_{today}" + efs_volume_name = "test_rclone_transfer_batch_efs" + + info = rclone_transfer_batch_job( + rclone_command=rclone_command, + job_name=job_name, + efs_volume_name=efs_volume_name, + rclone_config_file_path=rclone_config_file_path, + ) + + # Wait for AWS to process the job + time.sleep(60) + + job_id = info["job_submission_info"]["jobId"] + job = None + max_retries = 10 + retry = 0 + while retry < max_retries: + job_description_response = batch_client.describe_jobs(jobs=[job_id]) + assert job_description_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + jobs = job_description_response["jobs"] + assert len(jobs) == 1 + + job = jobs[0] + + if job["status"] in _RETRY_STATES: + retry += 1 + time.sleep(60) + else: + break + + # Check EFS specific details + efs_volumes = efs_client.describe_file_systems() + matching_efs_volumes = [ + file_system + for file_system in efs_volumes["FileSystems"] + for tag in file_system["Tags"] + if tag["Key"] == "Name" and tag["Value"] == efs_volume_name + ] + assert len(matching_efs_volumes) == 1 + efs_volume = matching_efs_volumes[0] + self.efs_id = efs_volume["FileSystemId"] + + # Check normal job completion + assert job["jobName"] == job_name + assert "neuroconv_batch_queue" in job["jobQueue"] + assert "fs-" in job["jobDefinition"] + assert job["status"] == "SUCCEEDED" + + status_tracker_table_name = "neuroconv_batch_status_tracker" + table = dynamodb_resource.Table(name=status_tracker_table_name) + table_submission_id = info["table_submission_info"]["id"] + + table_item_response = table.get_item(Key={"id": table_submission_id}) + assert table_item_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + table_item = table_item_response["Item"] + assert table_item["job_name"] == job_name + assert table_item["job_id"] == job_id + assert table_item["status"] == "Job submitted..." + + table.update_item( + Key={"id": table_submission_id}, + AttributeUpdates={"status": {"Action": "PUT", "Value": "Test passed - cleaning up..."}}, + ) + + table.update_item( + Key={"id": table_submission_id}, AttributeUpdates={"status": {"Action": "PUT", "Value": "Test passed."}} + ) From 54e6ea9a54a09b06400afabbd61069423c52059c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 6 Dec 2024 13:01:15 -0600 Subject: [PATCH 091/118] Fix live-services tests (dandi) on windows (#1151) --- CHANGELOG.md | 11 +- .../test_tools/dandi_transfer_tools.py | 111 +++++++----------- 2 files changed, 48 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a37093e39..5dbb44ec2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,4 @@ -# Upcoming - -## Features -* Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) - - - -## v0.6.4 +# v0.6.6 (Upcoming) ## Deprecations * Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) @@ -18,9 +11,11 @@ * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) +* Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) ## Improvements * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) +* Use pytest format for dandi tests to avoid window permission error on teardown [PR #1151](https://github.com/catalystneuro/neuroconv/pull/1151) # v0.6.5 (November 1, 2024) diff --git a/tests/test_minimal/test_tools/dandi_transfer_tools.py b/tests/test_minimal/test_tools/dandi_transfer_tools.py index df4226d10..da35725a0 100644 --- a/tests/test_minimal/test_tools/dandi_transfer_tools.py +++ b/tests/test_minimal/test_tools/dandi_transfer_tools.py @@ -1,13 +1,9 @@ import os import sys from datetime import datetime -from pathlib import Path from platform import python_version as get_python_version -from shutil import rmtree -from tempfile import mkdtemp import pytest -from hdmf.testing import TestCase from pynwb import NWBHDF5IO from neuroconv.tools.data_transfers import automatic_dandi_upload @@ -24,80 +20,63 @@ not HAVE_DANDI_KEY, reason="You must set your DANDI_API_KEY to run this test!", ) -class TestAutomaticDANDIUpload(TestCase): - def setUp(self): - self.tmpdir = Path(mkdtemp()) - self.nwb_folder_path = self.tmpdir / "test_nwb" - self.nwb_folder_path.mkdir() - metadata = get_default_nwbfile_metadata() - metadata["NWBFile"].update( - session_start_time=datetime.now().astimezone(), - session_id=f"test-automatic-upload-{sys.platform}-{get_python_version().replace('.', '-')}", - ) - metadata.update(Subject=dict(subject_id="foo", species="Mus musculus", age="P1D", sex="U")) - with NWBHDF5IO(path=self.nwb_folder_path / "test_nwb_1.nwb", mode="w") as io: - io.write(make_nwbfile_from_metadata(metadata=metadata)) +def test_automatic_dandi_upload(tmp_path): + nwb_folder_path = tmp_path / "test_nwb" + nwb_folder_path.mkdir() + metadata = get_default_nwbfile_metadata() + metadata["NWBFile"].update( + session_start_time=datetime.now().astimezone(), + session_id=f"test-automatic-upload-{sys.platform}-{get_python_version().replace('.', '-')}", + ) + metadata.update(Subject=dict(subject_id="foo", species="Mus musculus", age="P1D", sex="U")) + with NWBHDF5IO(path=nwb_folder_path / "test_nwb_1.nwb", mode="w") as io: + io.write(make_nwbfile_from_metadata(metadata=metadata)) - def tearDown(self): - rmtree(self.tmpdir) - - def test_automatic_dandi_upload(self): - automatic_dandi_upload(dandiset_id="200560", nwb_folder_path=self.nwb_folder_path, staging=True) + automatic_dandi_upload(dandiset_id="200560", nwb_folder_path=nwb_folder_path, staging=True) @pytest.mark.skipif( not HAVE_DANDI_KEY, reason="You must set your DANDI_API_KEY to run this test!", ) -class TestAutomaticDANDIUploadNonParallel(TestCase): - def setUp(self): - self.tmpdir = Path(mkdtemp()) - self.nwb_folder_path = self.tmpdir / "test_nwb" - self.nwb_folder_path.mkdir() - metadata = get_default_nwbfile_metadata() - metadata["NWBFile"].update( - session_start_time=datetime.now().astimezone(), - session_id=f"test-automatic-upload-{sys.platform}-{get_python_version().replace('.', '-')}-non-parallel", - ) - metadata.update(Subject=dict(subject_id="foo", species="Mus musculus", age="P1D", sex="U")) - with NWBHDF5IO(path=self.nwb_folder_path / "test_nwb_2.nwb", mode="w") as io: - io.write(make_nwbfile_from_metadata(metadata=metadata)) - - def tearDown(self): - rmtree(self.tmpdir) +def test_automatic_dandi_upload_non_parallel(tmp_path): + nwb_folder_path = tmp_path / "test_nwb" + nwb_folder_path.mkdir() + metadata = get_default_nwbfile_metadata() + metadata["NWBFile"].update( + session_start_time=datetime.now().astimezone(), + session_id=(f"test-automatic-upload-{sys.platform}-" f"{get_python_version().replace('.', '-')}-non-parallel"), + ) + metadata.update(Subject=dict(subject_id="foo", species="Mus musculus", age="P1D", sex="U")) + with NWBHDF5IO(path=nwb_folder_path / "test_nwb_2.nwb", mode="w") as io: + io.write(make_nwbfile_from_metadata(metadata=metadata)) - def test_automatic_dandi_upload_non_parallel(self): - automatic_dandi_upload( - dandiset_id="200560", nwb_folder_path=self.nwb_folder_path, staging=True, number_of_jobs=1 - ) + automatic_dandi_upload(dandiset_id="200560", nwb_folder_path=nwb_folder_path, staging=True, number_of_jobs=1) @pytest.mark.skipif( not HAVE_DANDI_KEY, reason="You must set your DANDI_API_KEY to run this test!", ) -class TestAutomaticDANDIUploadNonParallelNonThreaded(TestCase): - def setUp(self): - self.tmpdir = Path(mkdtemp()) - self.nwb_folder_path = self.tmpdir / "test_nwb" - self.nwb_folder_path.mkdir() - metadata = get_default_nwbfile_metadata() - metadata["NWBFile"].update( - session_start_time=datetime.now().astimezone(), - session_id=f"test-automatic-upload-{sys.platform}-{get_python_version().replace('.', '-')}-non-parallel-non-threaded", - ) - metadata.update(Subject=dict(subject_id="foo", species="Mus musculus", age="P1D", sex="U")) - with NWBHDF5IO(path=self.nwb_folder_path / "test_nwb_3.nwb", mode="w") as io: - io.write(make_nwbfile_from_metadata(metadata=metadata)) - - def tearDown(self): - rmtree(self.tmpdir) +def test_automatic_dandi_upload_non_parallel_non_threaded(tmp_path): + nwb_folder_path = tmp_path / "test_nwb" + nwb_folder_path.mkdir() + metadata = get_default_nwbfile_metadata() + metadata["NWBFile"].update( + session_start_time=datetime.now().astimezone(), + session_id=( + f"test-automatic-upload-{sys.platform}-" + f"{get_python_version().replace('.', '-')}-non-parallel-non-threaded" + ), + ) + metadata.update(Subject=dict(subject_id="foo", species="Mus musculus", age="P1D", sex="U")) + with NWBHDF5IO(path=nwb_folder_path / "test_nwb_3.nwb", mode="w") as io: + io.write(make_nwbfile_from_metadata(metadata=metadata)) - def test_automatic_dandi_upload_non_parallel_non_threaded(self): - automatic_dandi_upload( - dandiset_id="200560", - nwb_folder_path=self.nwb_folder_path, - staging=True, - number_of_jobs=1, - number_of_threads=1, - ) + automatic_dandi_upload( + dandiset_id="200560", + nwb_folder_path=nwb_folder_path, + staging=True, + number_of_jobs=1, + number_of_threads=1, + ) From 96dfdffc9a8a3fb9d9e1897fbfd16ad51a2f1994 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:02:54 -0500 Subject: [PATCH 092/118] [Cloud Deployment IVc] Deploy NeuroConv in AWS with EFS (#1086) Co-authored-by: Heberto Mayorquin --- .../neuroconv_deployment_aws_tests.yml | 46 ++++ CHANGELOG.md | 2 + src/neuroconv/tools/aws/__init__.py | 7 +- .../tools/aws/_deploy_neuroconv_batch_job.py | 241 ++++++++++++++++++ .../tools/aws/_submit_aws_batch_job.py | 20 +- .../_yaml_conversion_specification.py | 8 +- .../neuroconv_deployment_aws_tools_tests.py | 167 ++++++++++++ .../test_yaml/yaml_aws_tools_tests.py | 5 +- 8 files changed, 482 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/neuroconv_deployment_aws_tests.yml create mode 100644 src/neuroconv/tools/aws/_deploy_neuroconv_batch_job.py create mode 100644 tests/test_on_data/test_yaml/neuroconv_deployment_aws_tools_tests.py diff --git a/.github/workflows/neuroconv_deployment_aws_tests.yml b/.github/workflows/neuroconv_deployment_aws_tests.yml new file mode 100644 index 000000000..64aae5ec9 --- /dev/null +++ b/.github/workflows/neuroconv_deployment_aws_tests.yml @@ -0,0 +1,46 @@ +name: NeuroConv Deployment AWS Tests +on: + schedule: + - cron: "0 16 * * 3" # Weekly at noon on Wednesday + workflow_dispatch: + +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + RCLONE_DRIVE_ACCESS_TOKEN: ${{ secrets.RCLONE_DRIVE_ACCESS_TOKEN }} + RCLONE_DRIVE_REFRESH_TOKEN: ${{ secrets.RCLONE_DRIVE_REFRESH_TOKEN }} + RCLONE_EXPIRY_TOKEN: ${{ secrets.RCLONE_EXPIRY_TOKEN }} + DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + +jobs: + run: + name: ${{ matrix.os }} Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - run: git fetch --prune --unshallow --tags + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Global Setup + run: | + python -m pip install -U pip # Official recommended way + git config --global user.email "CI@example.com" + git config --global user.name "CI Almighty" + + - name: Install AWS requirements + run: pip install .[aws,test] + + - name: Run NeuroConv Deployment on AWS tests + run: pytest -rsx -n auto tests/test_on_data/test_yaml/neuroconv_deployment_aws_tools_tests.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dbb44ec2..31a0fd7db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) * Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) +* Added the `deploy_neuroconv_batch_job` helper function for deploying NeuroConv AWS Batch jobs. [PR #1086](https://github.com/catalystneuro/neuroconv/pull/1086) + ## Improvements * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) diff --git a/src/neuroconv/tools/aws/__init__.py b/src/neuroconv/tools/aws/__init__.py index 88144fb01..70a42cbf5 100644 --- a/src/neuroconv/tools/aws/__init__.py +++ b/src/neuroconv/tools/aws/__init__.py @@ -1,4 +1,9 @@ from ._submit_aws_batch_job import submit_aws_batch_job from ._rclone_transfer_batch_job import rclone_transfer_batch_job +from ._deploy_neuroconv_batch_job import deploy_neuroconv_batch_job -__all__ = ["submit_aws_batch_job", "rclone_transfer_batch_job"] +__all__ = [ + "submit_aws_batch_job", + "rclone_transfer_batch_job", + "deploy_neuroconv_batch_job", +] diff --git a/src/neuroconv/tools/aws/_deploy_neuroconv_batch_job.py b/src/neuroconv/tools/aws/_deploy_neuroconv_batch_job.py new file mode 100644 index 000000000..1df86d957 --- /dev/null +++ b/src/neuroconv/tools/aws/_deploy_neuroconv_batch_job.py @@ -0,0 +1,241 @@ +"""Collection of helper functions for deploying NeuroConv in EC2 Batch jobs on AWS.""" + +import os +import time +import uuid +import warnings +from typing import Optional + +import boto3 +from pydantic import FilePath, validate_call + +from ._rclone_transfer_batch_job import rclone_transfer_batch_job +from ._submit_aws_batch_job import submit_aws_batch_job + +_RETRY_STATES = ["RUNNABLE", "PENDING", "STARTING", "RUNNING"] + + +@validate_call +def deploy_neuroconv_batch_job( + *, + rclone_command: str, + yaml_specification_file_path: FilePath, + job_name: str, + efs_volume_name: str, + rclone_config_file_path: Optional[FilePath] = None, + status_tracker_table_name: str = "neuroconv_batch_status_tracker", + compute_environment_name: str = "neuroconv_batch_environment", + job_queue_name: str = "neuroconv_batch_queue", + job_definition_name: Optional[str] = None, + minimum_worker_ram_in_gib: int = 16, # Higher than previous recommendations for safer buffering room + minimum_worker_cpus: int = 4, + region: Optional[str] = None, +) -> dict[str, str]: + """ + Submit a job to AWS Batch for processing. + + Requires AWS credentials saved to files in the `~/.aws/` folder or set as environment variables. + + Parameters + ---------- + rclone_command : str + The command to pass directly to Rclone running on the EC2 instance. + E.g.: "rclone copy my_drive:testing_rclone /mnt/efs/source" + Must move data from or to '/mnt/efs/source'. + yaml_specification_file_path : FilePath + The path to the YAML file containing the NeuroConv specification. + job_name : str + The name of the job to submit. + efs_volume_name : str + The name of an EFS volume to be created and attached to the job. + The path exposed to the container will always be `/mnt/efs`. + rclone_config_file_path : FilePath, optional + The path to the Rclone configuration file to use for the job. + If unspecified, method will attempt to find the file in `~/.rclone` and will raise an error if it cannot. + status_tracker_table_name : str, default: "neuroconv_batch_status_tracker" + The name of the DynamoDB table to use for tracking job status. + compute_environment_name : str, default: "neuroconv_batch_environment" + The name of the compute environment to use for the job. + job_queue_name : str, default: "neuroconv_batch_queue" + The name of the job queue to use for the job. + job_definition_name : str, optional + The name of the job definition to use for the job. + If unspecified, a name starting with 'neuroconv_batch_' will be generated. + minimum_worker_ram_in_gib : int, default: 4 + The minimum amount of base worker memory required to run this job. + Determines the EC2 instance type selected by the automatic 'best fit' selector. + Recommended to be several GiB to allow comfortable buffer space for data chunk iterators. + minimum_worker_cpus : int, default: 4 + The minimum number of CPUs required to run this job. + A minimum of 4 is required, even if only one will be used in the actual process. + region : str, optional + The AWS region to use for the job. + If not provided, we will attempt to load the region from your local AWS configuration. + If that file is not found on your system, we will default to "us-east-2", the location of the DANDI Archive. + + Returns + ------- + info : dict + A dictionary containing information about this AWS Batch job. + + info["rclone_job_submission_info"] is the return value of `neuroconv.tools.aws.rclone_transfer_batch_job`. + info["neuroconv_job_submission_info"] is the return value of `neuroconv.tools.aws.submit_job`. + """ + efs_volume_name = efs_volume_name or f"neuroconv_batch_efs_volume_{uuid.uuid4().hex[:4]}" + region = region or "us-east-2" + + if "/mnt/efs/source" not in rclone_command: + message = ( + f"The Rclone command '{rclone_command}' does not contain a reference to '/mnt/efs/source'. " + "Without utilizing the EFS mount, the instance is unlikely to have enough local disk space. " + "The subfolder 'source' is also required to eliminate ambiguity in the transfer process." + ) + raise ValueError(message) + + rclone_job_name = f"{job_name}_rclone_transfer" + rclone_job_submission_info = rclone_transfer_batch_job( + rclone_command=rclone_command, + job_name=rclone_job_name, + efs_volume_name=efs_volume_name, + rclone_config_file_path=rclone_config_file_path, + region=region, + ) + rclone_job_id = rclone_job_submission_info["job_submission_info"]["jobId"] + + # Give the EFS and other aspects time to spin up before submitting next dependent job + # (Otherwise, good chance that duplicate EFS will be created) + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) + aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) + + batch_client = boto3.client( + service_name="batch", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + efs_client = boto3.client( + service_name="efs", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + + available_efs_volumes = efs_client.describe_file_systems() + matching_efs_volumes = [ + file_system + for file_system in available_efs_volumes["FileSystems"] + for tag in file_system["Tags"] + if tag["Key"] == "Name" and tag["Value"] == efs_volume_name + ] + max_iterations = 10 + iteration = 0 + while len(matching_efs_volumes) == 0 and iteration < max_iterations: + iteration += 1 + time.sleep(30) + + matching_efs_volumes = [ + file_system + for file_system in available_efs_volumes["FileSystems"] + for tag in file_system["Tags"] + if tag["Key"] == "Name" and tag["Value"] == efs_volume_name + ] + + if len(matching_efs_volumes) == 0: + message = f"Unable to create EFS volume '{efs_volume_name}' after {max_iterations} attempts!" + raise ValueError(message) + + docker_image = "ghcr.io/catalystneuro/neuroconv_yaml_variable:latest" + + with open(file=yaml_specification_file_path, mode="r") as io: + yaml_specification_file_stream = io.read() + + neuroconv_job_name = f"{job_name}_neuroconv_deployment" + job_dependencies = [{"jobId": rclone_job_id, "type": "SEQUENTIAL"}] + neuroconv_job_submission_info = submit_aws_batch_job( + job_name=neuroconv_job_name, + docker_image=docker_image, + environment_variables={ + "NEUROCONV_YAML": yaml_specification_file_stream, + "NEUROCONV_DATA_PATH": "/mnt/efs/source", + # TODO: would prefer this to use subfolders for source and output, but need logic for YAML + # related code to create them if missing (hard to send EFS this command directly) + # (the code was included in this PR, but a release cycle needs to complete for the docker images before + # it can be used here) + # "NEUROCONV_OUTPUT_PATH": "/mnt/efs/output", + "NEUROCONV_OUTPUT_PATH": "/mnt/efs", + }, + efs_volume_name=efs_volume_name, + job_dependencies=job_dependencies, + status_tracker_table_name=status_tracker_table_name, + compute_environment_name=compute_environment_name, + job_queue_name=job_queue_name, + job_definition_name=job_definition_name, + minimum_worker_ram_in_gib=minimum_worker_ram_in_gib, + minimum_worker_cpus=minimum_worker_cpus, + region=region, + ) + + info = { + "rclone_job_submission_info": rclone_job_submission_info, + "neuroconv_job_submission_info": neuroconv_job_submission_info, + } + + # TODO: would be better to spin up third dependent job to clean up EFS volume after neuroconv job completes + neuroconv_job_id = neuroconv_job_submission_info["job_submission_info"]["jobId"] + job = None + max_retries = 60 * 12 # roughly 12 hours max runtime (aside from internet loss) for checking cleanup + sleep_time = 60 # 1 minute + retry = 0.0 + time.sleep(sleep_time) + while retry < max_retries: + job_description_response = batch_client.describe_jobs(jobs=[neuroconv_job_id]) + if job_description_response["ResponseMetadata"]["HTTPStatusCode"] == 200: + # sleep but only increment retry by a small amount + # (really should only apply if internet connection is temporarily lost) + retry += 0.1 + time.sleep(sleep_time) + + job = job_description_response["jobs"][0] + if job["status"] in _RETRY_STATES: + retry += 1.0 + time.sleep(sleep_time) + elif job["status"] == "SUCCEEDED": + break + + if retry >= max_retries: + message = ( + "Maximum retries reached for checking job completion for automatic EFS cleanup! " + "Please delete the EFS volume manually." + ) + warnings.warn(message=message, stacklevel=2) + + return info + + # Cleanup EFS after job is complete - must clear mount targets first, then wait before deleting the volume + efs_volumes = efs_client.describe_file_systems() + matching_efs_volumes = [ + file_system + for file_system in efs_volumes["FileSystems"] + for tag in file_system["Tags"] + if tag["Key"] == "Name" and tag["Value"] == efs_volume_name + ] + if len(matching_efs_volumes) != 1: + message = ( + f"Expected to find exactly one EFS volume with name '{efs_volume_name}', " + f"but found {len(matching_efs_volumes)}\n\n{matching_efs_volumes=}\n\n!" + "You will have to delete these manually." + ) + warnings.warn(message=message, stacklevel=2) + + return info + + efs_volume = matching_efs_volumes[0] + efs_id = efs_volume["FileSystemId"] + mount_targets = efs_client.describe_mount_targets(FileSystemId=efs_id) + for mount_target in mount_targets["MountTargets"]: + efs_client.delete_mount_target(MountTargetId=mount_target["MountTargetId"]) + + time.sleep(sleep_time) + efs_client.delete_file_system(FileSystemId=efs_id) + + return info diff --git a/src/neuroconv/tools/aws/_submit_aws_batch_job.py b/src/neuroconv/tools/aws/_submit_aws_batch_job.py index 748f25399..cae25f3ce 100644 --- a/src/neuroconv/tools/aws/_submit_aws_batch_job.py +++ b/src/neuroconv/tools/aws/_submit_aws_batch_job.py @@ -464,11 +464,14 @@ def _create_or_get_efs_id( if tag["Key"] == "Name" and tag["Value"] == efs_volume_name ] - if len(matching_efs_volumes) > 1: + if len(matching_efs_volumes) == 1: efs_volume = matching_efs_volumes[0] efs_id = efs_volume["FileSystemId"] return efs_id + elif len(matching_efs_volumes) > 1: + message = f"Multiple EFS volumes with the name '{efs_volume_name}' were found!\n\n{matching_efs_volumes=}\n" + raise ValueError(message) # Existing volume not found - must create a fresh one and set mount targets on it efs_volume = efs_client.create_file_system( @@ -506,7 +509,7 @@ def _create_or_get_efs_id( return efs_id -def _generate_job_definition_name( +def generate_job_definition_name( *, docker_image: str, minimum_worker_ram_in_gib: int, @@ -515,9 +518,7 @@ def _generate_job_definition_name( ) -> str: # pragma: no cover """ Generate a job definition name for the AWS Batch job. - Note that Docker images don't strictly require a tag to be pulled or used - 'latest' is always used by default. - Parameters ---------- docker_image : str @@ -529,15 +530,13 @@ def _generate_job_definition_name( minimum_worker_cpus : int The minimum number of CPUs required to run this job. A minimum of 4 is required, even if only one will be used in the actual process. + efs_id : Optional[str] + The ID of the EFS filesystem to mount, if any. """ - docker_tags = docker_image.split(":")[1:] - docker_tag = docker_tags[0] if len(docker_tags) > 1 else None - # AWS Batch does not allow colons, slashes, or periods in job definition names parsed_docker_image_name = str(docker_image) - for disallowed_character in [":", r"/", "."]: + for disallowed_character in [":", "/", r"/", "."]: parsed_docker_image_name = parsed_docker_image_name.replace(disallowed_character, "-") - job_definition_name = f"neuroconv_batch" job_definition_name += f"_{parsed_docker_image_name}-image" job_definition_name += f"_{minimum_worker_ram_in_gib}-GiB-RAM" @@ -546,7 +545,6 @@ def _generate_job_definition_name( job_definition_name += f"_{efs_id}" if docker_tag is None or docker_tag == "latest": date = datetime.now().strftime("%Y-%m-%d") - return job_definition_name @@ -644,7 +642,7 @@ def _ensure_job_definition_exists_and_get_arn( }, }, ] - mountPoints = [{"containerPath": "/mnt/efs/", "readOnly": False, "sourceVolume": "neuroconv_batch_efs_mounted"}] + mountPoints = [{"containerPath": "/mnt/efs", "readOnly": False, "sourceVolume": "neuroconv_batch_efs_mounted"}] # batch_client.register_job_definition is not synchronous and so we need to wait a bit afterwards batch_client.register_job_definition( diff --git a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py index 10e33cbc8..0e2f05f74 100644 --- a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py +++ b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py @@ -73,10 +73,16 @@ def run_conversion_from_yaml( if data_folder_path is None: data_folder_path = Path(specification_file_path).parent + else: + data_folder_path = Path(data_folder_path) + data_folder_path.mkdir(exist_ok=True) + if output_folder_path is None: - output_folder_path = Path(specification_file_path).parent + output_folder_path = specification_file_path.parent else: output_folder_path = Path(output_folder_path) + output_folder_path.mkdir(exist_ok=True) + specification = load_dict_from_file(file_path=specification_file_path) schema_folder = Path(__file__).parent.parent.parent / "schemas" specification_schema = load_dict_from_file(file_path=schema_folder / "yaml_conversion_specification_schema.json") diff --git a/tests/test_on_data/test_yaml/neuroconv_deployment_aws_tools_tests.py b/tests/test_on_data/test_yaml/neuroconv_deployment_aws_tools_tests.py new file mode 100644 index 000000000..f58865d26 --- /dev/null +++ b/tests/test_on_data/test_yaml/neuroconv_deployment_aws_tools_tests.py @@ -0,0 +1,167 @@ +import os +import pathlib +import time +import unittest + +import boto3 + +from neuroconv.tools.aws import deploy_neuroconv_batch_job + +from ..setup_paths import OUTPUT_PATH + +_RETRY_STATES = ["RUNNABLE", "PENDING", "STARTING", "RUNNING"] + + +class TestNeuroConvDeploymentBatchJob(unittest.TestCase): + """ + To allow this test to work, the developer must create a folder on the outer level of their personal Google Drive + called 'testing_rclone_spikegl_and_phy' with the following structure: + + testing_rclone_spikeglx_and_phy + ├── ci_tests + ├──── spikeglx + ├────── Noise4Sam_g0 + ├──── phy + ├────── phy_example_0 + + Where 'Noise4Sam' is from the 'spikeglx/Noise4Sam_g0' GIN ephys dataset and 'phy_example_0' is likewise from the + 'phy' folder of the same dataset. + + Then the developer must install Rclone and call `rclone config` to generate tokens in their own `rclone.conf` file. + The developer can easily find the location of the config file on their system using `rclone config file`. + """ + + test_folder = OUTPUT_PATH / "aws_rclone_tests" + test_config_file_path = test_folder / "rclone.conf" + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) + aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) + region = "us-east-2" + + def setUp(self): + self.test_folder.mkdir(exist_ok=True) + + # Pretend as if .conf file already exists on the system (created via interactive `rclone config` command) + token_dictionary = dict( + access_token=os.environ["RCLONE_DRIVE_ACCESS_TOKEN"], + token_type="Bearer", + refresh_token=os.environ["RCLONE_DRIVE_REFRESH_TOKEN"], + expiry=os.environ["RCLONE_EXPIRY_TOKEN"], + ) + token_string = str(token_dictionary).replace("'", '"').replace(" ", "") + rclone_config_contents = [ + "[test_google_drive_remote]\n", + "type = drive\n", + "scope = drive\n", + f"token = {token_string}\n", + "team_drive = \n", + "\n", + ] + with open(file=self.test_config_file_path, mode="w") as io: + io.writelines(rclone_config_contents) + + def test_deploy_neuroconv_batch_job(self): + region = "us-east-2" + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) + aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) + + dynamodb_resource = boto3.resource( + service_name="dynamodb", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + batch_client = boto3.client( + service_name="batch", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + efs_client = boto3.client( + service_name="efs", + region_name=region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + # Assume no other tests of EFS volumes are fluctuating at the same time, otherwise make this more specific + efs_volumes_before = efs_client.describe_file_systems() + + rclone_command = ( + "rclone copy test_google_drive_remote:testing_rclone_spikeglx_and_phy/ci_tests /mnt/efs/source " + "--verbose --progress --config ./rclone.conf" # TODO: should just include this in helper function? + ) + + testing_base_folder_path = pathlib.Path(__file__).parent.parent.parent + yaml_specification_file_path = ( + testing_base_folder_path + / "test_on_data" + / "test_yaml" + / "conversion_specifications" + / "GIN_conversion_specification.yml" + ) + + rclone_config_file_path = self.test_config_file_path + + job_name = "test_deploy_neuroconv_batch_job" + efs_volume_name = "test_deploy_neuroconv_batch_job" + all_info = deploy_neuroconv_batch_job( + rclone_command=rclone_command, + yaml_specification_file_path=yaml_specification_file_path, + job_name=job_name, + efs_volume_name=efs_volume_name, + rclone_config_file_path=rclone_config_file_path, + ) + + # Wait additional time for AWS to clean up resources + time.sleep(120) + + info = all_info["neuroconv_job_submission_info"] + job_id = info["job_submission_info"]["jobId"] + job = None + max_retries = 10 + retry = 0 + while retry < max_retries: + job_description_response = batch_client.describe_jobs(jobs=[job_id]) + assert job_description_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + jobs = job_description_response["jobs"] + assert len(jobs) == 1 + + job = jobs[0] + + if job["status"] in _RETRY_STATES: + retry += 1 + time.sleep(60) + else: + break + + # Check EFS cleaned up automatically + efs_volumes_after = efs_client.describe_file_systems() + assert len(efs_volumes_after["FileSystems"]) == len(efs_volumes_before["FileSystems"]) + + # Check normal job completion + expected_job_name = f"{job_name}_neuroconv_deployment" + assert job["jobName"] == expected_job_name + assert "neuroconv_batch_queue" in job["jobQueue"] + assert "fs-" in job["jobDefinition"] + assert job["status"] == "SUCCEEDED" + + status_tracker_table_name = "neuroconv_batch_status_tracker" + table = dynamodb_resource.Table(name=status_tracker_table_name) + table_submission_id = info["table_submission_info"]["id"] + + table_item_response = table.get_item(Key={"id": table_submission_id}) + assert table_item_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + table_item = table_item_response["Item"] + assert table_item["job_name"] == expected_job_name + assert table_item["job_id"] == job_id + assert table_item["status"] == "Job submitted..." + + table.update_item( + Key={"id": table_submission_id}, + AttributeUpdates={"status": {"Action": "PUT", "Value": "Test passed - cleaning up..."}}, + ) + + table.update_item( + Key={"id": table_submission_id}, AttributeUpdates={"status": {"Action": "PUT", "Value": "Test passed."}} + ) diff --git a/tests/test_on_data/test_yaml/yaml_aws_tools_tests.py b/tests/test_on_data/test_yaml/yaml_aws_tools_tests.py index 7ea49e644..e767e516b 100644 --- a/tests/test_on_data/test_yaml/yaml_aws_tools_tests.py +++ b/tests/test_on_data/test_yaml/yaml_aws_tools_tests.py @@ -36,6 +36,7 @@ class TestRcloneTransferBatchJob(unittest.TestCase): aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None) region = "us-east-2" + efs_id = None def setUp(self): self.test_folder.mkdir(exist_ok=True) @@ -66,7 +67,9 @@ def setUp(self): aws_secret_access_key=self.aws_secret_access_key, ) - def tearDown(self): + def tearDown(self) -> None: + if self.efs_id is None: + return None efs_client = self.efs_client # Cleanup EFS after testing is complete - must clear mount targets first, then wait before deleting the volume From 4ba1e827d373153344266f325b002f4b6dffcaad Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 9 Dec 2024 08:31:58 -0600 Subject: [PATCH 093/118] Remove soon to be deprecated jsonschema.RefResolver (#1133) --- CHANGELOG.md | 3 ++- pyproject.toml | 3 ++- .../_yaml_conversion_specification.py | 15 +++++++++++---- .../test_yaml_conversion_specification.py | 17 ++++++++++------- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31a0fd7db..c49e995b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ # v0.6.6 (Upcoming) ## Deprecations -* Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) +* Removed use of `jsonschema.RefResolver` as it will be deprecated from the jsonschema library [PR #1133](https://github.com/catalystneuro/neuroconv/pull/1133) +* Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) diff --git a/pyproject.toml b/pyproject.toml index 5efd432f5..a4f391512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,8 @@ dependencies = [ "parse>=1.20.0", "click", "docstring-parser", - "packaging" # Issue 903 + "packaging", # Issue 903 + "referencing", ] diff --git a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py index 0e2f05f74..f8a4e8655 100644 --- a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py +++ b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py @@ -1,11 +1,11 @@ -import sys from importlib import import_module from pathlib import Path from typing import Optional import click -from jsonschema import RefResolver, validate +from jsonschema import validate from pydantic import DirectoryPath, FilePath +from referencing import Registry, Resource from ...nwbconverter import NWBConverter from ...utils import dict_deep_update, load_dict_from_file @@ -85,12 +85,19 @@ def run_conversion_from_yaml( specification = load_dict_from_file(file_path=specification_file_path) schema_folder = Path(__file__).parent.parent.parent / "schemas" + + # Load all required schemas specification_schema = load_dict_from_file(file_path=schema_folder / "yaml_conversion_specification_schema.json") - sys_uri_base = "file:/" if sys.platform.startswith("win32") else "file://" + metadata_schema = load_dict_from_file(file_path=schema_folder / "metadata_schema.json") + + # The yaml specification references the metadata schema, so we need to load it into the registry + registry = Registry().with_resource("metadata_schema.json", Resource.from_contents(metadata_schema)) + + # Validate using the registry validate( instance=specification, schema=specification_schema, - resolver=RefResolver(base_uri=sys_uri_base + str(schema_folder) + "/", referrer=specification_schema), + registry=registry, ) global_metadata = specification.get("metadata", dict()) diff --git a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py index 61c71cf86..e46e25352 100644 --- a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py @@ -1,12 +1,12 @@ -import sys import unittest from datetime import datetime from pathlib import Path import pytest from hdmf.testing import TestCase -from jsonschema import RefResolver, validate +from jsonschema import validate from pynwb import NWBHDF5IO +from referencing import Registry, Resource from neuroconv import run_conversion_from_yaml from neuroconv.utils import load_dict_from_file @@ -27,16 +27,19 @@ def test_validate_example_specifications(fname): path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" schema_folder = path_to_test_yml_files.parent.parent.parent.parent / "src" / "neuroconv" / "schemas" + + # Load schemas specification_schema = load_dict_from_file(file_path=schema_folder / "yaml_conversion_specification_schema.json") - sys_uri_base = "file://" - if sys.platform.startswith("win32"): - sys_uri_base = "file:/" + metadata_schema = load_dict_from_file(file_path=schema_folder / "metadata_schema.json") + + # The yaml specification references the metadata schema, so we need to load it into the registry + registry = Registry().with_resource("metadata_schema.json", Resource.from_contents(metadata_schema)) yaml_file_path = path_to_test_yml_files / fname validate( instance=load_dict_from_file(file_path=yaml_file_path), - schema=load_dict_from_file(file_path=schema_folder / "yaml_conversion_specification_schema.json"), - resolver=RefResolver(base_uri=sys_uri_base + str(schema_folder) + "/", referrer=specification_schema), + schema=specification_schema, + registry=registry, ) From 4b3172c25676eaebe3f3a388e41d4c37d91efd49 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 9 Dec 2024 12:06:45 -0500 Subject: [PATCH 094/118] Add DANDI upload to YAML spec (#1089) Co-authored-by: Heberto Mayorquin --- .github/workflows/deploy-tests.yml | 3 + .github/workflows/live-service-testing.yml | 16 +++++ CHANGELOG.md | 1 + .../yaml_conversion_specification_schema.json | 1 + .../_yaml_conversion_specification.py | 42 +++++++++++- tests/imports.py | 1 + ..._conversion_specification_dandi_upload.yml | 66 +++++++++++++++++++ .../test_yaml_conversion_specification.py | 1 + .../test_yaml/yaml_dandi_transfer_tools.py | 53 +++++++++++++++ 9 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_dandi_upload.yml create mode 100644 tests/test_on_data/test_yaml/yaml_dandi_transfer_tools.py diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index a18fe8310..a2e56b00a 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -69,6 +69,9 @@ jobs: if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} uses: ./.github/workflows/live-service-testing.yml secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} with: # Ternary operator: condition && value_if_true || value_if_false python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index 24eda7bc3..155438fb2 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -13,6 +13,12 @@ on: type: string secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + S3_GIN_BUCKET: + required: true DANDI_API_KEY: required: true @@ -45,7 +51,17 @@ jobs: - name: Install full requirements run: pip install .[test,full] + - name: Prepare data for tests + uses: ./.github/actions/load-data + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + s3-gin-bucket: ${{ secrets.S3_GIN_BUCKET }} + os: ${{ matrix.os }} + - name: Run subset of tests that use DANDI live services run: pytest -rsx -n auto tests/test_minimal/test_tools/dandi_transfer_tools.py + - name: Run subset of tests that use DANDI live services with YAML + run: pytest -rsx -n auto tests/test_on_data/test_yaml/yaml_dandi_transfer_tools.py - name: Run subset of tests that use Globus live services run: pytest -rsx -n auto tests/test_minimal/test_tools/globus_transfer_tools.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c49e995b9..ae105b907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) * Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) * Added the `deploy_neuroconv_batch_job` helper function for deploying NeuroConv AWS Batch jobs. [PR #1086](https://github.com/catalystneuro/neuroconv/pull/1086) +* YAML specification files now accept an outer keyword `upload_to_dandiset="< six-digit ID >"` to automatically upload the produced NWB files to the DANDI archive [PR #1089](https://github.com/catalystneuro/neuroconv/pull/1089) ## Improvements diff --git a/src/neuroconv/schemas/yaml_conversion_specification_schema.json b/src/neuroconv/schemas/yaml_conversion_specification_schema.json index c6526803b..039a1cf48 100644 --- a/src/neuroconv/schemas/yaml_conversion_specification_schema.json +++ b/src/neuroconv/schemas/yaml_conversion_specification_schema.json @@ -8,6 +8,7 @@ "required": ["experiments"], "additionalProperties": false, "properties": { + "upload_to_dandiset": {"type": "string"}, "metadata": {"$ref": "./metadata_schema.json#"}, "conversion_options": {"type": "object"}, "data_interfaces": { diff --git a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py index f8a4e8655..7cdec0d2c 100644 --- a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py +++ b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py @@ -1,3 +1,5 @@ +import json +import os from importlib import import_module from pathlib import Path from typing import Optional @@ -7,6 +9,7 @@ from pydantic import DirectoryPath, FilePath from referencing import Registry, Resource +from ..data_transfers import automatic_dandi_upload from ...nwbconverter import NWBConverter from ...utils import dict_deep_update, load_dict_from_file @@ -50,7 +53,7 @@ def run_conversion_from_yaml( data_folder_path: Optional[DirectoryPath] = None, output_folder_path: Optional[DirectoryPath] = None, overwrite: bool = False, -): +) -> None: """ Run conversion to NWB given a yaml specification file. @@ -100,6 +103,14 @@ def run_conversion_from_yaml( registry=registry, ) + upload_to_dandiset = "upload_to_dandiset" in specification + if upload_to_dandiset and "DANDI_API_KEY" not in os.environ: + message = ( + "The 'upload_to_dandiset' prompt was found in the YAML specification, " + "but the environment variable 'DANDI_API_KEY' was not set." + ) + raise ValueError(message) + global_metadata = specification.get("metadata", dict()) global_conversion_options = specification.get("conversion_options", dict()) data_interfaces_spec = specification.get("data_interfaces") @@ -115,6 +126,7 @@ def run_conversion_from_yaml( experiment_metadata = experiment.get("metadata", dict()) for session in experiment["sessions"]: file_counter += 1 + source_data = session["source_data"] for interface_name, interface_source_data in session["source_data"].items(): for key, value in interface_source_data.items(): @@ -122,21 +134,47 @@ def run_conversion_from_yaml( source_data[interface_name].update({key: [str(Path(data_folder_path) / x) for x in value]}) elif key in ("file_path", "folder_path"): source_data[interface_name].update({key: str(Path(data_folder_path) / value)}) + converter = CustomNWBConverter(source_data=source_data) + metadata = converter.get_metadata() for metadata_source in [global_metadata, experiment_metadata, session.get("metadata", dict())]: metadata = dict_deep_update(metadata, metadata_source) - nwbfile_name = session.get("nwbfile_name", f"temp_nwbfile_name_{file_counter}").strip(".nwb") + + session_id = session.get("metadata", dict()).get("NWBFile", dict()).get("session_id", None) + if upload_to_dandiset and session_id is None: + message = ( + "The 'upload_to_dandiset' prompt was found in the YAML specification, " + "but the 'session_id' was not found for session with info block: " + f"\n\n {json.dumps(obj=session, indent=2)}\n\n" + "File intended for DANDI upload must include a session ID." + ) + raise ValueError(message) + session_conversion_options = session.get("conversion_options", dict()) conversion_options = dict() for key in converter.data_interface_objects: conversion_options[key] = dict(session_conversion_options.get(key, dict()), **global_conversion_options) + + nwbfile_name = session.get("nwbfile_name", f"temp_nwbfile_name_{file_counter}").strip(".nwb") converter.run_conversion( nwbfile_path=output_folder_path / f"{nwbfile_name}.nwb", metadata=metadata, overwrite=overwrite, conversion_options=conversion_options, ) + + if upload_to_dandiset: + dandiset_id = specification["upload_to_dandiset"] + staging = int(dandiset_id) >= 200_000 + automatic_dandi_upload( + dandiset_id=dandiset_id, + nwb_folder_path=output_folder_path, + staging=staging, + ) + + return None # We can early return since organization below will occur within the upload step + # To properly mimic a true dandi organization, the full directory must be populated with NWBFiles. all_nwbfile_paths = [nwbfile_path for nwbfile_path in output_folder_path.iterdir() if nwbfile_path.suffix == ".nwb"] nwbfile_paths_to_set = [ diff --git a/tests/imports.py b/tests/imports.py index 5f8b65e72..7ac95713b 100644 --- a/tests/imports.py +++ b/tests/imports.py @@ -68,6 +68,7 @@ def test_tools(self): "get_package_version", "is_package_installed", "deploy_process", + "data_transfers", "LocalPathExpander", "get_module", ] diff --git a/tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_dandi_upload.yml b/tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_dandi_upload.yml new file mode 100644 index 000000000..adf590d3a --- /dev/null +++ b/tests/test_on_data/test_yaml/conversion_specifications/GIN_conversion_specification_dandi_upload.yml @@ -0,0 +1,66 @@ +metadata: + NWBFile: + lab: My Lab + institution: My Institution + +conversion_options: + stub_test: True + +data_interfaces: + ap: SpikeGLXRecordingInterface + lf: SpikeGLXRecordingInterface + phy: PhySortingInterface + +upload_to_dandiset: "200560" + +experiments: + ymaze: + metadata: + NWBFile: + session_description: Subject navigating a Y-shaped maze. + + sessions: + - nwbfile_name: example_converter_spec_1 + source_data: + ap: + file_path: spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin + metadata: + NWBFile: + session_start_time: "2020-10-09T21:19:09+00:00" + session_id: "test-yaml-1" + Subject: + subject_id: "yaml-1" + sex: F + age: P35D + species: Mus musculus + - nwbfile_name: example_converter_spec_2.nwb + metadata: + NWBFile: + session_start_time: "2020-10-10T21:19:09+00:00" + session_id: "test-yaml-2" + Subject: + subject_id: "yaml-002" + sex: F + age: P35D + species: Mus musculus + source_data: + lf: + file_path: spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.lf.bin + + open_explore: + sessions: + - nwbfile_name: example_converter_spec_3 + source_data: + lf: + file_path: spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.lf.bin + phy: + folder_path: phy/phy_example_0/ + metadata: + NWBFile: + session_start_time: "2020-10-11T21:19:09+00:00" + session_id: test YAML 3 + Subject: + subject_id: YAML Subject Name + sex: F + age: P35D + species: Mus musculus diff --git a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py index e46e25352..5a623d141 100644 --- a/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py +++ b/tests/test_on_data/test_yaml/test_yaml_conversion_specification.py @@ -19,6 +19,7 @@ "fname", [ "GIN_conversion_specification.yml", + "GIN_conversion_specification_dandi_upload.yml", "GIN_conversion_specification_missing_nwbfile_names.yml", "GIN_conversion_specification_no_nwbfile_name_or_other_metadata.yml", "GIN_conversion_specification_videos.yml", diff --git a/tests/test_on_data/test_yaml/yaml_dandi_transfer_tools.py b/tests/test_on_data/test_yaml/yaml_dandi_transfer_tools.py new file mode 100644 index 000000000..c36d072e7 --- /dev/null +++ b/tests/test_on_data/test_yaml/yaml_dandi_transfer_tools.py @@ -0,0 +1,53 @@ +import os +import platform +import time +from datetime import datetime, timedelta +from pathlib import Path + +import dandi.dandiapi +import pytest +from packaging.version import Version + +from neuroconv import run_conversion_from_yaml + +from ..setup_paths import ECEPHY_DATA_PATH, OUTPUT_PATH + +DANDI_API_KEY = os.getenv("DANDI_API_KEY") +HAVE_DANDI_KEY = DANDI_API_KEY is not None and DANDI_API_KEY != "" # can be "" from external forks +_PYTHON_VERSION = platform.python_version() + + +@pytest.mark.skipif( + not HAVE_DANDI_KEY or Version(".".join(_PYTHON_VERSION.split(".")[:2])) != Version("3.12"), + reason="You must set your DANDI_API_KEY to run this test!", +) +def test_run_conversion_from_yaml_with_dandi_upload(): + path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" + yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification_dandi_upload.yml" + run_conversion_from_yaml( + specification_file_path=yaml_file_path, + data_folder_path=ECEPHY_DATA_PATH, + output_folder_path=OUTPUT_PATH, + overwrite=True, + ) + + time.sleep(60) # Give some buffer room for server to process before making assertions against DANDI API + + client = dandi.dandiapi.DandiAPIClient(api_url="https://api-staging.dandiarchive.org/api") + dandiset = client.get_dandiset("200560") + + expected_asset_paths = [ + "sub-yaml-1/sub-yaml-1_ses-test-yaml-1_ecephys.nwb", + "sub-yaml-002/sub-yaml-002_ses-test-yaml-2_ecephys.nwb", + "sub-YAML-Subject-Name/sub-YAML-Subject-Name_ses-test-YAML-3_ecephys.nwb", + ] + for asset_path in expected_asset_paths: + test_asset = dandiset.get_asset_by_path(path=asset_path) # Will error if not found + test_asset_metadata = test_asset.get_raw_metadata() + + # Past uploads may have created the same apparent file, so look at the modification time to ensure + # this test is actually testing the most recent upload + date_modified = datetime.fromisoformat( + test_asset_metadata["dateModified"].split("Z")[0] # Timezones look a little messy + ) + assert datetime.now() - date_modified < timedelta(minutes=10) From 737468c63f17f206488cbba57484e6c9df200ff5 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 10 Dec 2024 09:38:59 -0600 Subject: [PATCH 095/118] Support arbitrary multi probe and multi trigger structures in spikeglx (#1150) --- CHANGELOG.md | 2 + .../ecephys/spikeglx/spikeglxconverter.py | 31 ++++------ .../ecephys/spikeglx/spikeglxdatainterface.py | 56 +++++++++++++------ .../ecephys/spikeglx/spikeglxnidqinterface.py | 27 ++++++--- .../spikeglx_single_probe_metadata.json | 12 ++-- tests/test_on_data/ecephys/test_lfp.py | 4 +- .../ecephys/test_recording_interfaces.py | 4 +- .../ecephys/test_spikeglx_converter.py | 31 +++++----- 8 files changed, 95 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae105b907..65d3515f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,13 @@ ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) * Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) +* `SpikeGLXConverterPipe` converter now accepts multi-probe structures with multi-trigger and does not assume a specific folder structure [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) * Added .csv support to DeepLabCutInterface [PR #1140](https://github.com/catalystneuro/neuroconv/pull/1140) +* `SpikeGLXRecordingInterface` now also accepts `folder_path` making its behavior equivalent to SpikeInterface [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) * Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) * Added the `deploy_neuroconv_batch_job` helper function for deploying NeuroConv AWS Batch jobs. [PR #1086](https://github.com/catalystneuro/neuroconv/pull/1086) * YAML specification files now accept an outer keyword `upload_to_dandiset="< six-digit ID >"` to automatically upload the produced NWB files to the DANDI archive [PR #1089](https://github.com/catalystneuro/neuroconv/pull/1089) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py index 007c3177c..029955d24 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py @@ -29,8 +29,10 @@ def get_source_schema(cls): @classmethod def get_streams(cls, folder_path: DirectoryPath) -> list[str]: + "Return the stream ids available in the folder." from spikeinterface.extractors import SpikeGLXRecordingExtractor + # The first entry is the stream ids the second is the stream names return SpikeGLXRecordingExtractor.get_streams(folder_path=folder_path)[0] @validate_call @@ -61,28 +63,17 @@ def __init__( """ folder_path = Path(folder_path) - streams = streams or self.get_streams(folder_path=folder_path) + streams_ids = streams or self.get_streams(folder_path=folder_path) data_interfaces = dict() - for stream in streams: - if "ap" in stream: - probe_name = stream[:5] - file_path = ( - folder_path / f"{folder_path.stem}_{probe_name}" / f"{folder_path.stem}_t0.{probe_name}.ap.bin" - ) - es_key = f"ElectricalSeriesAP{probe_name.capitalize()}" - interface = SpikeGLXRecordingInterface(file_path=file_path, es_key=es_key) - elif "lf" in stream: - probe_name = stream[:5] - file_path = ( - folder_path / f"{folder_path.stem}_{probe_name}" / f"{folder_path.stem}_t0.{probe_name}.lf.bin" - ) - es_key = f"ElectricalSeriesLF{probe_name.capitalize()}" - interface = SpikeGLXRecordingInterface(file_path=file_path, es_key=es_key) - elif "nidq" in stream: - file_path = folder_path / f"{folder_path.stem}_t0.nidq.bin" - interface = SpikeGLXNIDQInterface(file_path=file_path) - data_interfaces.update({str(stream): interface}) # Without str() casting, is a numpy string + + nidq_streams = [stream_id for stream_id in streams_ids if stream_id == "nidq"] + electrical_streams = [stream_id for stream_id in streams_ids if stream_id not in nidq_streams] + for stream_id in electrical_streams: + data_interfaces[stream_id] = SpikeGLXRecordingInterface(folder_path=folder_path, stream_id=stream_id) + + for stream_id in nidq_streams: + data_interfaces[stream_id] = SpikeGLXNIDQInterface(folder_path=folder_path) super().__init__(data_interfaces=data_interfaces, verbose=verbose) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index c15516431..00419e036 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -4,7 +4,7 @@ from typing import Optional import numpy as np -from pydantic import FilePath, validate_call +from pydantic import DirectoryPath, FilePath, validate_call from .spikeglx_utils import ( add_recording_extractor_properties, @@ -45,7 +45,6 @@ def get_source_schema(cls) -> dict: def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: extractor_kwargs = source_data.copy() - extractor_kwargs.pop("file_path") extractor_kwargs["folder_path"] = self.folder_path extractor_kwargs["all_annotations"] = True extractor_kwargs["stream_id"] = self.stream_id @@ -54,38 +53,59 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: @validate_call def __init__( self, - file_path: FilePath, + file_path: Optional[FilePath] = None, verbose: bool = True, es_key: Optional[str] = None, + folder_path: Optional[DirectoryPath] = None, + stream_id: Optional[str] = None, ): """ Parameters ---------- + folder_path: DirectoryPath + Folder path containing the binary files of the SpikeGLX recording. + stream_id: str, optional + Stream ID of the SpikeGLX recording. + Examples are 'nidq', 'imec0.ap', 'imec0.lf', 'imec1.ap', 'imec1.lf', etc. file_path : FilePathType Path to .bin file. Point to .ap.bin for SpikeGLXRecordingInterface and .lf.bin for SpikeGLXLFPInterface. verbose : bool, default: True Whether to output verbose text. - es_key : str, default: "ElectricalSeries" + es_key : str, the key to access the metadata of the ElectricalSeries. """ - self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) - if es_key is None: - if "lf" in self.stream_id: - es_key = "ElectricalSeriesLF" - elif "ap" in self.stream_id: - es_key = "ElectricalSeriesAP" - else: - raise ValueError("Cannot automatically determine es_key from path") - file_path = Path(file_path) - self.folder_path = file_path.parent + if file_path is not None and stream_id is None: + self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) + self.folder_path = Path(file_path).parent + + else: + self.stream_id = stream_id + self.folder_path = Path(folder_path) super().__init__( - file_path=file_path, + folder_path=folder_path, verbose=verbose, es_key=es_key, ) - self.source_data["file_path"] = str(file_path) - self.meta = self.recording_extractor.neo_reader.signals_info_dict[(0, self.stream_id)]["meta"] + + signal_info_key = (0, self.stream_id) # Key format is (segment_index, stream_id) + self._signals_info_dict = self.recording_extractor.neo_reader.signals_info_dict[signal_info_key] + self.meta = self._signals_info_dict["meta"] + + if es_key is None: + stream_kind = self._signals_info_dict["stream_kind"] # ap or lf + stream_kind_caps = stream_kind.upper() + device = self._signals_info_dict["device"].capitalize() # imec0, imec1, etc. + + electrical_series_name = f"ElectricalSeries{stream_kind_caps}" + + # Add imec{probe_index} to the electrical series name when there are multiple probes + # or undefined, `typeImEnabled` is present in the meta of all the production probes + self.probes_enabled_in_run = int(self.meta.get("typeImEnabled", 0)) + if self.probes_enabled_in_run != 1: + electrical_series_name += f"{device}" + + self.es_key = electrical_series_name # Set electrodes properties add_recording_extractor_properties(self.recording_extractor) @@ -100,7 +120,7 @@ def get_metadata(self) -> dict: device = get_device_metadata(self.meta) # Should follow pattern 'Imec0', 'Imec1', etc. - probe_name = self.stream_id[:5].capitalize() + probe_name = self._signals_info_dict["device"].capitalize() device["name"] = f"Neuropixel{probe_name}" # Add groups metadata diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 3cf50080a..1d7079716 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -1,7 +1,8 @@ from pathlib import Path +from typing import Optional import numpy as np -from pydantic import ConfigDict, FilePath, validate_call +from pydantic import ConfigDict, DirectoryPath, FilePath, validate_call from .spikeglx_utils import get_session_start_time from ..baserecordingextractorinterface import BaseRecordingExtractorInterface @@ -29,7 +30,6 @@ def get_source_schema(cls) -> dict: def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: extractor_kwargs = source_data.copy() - extractor_kwargs.pop("file_path") extractor_kwargs["folder_path"] = self.folder_path extractor_kwargs["stream_id"] = self.stream_id return extractor_kwargs @@ -37,10 +37,11 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, - file_path: FilePath, + file_path: Optional[FilePath] = None, verbose: bool = True, load_sync_channel: bool = False, es_key: str = "ElectricalSeriesNIDQ", + folder_path: Optional[DirectoryPath] = None, ): """ Read channel data from the NIDQ board for the SpikeGLX recording. @@ -49,6 +50,8 @@ def __init__( Parameters ---------- + folder_path : DirectoryPath + Path to the folder containing the .nidq.bin file. file_path : FilePathType Path to .nidq.bin file. verbose : bool, default: True @@ -59,10 +62,17 @@ def __init__( es_key : str, default: "ElectricalSeriesNIDQ" """ - self.file_path = Path(file_path) - self.folder_path = self.file_path.parent + if file_path is None and folder_path is None: + raise ValueError("Either 'file_path' or 'folder_path' must be provided.") + + if file_path is not None: + file_path = Path(file_path) + self.folder_path = file_path.parent + + if folder_path is not None: + self.folder_path = Path(folder_path) + super().__init__( - file_path=self.file_path, verbose=verbose, load_sync_channel=load_sync_channel, es_key=es_key, @@ -72,7 +82,10 @@ def __init__( self.recording_extractor.set_property( key="group_name", values=["NIDQChannelGroup"] * self.recording_extractor.get_num_channels() ) - self.meta = self.recording_extractor.neo_reader.signals_info_dict[(0, "nidq")]["meta"] + + signal_info_key = (0, self.stream_id) # Key format is (segment_index, stream_id) + self._signals_info_dict = self.recording_extractor.neo_reader.signals_info_dict[signal_info_key] + self.meta = self._signals_info_dict["meta"] def get_metadata(self) -> dict: metadata = super().get_metadata() diff --git a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json index 637124cd0..20f11742b 100644 --- a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json +++ b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json @@ -29,9 +29,9 @@ "device": "NIDQBoard" } ], - "ElectricalSeriesAPImec0": { - "name": "ElectricalSeriesAPImec0", - "description": "Acquisition traces for the ElectricalSeriesAPImec0." + "ElectricalSeriesAP": { + "name": "ElectricalSeriesAP", + "description": "Acquisition traces for the ElectricalSeriesAP." }, "Electrodes": [ { @@ -51,9 +51,9 @@ "name": "ElectricalSeriesNIDQ", "description": "Raw acquisition traces from the NIDQ (.nidq.bin) channels." }, - "ElectricalSeriesLFImec0": { - "name": "ElectricalSeriesLFImec0", - "description": "Acquisition traces for the ElectricalSeriesLFImec0." + "ElectricalSeriesLF": { + "name": "ElectricalSeriesLF", + "description": "Acquisition traces for the ElectricalSeriesLF." } } } diff --git a/tests/test_on_data/ecephys/test_lfp.py b/tests/test_on_data/ecephys/test_lfp.py index c46f8d297..010516d86 100644 --- a/tests/test_on_data/ecephys/test_lfp.py +++ b/tests/test_on_data/ecephys/test_lfp.py @@ -57,9 +57,7 @@ class TestEcephysLFPNwbConversions(unittest.TestCase): param( data_interface=SpikeGLXRecordingInterface, interface_kwargs=dict( - file_path=( - DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.lf.bin" - ) + folder_path=DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0", stream_id="imec0.lf" ), expected_write_module="raw", ), diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 7677ded22..0520b5b42 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -641,9 +641,7 @@ def test_extracted_metadata(self, setup_interface): class TestSpikeGLXRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = SpikeGLXRecordingInterface interface_kwargs = dict( - file_path=str( - ECEPHY_DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" / "Noise4Sam_g0_t0.imec0.ap.bin" - ) + folder_path=ECEPHY_DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_imec0", stream_id="imec0.ap" ) save_directory = OUTPUT_PATH diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index af98789c1..970a815af 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -33,10 +33,10 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() - assert nwbfile.session_start_time == expected_session_start_time + assert nwbfile.session_start_time.replace(tzinfo=None) == expected_session_start_time - assert "ElectricalSeriesAPImec0" in nwbfile.acquisition - assert "ElectricalSeriesLFImec0" in nwbfile.acquisition + assert "ElectricalSeriesAP" in nwbfile.acquisition + assert "ElectricalSeriesLF" in nwbfile.acquisition assert "ElectricalSeriesNIDQ" in nwbfile.acquisition assert len(nwbfile.acquisition) == 3 @@ -74,7 +74,7 @@ def test_single_probe_spikeglx_converter(self): nwbfile_path = self.tmpdir / "test_single_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) - expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10).astimezone() + expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) def test_in_converter_pipe(self): @@ -84,7 +84,7 @@ def test_in_converter_pipe(self): nwbfile_path = self.tmpdir / "test_spikeglx_converter_in_converter_pipe.nwb" converter_pipe.run_conversion(nwbfile_path=nwbfile_path) - expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10).astimezone() + expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) def test_in_nwbconverter(self): @@ -101,7 +101,7 @@ class TestConverter(NWBConverter): nwbfile_path = self.tmpdir / "test_spikeglx_converter_in_nwbconverter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path) - expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10).astimezone() + expected_session_start_time = datetime(2020, 11, 3, 10, 35, 10) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) @@ -118,7 +118,9 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ with NWBHDF5IO(path=nwbfile_path) as io: nwbfile = io.read() - assert nwbfile.session_start_time == expected_session_start_time + # Do the comparison without timezone information to avoid CI timezone issues + # The timezone is set by pynbw automatically + assert nwbfile.session_start_time.replace(tzinfo=None) == expected_session_start_time # TODO: improve name of segments using 'Segment{index}' for clarity assert "ElectricalSeriesAPImec00" in nwbfile.acquisition @@ -129,7 +131,7 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ assert "ElectricalSeriesLFImec01" in nwbfile.acquisition assert "ElectricalSeriesLFImec10" in nwbfile.acquisition assert "ElectricalSeriesLFImec11" in nwbfile.acquisition - assert len(nwbfile.acquisition) == 8 + assert len(nwbfile.acquisition) == 16 assert "NeuropixelImec0" in nwbfile.devices assert "NeuropixelImec1" in nwbfile.devices @@ -141,7 +143,7 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ def test_multi_probe_spikeglx_converter(self): converter = SpikeGLXConverterPipe( - folder_path=SPIKEGLX_PATH / "multi_trigger_multi_gate" / "SpikeGLX" / "5-19-2022-CI0" / "5-19-2022-CI0_g0" + folder_path=SPIKEGLX_PATH / "multi_trigger_multi_gate" / "SpikeGLX" / "5-19-2022-CI0" ) metadata = converter.get_metadata() @@ -161,13 +163,12 @@ def test_multi_probe_spikeglx_converter(self): expected_device_metadata = expected_ecephys_metadata.pop("Device") assert device_metadata == expected_device_metadata - assert test_ecephys_metadata == expected_ecephys_metadata nwbfile_path = self.tmpdir / "test_multi_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) - expected_session_start_time = datetime(2022, 5, 19, 17, 37, 47).astimezone() + expected_session_start_time = datetime(2022, 5, 19, 17, 37, 47) self.assertNWBFileStructure(nwbfile_path=nwbfile_path, expected_session_start_time=expected_session_start_time) @@ -193,7 +194,7 @@ def test_electrode_table_writing(tmp_path): np.testing.assert_array_equal(saved_channel_names, expected_channel_names_nidq) # Test AP - electrical_series = nwbfile.acquisition["ElectricalSeriesAPImec0"] + electrical_series = nwbfile.acquisition["ElectricalSeriesAP"] ap_electrodes_table_region = electrical_series.electrodes region_indices = ap_electrodes_table_region.data recording_extractor = converter.data_interface_objects["imec0.ap"].recording_extractor @@ -203,7 +204,7 @@ def test_electrode_table_writing(tmp_path): np.testing.assert_array_equal(saved_channel_names, expected_channel_names_ap) # Test LF - electrical_series = nwbfile.acquisition["ElectricalSeriesLFImec0"] + electrical_series = nwbfile.acquisition["ElectricalSeriesLF"] lf_electrodes_table_region = electrical_series.electrodes region_indices = lf_electrodes_table_region.data recording_extractor = converter.data_interface_objects["imec0.lf"].recording_extractor @@ -222,7 +223,7 @@ def test_electrode_table_writing(tmp_path): # Test round trip with spikeinterface recording_extractor_ap = NwbRecordingExtractor( file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesAPImec0", + electrical_series_name="ElectricalSeriesAP", ) channel_ids = recording_extractor_ap.get_channel_ids() @@ -230,7 +231,7 @@ def test_electrode_table_writing(tmp_path): recording_extractor_lf = NwbRecordingExtractor( file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesLFImec0", + electrical_series_name="ElectricalSeriesLF", ) channel_ids = recording_extractor_lf.get_channel_ids() From 1032e151d9515413d64f48c315c03cc3a84ffac8 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 10 Dec 2024 12:21:02 -0600 Subject: [PATCH 096/118] restore public status of `NWBMetaDataEncoder` encoder (#1142) --- CHANGELOG.md | 1 + docs/api/utils.rst | 2 + src/neuroconv/basedatainterface.py | 3 +- .../neuralynx/neuralynx_nvt_interface.py | 3 +- .../nwb_helpers/_metadata_and_file_helpers.py | 5 +-- .../tools/testing/data_interface_mixins.py | 2 +- src/neuroconv/utils/__init__.py | 2 +- src/neuroconv/utils/json_schema.py | 44 +++++++------------ .../test_tools/test_expand_paths.py | 2 +- .../test_utils/test_json_schema_utils.py | 2 +- 10 files changed, 28 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65d3515f1..728123253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) +* Make `NWBMetaDataEncoder` public again [PR #1142](https://github.com/catalystneuro/neuroconv/pull/1142) * Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) * `SpikeGLXConverterPipe` converter now accepts multi-probe structures with multi-trigger and does not assume a specific folder structure [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) diff --git a/docs/api/utils.rst b/docs/api/utils.rst index 4f19f7cee..c9b85b14c 100644 --- a/docs/api/utils.rst +++ b/docs/api/utils.rst @@ -8,6 +8,8 @@ Dictionaries JSON Schema ----------- .. automodule:: neuroconv.utils.json_schema + :members: + :exclude-members: NWBMetaDataEncoder Common Reused Types ------------------- diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 272abbd0c..530eec60c 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -19,12 +19,11 @@ ) from .tools.nwb_helpers._metadata_and_file_helpers import _resolve_backend from .utils import ( - _NWBMetaDataEncoder, get_json_schema_from_method_signature, load_dict_from_file, ) from .utils.dict import DeepDict -from .utils.json_schema import _NWBSourceDataEncoder +from .utils.json_schema import _NWBMetaDataEncoder, _NWBSourceDataEncoder class BaseDataInterface(ABC): diff --git a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py index e161387f0..213adf731 100644 --- a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py +++ b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py @@ -8,7 +8,8 @@ from .nvt_utils import read_data, read_header from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface -from ....utils import DeepDict, _NWBMetaDataEncoder, get_base_schema +from ....utils import DeepDict, get_base_schema +from ....utils.json_schema import _NWBMetaDataEncoder from ....utils.path import infer_path diff --git a/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py b/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py index c3aaea48d..355c86510 100644 --- a/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py +++ b/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py @@ -8,7 +8,6 @@ from datetime import datetime from pathlib import Path from typing import Literal, Optional -from warnings import warn from hdmf_zarr import NWBZarrIO from pydantic import FilePath @@ -26,7 +25,7 @@ def get_module(nwbfile: NWBFile, name: str, description: str = None): """Check if processing module exists. If not, create it. Then return module.""" if name in nwbfile.processing: if description is not None and nwbfile.processing[name].description != description: - warn( + warnings.warn( "Custom description given to get_module does not match existing module description! " "Ignoring custom description." ) @@ -157,7 +156,7 @@ def _attempt_cleanup_of_existing_nwbfile(nwbfile_path: Path) -> None: # Windows in particular can encounter errors at this step except PermissionError: # pragma: no cover message = f"Unable to remove NWB file located at {nwbfile_path.absolute()}! Please remove it manually." - warn(message=message, stacklevel=2) + warnings.warn(message=message, stacklevel=2) @contextmanager diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index fab049165..dc45cec53 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -33,7 +33,7 @@ configure_backend, get_default_backend_configuration, ) -from neuroconv.utils import _NWBMetaDataEncoder +from neuroconv.utils.json_schema import _NWBMetaDataEncoder class DataInterfaceTestMixin: diff --git a/src/neuroconv/utils/__init__.py b/src/neuroconv/utils/__init__.py index c0061a983..f7163f3ff 100644 --- a/src/neuroconv/utils/__init__.py +++ b/src/neuroconv/utils/__init__.py @@ -7,7 +7,7 @@ load_dict_from_file, ) from .json_schema import ( - _NWBMetaDataEncoder, + NWBMetaDataEncoder, fill_defaults, get_base_schema, get_metadata_schema_for_icephys, diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index 07dc3321f..6aa7a75d0 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -16,13 +16,8 @@ from pynwb.icephys import IntracellularElectrode -class _NWBMetaDataEncoder(json.JSONEncoder): - """ - Custom JSON encoder for NWB metadata. - - This encoder extends the default JSONEncoder class and provides custom serialization - for certain data types commonly used in NWB metadata. - """ +class _GenericNeuroconvEncoder(json.JSONEncoder): + """Generic JSON encoder for NeuroConv data.""" def default(self, obj): """ @@ -36,45 +31,38 @@ def default(self, obj): if isinstance(obj, np.generic): return obj.item() + # Numpy arrays should be converted to lists if isinstance(obj, np.ndarray): return obj.tolist() + # Over-write behaviors for Paths + if isinstance(obj, Path): + return str(obj) + # The base-class handles it return super().default(obj) -class _NWBSourceDataEncoder(_NWBMetaDataEncoder): +class _NWBMetaDataEncoder(_GenericNeuroconvEncoder): """ - Custom JSON encoder for data interface source data (i.e. kwargs). - - This encoder extends the default JSONEncoder class and provides custom serialization - for certain data types commonly used in interface source data. + Custom JSON encoder for NWB metadata. """ - def default(self, obj): - # Over-write behaviors for Paths - if isinstance(obj, Path): - return str(obj) - - return super().default(obj) +class _NWBSourceDataEncoder(_GenericNeuroconvEncoder): + """ + Custom JSON encoder for data interface source data (i.e. kwargs). + """ -class _NWBConversionOptionsEncoder(_NWBMetaDataEncoder): +class _NWBConversionOptionsEncoder(_GenericNeuroconvEncoder): """ Custom JSON encoder for conversion options of the data interfaces and converters (i.e. kwargs). - - This encoder extends the default JSONEncoder class and provides custom serialization - for certain data types commonly used in interface source data. """ - def default(self, obj): - - # Over-write behaviors for Paths - if isinstance(obj, Path): - return str(obj) - return super().default(obj) +# This is used in the Guide so we will keep it public. +NWBMetaDataEncoder = _NWBMetaDataEncoder def get_base_schema( diff --git a/tests/test_minimal/test_tools/test_expand_paths.py b/tests/test_minimal/test_tools/test_expand_paths.py index 9e7f03631..59924f93a 100644 --- a/tests/test_minimal/test_tools/test_expand_paths.py +++ b/tests/test_minimal/test_tools/test_expand_paths.py @@ -9,7 +9,7 @@ from neuroconv.tools import LocalPathExpander from neuroconv.tools.path_expansion import construct_path_template from neuroconv.tools.testing import generate_path_expander_demo_ibl -from neuroconv.utils import _NWBMetaDataEncoder +from neuroconv.utils.json_schema import _NWBMetaDataEncoder def create_test_directories_and_files( diff --git a/tests/test_minimal/test_utils/test_json_schema_utils.py b/tests/test_minimal/test_utils/test_json_schema_utils.py index 4edf1e724..5ce63ee56 100644 --- a/tests/test_minimal/test_utils/test_json_schema_utils.py +++ b/tests/test_minimal/test_utils/test_json_schema_utils.py @@ -6,12 +6,12 @@ from pynwb.ophys import ImagingPlane, TwoPhotonSeries from neuroconv.utils import ( - _NWBMetaDataEncoder, dict_deep_update, fill_defaults, get_schema_from_hdmf_class, load_dict_from_file, ) +from neuroconv.utils.json_schema import _NWBMetaDataEncoder def compare_dicts(a: dict, b: dict): From 535626302a6c85945beb07288c48b76887947517 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 11 Dec 2024 12:44:21 -0600 Subject: [PATCH 097/118] Support digital channels on NIDQ interface and use TimeSeries instead of ElecricalSeries for analog channels (#1152) Co-authored-by: Ben Dichter --- CHANGELOG.md | 7 +- .../recording/spikeglx.rst | 2 +- .../ecephys/spikeglx/spikeglxdatainterface.py | 8 +- .../ecephys/spikeglx/spikeglxnidqinterface.py | 271 +++++++++++++++--- .../tools/testing/mock_interfaces.py | 4 + .../test_ecephys/test_mock_nidq_interface.py | 52 +--- .../spikeglx_single_probe_metadata.json | 16 +- .../ecephys/test_aux_interfaces.py | 100 ------- .../ecephys/test_nidq_interface.py | 57 ++++ .../ecephys/test_spikeglx_converter.py | 27 +- .../test_temporal_alignment_methods.py | 1 - 11 files changed, 327 insertions(+), 218 deletions(-) delete mode 100644 tests/test_on_data/ecephys/test_aux_interfaces.py create mode 100644 tests/test_on_data/ecephys/test_nidq_interface.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 728123253..c9c9afcde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ * Make `NWBMetaDataEncoder` public again [PR #1142](https://github.com/catalystneuro/neuroconv/pull/1142) * Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) * `SpikeGLXConverterPipe` converter now accepts multi-probe structures with multi-trigger and does not assume a specific folder structure [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) +* `SpikeGLXNIDQInterface` is no longer written as an ElectricalSeries [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) + ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) @@ -17,7 +19,10 @@ * `SpikeGLXRecordingInterface` now also accepts `folder_path` making its behavior equivalent to SpikeInterface [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) * Added the `rclone_transfer_batch_job` helper function for executing Rclone data transfers in AWS Batch jobs. [PR #1085](https://github.com/catalystneuro/neuroconv/pull/1085) * Added the `deploy_neuroconv_batch_job` helper function for deploying NeuroConv AWS Batch jobs. [PR #1086](https://github.com/catalystneuro/neuroconv/pull/1086) -* YAML specification files now accept an outer keyword `upload_to_dandiset="< six-digit ID >"` to automatically upload the produced NWB files to the DANDI archive [PR #1089](https://github.com/catalystneuro/neuroconv/pull/1089) +* YAML specification files now accepts an outer keyword `upload_to_dandiset="< six-digit ID >"` to automatically upload the produced NWB files to the DANDI archive [PR #1089](https://github.com/catalystneuro/neuroconv/pull/1089) +*`SpikeGLXNIDQInterface` now handdles digital demuxed channels (`XD0`) [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) + + ## Improvements diff --git a/docs/conversion_examples_gallery/recording/spikeglx.rst b/docs/conversion_examples_gallery/recording/spikeglx.rst index 7f57470af..97b23bac9 100644 --- a/docs/conversion_examples_gallery/recording/spikeglx.rst +++ b/docs/conversion_examples_gallery/recording/spikeglx.rst @@ -24,7 +24,7 @@ We can easily convert all data stored in the native SpikeGLX folder structure to >>> >>> folder_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0" >>> converter = SpikeGLXConverterPipe(folder_path=folder_path) - >>> + Source data is valid! >>> # Extract what metadata we can from the source files >>> metadata = converter.get_metadata() >>> # For data provenance we add the time zone information to the conversion diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 00419e036..e8b6a78c9 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -66,7 +66,7 @@ def __init__( Folder path containing the binary files of the SpikeGLX recording. stream_id: str, optional Stream ID of the SpikeGLX recording. - Examples are 'nidq', 'imec0.ap', 'imec0.lf', 'imec1.ap', 'imec1.lf', etc. + Examples are 'imec0.ap', 'imec0.lf', 'imec1.ap', 'imec1.lf', etc. file_path : FilePathType Path to .bin file. Point to .ap.bin for SpikeGLXRecordingInterface and .lf.bin for SpikeGLXLFPInterface. verbose : bool, default: True @@ -74,10 +74,14 @@ def __init__( es_key : str, the key to access the metadata of the ElectricalSeries. """ + if stream_id == "nidq": + raise ValueError( + "SpikeGLXRecordingInterface is not designed to handle nidq files. Use SpikeGLXNIDQInterface instead" + ) + if file_path is not None and stream_id is None: self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) self.folder_path = Path(file_path).parent - else: self.stream_id = stream_id self.folder_path = Path(folder_path) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 1d7079716..5249dfe39 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -1,39 +1,36 @@ +import warnings from pathlib import Path -from typing import Optional +from typing import Literal, Optional import numpy as np from pydantic import ConfigDict, DirectoryPath, FilePath, validate_call +from pynwb import NWBFile +from pynwb.base import TimeSeries from .spikeglx_utils import get_session_start_time -from ..baserecordingextractorinterface import BaseRecordingExtractorInterface +from ....basedatainterface import BaseDataInterface from ....tools.signal_processing import get_rising_frames_from_ttl -from ....utils import get_json_schema_from_method_signature +from ....tools.spikeinterface.spikeinterface import _recording_traces_to_hdmf_iterator +from ....utils import ( + calculate_regular_series_rate, + get_json_schema_from_method_signature, +) -class SpikeGLXNIDQInterface(BaseRecordingExtractorInterface): +class SpikeGLXNIDQInterface(BaseDataInterface): """Primary data interface class for converting the high-pass (ap) SpikeGLX format.""" display_name = "NIDQ Recording" - keywords = BaseRecordingExtractorInterface.keywords + ("Neuropixels",) + keywords = ("Neuropixels", "nidq", "NIDQ", "SpikeGLX") associated_suffixes = (".nidq", ".meta", ".bin") info = "Interface for NIDQ board recording data." - ExtractorName = "SpikeGLXRecordingExtractor" - stream_id = "nidq" - @classmethod def get_source_schema(cls) -> dict: source_schema = get_json_schema_from_method_signature(method=cls.__init__, exclude=["x_pitch", "y_pitch"]) source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX .nidq file." return source_schema - def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: - - extractor_kwargs = source_data.copy() - extractor_kwargs["folder_path"] = self.folder_path - extractor_kwargs["stream_id"] = self.stream_id - return extractor_kwargs - @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def __init__( self, @@ -56,12 +53,18 @@ def __init__( Path to .nidq.bin file. verbose : bool, default: True Whether to output verbose text. - load_sync_channel : bool, default: False - Whether to load the last channel in the stream, which is typically used for synchronization. - If True, then the probe is not loaded. es_key : str, default: "ElectricalSeriesNIDQ" """ + if load_sync_channel: + + warnings.warn( + "The 'load_sync_channel' parameter is deprecated and will be removed in June 2025. " + "The sync channel data is only available the raw files of spikeglx`.", + DeprecationWarning, + stacklevel=2, + ) + if file_path is None and folder_path is None: raise ValueError("Either 'file_path' or 'folder_path' must be provided.") @@ -72,18 +75,36 @@ def __init__( if folder_path is not None: self.folder_path = Path(folder_path) + from spikeinterface.extractors import SpikeGLXRecordingExtractor + + self.recording_extractor = SpikeGLXRecordingExtractor( + folder_path=self.folder_path, + stream_id="nidq", + all_annotations=True, + ) + + channel_ids = self.recording_extractor.get_channel_ids() + analog_channel_signatures = ["XA", "MA"] + self.analog_channel_ids = [ch for ch in channel_ids if "XA" in ch or "MA" in ch] + self.has_analog_channels = len(self.analog_channel_ids) > 0 + self.has_digital_channels = len(self.analog_channel_ids) < len(channel_ids) + if self.has_digital_channels: + import ndx_events # noqa: F401 + from spikeinterface.extractors import SpikeGLXEventExtractor + + self.event_extractor = SpikeGLXEventExtractor(folder_path=self.folder_path) + super().__init__( verbose=verbose, load_sync_channel=load_sync_channel, es_key=es_key, + folder_path=self.folder_path, + file_path=file_path, ) - self.source_data.update(file_path=str(file_path)) - self.recording_extractor.set_property( - key="group_name", values=["NIDQChannelGroup"] * self.recording_extractor.get_num_channels() - ) + self.subset_channels = None - signal_info_key = (0, self.stream_id) # Key format is (segment_index, stream_id) + signal_info_key = (0, "nidq") # Key format is (segment_index, stream_id) self._signals_info_dict = self.recording_extractor.neo_reader.signals_info_dict[signal_info_key] self.meta = self._signals_info_dict["meta"] @@ -101,24 +122,206 @@ def get_metadata(self) -> dict: manufacturer="National Instruments", ) - # Add groups metadata - metadata["Ecephys"]["Device"] = [device] + metadata["Devices"] = [device] - metadata["Ecephys"]["ElectrodeGroup"][0].update( - name="NIDQChannelGroup", description="A group representing the NIDQ channels.", device=device["name"] - ) - metadata["Ecephys"]["Electrodes"] = [ - dict(name="group_name", description="Name of the ElectrodeGroup this electrode is a part of."), - ] - metadata["Ecephys"]["ElectricalSeriesNIDQ"][ - "description" - ] = "Raw acquisition traces from the NIDQ (.nidq.bin) channels." return metadata def get_channel_names(self) -> list[str]: """Return a list of channel names as set in the recording extractor.""" return list(self.recording_extractor.get_channel_ids()) + def add_to_nwbfile( + self, + nwbfile: NWBFile, + metadata: Optional[dict] = None, + stub_test: bool = False, + starting_time: Optional[float] = None, + write_as: Literal["raw", "lfp", "processed"] = "raw", + write_electrical_series: bool = True, + iterator_type: Optional[str] = "v2", + iterator_opts: Optional[dict] = None, + always_write_timestamps: bool = False, + ): + """ + Add NIDQ board data to an NWB file, including both analog and digital channels if present. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file to which the NIDQ data will be added + metadata : Optional[dict], default: None + Metadata dictionary with device information. If None, uses default metadata + stub_test : bool, default: False + If True, only writes a small amount of data for testing + starting_time : Optional[float], default: None + DEPRECATED: Will be removed in June 2025. Starting time offset for the TimeSeries + write_as : Literal["raw", "lfp", "processed"], default: "raw" + DEPRECATED: Will be removed in June 2025. Specifies how to write the data + write_electrical_series : bool, default: True + DEPRECATED: Will be removed in June 2025. Whether to write electrical series data + iterator_type : Optional[str], default: "v2" + Type of iterator to use for data streaming + iterator_opts : Optional[dict], default: None + Additional options for the iterator + always_write_timestamps : bool, default: False + If True, always writes timestamps instead of using sampling rate + """ + + if starting_time is not None: + warnings.warn( + "The 'starting_time' parameter is deprecated and will be removed in June 2025. " + "Use the time alignment methods for modifying the starting time or timestamps " + "of the data if needed: " + "https://neuroconv.readthedocs.io/en/main/user_guide/temporal_alignment.html", + DeprecationWarning, + stacklevel=2, + ) + + if write_as != "raw": + warnings.warn( + "The 'write_as' parameter is deprecated and will be removed in June 2025. " + "NIDQ should always be written in the acquisition module of NWB. " + "Writing data as LFP or processed data is not supported.", + DeprecationWarning, + stacklevel=2, + ) + + if write_electrical_series is not True: + warnings.warn( + "The 'write_electrical_series' parameter is deprecated and will be removed in June 2025. " + "The option to skip the addition of the data is no longer supported. " + "This option was used in ElectricalSeries to write the electrode and electrode group " + "metadata without the raw data.", + DeprecationWarning, + stacklevel=2, + ) + + if stub_test or self.subset_channels is not None: + recording = self.subset_recording(stub_test=stub_test) + else: + recording = self.recording_extractor + + if metadata is None: + metadata = self.get_metadata() + + # Add devices + device_metadata = metadata.get("Devices", []) + for device in device_metadata: + if device["name"] not in nwbfile.devices: + nwbfile.create_device(**device) + + # Add analog and digital channels + if self.has_analog_channels: + self._add_analog_channels( + nwbfile=nwbfile, + recording=recording, + iterator_type=iterator_type, + iterator_opts=iterator_opts, + always_write_timestamps=always_write_timestamps, + ) + + if self.has_digital_channels: + self._add_digital_channels(nwbfile=nwbfile) + + def _add_analog_channels( + self, + nwbfile: NWBFile, + recording, + iterator_type: Optional[str], + iterator_opts: Optional[dict], + always_write_timestamps: bool, + ): + """ + Add analog channels from the NIDQ board to the NWB file. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file to add the analog channels to + recording : BaseRecording + The recording extractor containing the analog channels + iterator_type : Optional[str] + Type of iterator to use for data streaming + iterator_opts : Optional[dict] + Additional options for the iterator + always_write_timestamps : bool + If True, always writes timestamps instead of using sampling rate + """ + analog_recorder = recording.select_channels(channel_ids=self.analog_channel_ids) + channel_names = analog_recorder.get_property(key="channel_names") + segment_index = 0 + analog_data_iterator = _recording_traces_to_hdmf_iterator( + recording=analog_recorder, + segment_index=segment_index, + iterator_type=iterator_type, + iterator_opts=iterator_opts, + ) + + name = "TimeSeriesNIDQ" + description = f"Analog data from the NIDQ board. Channels are {channel_names} in that order." + time_series_kwargs = dict(name=name, data=analog_data_iterator, unit="a.u.", description=description) + + if always_write_timestamps: + timestamps = recording.get_times(segment_index=segment_index) + shifted_timestamps = timestamps + time_series_kwargs.update(timestamps=shifted_timestamps) + else: + recording_has_timestamps = recording.has_time_vector(segment_index=segment_index) + if recording_has_timestamps: + timestamps = recording.get_times(segment_index=segment_index) + rate = calculate_regular_series_rate(series=timestamps) + recording_t_start = timestamps[0] + else: + rate = recording.get_sampling_frequency() + recording_t_start = recording._recording_segments[segment_index].t_start or 0 + + if rate: + starting_time = float(recording_t_start) + time_series_kwargs.update(starting_time=starting_time, rate=recording.get_sampling_frequency()) + else: + shifted_timestamps = timestamps + time_series_kwargs.update(timestamps=shifted_timestamps) + + time_series = TimeSeries(**time_series_kwargs) + nwbfile.add_acquisition(time_series) + + def _add_digital_channels(self, nwbfile: NWBFile): + """ + Add digital channels from the NIDQ board to the NWB file as events. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file to add the digital channels to + """ + from ndx_events import LabeledEvents + + event_channels = self.event_extractor.channel_ids + for channel_id in event_channels: + events_structure = self.event_extractor.get_events(channel_id=channel_id) + timestamps = events_structure["time"] + labels = events_structure["label"] + + # Some channels have no events + if timestamps.size > 0: + + # Timestamps are not ordered, the ones for off are first and then the ones for on + ordered_indices = np.argsort(timestamps) + ordered_timestamps = timestamps[ordered_indices] + ordered_labels = labels[ordered_indices] + + unique_labels = np.unique(ordered_labels) + label_to_index = {label: index for index, label in enumerate(unique_labels)} + data = [label_to_index[label] for label in ordered_labels] + + channel_name = channel_id.split("#")[-1] + description = f"On and Off Events from channel {channel_name}" + name = f"EventsNIDQDigitalChannel{channel_name}" + labeled_events = LabeledEvents( + name=name, description=description, timestamps=ordered_timestamps, data=data, labels=unique_labels + ) + nwbfile.add_acquisition(labeled_events) + def get_event_times_from_ttl(self, channel_name: str) -> np.ndarray: """ Return the start of event times from the rising part of TTL pulses on one of the NIDQ channels. diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 0652284e7..5a96b4b68 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -119,6 +119,9 @@ def __init__( """ from spikeinterface.extractors import NumpyRecording + self.has_analog_channels = True + self.has_digital_channels = False + if ttl_times is None: # Begin in 'off' state number_of_periods = int(np.ceil((signal_duration - ttl_duration) / (ttl_duration * 2))) @@ -127,6 +130,7 @@ def __init__( number_of_channels = len(ttl_times) channel_ids = [f"nidq#XA{channel_index}" for channel_index in range(number_of_channels)] # NIDQ channel IDs channel_groups = ["NIDQChannelGroup"] * number_of_channels + self.analog_channel_ids = channel_ids sampling_frequency = 25_000.0 # NIDQ sampling rate number_of_frames = int(signal_duration * sampling_frequency) diff --git a/tests/test_ecephys/test_mock_nidq_interface.py b/tests/test_ecephys/test_mock_nidq_interface.py index c0fb4eed2..1b098e1bb 100644 --- a/tests/test_ecephys/test_mock_nidq_interface.py +++ b/tests/test_ecephys/test_mock_nidq_interface.py @@ -1,4 +1,3 @@ -import pathlib from datetime import datetime from numpy.testing import assert_array_almost_equal @@ -46,47 +45,26 @@ def test_mock_metadata(): metadata = interface.get_metadata() - expected_ecephys_metadata = { - "Ecephys": { - "Device": [ - { - "name": "NIDQBoard", - "description": "A NIDQ board used in conjunction with SpikeGLX.", - "manufacturer": "National Instruments", - }, - ], - "ElectrodeGroup": [ - { - "name": "NIDQChannelGroup", - "description": "A group representing the NIDQ channels.", - "device": "NIDQBoard", - "location": "unknown", - }, - ], - "Electrodes": [ - {"name": "group_name", "description": "Name of the ElectrodeGroup this electrode is a part of."} - ], - "ElectricalSeriesNIDQ": { - "name": "ElectricalSeriesNIDQ", - "description": "Raw acquisition traces from the NIDQ (.nidq.bin) channels.", - }, - } - } - - assert metadata["Ecephys"] == expected_ecephys_metadata["Ecephys"] + expected_devices_metadata = [ + { + "name": "NIDQBoard", + "description": "A NIDQ board used in conjunction with SpikeGLX.", + "manufacturer": "National Instruments", + }, + ] + + assert metadata["Devices"] == expected_devices_metadata expected_start_time = datetime(2020, 11, 3, 10, 35, 10) assert metadata["NWBFile"]["session_start_time"] == expected_start_time -def test_mock_run_conversion(tmpdir: pathlib.Path): +def test_mock_run_conversion(tmp_path): interface = MockSpikeGLXNIDQInterface() metadata = interface.get_metadata() - test_directory = pathlib.Path(tmpdir) / "TestMockSpikeGLXNIDQInterface" - test_directory.mkdir(exist_ok=True) - nwbfile_path = test_directory / "test_mock_run_conversion.nwb" + nwbfile_path = tmp_path / "test_mock_run_conversion.nwb" interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) with NWBHDF5IO(path=nwbfile_path, mode="r") as io: @@ -94,11 +72,3 @@ def test_mock_run_conversion(tmpdir: pathlib.Path): assert "NIDQBoard" in nwbfile.devices assert len(nwbfile.devices) == 1 - - assert "NIDQChannelGroup" in nwbfile.electrode_groups - assert len(nwbfile.electrode_groups) == 1 - - assert list(nwbfile.electrodes.id[:]) == [0, 1, 2, 3, 4, 5, 6, 7] - - assert "ElectricalSeriesNIDQ" in nwbfile.acquisition - assert len(nwbfile.acquisition) == 1 diff --git a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json index 20f11742b..f3f5fb595 100644 --- a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json +++ b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json @@ -8,11 +8,6 @@ "name": "NeuropixelImec0", "description": "{\"probe_type\": \"0\", \"probe_type_description\": \"NP1.0\", \"flex_part_number\": \"NP2_FLEX_0\", \"connected_base_station_part_number\": \"NP2_QBSC_00\"}", "manufacturer": "Imec" - }, - { - "name": "NIDQBoard", - "description": "A NIDQ board used in conjunction with SpikeGLX.", - "manufacturer": "National Instruments" } ], "ElectrodeGroup": [ @@ -21,13 +16,8 @@ "description": "A group representing probe/shank 'Imec0'.", "location": "unknown", "device": "NeuropixelImec0" - }, - { - "name": "NIDQChannelGroup", - "description": "A group representing the NIDQ channels.", - "location": "unknown", - "device": "NIDQBoard" } + ], "ElectricalSeriesAP": { "name": "ElectricalSeriesAP", @@ -47,10 +37,6 @@ "description": "The id of the contact on the electrode" } ], - "ElectricalSeriesNIDQ": { - "name": "ElectricalSeriesNIDQ", - "description": "Raw acquisition traces from the NIDQ (.nidq.bin) channels." - }, "ElectricalSeriesLF": { "name": "ElectricalSeriesLF", "description": "Acquisition traces for the ElectricalSeriesLF." diff --git a/tests/test_on_data/ecephys/test_aux_interfaces.py b/tests/test_on_data/ecephys/test_aux_interfaces.py deleted file mode 100644 index 7934e29a1..000000000 --- a/tests/test_on_data/ecephys/test_aux_interfaces.py +++ /dev/null @@ -1,100 +0,0 @@ -import unittest -from datetime import datetime - -import pytest -from parameterized import param, parameterized -from spikeinterface.core.testing import check_recordings_equal -from spikeinterface.extractors import NwbRecordingExtractor - -from neuroconv import NWBConverter -from neuroconv.datainterfaces import SpikeGLXNIDQInterface - -# enable to run locally in interactive mode -try: - from ..setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from ..setup_paths import OUTPUT_PATH -except ImportError: - from setup_paths import ECEPHY_DATA_PATH as DATA_PATH - from setup_paths import OUTPUT_PATH - -if not DATA_PATH.exists(): - pytest.fail(f"No folder found in location: {DATA_PATH}!") - - -def custom_name_func(testcase_func, param_num, param): - interface_name = param.kwargs["data_interface"].__name__ - reduced_interface_name = interface_name.replace("Interface", "") - - return ( - f"{testcase_func.__name__}_{param_num}_" - f"{parameterized.to_safe_name(reduced_interface_name)}" - f"_{param.kwargs.get('case_name', '')}" - ) - - -class TestEcephysAuxNwbConversions(unittest.TestCase): - savedir = OUTPUT_PATH - - parameterized_aux_list = [ - param( - data_interface=SpikeGLXNIDQInterface, - interface_kwargs=dict(file_path=str(DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_t0.nidq.bin")), - case_name="load_sync_channel_False", - ), - param( - data_interface=SpikeGLXNIDQInterface, - interface_kwargs=dict( - file_path=str(DATA_PATH / "spikeglx" / "Noise4Sam_g0" / "Noise4Sam_g0_t0.nidq.bin"), - load_sync_channel=True, - ), - case_name="load_sync_channel_True", - ), - ] - - @parameterized.expand(input=parameterized_aux_list, name_func=custom_name_func) - def test_aux_recording_extractor_to_nwb(self, data_interface, interface_kwargs, case_name=""): - nwbfile_path = str(self.savedir / f"{data_interface.__name__}_{case_name}.nwb") - - class TestConverter(NWBConverter): - data_interface_classes = dict(TestAuxRecording=data_interface) - - converter = TestConverter(source_data=dict(TestAuxRecording=interface_kwargs)) - - for interface_kwarg in interface_kwargs: - if interface_kwarg in ["file_path", "folder_path"]: - self.assertIn( - member=interface_kwarg, container=converter.data_interface_objects["TestAuxRecording"].source_data - ) - - metadata = converter.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - converter.run_conversion(nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata) - recording = converter.data_interface_objects["TestAuxRecording"].recording_extractor - - electrical_series_name = metadata["Ecephys"][converter.data_interface_objects["TestAuxRecording"].es_key][ - "name" - ] - - # NWBRecordingExtractor on spikeinterface does not yet support loading data written from multiple segments. - if recording.get_num_segments() == 1: - # Spikeinterface behavior is to load the electrode table channel_name property as a channel_id - nwb_recording = NwbRecordingExtractor(file_path=nwbfile_path, electrical_series_name=electrical_series_name) - if "channel_name" in recording.get_property_keys(): - renamed_channel_ids = recording.get_property("channel_name") - else: - renamed_channel_ids = recording.get_channel_ids().astype("str") - recording = recording.channel_slice( - channel_ids=recording.get_channel_ids(), renamed_channel_ids=renamed_channel_ids - ) - - # Edge case that only occurs in testing; I think it's fixed in > 0.96.1 versions (unreleased as of 1/11/23) - # The NwbRecordingExtractor on spikeinterface experiences an issue when duplicated channel_ids - # are specified, which occurs during check_recordings_equal when there is only one channel - if nwb_recording.get_channel_ids()[0] != nwb_recording.get_channel_ids()[-1]: - check_recordings_equal(RX1=recording, RX2=nwb_recording, return_scaled=False) - if recording.has_scaled_traces() and nwb_recording.has_scaled_traces(): - check_recordings_equal(RX1=recording, RX2=nwb_recording, return_scaled=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_on_data/ecephys/test_nidq_interface.py b/tests/test_on_data/ecephys/test_nidq_interface.py new file mode 100644 index 000000000..6d6517323 --- /dev/null +++ b/tests/test_on_data/ecephys/test_nidq_interface.py @@ -0,0 +1,57 @@ +import numpy as np +import pytest +from pynwb import NWBHDF5IO + +from neuroconv.datainterfaces import SpikeGLXNIDQInterface + +# enable to run locally in interactive mode +try: + from ..setup_paths import ECEPHY_DATA_PATH +except ImportError: + from setup_paths import ECEPHY_DATA_PATH + +if not ECEPHY_DATA_PATH.exists(): + pytest.fail(f"No folder found in location: {ECEPHY_DATA_PATH}!") + + +def test_nidq_interface_digital_data(tmp_path): + + nwbfile_path = tmp_path / "nidq_test_digital.nwb" + folder_path = ECEPHY_DATA_PATH / "spikeglx" / "DigitalChannelTest_g0" + interface = SpikeGLXNIDQInterface(folder_path=folder_path) + interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True) + + with NWBHDF5IO(nwbfile_path, "r") as io: + nwbfile = io.read() + assert len(nwbfile.acquisition) == 1 # Only one channel has data for this set + events = nwbfile.acquisition["EventsNIDQDigitalChannelXD0"] + assert events.name == "EventsNIDQDigitalChannelXD0" + assert events.timestamps.size == 326 + assert len(nwbfile.devices) == 1 + + data = events.data + # Check that there is one followed by 0 + np.sum(data == 1) == 163 + np.sum(data == 0) == 163 + + +def test_nidq_interface_analog_data(tmp_path): + + nwbfile_path = tmp_path / "nidq_test_analog.nwb" + folder_path = ECEPHY_DATA_PATH / "spikeglx" / "Noise4Sam_g0" + interface = SpikeGLXNIDQInterface(folder_path=folder_path) + interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True) + + with NWBHDF5IO(nwbfile_path, "r") as io: + nwbfile = io.read() + assert len(nwbfile.acquisition) == 1 # The time series object + time_series = nwbfile.acquisition["TimeSeriesNIDQ"] + assert time_series.name == "TimeSeriesNIDQ" + expected_description = "Analog data from the NIDQ board. Channels are ['XA0' 'XA1' 'XA2' 'XA3' 'XA4' 'XA5' 'XA6' 'XA7'] in that order." + assert time_series.description == expected_description + number_of_samples = time_series.data.shape[0] + assert number_of_samples == 60_864 + number_of_channels = time_series.data.shape[1] + assert number_of_channels == 8 + + assert len(nwbfile.devices) == 1 diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index 970a815af..93b228053 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -37,16 +37,16 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ assert "ElectricalSeriesAP" in nwbfile.acquisition assert "ElectricalSeriesLF" in nwbfile.acquisition - assert "ElectricalSeriesNIDQ" in nwbfile.acquisition + assert "TimeSeriesNIDQ" in nwbfile.acquisition + assert len(nwbfile.acquisition) == 3 assert "NeuropixelImec0" in nwbfile.devices assert "NIDQBoard" in nwbfile.devices assert len(nwbfile.devices) == 2 - assert "NIDQChannelGroup" in nwbfile.electrode_groups assert "Imec0" in nwbfile.electrode_groups - assert len(nwbfile.electrode_groups) == 2 + assert len(nwbfile.electrode_groups) == 1 def test_single_probe_spikeglx_converter(self): converter = SpikeGLXConverterPipe(folder_path=SPIKEGLX_PATH / "Noise4Sam_g0") @@ -63,14 +63,13 @@ def test_single_probe_spikeglx_converter(self): expected_ecephys_metadata = expected_metadata["Ecephys"] test_ecephys_metadata = test_metadata["Ecephys"] + assert test_ecephys_metadata == expected_ecephys_metadata device_metadata = test_ecephys_metadata.pop("Device") expected_device_metadata = expected_ecephys_metadata.pop("Device") assert device_metadata == expected_device_metadata - assert test_ecephys_metadata == expected_ecephys_metadata - nwbfile_path = self.tmpdir / "test_single_probe_spikeglx_converter.nwb" converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) @@ -183,16 +182,6 @@ def test_electrode_table_writing(tmp_path): electrodes_table = nwbfile.electrodes - # Test NIDQ - electrical_series = nwbfile.acquisition["ElectricalSeriesNIDQ"] - nidq_electrodes_table_region = electrical_series.electrodes - region_indices = nidq_electrodes_table_region.data - recording_extractor = converter.data_interface_objects["nidq"].recording_extractor - - saved_channel_names = electrodes_table[region_indices]["channel_name"] - expected_channel_names_nidq = recording_extractor.get_property("channel_name") - np.testing.assert_array_equal(saved_channel_names, expected_channel_names_nidq) - # Test AP electrical_series = nwbfile.acquisition["ElectricalSeriesAP"] ap_electrodes_table_region = electrical_series.electrodes @@ -236,11 +225,3 @@ def test_electrode_table_writing(tmp_path): channel_ids = recording_extractor_lf.get_channel_ids() np.testing.assert_array_equal(channel_ids, expected_channel_names_lf) - - recording_extractor_nidq = NwbRecordingExtractor( - file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesNIDQ", - ) - - channel_ids = recording_extractor_nidq.get_channel_ids() - np.testing.assert_array_equal(channel_ids, expected_channel_names_nidq) diff --git a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py index c8c6bad09..081cd172d 100644 --- a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py +++ b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py @@ -75,7 +75,6 @@ def assertNWBFileTimesAligned(self, nwbfile_path: Union[str, Path]): # High level groups were written to file assert "BehaviorEvents" in nwbfile.acquisition - assert "ElectricalSeriesNIDQ" in nwbfile.acquisition assert "trials" in nwbfile.intervals # Aligned data was written From 43477de78ca23b3bb0dba0a7bcd66b4e9a68869b Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 11 Dec 2024 16:45:16 -0600 Subject: [PATCH 098/118] Add more docstrings to public methods (#1063) Co-authored-by: Paul Adkisson --- CHANGELOG.md | 5 +- pyproject.toml | 2 +- src/neuroconv/basedatainterface.py | 9 +- .../lightningpose/lightningposeconverter.py | 49 ++++++++++ .../behavior/medpc/medpcdatainterface.py | 1 + .../ecephys/basesortingextractorinterface.py | 13 +++ .../cellexplorer/cellexplorerdatainterface.py | 27 ++++++ .../ophys/baseimagingextractorinterface.py | 21 +++++ .../basesegmentationextractorinterface.py | 21 +++++ .../ophys/brukertiff/brukertiffconverter.py | 67 ++++++++++++++ .../brukertiff/brukertiffdatainterface.py | 34 +++++++ .../ophys/caiman/caimandatainterface.py | 1 + .../micromanagertiffdatainterface.py | 2 + .../ophys/miniscope/miniscopeconverter.py | 35 ++++++++ .../miniscopeimagingdatainterface.py | 20 +++++ .../ophys/sbx/sbxdatainterface.py | 1 + .../scanimage/scanimageimaginginterfaces.py | 7 ++ .../ophys/suite2p/suite2pdatainterface.py | 36 ++++++++ .../tdt_fp/tdtfiberphotometrydatainterface.py | 2 + .../ophys/tiff/tiffdatainterface.py | 1 + .../text/timeintervalsinterface.py | 89 ++++++++++++++++++- src/neuroconv/nwbconverter.py | 16 +++- src/neuroconv/tools/hdmf.py | 1 + .../tools/testing/mock_interfaces.py | 50 ++++++++++- src/neuroconv/utils/dict.py | 17 ++++ 25 files changed, 516 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9c9afcde..11dce4e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,15 +28,16 @@ ## Improvements * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) * Use pytest format for dandi tests to avoid window permission error on teardown [PR #1151](https://github.com/catalystneuro/neuroconv/pull/1151) +* Added many docstrings for public functions [PR #1063](https://github.com/catalystneuro/neuroconv/pull/1063) # v0.6.5 (November 1, 2024) -## Deprecations - ## Bug Fixes * Fixed formatwise installation from pipy [PR #1118](https://github.com/catalystneuro/neuroconv/pull/1118) * Fixed dailies [PR #1113](https://github.com/catalystneuro/neuroconv/pull/1113) +## Deprecations + ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) * Data interfaces now perform source (argument inputs) validation with the json schema [PR #1020](https://github.com/catalystneuro/neuroconv/pull/1020) diff --git a/pyproject.toml b/pyproject.toml index a4f391512..e318cc9c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -356,7 +356,7 @@ doctest_optionflags = "ELLIPSIS" [tool.black] line-length = 120 -target-version = ['py38', 'py39', 'py310'] +target-version = ['py39', 'py310'] include = '\.pyi?$' extend-exclude = ''' /( diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 530eec60c..d9e9dc11e 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -36,7 +36,14 @@ class BaseDataInterface(ABC): @classmethod def get_source_schema(cls) -> dict: - """Infer the JSON schema for the source_data from the method signature (annotation typing).""" + """ + Infer the JSON schema for the source_data from the method signature (annotation typing). + + Returns + ------- + dict + The JSON schema for the source_data. + """ return get_json_schema_from_method_signature(cls, exclude=["source_data"]) @classmethod diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py index 505aa144d..62edaf140 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py @@ -111,6 +111,28 @@ def add_to_nwbfile( starting_frames_labeled_videos: Optional[list[int]] = None, stub_test: bool = False, ): + """ + Add behavior and pose estimation data, including original and labeled videos, to the specified NWBFile. + + Parameters + ---------- + nwbfile : NWBFile + The NWBFile object to which the data will be added. + metadata : dict + Metadata dictionary containing information about the behavior and videos. + reference_frame : str, optional + Description of the reference frame for pose estimation, by default None. + confidence_definition : str, optional + Definition for the confidence levels in pose estimation, by default None. + external_mode : bool, optional + If True, the videos will be referenced externally rather than embedded within the NWB file, by default True. + starting_frames_original_videos : list of int, optional + List of starting frames for the original videos, by default None. + starting_frames_labeled_videos : list of int, optional + List of starting frames for the labeled videos, by default None. + stub_test : bool, optional + If True, only a subset of the data will be added for testing purposes, by default False. + """ original_video_interface = self.data_interface_objects["OriginalVideo"] original_video_metadata = next( @@ -172,6 +194,33 @@ def run_conversion( starting_frames_labeled_videos: Optional[list] = None, stub_test: bool = False, ) -> None: + """ + Run the full conversion process, adding behavior, video, and pose estimation data to an NWB file. + + Parameters + ---------- + nwbfile_path : FilePath, optional + The file path where the NWB file will be saved. If None, the file is handled in memory. + nwbfile : NWBFile, optional + An in-memory NWBFile object. If None, a new NWBFile object will be created. + metadata : dict, optional + Metadata dictionary for describing the NWB file contents. If None, it is auto-generated. + overwrite : bool, optional + If True, overwrites the NWB file at `nwbfile_path` if it exists. If False, appends to the file, by default False. + reference_frame : str, optional + Description of the reference frame for pose estimation, by default None. + confidence_definition : str, optional + Definition for confidence levels in pose estimation, by default None. + external_mode : bool, optional + If True, the videos will be referenced externally rather than embedded within the NWB file, by default True. + starting_frames_original_videos : list of int, optional + List of starting frames for the original videos, by default None. + starting_frames_labeled_videos : list of int, optional + List of starting frames for the labeled videos, by default None. + stub_test : bool, optional + If True, only a subset of the data will be added for testing purposes, by default False. + + """ if metadata is None: metadata = self.get_metadata() diff --git a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py index 09f9111d7..d519dc71a 100644 --- a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py @@ -187,6 +187,7 @@ def add_to_nwbfile( nwbfile: NWBFile, metadata: dict, ) -> None: + ndx_events = get_package(package_name="ndx_events", installation_instructions="pip install ndx-events") medpc_name_to_info_dict = metadata["MedPC"].get("medpc_name_to_info_dict", None) assert medpc_name_to_info_dict is not None, "medpc_name_to_info_dict must be provided in metadata" diff --git a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py index dca2dea5f..8eeb59324 100644 --- a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py @@ -220,6 +220,19 @@ def set_aligned_segment_starting_times(self, aligned_segment_starting_times: lis sorting_segment._t_start = aligned_segment_starting_time def subset_sorting(self): + """ + Generate a subset of the sorting extractor based on spike timing data. + + This method identifies the earliest spike time across all units in the sorting extractor and creates a + subset of the sorting data up to 110% of the earliest spike time. If the sorting extractor is associated + with a recording, the subset is further limited by the total number of samples in the recording. + + Returns + ------- + SortingExtractor + A new `SortingExtractor` object representing the subset of the original sorting data, + sliced from the start frame to the calculated end frame. + """ max_min_spike_time = max( [ min(x) diff --git a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py index 46e825fb5..9ffeb78a2 100644 --- a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py @@ -518,6 +518,33 @@ def __init__(self, file_path: FilePath, verbose: bool = True): ) def generate_recording_with_channel_metadata(self): + """ + Generate a dummy recording extractor with channel metadata from session data. + + This method reads session data from a `.session.mat` file (if available) and generates a dummy recording + extractor. The recording extractor is then populated with channel metadata extracted from the session file. + + Returns + ------- + NumpyRecording + A `NumpyRecording` object representing the dummy recording extractor, containing the channel metadata. + + Notes + ----- + - The method reads the `.session.mat` file using `pymatreader` and extracts `extracellular` data. + - It creates a dummy recording extractor using `spikeinterface.core.numpyextractors.NumpyRecording`. + - The generated extractor includes channel IDs and other relevant metadata such as number of channels, + number of samples, and sampling frequency. + - Channel metadata is added to the dummy extractor using the `add_channel_metadata_to_recoder` function. + - If the `.session.mat` file is not found, no extractor is returned. + + Warnings + -------- + Ensure that the `.session.mat` file is correctly located in the expected session path, or the method will not generate + a recording extractor. The expected session is self.session_path / f"{self.session_id}.session.mat" + + """ + session_data_file_path = self.session_path / f"{self.session_id}.session.mat" if session_data_file_path.is_file(): from pymatreader import read_mat diff --git a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py index 0019b8bd7..04407a3d4 100644 --- a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py @@ -47,6 +47,17 @@ def __init__( def get_metadata_schema( self, ) -> dict: + """ + Retrieve the metadata schema for the optical physiology (Ophys) data, with optional handling of photon series type. + + Parameters + ---------- + photon_series_type : {"OnePhotonSeries", "TwoPhotonSeries"}, optional + The type of photon series to include in the schema. If None, the value from the instance is used. + This argument is deprecated and will be removed in a future version. Set `photon_series_type` during + the initialization of the `BaseImagingExtractorInterface` instance. + + """ metadata_schema = super().get_metadata_schema() @@ -93,6 +104,16 @@ def get_metadata_schema( def get_metadata( self, ) -> DeepDict: + """ + Retrieve the metadata for the imaging data, with optional handling of photon series type. + + Parameters + ---------- + photon_series_type : {"OnePhotonSeries", "TwoPhotonSeries"}, optional + The type of photon series to include in the metadata. If None, the value from the instance is used. + This argument is deprecated and will be removed in a future version. Instead, set `photon_series_type` + during the initialization of the `BaseImagingExtractorInterface` instance. + """ from ...tools.roiextractors import get_nwb_imaging_metadata diff --git a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py index 0f2e41bb9..66d35f57a 100644 --- a/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py @@ -24,6 +24,27 @@ def __init__(self, verbose: bool = False, **source_data): self.segmentation_extractor = self.get_extractor()(**source_data) def get_metadata_schema(self) -> dict: + """ + Generate the metadata schema for Ophys data, updating required fields and properties. + + This method builds upon the base schema and customizes it for Ophys-specific metadata, including required + components such as devices, fluorescence data, imaging planes, and two-photon series. It also applies + temporary schema adjustments to handle certain use cases until a centralized metadata schema definition + is available. + + Returns + ------- + dict + A dictionary representing the updated Ophys metadata schema. + + Notes + ----- + - Ensures that `Device` and `ImageSegmentation` are marked as required. + - Updates various properties, including ensuring arrays for `ImagingPlane` and `TwoPhotonSeries`. + - Adjusts the schema for `Fluorescence`, including required fields and pattern properties. + - Adds schema definitions for `DfOverF`, segmentation images, and summary images. + - Applies temporary fixes, such as setting additional properties for `ImageSegmentation` to True. + """ metadata_schema = super().get_metadata_schema() metadata_schema["required"] = ["Ophys"] metadata_schema["properties"]["Ophys"] = get_base_schema() diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py index 2a67da720..1fc854b81 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py @@ -31,6 +31,7 @@ def get_source_schema(cls): return source_schema def get_conversion_options_schema(self): + """get the conversion options schema.""" interface_name = list(self.data_interface_objects.keys())[0] return self.data_interface_objects[interface_name].get_conversion_options_schema() @@ -91,6 +92,20 @@ def add_to_nwbfile( stub_test: bool = False, stub_frames: int = 100, ): + """ + Add data from multiple data interfaces to the specified NWBFile. + + Parameters + ---------- + nwbfile : NWBFile + The NWBFile object to which the data will be added. + metadata : dict + Metadata dictionary containing information to describe the data being added to the NWB file. + stub_test : bool, optional + If True, only a subset of the data (up to `stub_frames`) will be added for testing purposes. Default is False. + stub_frames : int, optional + The number of frames to include in the subset if `stub_test` is True. Default is 100. + """ for photon_series_index, (interface_name, data_interface) in enumerate(self.data_interface_objects.items()): data_interface.add_to_nwbfile( nwbfile=nwbfile, @@ -109,6 +124,24 @@ def run_conversion( stub_test: bool = False, stub_frames: int = 100, ) -> None: + """ + Run the conversion process for the instantiated data interfaces and add data to the NWB file. + + Parameters + ---------- + nwbfile_path : FilePath, optional + Path where the NWB file will be written. If None, the file will be handled in-memory. + nwbfile : NWBFile, optional + An in-memory NWBFile object. If None, a new NWBFile object will be created. + metadata : dict, optional + Metadata dictionary for describing the NWB file. If None, it will be auto-generated using the `get_metadata()` method. + overwrite : bool, optional + If True, overwrites the existing NWB file at `nwbfile_path`. If False, appends to the file (default is False). + stub_test : bool, optional + If True, only a subset of the data (up to `stub_frames`) will be added for testing purposes, by default False. + stub_frames : int, optional + The number of frames to include in the subset if `stub_test` is True, by default 100. + """ if metadata is None: metadata = self.get_metadata() @@ -141,6 +174,7 @@ def get_source_schema(cls): return get_json_schema_from_method_signature(cls) def get_conversion_options_schema(self): + """Get the conversion options schema.""" interface_name = list(self.data_interface_objects.keys())[0] return self.data_interface_objects[interface_name].get_conversion_options_schema() @@ -187,6 +221,21 @@ def add_to_nwbfile( stub_test: bool = False, stub_frames: int = 100, ): + """ + Add data from all instantiated data interfaces to the provided NWBFile. + + Parameters + ---------- + nwbfile : NWBFile + The NWBFile object to which the data will be added. + metadata : dict + Metadata dictionary containing information about the data to be added. + stub_test : bool, optional + If True, only a subset of the data (defined by `stub_frames`) will be added for testing purposes, + by default False. + stub_frames : int, optional + The number of frames to include in the subset if `stub_test` is True, by default 100. + """ for photon_series_index, (interface_name, data_interface) in enumerate(self.data_interface_objects.items()): data_interface.add_to_nwbfile( nwbfile=nwbfile, @@ -205,6 +254,24 @@ def run_conversion( stub_test: bool = False, stub_frames: int = 100, ) -> None: + """ + Run the NWB conversion process for all instantiated data interfaces. + + Parameters + ---------- + nwbfile_path : FilePath, optional + The file path where the NWB file will be written. If None, the file is handled in-memory. + nwbfile : NWBFile, optional + An existing in-memory NWBFile object. If None, a new NWBFile object will be created. + metadata : dict, optional + Metadata dictionary used to create or validate the NWBFile. If None, metadata is automatically generated. + overwrite : bool, optional + If True, the NWBFile at `nwbfile_path` is overwritten if it exists. If False (default), data is appended. + stub_test : bool, optional + If True, only a subset of the data (up to `stub_frames`) is used for testing purposes. By default False. + stub_frames : int, optional + The number of frames to include in the subset if `stub_test` is True. By default 100. + """ if metadata is None: metadata = self.get_metadata() diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py index 9742711e1..f7e7bee1b 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py @@ -16,6 +16,7 @@ class BrukerTiffMultiPlaneImagingInterface(BaseImagingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """Get the source schema for the Bruker TIFF imaging data.""" source_schema = super().get_source_schema() source_schema["properties"]["folder_path"][ "description" @@ -28,6 +29,23 @@ def get_streams( folder_path: DirectoryPath, plane_separation_type: Literal["contiguous", "disjoint"] = None, ) -> dict: + """ + Get streams for the Bruker TIFF imaging data. + + Parameters + ---------- + folder_path : DirectoryPath + Path to the folder containing the Bruker TIFF files. + plane_separation_type : Literal["contiguous", "disjoint"], optional + Type of plane separation to apply. If "contiguous", only the first plane stream for each channel is retained. + + Returns + ------- + dict + A dictionary containing the streams for the Bruker TIFF imaging data. The dictionary has the following keys: + - "channel_streams": List of channel stream names. + - "plane_streams": Dictionary where keys are channel stream names and values are lists of plane streams. + """ from roiextractors import BrukerTiffMultiPlaneImagingExtractor streams = BrukerTiffMultiPlaneImagingExtractor.get_streams(folder_path=folder_path) @@ -117,6 +135,7 @@ def _determine_position_current(self) -> list[float]: return position_values def get_metadata(self) -> DeepDict: + """get metadata for the Bruker TIFF imaging data.""" metadata = super().get_metadata() xml_metadata = self.imaging_extractor.xml_metadata @@ -183,6 +202,7 @@ class BrukerTiffSinglePlaneImagingInterface(BaseImagingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """Get the source schema for the Bruker TIFF imaging data.""" source_schema = super().get_source_schema() source_schema["properties"]["folder_path"][ "description" @@ -191,6 +211,19 @@ def get_source_schema(cls) -> dict: @classmethod def get_streams(cls, folder_path: DirectoryPath) -> dict: + """ + Get streams for the Bruker TIFF imaging data. + + Parameters + ---------- + folder_path : DirectoryPath + Path to the folder containing the Bruker TIFF files. + + Returns + ------- + dict + A dictionary containing the streams extracted from the Bruker TIFF files. + """ from roiextractors import BrukerTiffMultiPlaneImagingExtractor streams = BrukerTiffMultiPlaneImagingExtractor.get_streams(folder_path=folder_path) @@ -263,6 +296,7 @@ def _determine_position_current(self) -> list[float]: return position_values def get_metadata(self) -> DeepDict: + """get metadata for the Bruker TIFF imaging data.""" metadata = super().get_metadata() xml_metadata = self.imaging_extractor.xml_metadata diff --git a/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py b/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py index 386c03d3c..802645139 100644 --- a/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py @@ -12,6 +12,7 @@ class CaimanSegmentationInterface(BaseSegmentationExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """Get the source schema for the Caiman segmentation interface.""" source_metadata = super().get_source_schema() source_metadata["properties"]["file_path"]["description"] = "Path to .hdf5 file." return source_metadata diff --git a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py index 17cbc95ed..5373b7004 100644 --- a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py @@ -13,6 +13,7 @@ class MicroManagerTiffImagingInterface(BaseImagingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """get the source schema for the Micro-Manager TIFF imaging interface.""" source_schema = super().get_source_schema() source_schema["properties"]["folder_path"]["description"] = "The folder containing the OME-TIF image files." @@ -37,6 +38,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True): self.imaging_extractor._channel_names = [f"OpticalChannel{channel_name}"] def get_metadata(self) -> dict: + """Get metadata for the Micro-Manager TIFF imaging data.""" metadata = super().get_metadata() micromanager_metadata = self.imaging_extractor.micromanager_metadata diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py index d1a0fb701..59424d7a5 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py @@ -61,6 +61,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True): ) def get_conversion_options_schema(self) -> dict: + """get the conversion options schema.""" return self.data_interface_objects["MiniscopeImaging"].get_conversion_options_schema() def add_to_nwbfile( @@ -70,6 +71,21 @@ def add_to_nwbfile( stub_test: bool = False, stub_frames: int = 100, ): + """ + Add Miniscope imaging and behavioral camera data to the specified NWBFile. + + Parameters + ---------- + nwbfile : NWBFile + The NWBFile object to which the imaging and behavioral data will be added. + metadata : dict + Metadata dictionary containing information about the imaging and behavioral recordings. + stub_test : bool, optional + If True, only a subset of the data (defined by `stub_frames`) will be added for testing purposes, + by default False. + stub_frames : int, optional + The number of frames to include in the subset if `stub_test` is True, by default 100. + """ self.data_interface_objects["MiniscopeImaging"].add_to_nwbfile( nwbfile=nwbfile, metadata=metadata, @@ -90,6 +106,25 @@ def run_conversion( stub_test: bool = False, stub_frames: int = 100, ) -> None: + """ + Run the NWB conversion process for the instantiated data interfaces. + + Parameters + ---------- + nwbfile_path : str, optional + Path where the NWBFile will be written. If None, the file is handled in-memory. + nwbfile : NWBFile, optional + An in-memory NWBFile object to be written to the file. If None, a new NWBFile is created. + metadata : dict, optional + Metadata dictionary with information to create the NWBFile. If None, metadata is auto-generated. + overwrite : bool, optional + If True, overwrites the existing NWBFile at `nwbfile_path`. If False (default), data is appended. + stub_test : bool, optional + If True, only a subset of the data (up to `stub_frames`) is written for testing purposes, + by default False. + stub_frames : int, optional + The number of frames to include in the subset if `stub_test` is True, by default 100. + """ if metadata is None: metadata = self.get_metadata() diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py index 64a180c46..5a1f6d521 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeimagingdatainterface.py @@ -19,6 +19,7 @@ class MiniscopeImagingInterface(BaseImagingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """Get the source schema for the Miniscope imaging interface.""" source_schema = super().get_source_schema() source_schema["properties"]["folder_path"][ "description" @@ -49,6 +50,7 @@ def __init__(self, folder_path: DirectoryPath): self.photon_series_type = "OnePhotonSeries" def get_metadata(self) -> DeepDict: + """Get metadata for the Miniscope imaging data.""" from ....tools.roiextractors import get_nwb_imaging_metadata metadata = super().get_metadata() @@ -74,6 +76,7 @@ def get_metadata(self) -> DeepDict: return metadata def get_metadata_schema(self) -> dict: + """Get the metadata schema for the Miniscope imaging data.""" metadata_schema = super().get_metadata_schema() metadata_schema["properties"]["Ophys"]["definitions"]["Device"]["additionalProperties"] = True return metadata_schema @@ -92,6 +95,23 @@ def add_to_nwbfile( stub_test: bool = False, stub_frames: int = 100, ): + """ + Add imaging data to the specified NWBFile, including device and photon series information. + + Parameters + ---------- + nwbfile : NWBFile + The NWBFile object to which the imaging data will be added. + metadata : dict, optional + Metadata containing information about the imaging device and photon series. If None, default metadata is used. + photon_series_type : {"TwoPhotonSeries", "OnePhotonSeries"}, optional + The type of photon series to be added, either "TwoPhotonSeries" or "OnePhotonSeries", by default "OnePhotonSeries". + stub_test : bool, optional + If True, only a subset of the data (defined by `stub_frames`) will be added for testing purposes, + by default False. + stub_frames : int, optional + The number of frames to include if `stub_test` is True, by default 100. + """ from ndx_miniscope.utils import add_miniscope_device from ....tools.roiextractors import add_photon_series_to_nwbfile diff --git a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py index 554cc5aba..49e556d06 100644 --- a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py @@ -37,6 +37,7 @@ def __init__( ) def get_metadata(self) -> dict: + """Get metadata for the Scanbox imaging data.""" metadata = super().get_metadata() metadata["Ophys"]["Device"][0]["description"] = "Scanbox imaging" return metadata diff --git a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py index c74161e55..7d9d7003b 100644 --- a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py +++ b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py @@ -28,6 +28,7 @@ class ScanImageImagingInterface(BaseImagingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """Get the source schema for the ScanImage imaging interface.""" source_schema = super().get_source_schema() source_schema["properties"]["file_path"]["description"] = "Path to Tiff file." return source_schema @@ -139,6 +140,7 @@ def __init__( super().__init__(file_path=file_path, fallback_sampling_frequency=fallback_sampling_frequency, verbose=verbose) def get_metadata(self) -> dict: + """get metadata for the ScanImage imaging data""" device_number = 0 # Imaging plane metadata is a list with metadata for each plane metadata = super().get_metadata() @@ -174,6 +176,7 @@ class ScanImageMultiFileImagingInterface(BaseImagingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """get the source schema for the ScanImage multi-file imaging interface.""" source_schema = super().get_source_schema() source_schema["properties"]["folder_path"]["description"] = "Path to the folder containing the TIFF files." return source_schema @@ -304,6 +307,7 @@ def __init__( ) def get_metadata(self) -> dict: + """get metadata for the ScanImage imaging data""" metadata = super().get_metadata() extracted_session_start_time = datetime.datetime.strptime( @@ -421,6 +425,7 @@ def __init__( ) def get_metadata(self) -> dict: + """get metadata for the ScanImage imaging data""" metadata = super().get_metadata() extracted_session_start_time = datetime.datetime.strptime( @@ -548,6 +553,7 @@ def __init__( ) def get_metadata(self) -> dict: + """get metadata for the ScanImage imaging data""" metadata = super().get_metadata() extracted_session_start_time = datetime.datetime.strptime( @@ -677,6 +683,7 @@ def __init__( ) def get_metadata(self) -> dict: + """get metadata for the ScanImage imaging data""" metadata = super().get_metadata() extracted_session_start_time = datetime.datetime.strptime( diff --git a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py index 056616ce5..8a3f876c2 100644 --- a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py @@ -50,6 +50,7 @@ class Suite2pSegmentationInterface(BaseSegmentationExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """Get the source schema for the Suite2p segmentation interface.""" schema = super().get_source_schema() schema["properties"]["folder_path"][ "description" @@ -113,6 +114,7 @@ def __init__( self.verbose = verbose def get_metadata(self) -> DeepDict: + """get metadata for the Suite2p segmentation data""" metadata = super().get_metadata() # No need to update the metadata links for the default plane segmentation name @@ -140,6 +142,40 @@ def add_to_nwbfile( iterator_options: Optional[dict] = None, compression_options: Optional[dict] = None, ): + """ + Add segmentation data to the specified NWBFile. + + Parameters + ---------- + nwbfile : NWBFile + The NWBFile object to which the segmentation data will be added. + metadata : dict, optional + Metadata containing information about the segmentation. If None, default metadata is used. + stub_test : bool, optional + If True, only a subset of the data (defined by `stub_frames`) will be added for testing purposes, + by default False. + stub_frames : int, optional + The number of frames to include in the subset if `stub_test` is True, by default 100. + include_roi_centroids : bool, optional + Whether to include the centroids of regions of interest (ROIs) in the data, by default True. + include_roi_acceptance : bool, optional + Whether to include acceptance status of ROIs, by default True. + mask_type : str, default: 'image' + There are three types of ROI masks in NWB, 'image', 'pixel', and 'voxel'. + + * 'image' masks have the same shape as the reference images the segmentation was applied to, and weight each pixel + by its contribution to the ROI (typically boolean, with 0 meaning 'not in the ROI'). + * 'pixel' masks are instead indexed by ROI, with the data at each index being the shape of the image by the number + of pixels in each ROI. + * 'voxel' masks are instead indexed by ROI, with the data at each index being the shape of the volume by the number + of voxels in each ROI. + + Specify your choice between these two as mask_type='image', 'pixel', 'voxel', or None. + plane_segmentation_name : str, optional + The name of the plane segmentation object, by default None. + iterator_options : dict, optional + Additional options for iterating over the data, by default None. + """ super().add_to_nwbfile( nwbfile=nwbfile, metadata=metadata, diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py index 8b092464e..0c6b90aea 100644 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py @@ -47,6 +47,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True): import ndx_fiber_photometry # noqa: F401 def get_metadata(self) -> DeepDict: + """Get metadata for the TDTFiberPhotometryInterface.""" metadata = super().get_metadata() tdt_photometry = self.load(evtype=["scalars"]) # This evtype quickly loads info without loading all the data. start_timestamp = tdt_photometry.info.start_date.timestamp() @@ -55,6 +56,7 @@ def get_metadata(self) -> DeepDict: return metadata def get_metadata_schema(self) -> dict: + """Get the metadata schema for the TDTFiberPhotometryInterface.""" metadata_schema = super().get_metadata_schema() return metadata_schema diff --git a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py index 1eaa3b55e..ce98561de 100644 --- a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py @@ -14,6 +14,7 @@ class TiffImagingInterface(BaseImagingExtractorInterface): @classmethod def get_source_schema(cls) -> dict: + """ "Get the source schema for the TIFF imaging interface.""" source_schema = super().get_source_schema() source_schema["properties"]["file_path"]["description"] = "Path to Tiff file." return source_schema diff --git a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py index 5f5b1107d..a1de63a07 100644 --- a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py @@ -24,16 +24,20 @@ def __init__( verbose: bool = True, ): """ + Initialize the TimeIntervalsInterface. + Parameters ---------- file_path : FilePath + The path to the file containing time intervals data. read_kwargs : dict, optional - verbose : bool, default: True + Additional arguments for reading the file, by default None. + verbose : bool, optional + If True, provides verbose output, by default True. """ read_kwargs = read_kwargs or dict() super().__init__(file_path=file_path) self.verbose = verbose - self._read_kwargs = read_kwargs self.dataframe = self._read_file(file_path, **read_kwargs) self.time_intervals = None @@ -50,22 +54,74 @@ def get_metadata(self) -> dict: return metadata def get_metadata_schema(self) -> dict: + """ + Get the metadata schema for the time intervals. + + Returns + ------- + dict + The schema dictionary for time intervals metadata. + """ fpath = Path(__file__).parent.parent.parent / "schemas" / "timeintervals_schema.json" return load_dict_from_file(fpath) def get_original_timestamps(self, column: str) -> np.ndarray: + """ + Get the original timestamps for a given column. + + Parameters + ---------- + column : str + The name of the column containing timestamps. + + Returns + ------- + np.ndarray + The original timestamps from the specified column. + + Raises + ------ + ValueError + If the column name does not end with '_time'. + """ if not column.endswith("_time"): raise ValueError("Timing columns on a TimeIntervals table need to end with '_time'!") return self._read_file(**self.source_data, **self._read_kwargs)[column].values def get_timestamps(self, column: str) -> np.ndarray: + """ + Get the current timestamps for a given column. + + Parameters + ---------- + column : str + The name of the column containing timestamps. + + Returns + ------- + np.ndarray + The current timestamps from the specified column. + + Raises + ------ + ValueError + If the column name does not end with '_time'. + """ if not column.endswith("_time"): raise ValueError("Timing columns on a TimeIntervals table need to end with '_time'!") return self.dataframe[column].values def set_aligned_starting_time(self, aligned_starting_time: float): + """ + Align the starting time by shifting all timestamps by the given value. + + Parameters + ---------- + aligned_starting_time : float + The aligned starting time to shift all timestamps by. + """ timing_columns = [column for column in self.dataframe.columns if column.endswith("_time")] for column in timing_columns: @@ -74,6 +130,23 @@ def set_aligned_starting_time(self, aligned_starting_time: float): def set_aligned_timestamps( self, aligned_timestamps: np.ndarray, column: str, interpolate_other_columns: bool = False ): + """ + Set aligned timestamps for the given column and optionally interpolate other columns. + + Parameters + ---------- + aligned_timestamps : np.ndarray + The aligned timestamps to set for the given column. + column : str + The name of the column to update with the aligned timestamps. + interpolate_other_columns : bool, optional + If True, interpolate the timestamps in other columns, by default False. + + Raises + ------ + ValueError + If the column name does not end with '_time'. + """ if not column.endswith("_time"): raise ValueError("Timing columns on a TimeIntervals table need to end with '_time'!") @@ -96,6 +169,18 @@ def set_aligned_timestamps( ) def align_by_interpolation(self, unaligned_timestamps: np.ndarray, aligned_timestamps: np.ndarray, column: str): + """ + Align timestamps using linear interpolation. + + Parameters + ---------- + unaligned_timestamps : np.ndarray + The original unaligned timestamps that map to the aligned timestamps. + aligned_timestamps : np.ndarray + The target aligned timestamps corresponding to the unaligned timestamps. + column : str + The name of the column containing the timestamps to be aligned. + """ current_timestamps = self.get_timestamps(column=column) assert ( current_timestamps[1] >= unaligned_timestamps[0] diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index fe1b09915..2d70cf8ee 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -177,7 +177,21 @@ def create_nwbfile(self, metadata: Optional[dict] = None, conversion_options: Op self.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, conversion_options=conversion_options) return nwbfile - def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optional[dict] = None) -> None: + def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optional[dict] = None): + """ + Add data from the instantiated data interfaces to the given NWBFile. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file object to which the data from the data interfaces will be added. + metadata : dict + The metadata dictionary that contains information used to describe the data. + conversion_options : dict, optional + A dictionary containing conversion options for each interface, where non-default behavior is requested. + Each key corresponds to a data interface name, and the values are dictionaries with options for that interface. + By default, None. + """ conversion_options = conversion_options or dict() for interface_name, data_interface in self.data_interface_objects.items(): data_interface.add_to_nwbfile( diff --git a/src/neuroconv/tools/hdmf.py b/src/neuroconv/tools/hdmf.py index 660971df5..f32ea23a0 100644 --- a/src/neuroconv/tools/hdmf.py +++ b/src/neuroconv/tools/hdmf.py @@ -50,6 +50,7 @@ def estimate_default_chunk_shape(chunk_mb: float, maxshape: tuple[int, ...], dty def estimate_default_buffer_shape( buffer_gb: float, chunk_shape: tuple[int, ...], maxshape: tuple[int, ...], dtype: np.dtype ) -> tuple[int, ...]: + # TODO: Ad ddocstring to this once someone understands it better # Elevate any overflow warnings to trigger error. # This is usually an indicator of something going terribly wrong with the estimation calculations and should be # avoided at all costs. diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 5a96b4b68..0350806bb 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -57,31 +57,70 @@ def get_source_schema(cls) -> dict: def __init__(self, event_times: Optional[ArrayType] = None): """ - Define event times for some behavior. + Initialize the interface with event times for behavior. Parameters ---------- event_times : list of floats, optional The event times to set as timestamps for this interface. - The default is the array [1.2, 2.3, 3.4] for similarity to the timescale of the MockSpikeGLXNIDQInterface. + The default is the array [1.2, 2.3, 3.4] to simulate a time series similar to the + MockSpikeGLXNIDQInterface. """ event_times = event_times or [1.2, 2.3, 3.4] self.event_times = np.array(event_times) self.original_event_times = np.array(event_times) # Make a copy of the initial loaded timestamps def get_original_timestamps(self) -> np.ndarray: + """ + Get the original event times before any alignment or transformation. + + Returns + ------- + np.ndarray + The original event times as a NumPy array. + """ return self.original_event_times def get_timestamps(self) -> np.ndarray: + """ + Get the current (possibly aligned) event times. + + Returns + ------- + np.ndarray + The current event times as a NumPy array, possibly modified after alignment. + """ return self.event_times def set_aligned_timestamps(self, aligned_timestamps: np.ndarray): + """ + Set the event times after alignment. + + Parameters + ---------- + aligned_timestamps : np.ndarray + The aligned event timestamps to update the internal event times. + """ self.event_times = aligned_timestamps def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict): + """ + Add the event times to an NWBFile as a DynamicTable. + + Parameters + ---------- + nwbfile : NWBFile + The NWB file to which the event times will be added. + metadata : dict + Metadata to describe the event times in the NWB file. + + Notes + ----- + This method creates a DynamicTable to store event times and adds it to the NWBFile's acquisition. + """ table = DynamicTable(name="BehaviorEvents", description="Times of various classified behaviors.") table.add_column(name="event_time", description="Time of each event.") - for timestamp in self.get_timestamps(): # adding data by column gives error + for timestamp in self.get_timestamps(): table.add_row(event_time=timestamp) nwbfile.add_acquisition(table) @@ -182,6 +221,9 @@ def __init__( ) def get_metadata(self) -> dict: + """ + Returns the metadata dictionary for the current object. + """ metadata = super().get_metadata() session_start_time = datetime.now().astimezone() metadata["NWBFile"]["session_start_time"] = session_start_time @@ -229,7 +271,7 @@ def __init__( verbose=verbose, ) - def get_metadata(self) -> dict: # noqa D102 + def get_metadata(self) -> dict: metadata = super().get_metadata() session_start_time = datetime.now().astimezone() metadata["NWBFile"]["session_start_time"] = session_start_time diff --git a/src/neuroconv/utils/dict.py b/src/neuroconv/utils/dict.py index f0507b653..a6cef630a 100644 --- a/src/neuroconv/utils/dict.py +++ b/src/neuroconv/utils/dict.py @@ -209,12 +209,29 @@ class DeepDict(defaultdict): """A defaultdict of defaultdicts""" def __init__(self, *args: Any, **kwargs: Any) -> None: + """A defaultdict of defaultdicts""" super().__init__(lambda: DeepDict(), *args, **kwargs) for key, value in self.items(): if isinstance(value, dict): self[key] = DeepDict(value) def deep_update(self, other: Optional[Union[dict, "DeepDict"]] = None, **kwargs) -> None: + """ + Recursively update the DeepDict with another dictionary or DeepDict. + + Parameters + ---------- + other : dict or DeepDict, optional + The dictionary or DeepDict to update the current instance with. + **kwargs : Any + Additional keyword arguments representing key-value pairs to update the DeepDict. + + Notes + ----- + For any keys that exist in both the current instance and the provided dictionary, the values are merged + recursively if both are dictionaries. Otherwise, the value from `other` or `kwargs` will overwrite the + existing value. + """ for key, value in (other or kwargs).items(): if key in self and isinstance(self[key], dict) and isinstance(value, dict): self[key].deep_update(value) From 05423d16f0d55baf6b2b2679f89eddc881a6581c Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 13 Dec 2024 13:25:51 -0600 Subject: [PATCH 099/118] Soft deprecate `file_path` for `SpikeGLXRecordingInterface` and `SpikeGLXNIDQInterface` (#1155) Co-authored-by: Ben Dichter --- CHANGELOG.md | 8 +++----- .../combinations/spikeglx_and_phy.rst | 4 ++-- .../recording/spikeglx.rst | 7 ++++--- .../ecephys/spikeglx/spikeglxdatainterface.py | 10 ++++++++++ .../ecephys/spikeglx/spikeglxnidqinterface.py | 13 +++++++++++-- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11dce4e7a..9e173d0e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Deprecations * Removed use of `jsonschema.RefResolver` as it will be deprecated from the jsonschema library [PR #1133](https://github.com/catalystneuro/neuroconv/pull/1133) -* Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) +* Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) +* Soft deprecation for `file_path` as an argument of `SpikeGLXNIDQInterface` and `SpikeGLXRecordingInterface` [PR #1155](https://github.com/catalystneuro/neuroconv/pull/1155) ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) @@ -11,7 +12,6 @@ * `SpikeGLXConverterPipe` converter now accepts multi-probe structures with multi-trigger and does not assume a specific folder structure [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) * `SpikeGLXNIDQInterface` is no longer written as an ElectricalSeries [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) - ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) @@ -22,14 +22,12 @@ * YAML specification files now accepts an outer keyword `upload_to_dandiset="< six-digit ID >"` to automatically upload the produced NWB files to the DANDI archive [PR #1089](https://github.com/catalystneuro/neuroconv/pull/1089) *`SpikeGLXNIDQInterface` now handdles digital demuxed channels (`XD0`) [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) - - - ## Improvements * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) * Use pytest format for dandi tests to avoid window permission error on teardown [PR #1151](https://github.com/catalystneuro/neuroconv/pull/1151) * Added many docstrings for public functions [PR #1063](https://github.com/catalystneuro/neuroconv/pull/1063) + # v0.6.5 (November 1, 2024) ## Bug Fixes diff --git a/docs/conversion_examples_gallery/combinations/spikeglx_and_phy.rst b/docs/conversion_examples_gallery/combinations/spikeglx_and_phy.rst index 2c6de07f6..d8eef8844 100644 --- a/docs/conversion_examples_gallery/combinations/spikeglx_and_phy.rst +++ b/docs/conversion_examples_gallery/combinations/spikeglx_and_phy.rst @@ -16,8 +16,8 @@ were are combining a SpikeGLX recording with Phy sorting results using the >>> from neuroconv.datainterfaces import SpikeGLXRecordingInterface, PhySortingInterface >>> >>> # For this interface we need to pass the location of the ``.bin`` file. Change the file_path to the location in your system - >>> file_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin" - >>> interface_spikeglx = SpikeGLXRecordingInterface(file_path=file_path, verbose=False) + >>> folder_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0" + >>> interface_spikeglx = SpikeGLXRecordingInterface(folder_path=folder_path, stream_id="imec0.ap", verbose=False) >>> >>> folder_path = f"{ECEPHY_DATA_PATH}/phy/phy_example_0" # Change the folder_path to the location of the data in your system >>> interface_phy = PhySortingInterface(folder_path=folder_path, verbose=False) diff --git a/docs/conversion_examples_gallery/recording/spikeglx.rst b/docs/conversion_examples_gallery/recording/spikeglx.rst index 97b23bac9..0bc67fc1d 100644 --- a/docs/conversion_examples_gallery/recording/spikeglx.rst +++ b/docs/conversion_examples_gallery/recording/spikeglx.rst @@ -51,9 +51,10 @@ Defining a 'stream' as a single band on a single NeuroPixels probe, we can conve >>> from neuroconv.datainterfaces import SpikeGLXRecordingInterface >>> >>> # For this interface we need to pass the location of the ``.bin`` file - >>> file_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin" - >>> # Change the file_path to the location in your system - >>> interface = SpikeGLXRecordingInterface(file_path=file_path, verbose=False) + >>> folder_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0" + >>> # Options for the streams are "imec0.ap", "imec0.lf", "imec1.ap", "imec1.lf", etc. + >>> # Depending on the device and the band of interest, choose the appropriate stream + >>> interface = SpikeGLXRecordingInterface(folder_path=folder_path, stream_id="imec0.ap", verbose=False) >>> >>> # Extract what metadata we can from the source files >>> metadata = interface.get_metadata() diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index e8b6a78c9..e375ec351 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -1,5 +1,6 @@ """DataInterfaces for SpikeGLX.""" +import warnings from pathlib import Path from typing import Optional @@ -79,6 +80,15 @@ def __init__( "SpikeGLXRecordingInterface is not designed to handle nidq files. Use SpikeGLXNIDQInterface instead" ) + if file_path is not None: + warnings.warn( + "file_path is deprecated and will be removed by the end of 2025. " + "The first argument of this interface will be `folder_path` afterwards. " + "Use folder_path and stream_id instead.", + DeprecationWarning, + stacklevel=2, + ) + if file_path is not None and stream_id is None: self.stream_id = fetch_stream_id_for_spikelgx_file(file_path) self.folder_path = Path(file_path).parent diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 5249dfe39..4c7e5a6d9 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -36,7 +36,7 @@ def __init__( self, file_path: Optional[FilePath] = None, verbose: bool = True, - load_sync_channel: bool = False, + load_sync_channel: Optional[bool] = None, es_key: str = "ElectricalSeriesNIDQ", folder_path: Optional[DirectoryPath] = None, ): @@ -56,7 +56,7 @@ def __init__( es_key : str, default: "ElectricalSeriesNIDQ" """ - if load_sync_channel: + if load_sync_channel is not None: warnings.warn( "The 'load_sync_channel' parameter is deprecated and will be removed in June 2025. " @@ -65,6 +65,15 @@ def __init__( stacklevel=2, ) + if file_path is not None: + warnings.warn( + "file_path is deprecated and will be removed by the end of 2025. " + "The first argument of this interface will be `folder_path` afterwards. " + "Use folder_path and stream_id instead.", + DeprecationWarning, + stacklevel=2, + ) + if file_path is None and folder_path is None: raise ValueError("Either 'file_path' or 'folder_path' must be provided.") From 160c5c195c700ac2fd81f7989038768d87b42c49 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 16 Dec 2024 15:47:09 -0600 Subject: [PATCH 100/118] Ecephys warnings removal and clean up (#1158) --- CHANGELOG.md | 3 + .../sorting/blackrock.rst | 2 +- .../sorting/neuralynx.rst | 3 +- .../blackrock/blackrockdatainterface.py | 4 +- .../neuralynx/neuralynxdatainterface.py | 15 +- .../tools/spikeinterface/spikeinterface.py | 10 ++ .../tools/testing/data_interface_mixins.py | 22 +-- .../tools/testing/mock_interfaces.py | 4 + tests/test_ecephys/test_ecephys_interfaces.py | 139 +++++------------- .../test_configure_backend_defaults.py | 6 +- .../test_configure_backend_equivalency.py | 2 +- .../test_configure_backend_overrides.py | 4 +- ...test_configure_backend_zero_length_axes.py | 2 +- ...ataset_io_configurations_appended_files.py | 2 +- tests/test_on_data/ecephys/test_lfp.py | 2 +- .../ecephys/test_raw_recordings.py | 6 +- .../ecephys/test_sorting_interfaces.py | 10 +- .../ecephys/test_spikeglx_converter.py | 4 +- .../ecephys/test_spikeglx_metadata.py | 22 +-- 19 files changed, 103 insertions(+), 159 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e173d0e4..68f5c9868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ * Removed use of `jsonschema.RefResolver` as it will be deprecated from the jsonschema library [PR #1133](https://github.com/catalystneuro/neuroconv/pull/1133) * Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) * Soft deprecation for `file_path` as an argument of `SpikeGLXNIDQInterface` and `SpikeGLXRecordingInterface` [PR #1155](https://github.com/catalystneuro/neuroconv/pull/1155) +* `starting_time` in RecordingInterfaces has given a soft deprecation in favor of time alignment methods [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) + ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) @@ -26,6 +28,7 @@ * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) * Use pytest format for dandi tests to avoid window permission error on teardown [PR #1151](https://github.com/catalystneuro/neuroconv/pull/1151) * Added many docstrings for public functions [PR #1063](https://github.com/catalystneuro/neuroconv/pull/1063) +* Clean up with warnings and deprecations in the testing framework [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) # v0.6.5 (November 1, 2024) diff --git a/docs/conversion_examples_gallery/sorting/blackrock.rst b/docs/conversion_examples_gallery/sorting/blackrock.rst index 859a2867e..fdbf299f8 100644 --- a/docs/conversion_examples_gallery/sorting/blackrock.rst +++ b/docs/conversion_examples_gallery/sorting/blackrock.rst @@ -19,7 +19,7 @@ Convert Blackrock sorting data to NWB using >>> >>> file_path = f"{ECEPHY_DATA_PATH}/blackrock/FileSpec2.3001.nev" >>> # Change the file_path to the location of the file in your system - >>> interface = BlackrockSortingInterface(file_path=file_path, verbose=False) + >>> interface = BlackrockSortingInterface(file_path=file_path, sampling_frequency=30000.0, verbose=False) >>> >>> # Extract what metadata we can from the source files >>> metadata = interface.get_metadata() diff --git a/docs/conversion_examples_gallery/sorting/neuralynx.rst b/docs/conversion_examples_gallery/sorting/neuralynx.rst index 96c3f4f21..c1015667f 100644 --- a/docs/conversion_examples_gallery/sorting/neuralynx.rst +++ b/docs/conversion_examples_gallery/sorting/neuralynx.rst @@ -20,7 +20,8 @@ Convert Neuralynx data to NWB using >>> >>> folder_path = f"{ECEPHY_DATA_PATH}/neuralynx/Cheetah_v5.5.1/original_data" >>> # Change the folder_path to the location of the data in your system - >>> interface = NeuralynxSortingInterface(folder_path=folder_path, verbose=False) + >>> # The stream is optional but is used to specify the sampling frequency of the data + >>> interface = NeuralynxSortingInterface(folder_path=folder_path, verbose=False, stream_id="0") >>> >>> metadata = interface.get_metadata() >>> session_start_time = datetime(2020, 1, 1, 12, 30, 0, tzinfo=ZoneInfo("US/Pacific")).isoformat() diff --git a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py index 7e6e499d1..bc9f9b51b 100644 --- a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py @@ -98,8 +98,8 @@ def __init__(self, file_path: FilePath, sampling_frequency: Optional[float] = No The file path to the ``.nev`` data sampling_frequency: float, optional The sampling frequency for the sorting extractor. When the signal data is available (.ncs) those files will be - used to extract the frequency automatically. Otherwise, the sampling frequency needs to be specified for - this extractor to be initialized. + used to extract the frequency automatically. Otherwise, the sampling frequency needs to be specified for + this extractor to be initialized. verbose : bool, default: True Enables verbosity """ diff --git a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py index 446c06302..d87c8f0bd 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py @@ -112,7 +112,13 @@ class NeuralynxSortingInterface(BaseSortingExtractorInterface): associated_suffixes = (".nse", ".ntt", ".nse", ".nev") info = "Interface for Neuralynx sorting data." - def __init__(self, folder_path: DirectoryPath, sampling_frequency: Optional[float] = None, verbose: bool = True): + def __init__( + self, + folder_path: DirectoryPath, + sampling_frequency: Optional[float] = None, + verbose: bool = True, + stream_id: Optional[str] = None, + ): """_summary_ Parameters @@ -123,9 +129,14 @@ def __init__(self, folder_path: DirectoryPath, sampling_frequency: Optional[floa If a specific sampling_frequency is desired it can be set with this argument. verbose : bool, default: True Enables verbosity + stream_id: str, optional + Used by Spikeinterface and neo to calculate the t_start, if not provided and the stream is unique + it will be chosen automatically """ - super().__init__(folder_path=folder_path, sampling_frequency=sampling_frequency, verbose=verbose) + super().__init__( + folder_path=folder_path, sampling_frequency=sampling_frequency, stream_id=stream_id, verbose=verbose + ) def extract_neo_header_metadata(neo_reader) -> dict: diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index fa00d58ed..d31e6032c 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -862,6 +862,16 @@ def add_electrical_series_to_nwbfile( whenever possible. """ + if starting_time is not None: + warnings.warn( + "The 'starting_time' parameter is deprecated and will be removed in June 2025. " + "Use the time alignment methods or set the recording times directlyfor modifying the starting time or timestamps " + "of the data if needed: " + "https://neuroconv.readthedocs.io/en/main/user_guide/temporal_alignment.html", + DeprecationWarning, + stacklevel=2, + ) + assert write_as in [ "raw", "processed", diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index dc45cec53..d8586f44f 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -1,4 +1,3 @@ -import inspect import json import tempfile from abc import abstractmethod @@ -407,7 +406,7 @@ def check_read_nwb(self, nwbfile_path: str): # Spikeinterface behavior is to load the electrode table channel_name property as a channel_id self.nwb_recording = NwbRecordingExtractor( file_path=nwbfile_path, - electrical_series_name=electrical_series_name, + electrical_series_path=f"acquisition/{electrical_series_name}", use_pynwb=True, ) @@ -439,7 +438,7 @@ def check_read_nwb(self, nwbfile_path: str): assert_array_equal( recording.get_property(property_name), self.nwb_recording.get_property(property_name) ) - if recording.has_scaled_traces() and self.nwb_recording.has_scaled_traces(): + if recording.has_scaleable_traces() and self.nwb_recording.has_scaleable_traces(): check_recordings_equal(RX1=recording, RX2=self.nwb_recording, return_scaled=True) # Compare channel groups @@ -625,29 +624,22 @@ def check_read_nwb(self, nwbfile_path: str): # NWBSortingExtractor on spikeinterface does not yet support loading data written from multiple segment. if sorting.get_num_segments() == 1: - # TODO after 0.100 release remove this if - signature = inspect.signature(NwbSortingExtractor) - if "t_start" in signature.parameters: - nwb_sorting = NwbSortingExtractor(file_path=nwbfile_path, sampling_frequency=sf, t_start=0.0) - else: - nwb_sorting = NwbSortingExtractor(file_path=nwbfile_path, sampling_frequency=sf) + nwb_sorting = NwbSortingExtractor(file_path=nwbfile_path, sampling_frequency=sf, t_start=0.0) + # In the NWBSortingExtractor, since unit_names could be not unique, # table "ids" are loaded as unit_ids. Here we rename the original sorting accordingly if "unit_name" in sorting.get_property_keys(): renamed_unit_ids = sorting.get_property("unit_name") - # sorting_renamed = sorting.rename_units(new_unit_ids=renamed_unit_ids) #TODO after 0.100 release use this - sorting_renamed = sorting.select_units(unit_ids=sorting.unit_ids, renamed_unit_ids=renamed_unit_ids) + sorting_renamed = sorting.rename_units(new_unit_ids=renamed_unit_ids) else: nwb_has_ids_as_strings = all(isinstance(id, str) for id in nwb_sorting.unit_ids) if nwb_has_ids_as_strings: - renamed_unit_ids = sorting.get_unit_ids() - renamed_unit_ids = [str(id) for id in renamed_unit_ids] + renamed_unit_ids = [str(id) for id in sorting.get_unit_ids()] else: renamed_unit_ids = np.arange(len(sorting.unit_ids)) - # sorting_renamed = sorting.rename_units(new_unit_ids=sorting.unit_ids) #TODO after 0.100 release use this - sorting_renamed = sorting.select_units(unit_ids=sorting.unit_ids, renamed_unit_ids=renamed_unit_ids) + sorting_renamed = sorting.rename_units(new_unit_ids=renamed_unit_ids) check_sortings_equal(SX1=sorting_renamed, SX2=nwb_sorting) def check_interface_set_aligned_segment_timestamps(self): diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 0350806bb..38cc750ab 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -271,6 +271,10 @@ def __init__( verbose=verbose, ) + # Sorting extractor to have string unit ids until is changed in SpikeInterface + string_unit_ids = [str(id) for id in self.sorting_extractor.unit_ids] + self.sorting_extractor = self.sorting_extractor.rename_units(new_unit_ids=string_unit_ids) + def get_metadata(self) -> dict: metadata = super().get_metadata() session_start_time = datetime.now().astimezone() diff --git a/tests/test_ecephys/test_ecephys_interfaces.py b/tests/test_ecephys/test_ecephys_interfaces.py index e036ccb81..168ce5068 100644 --- a/tests/test_ecephys/test_ecephys_interfaces.py +++ b/tests/test_ecephys/test_ecephys_interfaces.py @@ -1,24 +1,12 @@ -import shutil -import unittest -from datetime import datetime -from pathlib import Path from platform import python_version as get_python_version -from tempfile import mkdtemp -from warnings import warn import jsonschema import numpy as np import pytest from hdmf.testing import TestCase from packaging.version import Version -from pynwb import NWBHDF5IO -from spikeinterface.extractors import NumpySorting -from neuroconv import NWBConverter from neuroconv.datainterfaces import Spike2RecordingInterface -from neuroconv.datainterfaces.ecephys.basesortingextractorinterface import ( - BaseSortingExtractorInterface, -) from neuroconv.tools.nwb_helpers import get_module from neuroconv.tools.testing.mock_interfaces import ( MockRecordingInterface, @@ -54,6 +42,45 @@ def test_propagate_conversion_options(self, setup_interface): assert nwbfile.units is None assert "processed_units" in ecephys.data_interfaces + def test_stub(self): + + interface = MockSortingInterface(num_units=4, durations=[1.0]) + sorting_extractor = interface.sorting_extractor + unit_ids = sorting_extractor.unit_ids + first_unit_spike = { + unit_id: sorting_extractor.get_unit_spike_train(unit_id=unit_id, return_times=True)[0] + for unit_id in unit_ids + } + + nwbfile = interface.create_nwbfile(stub_test=True) + units_table = nwbfile.units.to_dataframe() + + for unit_id, first_spike_time in first_unit_spike.items(): + unit_row = units_table[units_table["unit_name"] == unit_id] + unit_spike_times = unit_row["spike_times"].values[0] + np.testing.assert_almost_equal(unit_spike_times[0], first_spike_time, decimal=6) + + def test_stub_with_recording(self): + interface = MockSortingInterface(num_units=4, durations=[1.0]) + + recording_interface = MockRecordingInterface(num_channels=4, durations=[2.0]) + interface.register_recording(recording_interface) + + sorting_extractor = interface.sorting_extractor + unit_ids = sorting_extractor.unit_ids + first_unit_spike = { + unit_id: sorting_extractor.get_unit_spike_train(unit_id=unit_id, return_times=True)[0] + for unit_id in unit_ids + } + + nwbfile = interface.create_nwbfile(stub_test=True) + units_table = nwbfile.units.to_dataframe() + + for unit_id, first_spike_time in first_unit_spike.items(): + unit_row = units_table[units_table["unit_name"] == unit_id] + unit_spike_times = unit_row["spike_times"].values[0] + np.testing.assert_almost_equal(unit_spike_times[0], first_spike_time, decimal=6) + def test_electrode_indices(self, setup_interface): recording_interface = MockRecordingInterface(num_channels=4, durations=[0.100]) @@ -136,91 +163,3 @@ def test_spike2_import_assertions_3_11(self): exc_msg="\nThe package 'sonpy' is not available for Python version 3.11!", ): Spike2RecordingInterface.get_all_channels_info(file_path="does_not_matter.smrx") - - -class TestSortingInterfaceOld(unittest.TestCase): - """Old-style tests for the SortingInterface. Remove once we we are sure all the behaviors are covered by the mock.""" - - @classmethod - def setUpClass(cls) -> None: - cls.test_dir = Path(mkdtemp()) - cls.sorting_start_frames = [100, 200, 300] - cls.num_frames = 1000 - cls.sampling_frequency = 3000.0 - times = np.array([], dtype="int") - labels = np.array([], dtype="int") - for i, start_frame in enumerate(cls.sorting_start_frames): - times_i = np.arange(start_frame, cls.num_frames, dtype="int") - labels_i = (i + 1) * np.ones_like(times_i, dtype="int") - times = np.concatenate((times, times_i)) - labels = np.concatenate((labels, labels_i)) - sorting = NumpySorting.from_times_labels(times, labels, sampling_frequency=cls.sampling_frequency) - - class TestSortingInterface(BaseSortingExtractorInterface): - ExtractorName = "NumpySorting" - - def __init__(self, verbose: bool = True): - self.sorting_extractor = sorting - self.source_data = dict() - self.verbose = verbose - - class TempConverter(NWBConverter): - data_interface_classes = dict(TestSortingInterface=TestSortingInterface) - - source_data = dict(TestSortingInterface=dict()) - cls.test_sorting_interface = TempConverter(source_data) - - @classmethod - def tearDownClass(cls): - try: - shutil.rmtree(cls.test_dir) - except PermissionError: # Windows CI bug - warn(f"Unable to fully clean the temporary directory: {cls.test_dir}\n\nPlease remove it manually.") - - def test_sorting_stub(self): - minimal_nwbfile = self.test_dir / "stub_temp.nwb" - conversion_options = dict(TestSortingInterface=dict(stub_test=True)) - metadata = self.test_sorting_interface.get_metadata() - metadata["NWBFile"]["session_start_time"] = datetime.now().astimezone() - self.test_sorting_interface.run_conversion( - nwbfile_path=minimal_nwbfile, metadata=metadata, conversion_options=conversion_options - ) - with NWBHDF5IO(minimal_nwbfile, "r") as io: - nwbfile = io.read() - start_frame_max = np.max(self.sorting_start_frames) - for i, start_times in enumerate(self.sorting_start_frames): - assert len(nwbfile.units["spike_times"][i]) == (start_frame_max * 1.1) - start_times - - def test_sorting_stub_with_recording(self): - subset_end_frame = int(np.max(self.sorting_start_frames) * 1.1 - 1) - sorting_interface = self.test_sorting_interface.data_interface_objects["TestSortingInterface"] - sorting_interface.sorting_extractor = sorting_interface.sorting_extractor.frame_slice( - start_frame=0, end_frame=subset_end_frame - ) - recording_interface = MockRecordingInterface( - durations=[subset_end_frame / self.sampling_frequency], - sampling_frequency=self.sampling_frequency, - ) - sorting_interface.register_recording(recording_interface) - - minimal_nwbfile = self.test_dir / "stub_temp_recording.nwb" - conversion_options = dict(TestSortingInterface=dict(stub_test=True)) - metadata = self.test_sorting_interface.get_metadata() - metadata["NWBFile"]["session_start_time"] = datetime.now().astimezone() - self.test_sorting_interface.run_conversion( - nwbfile_path=minimal_nwbfile, metadata=metadata, conversion_options=conversion_options - ) - with NWBHDF5IO(minimal_nwbfile, "r") as io: - nwbfile = io.read() - for i, start_times in enumerate(self.sorting_start_frames): - assert len(nwbfile.units["spike_times"][i]) == subset_end_frame - start_times - - def test_sorting_full(self): - minimal_nwbfile = self.test_dir / "temp.nwb" - metadata = self.test_sorting_interface.get_metadata() - metadata["NWBFile"]["session_start_time"] = datetime.now().astimezone() - self.test_sorting_interface.run_conversion(nwbfile_path=minimal_nwbfile, metadata=metadata) - with NWBHDF5IO(minimal_nwbfile, "r") as io: - nwbfile = io.read() - for i, start_times in enumerate(self.sorting_start_frames): - assert len(nwbfile.units["spike_times"][i]) == self.num_frames - start_times diff --git a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_defaults.py b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_defaults.py index 693d8e4d0..a2f267c5d 100644 --- a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_defaults.py +++ b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_defaults.py @@ -66,7 +66,7 @@ def test_simple_time_series( dataset_configuration = backend_configuration.dataset_configurations["acquisition/TestTimeSeries/data"] configure_backend(nwbfile=nwbfile, backend_configuration=backend_configuration) - nwbfile_path = str(tmpdir / f"test_configure_defaults_{case_name}_time_series.nwb.{backend}") + nwbfile_path = str(tmpdir / f"test_configure_defaults_{case_name}_time_series.nwb") with BACKEND_NWB_IO[backend](path=nwbfile_path, mode="w") as io: io.write(nwbfile) @@ -98,7 +98,7 @@ def test_simple_dynamic_table(tmpdir: Path, integer_array: np.ndarray, backend: dataset_configuration = backend_configuration.dataset_configurations["acquisition/TestDynamicTable/TestColumn/data"] configure_backend(nwbfile=nwbfile, backend_configuration=backend_configuration) - nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb.{backend}") + nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb") NWB_IO = BACKEND_NWB_IO[backend] with NWB_IO(path=nwbfile_path, mode="w") as io: io.write(nwbfile) @@ -164,7 +164,7 @@ def test_time_series_timestamps_linkage( assert nwbfile.acquisition["TestTimeSeries1"].timestamps assert nwbfile.acquisition["TestTimeSeries2"].timestamps - nwbfile_path = str(tmpdir / f"test_time_series_timestamps_linkage_{case_name}_data.nwb.{backend}") + nwbfile_path = str(tmpdir / f"test_time_series_timestamps_linkage_{case_name}_data.nwb") with BACKEND_NWB_IO[backend](path=nwbfile_path, mode="w") as io: io.write(nwbfile) diff --git a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_equivalency.py b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_equivalency.py index dd938721f..225e0af66 100644 --- a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_equivalency.py +++ b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_equivalency.py @@ -69,7 +69,7 @@ def test_configure_backend_equivalency( dataset_configuration.compression_options = {"level": 2} configure_backend(nwbfile=nwbfile_1, backend_configuration=backend_configuration_2) - nwbfile_path = str(tmpdir / f"test_configure_backend_equivalency.nwb.{backend}") + nwbfile_path = str(tmpdir / f"test_configure_backend_equivalency.nwb") with BACKEND_NWB_IO[backend](path=nwbfile_path, mode="w") as io: io.write(nwbfile_1) diff --git a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_overrides.py b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_overrides.py index dba8bb891..15cc9ee48 100644 --- a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_overrides.py +++ b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_overrides.py @@ -58,7 +58,7 @@ def test_simple_time_series_override( if case_name != "unwrapped": # TODO: eventually, even this case will be buffered automatically assert nwbfile.acquisition["TestTimeSeries"].data - nwbfile_path = str(tmpdir / f"test_configure_defaults_{case_name}_data.nwb.{backend}") + nwbfile_path = str(tmpdir / f"test_configure_defaults_{case_name}_data.nwb") with BACKEND_NWB_IO[backend](path=nwbfile_path, mode="w") as io: io.write(nwbfile) @@ -99,7 +99,7 @@ def test_simple_dynamic_table_override(tmpdir: Path, backend: Literal["hdf5", "z configure_backend(nwbfile=nwbfile, backend_configuration=backend_configuration) - nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb.{backend}") + nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb") NWB_IO = BACKEND_NWB_IO[backend] with NWB_IO(path=nwbfile_path, mode="w") as io: io.write(nwbfile) diff --git a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_zero_length_axes.py b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_zero_length_axes.py index 39980f22e..c0c63d6c7 100644 --- a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_zero_length_axes.py +++ b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_configure_backend_zero_length_axes.py @@ -114,7 +114,7 @@ def test_dynamic_table_skip_zero_length_axis( dataset_configuration = backend_configuration.dataset_configurations["acquisition/TestDynamicTable/TestColumn/data"] configure_backend(nwbfile=nwbfile, backend_configuration=backend_configuration) - nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb.{backend}") + nwbfile_path = str(tmpdir / f"test_configure_defaults_dynamic_table.nwb") NWB_IO = BACKEND_NWB_IO[backend] with NWB_IO(path=nwbfile_path, mode="w") as io: io.write(nwbfile) diff --git a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_get_default_dataset_io_configurations_appended_files.py b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_get_default_dataset_io_configurations_appended_files.py index dca727f03..0dfd4650f 100644 --- a/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_get_default_dataset_io_configurations_appended_files.py +++ b/tests/test_minimal/test_tools/test_backend_and_dataset_configuration/test_helpers/test_get_default_dataset_io_configurations_appended_files.py @@ -31,7 +31,7 @@ def generate_nwbfile_with_existing_time_series() -> NWBFile: @pytest.fixture(scope="session") def hdf5_nwbfile_path(tmpdir_factory): - nwbfile_path = tmpdir_factory.mktemp("data").join("test_default_dataset_configurations_hdf5_nwbfile_.nwb.h5") + nwbfile_path = tmpdir_factory.mktemp("data").join("test_default_dataset_configurations_hdf5_nwbfile_.nwb") if not Path(nwbfile_path).exists(): nwbfile = generate_nwbfile_with_existing_time_series() with NWBHDF5IO(path=str(nwbfile_path), mode="w") as io: diff --git a/tests/test_on_data/ecephys/test_lfp.py b/tests/test_on_data/ecephys/test_lfp.py index 010516d86..1c537238f 100644 --- a/tests/test_on_data/ecephys/test_lfp.py +++ b/tests/test_on_data/ecephys/test_lfp.py @@ -99,7 +99,7 @@ class TestConverter(NWBConverter): npt.assert_array_equal(x=recording.get_traces(return_scaled=False), y=nwb_lfp_unscaled) # This can only be tested if both gain and offset are present - if recording.has_scaled_traces(): + if recording.has_scaleable_traces(): channel_conversion = nwb_lfp_electrical_series.channel_conversion nwb_lfp_conversion_vector = ( channel_conversion[:] diff --git a/tests/test_on_data/ecephys/test_raw_recordings.py b/tests/test_on_data/ecephys/test_raw_recordings.py index f892d6457..08826e3ba 100644 --- a/tests/test_on_data/ecephys/test_raw_recordings.py +++ b/tests/test_on_data/ecephys/test_raw_recordings.py @@ -97,16 +97,14 @@ class TestConverter(NWBConverter): renamed_channel_ids = recording.get_property("channel_name") else: renamed_channel_ids = recording.get_channel_ids().astype("str") - recording = recording.channel_slice( - channel_ids=recording.get_channel_ids(), renamed_channel_ids=renamed_channel_ids - ) + recording = recording.rename_channels(new_channel_ids=renamed_channel_ids) # Edge case that only occurs in testing, but should eventually be fixed nonetheless # The NwbRecordingExtractor on spikeinterface experiences an issue when duplicated channel_ids # are specified, which occurs during check_recordings_equal when there is only one channel if nwb_recording.get_channel_ids()[0] != nwb_recording.get_channel_ids()[-1]: check_recordings_equal(RX1=recording, RX2=nwb_recording, return_scaled=False) - if recording.has_scaled_traces() and nwb_recording.has_scaled_traces(): + if recording.has_scaleable_traces() and nwb_recording.has_scaleable_traces(): check_recordings_equal(RX1=recording, RX2=nwb_recording, return_scaled=True) diff --git a/tests/test_on_data/ecephys/test_sorting_interfaces.py b/tests/test_on_data/ecephys/test_sorting_interfaces.py index 492c4ad9f..054f08391 100644 --- a/tests/test_on_data/ecephys/test_sorting_interfaces.py +++ b/tests/test_on_data/ecephys/test_sorting_interfaces.py @@ -27,7 +27,7 @@ class TestBlackrockSortingInterface(SortingExtractorInterfaceTestMixin): data_interface_cls = BlackrockSortingInterface - interface_kwargs = dict(file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.nev")) + interface_kwargs = dict(file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.nev"), sampling_frequency=30_000.0) associated_recording_cls = BlackrockRecordingInterface associated_recording_kwargs = dict(file_path=str(DATA_PATH / "blackrock" / "FileSpec2.3001.ns5")) @@ -154,13 +154,17 @@ def test_writing_channel_metadata(self, setup_interface): class TestNeuralynxSortingInterfaceCheetahV551(SortingExtractorInterfaceTestMixin): data_interface_cls = NeuralynxSortingInterface - interface_kwargs = dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.5.1" / "original_data")) + interface_kwargs = dict( + folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.5.1" / "original_data"), stream_id="0" + ) save_directory = OUTPUT_PATH class TestNeuralynxSortingInterfaceCheetah563(SortingExtractorInterfaceTestMixin): data_interface_cls = NeuralynxSortingInterface - interface_kwargs = dict(folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data")) + interface_kwargs = dict( + folder_path=str(DATA_PATH / "neuralynx" / "Cheetah_v5.6.3" / "original_data"), stream_id="0" + ) save_directory = OUTPUT_PATH diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index 93b228053..27a8ed0c5 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -212,7 +212,7 @@ def test_electrode_table_writing(tmp_path): # Test round trip with spikeinterface recording_extractor_ap = NwbRecordingExtractor( file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesAP", + electrical_series_path="acquisition/ElectricalSeriesAP", ) channel_ids = recording_extractor_ap.get_channel_ids() @@ -220,7 +220,7 @@ def test_electrode_table_writing(tmp_path): recording_extractor_lf = NwbRecordingExtractor( file_path=nwbfile_path, - electrical_series_name="ElectricalSeriesLF", + electrical_series_path="acquisition/ElectricalSeriesLF", ) channel_ids = recording_extractor_lf.get_channel_ids() diff --git a/tests/test_on_data/ecephys/test_spikeglx_metadata.py b/tests/test_on_data/ecephys/test_spikeglx_metadata.py index 29d4cb5ad..9d57f049d 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_metadata.py +++ b/tests/test_on_data/ecephys/test_spikeglx_metadata.py @@ -1,7 +1,6 @@ import datetime import probeinterface as pi -import pytest from numpy.testing import assert_array_equal from spikeinterface.extractors import SpikeGLXRecordingExtractor @@ -38,7 +37,8 @@ def test_spikelgx_recording_property_addition(): expected_contact_ids = probe.contact_ids # Initialize the interface and get the added properties - interface = SpikeGLXRecordingInterface(file_path=ap_file_path) + folder_path = ap_file_path.parent + interface = SpikeGLXRecordingInterface(folder_path=folder_path, stream_id="imec0.ap") group_name = interface.recording_extractor.get_property("group_name") contact_shapes = interface.recording_extractor.get_property("contact_shapes") shank_ids = interface.recording_extractor.get_property("shank_ids") @@ -48,21 +48,3 @@ def test_spikelgx_recording_property_addition(): assert_array_equal(contact_shapes, expected_contact_shapes) assert_array_equal(shank_ids, expected_shank_ids) assert_array_equal(contact_ids, expected_contact_ids) - - -@pytest.mark.skip(reason="Legacy spikeextractors cannot read new GIN file.") -def test_matching_recording_property_addition_between_backends(): - """Test that the extracted properties match with both backends""" - folder_path = SPIKEGLX_PATH / "Noise4Sam_g0" / "Noise4Sam_g0_imec0" - ap_file_path = folder_path / "Noise4Sam_g0_t0.imec0.ap.bin" - - interface_new = SpikeGLXRecordingInterface(file_path=ap_file_path) - shank_electrode_number_new = interface_new.recording_extractor.get_property("shank_electrode_number") - group_name_new = interface_new.recording_extractor.get_property("group_name") - - interface_old = SpikeGLXRecordingInterface(file_path=ap_file_path, spikeextractors_backend=True) - shank_electrode_number_old = interface_old.recording_extractor.get_property("shank_electrode_number") - group_name_old = interface_old.recording_extractor.get_property("group_name") - - assert_array_equal(shank_electrode_number_new, shank_electrode_number_old) - assert_array_equal(group_name_new, group_name_old) From 2084d350001e8821a5a05bc4e2e4847d72d82199 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 19 Dec 2024 09:52:20 -0600 Subject: [PATCH 101/118] Fix extra electrode groups ephys (#1164) --- CHANGELOG.md | 2 ++ .../ecephys/baserecordingextractorinterface.py | 4 +++- tests/test_ecephys/test_ecephys_interfaces.py | 14 +++++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68f5c9868..7488d5834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ * Fix a bug where data in `DeepLabCutInterface` failed to write when `ndx-pose` was not imported. [#1144](https://github.com/catalystneuro/neuroconv/pull/1144) * `SpikeGLXConverterPipe` converter now accepts multi-probe structures with multi-trigger and does not assume a specific folder structure [#1150](https://github.com/catalystneuro/neuroconv/pull/1150) * `SpikeGLXNIDQInterface` is no longer written as an ElectricalSeries [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) +* Fix a bug on ecephys interfaces where extra electrode group and devices were written if the property of the "group_name" was set in the recording extractor [#1164](https://github.com/catalystneuro/neuroconv/pull/1164) + ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index 6d0df14c1..c9df3ba52 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -88,7 +88,9 @@ def get_metadata_schema(self) -> dict: def get_metadata(self) -> DeepDict: metadata = super().get_metadata() - channel_groups_array = self.recording_extractor.get_channel_groups() + from ...tools.spikeinterface.spikeinterface import _get_group_name + + channel_groups_array = _get_group_name(recording=self.recording_extractor) unique_channel_groups = set(channel_groups_array) if channel_groups_array is not None else ["ElectrodeGroup"] electrode_metadata = [ dict(name=str(group_id), description="no description", location="unknown", device="DeviceEcephys") diff --git a/tests/test_ecephys/test_ecephys_interfaces.py b/tests/test_ecephys/test_ecephys_interfaces.py index 168ce5068..d2fdd68ba 100644 --- a/tests/test_ecephys/test_ecephys_interfaces.py +++ b/tests/test_ecephys/test_ecephys_interfaces.py @@ -119,7 +119,7 @@ def test_electrode_indices_assertion_error_when_missing_table(self, setup_interf class TestRecordingInterface(RecordingExtractorInterfaceTestMixin): data_interface_cls = MockRecordingInterface - interface_kwargs = dict(durations=[0.100]) + interface_kwargs = dict(num_channels=4, durations=[0.100]) def test_stub(self, setup_interface): interface = self.interface @@ -146,6 +146,18 @@ def test_always_write_timestamps(self, setup_interface): expected_timestamps = self.interface.recording_extractor.get_times() np.testing.assert_array_equal(electrical_series.timestamps[:], expected_timestamps) + def test_group_naming_not_adding_extra_devices(self, setup_interface): + + interface = self.interface + recording_extractor = interface.recording_extractor + recording_extractor.set_channel_groups(groups=[0, 1, 2, 3]) + recording_extractor.set_property(key="group_name", values=["group1", "group2", "group3", "group4"]) + + nwbfile = interface.create_nwbfile() + + assert len(nwbfile.devices) == 1 + assert len(nwbfile.electrode_groups) == 4 + class TestAssertions(TestCase): @pytest.mark.skipif(python_version.minor != 10, reason="Only testing with Python 3.10!") From 74ca5b8a059a76e8456785c5592be24b2ca597b0 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 19 Dec 2024 13:17:31 -0600 Subject: [PATCH 102/118] Add missing typing to `NWBConverter` backend and backend options (#1160) --- CHANGELOG.md | 2 +- .../fiberphotometry/tdt_fp.rst | 6 ++---- src/neuroconv/nwbconverter.py | 11 ++++------- tests/test_on_data/ophys/test_miniscope_converter.py | 4 ++-- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7488d5834..389b23697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ * Use pytest format for dandi tests to avoid window permission error on teardown [PR #1151](https://github.com/catalystneuro/neuroconv/pull/1151) * Added many docstrings for public functions [PR #1063](https://github.com/catalystneuro/neuroconv/pull/1063) * Clean up with warnings and deprecations in the testing framework [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) - +* Enhance the typing of the signature on the `NWBConverter` by adding zarr as a literal option on the backend and backend configuration [PR #1160](https://github.com/catalystneuro/neuroconv/pull/1160) # v0.6.5 (November 1, 2024) diff --git a/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst b/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst index 52ceb866c..17fc33e40 100644 --- a/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst +++ b/docs/conversion_examples_gallery/fiberphotometry/tdt_fp.rst @@ -207,15 +207,13 @@ Convert TDT Fiber Photometry data to NWB using >>> LOCAL_PATH = Path(".") # Path to neuroconv >>> editable_metadata_path = LOCAL_PATH / "tests" / "test_on_data" / "ophys" / "fiber_photometry_metadata.yaml" - >>> interface = TDTFiberPhotometryInterface(folder_path=folder_path, verbose=True) - Source data is valid! + >>> interface = TDTFiberPhotometryInterface(folder_path=folder_path, verbose=False) >>> metadata = interface.get_metadata() >>> metadata["NWBFile"]["session_start_time"] = datetime.now(tz=ZoneInfo("US/Pacific")) >>> editable_metadata = load_dict_from_file(editable_metadata_path) >>> metadata = dict_deep_update(metadata, editable_metadata) >>> # Choose a path for saving the nwb file and run the conversion - >>> nwbfile_path = LOCAL_PATH / "example_tdtfp.nwb" + >>> nwbfile_path = f"{path_to_save_nwbfile}" >>> # t1 and t2 are optional arguments to specify the start and end times for the conversion >>> interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, t1=0.0, t2=1.0) - NWB file saved at example_tdtfp.nwb! diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index 2d70cf8ee..ff066ad6b 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -204,11 +204,8 @@ def run_conversion( nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - # TODO: when all H5DataIO prewraps are gone, introduce Zarr safely - # backend: Union[Literal["hdf5", "zarr"]], - # backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None, - backend: Optional[Literal["hdf5"]] = None, - backend_configuration: Optional[HDF5BackendConfiguration] = None, + backend: Optional[Literal["hdf5", "zarr"]] = None, + backend_configuration: Optional[Union[HDF5BackendConfiguration, ZarrBackendConfiguration]] = None, conversion_options: Optional[dict] = None, ) -> None: """ @@ -226,11 +223,11 @@ def run_conversion( overwrite : bool, default: False Whether to overwrite the NWBFile if one exists at the nwbfile_path. The default is False (append mode). - backend : "hdf5", optional + backend : {"hdf5", "zarr"}, optional The type of backend to use when writing the file. If a `backend_configuration` is not specified, the default type will be "hdf5". If a `backend_configuration` is specified, then the type will be auto-detected. - backend_configuration : HDF5BackendConfiguration, optional + backend_configuration : HDF5BackendConfiguration or ZarrBackendConfiguration, optional The configuration model to use when configuring the datasets for this backend. To customize, call the `.get_default_backend_configuration(...)` method, modify the returned BackendConfiguration object, and pass that instead. diff --git a/tests/test_on_data/ophys/test_miniscope_converter.py b/tests/test_on_data/ophys/test_miniscope_converter.py index a1e02ac1d..64acd899f 100644 --- a/tests/test_on_data/ophys/test_miniscope_converter.py +++ b/tests/test_on_data/ophys/test_miniscope_converter.py @@ -159,8 +159,8 @@ def assertNWBFileStructure(self, nwbfile_path: str): nwbfile = io.read() self.assertEqual( - nwbfile.session_start_time, - datetime(2021, 10, 7, 15, 3, 28, 635).astimezone(), + nwbfile.session_start_time.replace(tzinfo=None), + datetime(2021, 10, 7, 15, 3, 28, 635), ) self.assertIn(self.device_name, nwbfile.devices) From 612e6ff459b6eccfac1388beb618c6c8c8fcc478 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 20 Dec 2024 12:49:50 -0600 Subject: [PATCH 103/118] release 0.6.6 --- CHANGELOG.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 389b23697..82e6f2cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ -# v0.6.6 (Upcoming) +# v0.7.0 (Upcoming) +Breaking changes should be here + +# v0.6.9 (Upcoming) +Small fixes should be here. + + +# v0.6.6 (December 20, 2024) ## Deprecations * Removed use of `jsonschema.RefResolver` as it will be deprecated from the jsonschema library [PR #1133](https://github.com/catalystneuro/neuroconv/pull/1133) @@ -6,7 +13,6 @@ * Soft deprecation for `file_path` as an argument of `SpikeGLXNIDQInterface` and `SpikeGLXRecordingInterface` [PR #1155](https://github.com/catalystneuro/neuroconv/pull/1155) * `starting_time` in RecordingInterfaces has given a soft deprecation in favor of time alignment methods [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) - ## Bug Fixes * datetime objects now can be validated as conversion options [#1139](https://github.com/catalystneuro/neuroconv/pull/1126) * Make `NWBMetaDataEncoder` public again [PR #1142](https://github.com/catalystneuro/neuroconv/pull/1142) @@ -15,7 +21,6 @@ * `SpikeGLXNIDQInterface` is no longer written as an ElectricalSeries [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) * Fix a bug on ecephys interfaces where extra electrode group and devices were written if the property of the "group_name" was set in the recording extractor [#1164](https://github.com/catalystneuro/neuroconv/pull/1164) - ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) @@ -33,6 +38,7 @@ * Clean up with warnings and deprecations in the testing framework [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) * Enhance the typing of the signature on the `NWBConverter` by adding zarr as a literal option on the backend and backend configuration [PR #1160](https://github.com/catalystneuro/neuroconv/pull/1160) + # v0.6.5 (November 1, 2024) ## Bug Fixes @@ -55,8 +61,6 @@ * Avoid running link test when the PR is on draft [PR #1093](https://github.com/catalystneuro/neuroconv/pull/1093) * Centralize gin data preparation in a github action [PR #1095](https://github.com/catalystneuro/neuroconv/pull/1095) - - # v0.6.4 (September 17, 2024) ## Bug Fixes @@ -82,11 +86,8 @@ * Consolidated daily workflows into one workflow and added email notifications [PR #1081](https://github.com/catalystneuro/neuroconv/pull/1081) * Added zarr tests for the test on data with checking equivalent backends [PR #1083](https://github.com/catalystneuro/neuroconv/pull/1083) - - # v0.6.3 - # v0.6.2 (September 10, 2024) ## Bug Fixes From b5c59ea40a3581ef0cd91a9b397570c08adcae64 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 20 Dec 2024 12:56:48 -0600 Subject: [PATCH 104/118] post release --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 17 ++++++++++++++++- pyproject.toml | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a6977633..ae61219c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.8.3 hooks: - id: ruff args: [ --fix ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 82e6f2cce..ba0e4068f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,24 @@ # v0.7.0 (Upcoming) -Breaking changes should be here + +## Deprecations + +## Bug Fixes + +## Features + +## Improvements # v0.6.9 (Upcoming) Small fixes should be here. +## Deprecations + +## Bug Fixes + +## Features + +## Improvements + # v0.6.6 (December 20, 2024) diff --git a/pyproject.toml b/pyproject.toml index e318cc9c1..cd92852ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.6" +version = "0.6.7" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ From cb70df3c6d11b41e9702b487b041a3c1df65deb7 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Wed, 8 Jan 2025 02:28:56 +1100 Subject: [PATCH 105/118] Removed requirements.txt files from all datainterfaces (#1171) --- src/neuroconv/datainterfaces/behavior/audio/requirements.txt | 1 - .../datainterfaces/behavior/deeplabcut/requirements.txt | 3 --- src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt | 0 .../datainterfaces/behavior/lightningpose/requirements.txt | 2 -- src/neuroconv/datainterfaces/behavior/medpc/requirements.txt | 1 - .../datainterfaces/behavior/miniscope/requirements.txt | 2 -- src/neuroconv/datainterfaces/behavior/sleap/requirements.txt | 3 --- src/neuroconv/datainterfaces/behavior/video/requirements.txt | 1 - .../datainterfaces/ecephys/cellexplorer/requirements.txt | 2 -- src/neuroconv/datainterfaces/ecephys/edf/requirements.txt | 1 - src/neuroconv/datainterfaces/ecephys/intan/requirements.txt | 0 src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt | 1 - .../datainterfaces/ecephys/neuralynx/requirements.txt | 1 - .../datainterfaces/ecephys/neuroscope/requirements.txt | 1 - .../datainterfaces/ecephys/openephys/requirements.txt | 1 - src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt | 1 - src/neuroconv/datainterfaces/ecephys/requirements.txt | 2 -- src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt | 1 - src/neuroconv/datainterfaces/icephys/abf/requirements.txt | 1 - src/neuroconv/datainterfaces/icephys/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt | 1 - .../datainterfaces/ophys/micromanagertiff/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt | 2 -- src/neuroconv/datainterfaces/ophys/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt | 2 -- src/neuroconv/datainterfaces/ophys/tiff/requirements.txt | 1 - src/neuroconv/datainterfaces/text/excel/requirements.txt | 2 -- 28 files changed, 37 deletions(-) delete mode 100644 src/neuroconv/datainterfaces/behavior/audio/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/medpc/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/sleap/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/video/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/edf/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/intan/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/icephys/abf/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/icephys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/tiff/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/text/excel/requirements.txt diff --git a/src/neuroconv/datainterfaces/behavior/audio/requirements.txt b/src/neuroconv/datainterfaces/behavior/audio/requirements.txt deleted file mode 100644 index 5f5d607ea..000000000 --- a/src/neuroconv/datainterfaces/behavior/audio/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ndx-sound>=0.2.0 diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt b/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt deleted file mode 100644 index 3a5f2fd19..000000000 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -tables>=3.10.1; platform_system == 'Darwin' and python_version >= '3.10' -tables; platform_system != 'Darwin' -ndx-pose==0.1.1 diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt b/src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt b/src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt deleted file mode 100644 index 73aa32089..000000000 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ndx-pose==0.1.1 -neuroconv[video] diff --git a/src/neuroconv/datainterfaces/behavior/medpc/requirements.txt b/src/neuroconv/datainterfaces/behavior/medpc/requirements.txt deleted file mode 100644 index f0f292cab..000000000 --- a/src/neuroconv/datainterfaces/behavior/medpc/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ndx-events==0.2.0 diff --git a/src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt b/src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt deleted file mode 100644 index 90ab7d140..000000000 --- a/src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ndx-miniscope>=0.5.1 -natsort>=8.3.1 diff --git a/src/neuroconv/datainterfaces/behavior/sleap/requirements.txt b/src/neuroconv/datainterfaces/behavior/sleap/requirements.txt deleted file mode 100644 index a62a14965..000000000 --- a/src/neuroconv/datainterfaces/behavior/sleap/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sleap-io>=0.0.2; python_version>='3.9' -sleap-io>=0.0.2,<0.0.12; python_version<'3.9' -av>=10.0.0 diff --git a/src/neuroconv/datainterfaces/behavior/video/requirements.txt b/src/neuroconv/datainterfaces/behavior/video/requirements.txt deleted file mode 100644 index 650f0426f..000000000 --- a/src/neuroconv/datainterfaces/behavior/video/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -opencv-python-headless>=4.8.1.78 diff --git a/src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt b/src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt deleted file mode 100644 index c7417a085..000000000 --- a/src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -hdf5storage>=0.1.18 -pymatreader>=0.0.32 diff --git a/src/neuroconv/datainterfaces/ecephys/edf/requirements.txt b/src/neuroconv/datainterfaces/ecephys/edf/requirements.txt deleted file mode 100644 index 2ca43feaa..000000000 --- a/src/neuroconv/datainterfaces/ecephys/edf/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pyedflib>=0.1.36 diff --git a/src/neuroconv/datainterfaces/ecephys/intan/requirements.txt b/src/neuroconv/datainterfaces/ecephys/intan/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt b/src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt deleted file mode 100644 index 6cc8b3266..000000000 --- a/src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -MEArec>=1.8.0 diff --git a/src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt b/src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt deleted file mode 100644 index 8e45f67d0..000000000 --- a/src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -natsort>=7.1.1 diff --git a/src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt b/src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt deleted file mode 100644 index ae68fd6f2..000000000 --- a/src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -lxml>=4.6.5 diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt b/src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt deleted file mode 100644 index 2378cff71..000000000 --- a/src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -lxml>=4.9.4 diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt b/src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt deleted file mode 100644 index 1f3b3018c..000000000 --- a/src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -zugbruecke >= 0.2.1; platform_system != "Windows" diff --git a/src/neuroconv/datainterfaces/ecephys/requirements.txt b/src/neuroconv/datainterfaces/ecephys/requirements.txt deleted file mode 100644 index bcd36de3f..000000000 --- a/src/neuroconv/datainterfaces/ecephys/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -spikeinterface>=0.101.0 -neo>=0.13.3 diff --git a/src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt b/src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt deleted file mode 100644 index 7c00e91a8..000000000 --- a/src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sonpy>=1.7.1;python_version=='3.9' diff --git a/src/neuroconv/datainterfaces/icephys/abf/requirements.txt b/src/neuroconv/datainterfaces/icephys/abf/requirements.txt deleted file mode 100644 index b459ee3b4..000000000 --- a/src/neuroconv/datainterfaces/icephys/abf/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ndx-dandi-icephys>=0.4.0 diff --git a/src/neuroconv/datainterfaces/icephys/requirements.txt b/src/neuroconv/datainterfaces/icephys/requirements.txt deleted file mode 100644 index 7c4979c99..000000000 --- a/src/neuroconv/datainterfaces/icephys/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -neo>=0.13.2 diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt b/src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt deleted file mode 100644 index 1022c0578..000000000 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tifffile>=2023.3.21 diff --git a/src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt b/src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt deleted file mode 100644 index 1022c0578..000000000 --- a/src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tifffile>=2023.3.21 diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt b/src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt deleted file mode 100644 index 90ab7d140..000000000 --- a/src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ndx-miniscope>=0.5.1 -natsort>=8.3.1 diff --git a/src/neuroconv/datainterfaces/ophys/requirements.txt b/src/neuroconv/datainterfaces/ophys/requirements.txt deleted file mode 100644 index 1aac92108..000000000 --- a/src/neuroconv/datainterfaces/ophys/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -roiextractors>=0.5.7 diff --git a/src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt b/src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt deleted file mode 100644 index 0c734c8f4..000000000 --- a/src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -scanimage-tiff-reader>=1.4.1 diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt b/src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt deleted file mode 100644 index 268db6599..000000000 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -tdt -ndx-fiber-photometry diff --git a/src/neuroconv/datainterfaces/ophys/tiff/requirements.txt b/src/neuroconv/datainterfaces/ophys/tiff/requirements.txt deleted file mode 100644 index 530ee6f2d..000000000 --- a/src/neuroconv/datainterfaces/ophys/tiff/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tiffile>=2018.10.18 diff --git a/src/neuroconv/datainterfaces/text/excel/requirements.txt b/src/neuroconv/datainterfaces/text/excel/requirements.txt deleted file mode 100644 index 95079672f..000000000 --- a/src/neuroconv/datainterfaces/text/excel/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -openpyxl -xlrd From cd7764e699ea77bb279ba7f04c974d5e27f12393 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 14 Jan 2025 09:00:41 -0600 Subject: [PATCH 106/118] Set ceiling for HDMF to avoid failures with the latest release (#1175) --- .gitignore | 7 +++++++ CHANGELOG.md | 1 + pyproject.toml | 2 +- tests/test_on_data/setup_paths.py | 9 ++++----- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 04504e9e0..c1e9ea2b8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,13 @@ __pycache__/ # C extensions *.so + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +uv.lock + # Distribution / packaging .Python build/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ba0e4068f..f20217c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Small fixes should be here. ## Deprecations ## Bug Fixes +* Temporary set a ceiling for hdmf to avoid a chunking bug [PR #1175](https://github.com/catalystneuro/neuroconv/pull/1175) ## Features diff --git a/pyproject.toml b/pyproject.toml index cd92852ec..9debeb5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "PyYAML>=5.4", "scipy>=1.4.1", "h5py>=3.9.0", - "hdmf>=3.13.0", + "hdmf>=3.13.0,<=3.14.5", # Chunking bug "hdmf_zarr>=0.7.0", "pynwb>=2.7.0", "pydantic>=2.0.0", diff --git a/tests/test_on_data/setup_paths.py b/tests/test_on_data/setup_paths.py index 3f7bf4123..939a9d74b 100644 --- a/tests/test_on_data/setup_paths.py +++ b/tests/test_on_data/setup_paths.py @@ -10,7 +10,7 @@ # Load the configuration for the data tests - +project_root_path = Path(__file__).parent.parent.parent if os.getenv("CI"): LOCAL_PATH = Path(".") # Must be set to "." for CI @@ -18,12 +18,11 @@ else: # Override LOCAL_PATH in the `gin_test_config.json` file to a point on your system that contains the dataset folder # Use DANDIHub at hub.dandiarchive.org for open, free use of data found in the /shared/catalystneuro/ directory - test_config_path = Path(__file__).parent / "gin_test_config.json" + test_config_path = project_root_path / "tests" / "test_on_data" / "gin_test_config.json" config_file_exists = test_config_path.exists() if not config_file_exists: - root = test_config_path.parent.parent - base_test_config_path = root / "base_gin_test_config.json" + base_test_config_path = project_root_path / "base_gin_test_config.json" test_config_path.parent.mkdir(parents=True, exist_ok=True) copy(src=base_test_config_path, dst=test_config_path) @@ -40,4 +39,4 @@ ECEPHY_DATA_PATH = LOCAL_PATH / "ephy_testing_data" OPHYS_DATA_PATH = LOCAL_PATH / "ophys_testing_data" -TEXT_DATA_PATH = Path(__file__).parent.parent.parent / "tests" / "test_text" +TEXT_DATA_PATH = project_root_path / "tests" / "test_text" From fe5bfb4a7922a9f7f3ae11fd822dd5d618e2b0ab Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 14 Jan 2025 12:37:20 -0600 Subject: [PATCH 107/118] Remove source validation from converters and interfaces (#1168) --- CHANGELOG.md | 1 + docs/conversion_examples_gallery/behavior/lightningpose.rst | 1 - docs/conversion_examples_gallery/recording/spikeglx.rst | 1 - src/neuroconv/basedatainterface.py | 2 -- src/neuroconv/nwbconverter.py | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f20217c6c..1e7dfc4df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bug Fixes ## Features +* Source validation is no longer performed when initializing interfaces or converters [PR #1168](https://github.com/catalystneuro/neuroconv/pull/1168) ## Improvements diff --git a/docs/conversion_examples_gallery/behavior/lightningpose.rst b/docs/conversion_examples_gallery/behavior/lightningpose.rst index a024a6799..f0b91e804 100644 --- a/docs/conversion_examples_gallery/behavior/lightningpose.rst +++ b/docs/conversion_examples_gallery/behavior/lightningpose.rst @@ -23,7 +23,6 @@ Convert LightningPose pose estimation data to NWB using :py:class:`~neuroconv.da >>> labeled_video_file_path = str(folder_path / "labeled_videos/test_vid_labeled.mp4") >>> converter = LightningPoseConverter(file_path=file_path, original_video_file_path=original_video_file_path, labeled_video_file_path=labeled_video_file_path, verbose=False) - Source data is valid! >>> metadata = converter.get_metadata() >>> # For data provenance we add the time zone information to the conversion >>> session_start_time = metadata["NWBFile"]["session_start_time"] diff --git a/docs/conversion_examples_gallery/recording/spikeglx.rst b/docs/conversion_examples_gallery/recording/spikeglx.rst index 0bc67fc1d..e6ff6bf03 100644 --- a/docs/conversion_examples_gallery/recording/spikeglx.rst +++ b/docs/conversion_examples_gallery/recording/spikeglx.rst @@ -24,7 +24,6 @@ We can easily convert all data stored in the native SpikeGLX folder structure to >>> >>> folder_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0" >>> converter = SpikeGLXConverterPipe(folder_path=folder_path) - Source data is valid! >>> # Extract what metadata we can from the source files >>> metadata = converter.get_metadata() >>> # For data provenance we add the time zone information to the conversion diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index d9e9dc11e..64af908e3 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -68,8 +68,6 @@ def __init__(self, verbose: bool = False, **source_data): self.verbose = verbose self.source_data = source_data - self._validate_source_data(source_data=source_data, verbose=verbose) - def get_metadata_schema(self) -> dict: """Retrieve JSON schema for metadata.""" metadata_schema = load_dict_from_file(Path(__file__).parent / "schemas" / "base_metadata_schema.json") diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index ff066ad6b..5b024f126 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -80,7 +80,6 @@ def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = Tr def __init__(self, source_data: dict[str, dict], verbose: bool = True): """Validate source_data against source_schema and initialize all data interfaces.""" self.verbose = verbose - self._validate_source_data(source_data=source_data, verbose=self.verbose) self.data_interface_objects = { name: data_interface(**source_data[name]) for name, data_interface in self.data_interface_classes.items() From 935bf6a41fb5505deb6b09f8d266494219d7f63d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 14 Jan 2025 15:58:27 -0600 Subject: [PATCH 108/118] Avoid writing extra devices when using `IntanRecordingInterface` with multiple electrode groups (#1167) Co-authored-by: Szonja Weigl --- CHANGELOG.md | 1 + .../ecephys/intan/intandatainterface.py | 4 +-- .../ecephys/test_recording_interfaces.py | 26 +++++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7dfc4df..c48a29283 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Small fixes should be here. ## Features ## Improvements +* Fix metadata bug in `IntanRecordingInterface` where extra devices were added incorrectly if the recording contained multiple electrode groups or names [#1166](https://github.com/catalystneuro/neuroconv/pull/1166) # v0.6.6 (December 20, 2024) diff --git a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py index 2d7c849f2..126bea3b4 100644 --- a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py @@ -91,8 +91,8 @@ def get_metadata(self) -> dict: ecephys_metadata.update(Device=device_list) electrode_group_metadata = ecephys_metadata["ElectrodeGroup"] - electrode_group_metadata[0]["device"] = intan_device["name"] - + for electrode_group in electrode_group_metadata: + electrode_group["device"] = intan_device["name"] # Add electrodes and electrode groups ecephys_metadata.update( ElectricalSeriesRaw=dict(name="ElectricalSeriesRaw", description="Raw acquisition traces."), diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 0520b5b42..8978efa93 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -8,6 +8,7 @@ from numpy.testing import assert_array_equal from packaging import version from pynwb import NWBHDF5IO +from pynwb.testing.mock.file import mock_NWBFile from neuroconv.datainterfaces import ( AlphaOmegaRecordingInterface, @@ -258,8 +259,6 @@ def setup_interface(self, request): def test_devices_written_correctly(self, setup_interface): - from pynwb.testing.mock.file import mock_NWBFile - nwbfile = mock_NWBFile() self.interface.add_to_nwbfile(nwbfile=nwbfile) @@ -268,6 +267,29 @@ def test_devices_written_correctly(self, setup_interface): nwbfile.devices["Intan"].description == "RHD Recording System" + def test_not_adding_extra_devices_when_recording_has_groups(self, setup_interface): + # Test that no extra-devices are added when the recording has groups + + nwbfile = mock_NWBFile() + recording = self.interface.recording_extractor + num_channels = recording.get_num_channels() + channel_groups = np.full(shape=num_channels, fill_value=0, dtype=int) + channel_groups[::2] = 1 # Every other channel is in group 1, the rest are in group 0 + recording.set_channel_groups(groups=channel_groups) + + self.interface.add_to_nwbfile(nwbfile=nwbfile) + assert len(nwbfile.devices) == 1 + + nwbfile = mock_NWBFile() + recording = self.interface.recording_extractor + num_channels = recording.get_num_channels() + group_names = np.full(shape=num_channels, fill_value="A", dtype="str") + group_names[::2] = "B" # Every other channel group is named B, the rest are named A + recording.set_property("group_name", group_names) + + self.interface.add_to_nwbfile(nwbfile=nwbfile) + assert len(nwbfile.devices) == 1 + @pytest.mark.skip(reason="This interface fails to load the necessary plugin sometimes.") class TestMaxOneRecordingInterface(RecordingExtractorInterfaceTestMixin): From a66fd208252a4b2924689404d0bd85255cc89515 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 14 Jan 2025 20:13:09 -0600 Subject: [PATCH 109/118] Detect mismatch errors between group and group names when writing `ElectrodeGroups` (#1165) --- CHANGELOG.md | 1 + .../tools/spikeinterface/spikeinterface.py | 27 ++++++++++++++++++- .../tools/testing/mock_interfaces.py | 7 +++++ .../test_ecephys/test_tools_spikeinterface.py | 24 +++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c48a29283..12c1ac3ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Small fixes should be here. ## Features ## Improvements +* Detect mismatch errors between group and group names when writing ElectrodeGroups [PR #1165](https://github.com/catalystneuro/neuroconv/pull/1165) * Fix metadata bug in `IntanRecordingInterface` where extra devices were added incorrectly if the recording contained multiple electrode groups or names [#1166](https://github.com/catalystneuro/neuroconv/pull/1166) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index d31e6032c..7df25c703 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -245,11 +245,19 @@ def _get_group_name(recording: BaseRecording) -> np.ndarray: An array containing the group names. If the `group_name` property is not available, the channel groups will be returned. If the group names are empty, a default value 'ElectrodeGroup' will be used. + + Raises + ------ + ValueError + If the number of unique group names doesn't match the number of unique groups, + or if the mapping between group names and group numbers is inconsistent. """ default_value = "ElectrodeGroup" group_names = recording.get_property("group_name") + groups = recording.get_channel_groups() + if group_names is None: - group_names = recording.get_channel_groups() + group_names = groups if group_names is None: group_names = np.full(recording.get_num_channels(), fill_value=default_value) @@ -259,6 +267,23 @@ def _get_group_name(recording: BaseRecording) -> np.ndarray: # If for any reason the group names are empty, fill them with the default group_names[group_names == ""] = default_value + # Validate group names against groups + if groups is not None: + unique_groups = set(groups) + unique_names = set(group_names) + + if len(unique_names) != len(unique_groups): + raise ValueError("The number of group names must match the number of groups") + + # Check consistency of group name to group number mapping + group_to_name_map = {} + for group, name in zip(groups, group_names): + if group in group_to_name_map: + if group_to_name_map[group] != name: + raise ValueError("Inconsistent mapping between group numbers and group names") + else: + group_to_name_map[group] = name + return group_names diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 38cc750ab..42bf699df 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -220,6 +220,12 @@ def __init__( es_key=es_key, ) + # Adding this as a safeguard before the spikeinterface changes are merged: + # https://github.com/SpikeInterface/spikeinterface/pull/3588 + channel_ids = self.recording_extractor.get_channel_ids() + channel_ids_as_strings = [str(id) for id in channel_ids] + self.recording_extractor = self.recording_extractor.rename_channels(new_channel_ids=channel_ids_as_strings) + def get_metadata(self) -> dict: """ Returns the metadata dictionary for the current object. @@ -272,6 +278,7 @@ def __init__( ) # Sorting extractor to have string unit ids until is changed in SpikeInterface + # https://github.com/SpikeInterface/spikeinterface/pull/3588 string_unit_ids = [str(id) for id in self.sorting_extractor.unit_ids] self.sorting_extractor = self.sorting_extractor.rename_units(new_unit_ids=string_unit_ids) diff --git a/tests/test_ecephys/test_tools_spikeinterface.py b/tests/test_ecephys/test_tools_spikeinterface.py index 3436a2e70..68c681f46 100644 --- a/tests/test_ecephys/test_tools_spikeinterface.py +++ b/tests/test_ecephys/test_tools_spikeinterface.py @@ -24,6 +24,7 @@ from neuroconv.tools.nwb_helpers import get_module from neuroconv.tools.spikeinterface import ( add_electrical_series_to_nwbfile, + add_electrode_groups_to_nwbfile, add_electrodes_to_nwbfile, add_recording_to_nwbfile, add_sorting_to_nwbfile, @@ -1071,6 +1072,29 @@ def test_missing_bool_values(self): assert np.array_equal(extracted_incomplete_property, expected_incomplete_property) +class TestAddElectrodeGroups: + def test_group_naming_not_matching_group_number(self): + recording = generate_recording(num_channels=4) + recording.set_channel_groups(groups=[0, 1, 2, 3]) + recording.set_property(key="group_name", values=["A", "A", "A", "A"]) + + nwbfile = mock_NWBFile() + with pytest.raises(ValueError, match="The number of group names must match the number of groups"): + add_electrode_groups_to_nwbfile(nwbfile=nwbfile, recording=recording) + + def test_inconsistent_group_name_mapping(self): + recording = generate_recording(num_channels=3) + # Set up groups where the same group name is used for different group numbers + recording.set_channel_groups(groups=[0, 1, 0]) + recording.set_property( + key="group_name", values=["A", "B", "B"] # Inconsistent: group 0 maps to names "A" and "B" + ) + + nwbfile = mock_NWBFile() + with pytest.raises(ValueError, match="Inconsistent mapping between group numbers and group names"): + add_electrode_groups_to_nwbfile(nwbfile=nwbfile, recording=recording) + + class TestAddUnitsTable(TestCase): @classmethod def setUpClass(cls): From 32f683426663085d654fbff7e68c3aa812d70f47 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 16 Jan 2025 10:38:08 -0600 Subject: [PATCH 110/118] SpikeGLX: Add inter-sample shift description and fix ElectrodeGroup naming (#1177) --- CHANGELOG.md | 4 +++- .../ecephys/spikeglx/spikeglx_utils.py | 4 ++-- .../ecephys/spikeglx/spikeglxdatainterface.py | 17 +++++++------- .../spikeglx_multi_probe_metadata.json | 20 ++++++++++------- .../spikeglx_single_probe_metadata.json | 12 ++++++---- .../ecephys/test_recording_interfaces.py | 4 ++-- .../ecephys/test_spikeglx_converter.py | 22 +++++++++++++------ .../ecephys/test_spikeglx_metadata.py | 3 ++- 8 files changed, 52 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c1ac3ca..77b932e6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,10 @@ Small fixes should be here. * Temporary set a ceiling for hdmf to avoid a chunking bug [PR #1175](https://github.com/catalystneuro/neuroconv/pull/1175) ## Features +* Add description to inter-sample-shift for `SpikeGLXRecordingInterface` [PR #1177](https://github.com/catalystneuro/neuroconv/pull/1177) ## Improvements +* Improve the naming of ElectrodeGroups in the `SpikeGLXRecordingInterface` when multi probes are present [PR #1177](https://github.com/catalystneuro/neuroconv/pull/1177) * Detect mismatch errors between group and group names when writing ElectrodeGroups [PR #1165](https://github.com/catalystneuro/neuroconv/pull/1165) * Fix metadata bug in `IntanRecordingInterface` where extra devices were added incorrectly if the recording contained multiple electrode groups or names [#1166](https://github.com/catalystneuro/neuroconv/pull/1166) @@ -54,7 +56,7 @@ Small fixes should be here. * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) * Use pytest format for dandi tests to avoid window permission error on teardown [PR #1151](https://github.com/catalystneuro/neuroconv/pull/1151) * Added many docstrings for public functions [PR #1063](https://github.com/catalystneuro/neuroconv/pull/1063) -* Clean up with warnings and deprecations in the testing framework [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) +* Clean up warnings and deprecations in the testing framework [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) * Enhance the typing of the signature on the `NWBConverter` by adding zarr as a literal option on the backend and backend configuration [PR #1160](https://github.com/catalystneuro/neuroconv/pull/1160) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py index ba83d296d..5c7aa66d9 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglx_utils.py @@ -18,9 +18,9 @@ def add_recording_extractor_properties(recording_extractor) -> None: if probe.get_shank_count() > 1: shank_ids = probe.shank_ids recording_extractor.set_property(key="shank_ids", values=shank_ids) - group_name = [f"{probe_name}{shank_id}" for shank_id in shank_ids] + group_name = [f"Neuropixels{probe_name}Shank{shank_id}" for shank_id in shank_ids] else: - group_name = [f"{probe_name}"] * len(channel_ids) + group_name = [f"Neuropixels{probe_name}"] * len(channel_ids) recording_extractor.set_property(key="group_name", ids=channel_ids, values=group_name) diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index e375ec351..9ebe8ac48 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -27,14 +27,6 @@ class SpikeGLXRecordingInterface(BaseRecordingExtractorInterface): associated_suffixes = (".imec{probe_index}", ".ap", ".lf", ".meta", ".bin") info = "Interface for SpikeGLX recording data." - # TODO: Add probe_index to probeinterface and propagate it from there - # Note to developer. - # In a conversion with Jennifer Colonell she refers to the number after imec as the probe index - # Quoting here: - # imec0 is the probe in the lowest slot and port number, imec1 in the next highest, and so on. - # If you have probes in {slot 2, port 3}, {slot 3, port1} and {slot3, port2}, - # the probe indices in the SGLX output will be 0, 1, and 2, respectively. - ExtractorName = "SpikeGLXRecordingExtractor" @classmethod @@ -135,7 +127,7 @@ def get_metadata(self) -> dict: # Should follow pattern 'Imec0', 'Imec1', etc. probe_name = self._signals_info_dict["device"].capitalize() - device["name"] = f"Neuropixel{probe_name}" + device["name"] = f"Neuropixels{probe_name}" # Add groups metadata metadata["Ecephys"]["Device"] = [device] @@ -155,6 +147,13 @@ def get_metadata(self) -> dict: dict(name="group_name", description="Name of the ElectrodeGroup this electrode is a part of."), dict(name="contact_shapes", description="The shape of the electrode"), dict(name="contact_ids", description="The id of the contact on the electrode"), + dict( + name="inter_sample_shift", + description=( + "Array of relative phase shifts for each channel, with values ranging from 0 to 1, " + "representing the fractional delay within the sampling period due to sequential ADC." + ), + ), ] if self.recording_extractor.get_probe().get_shank_count() > 1: diff --git a/tests/test_on_data/ecephys/spikeglx_multi_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_multi_probe_metadata.json index ad4f99937..d5eaeffa6 100644 --- a/tests/test_on_data/ecephys/spikeglx_multi_probe_metadata.json +++ b/tests/test_on_data/ecephys/spikeglx_multi_probe_metadata.json @@ -5,28 +5,28 @@ "Ecephys": { "Device": [ { - "name": "NeuropixelImec0", + "name": "NeuropixelsImec0", "description": "{\"probe_type\": \"0\", \"probe_type_description\": \"NP1.0\", \"flex_part_number\": \"NP2_FLEX_0\", \"connected_base_station_part_number\": \"NP2_QBSC_00\"}", "manufacturer": "Imec" }, { - "name": "NeuropixelImec1", + "name": "NeuropixelsImec1", "description": "{\"probe_type\": \"0\", \"probe_type_description\": \"NP1.0\", \"flex_part_number\": \"NP2_FLEX_0\", \"connected_base_station_part_number\": \"NP2_QBSC_00\"}", "manufacturer": "Imec" } ], "ElectrodeGroup": [ { - "name": "Imec0", - "description": "A group representing probe/shank 'Imec0'.", + "name": "NeuropixelsImec0", + "description": "A group representing probe/shank 'NeuropixelsImec0'.", "location": "unknown", - "device": "NeuropixelImec0" + "device": "NeuropixelsImec0" }, { - "name": "Imec1", - "description": "A group representing probe/shank 'Imec1'.", + "name": "NeuropixelsImec1", + "description": "A group representing probe/shank 'NeuropixelsImec1'.", "location": "unknown", - "device": "NeuropixelImec1" + "device": "NeuropixelsImec1" } ], "ElectricalSeriesAPImec0": { @@ -45,6 +45,10 @@ { "name": "contact_ids", "description": "The id of the contact on the electrode" + }, + { + "name": "inter_sample_shift", + "description": "Array of relative phase shifts for each channel, with values ranging from 0 to 1, representing the fractional delay within the sampling period due to sequential ADC." } ], "ElectricalSeriesAPImec1": { diff --git a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json index f3f5fb595..b11c1a628 100644 --- a/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json +++ b/tests/test_on_data/ecephys/spikeglx_single_probe_metadata.json @@ -5,17 +5,17 @@ "Ecephys": { "Device": [ { - "name": "NeuropixelImec0", + "name": "NeuropixelsImec0", "description": "{\"probe_type\": \"0\", \"probe_type_description\": \"NP1.0\", \"flex_part_number\": \"NP2_FLEX_0\", \"connected_base_station_part_number\": \"NP2_QBSC_00\"}", "manufacturer": "Imec" } ], "ElectrodeGroup": [ { - "name": "Imec0", - "description": "A group representing probe/shank 'Imec0'.", + "name": "NeuropixelsImec0", + "description": "A group representing probe/shank 'NeuropixelsImec0'.", "location": "unknown", - "device": "NeuropixelImec0" + "device": "NeuropixelsImec0" } ], @@ -35,6 +35,10 @@ { "name": "contact_ids", "description": "The id of the contact on the electrode" + }, + { + "name": "inter_sample_shift", + "description": "Array of relative phase shifts for each channel, with values ranging from 0 to 1, representing the fractional delay within the sampling period due to sequential ADC." } ], "ElectricalSeriesLF": { diff --git a/tests/test_on_data/ecephys/test_recording_interfaces.py b/tests/test_on_data/ecephys/test_recording_interfaces.py index 8978efa93..e4bd126f4 100644 --- a/tests/test_on_data/ecephys/test_recording_interfaces.py +++ b/tests/test_on_data/ecephys/test_recording_interfaces.py @@ -670,7 +670,7 @@ class TestSpikeGLXRecordingInterface(RecordingExtractorInterfaceTestMixin): def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2020, 11, 3, 10, 35, 10) assert metadata["Ecephys"]["Device"][-1] == dict( - name="NeuropixelImec0", + name="NeuropixelsImec0", description="{" '"probe_type": "0", ' '"probe_type_description": "NP1.0", ' @@ -698,7 +698,7 @@ class TestSpikeGLXRecordingInterfaceLongNHP(RecordingExtractorInterfaceTestMixin def check_extracted_metadata(self, metadata: dict): assert metadata["NWBFile"]["session_start_time"] == datetime(2024, 1, 3, 11, 51, 51) assert metadata["Ecephys"]["Device"][-1] == dict( - name="NeuropixelImec0", + name="NeuropixelsImec0", description="{" '"probe_type": "1030", ' '"probe_type_description": "NP1.0 NHP", ' diff --git a/tests/test_on_data/ecephys/test_spikeglx_converter.py b/tests/test_on_data/ecephys/test_spikeglx_converter.py index 27a8ed0c5..378e03c59 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_converter.py +++ b/tests/test_on_data/ecephys/test_spikeglx_converter.py @@ -41,11 +41,11 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ assert len(nwbfile.acquisition) == 3 - assert "NeuropixelImec0" in nwbfile.devices + assert "NeuropixelsImec0" in nwbfile.devices assert "NIDQBoard" in nwbfile.devices assert len(nwbfile.devices) == 2 - assert "Imec0" in nwbfile.electrode_groups + assert "NeuropixelsImec0" in nwbfile.electrode_groups assert len(nwbfile.electrode_groups) == 1 def test_single_probe_spikeglx_converter(self): @@ -132,12 +132,12 @@ def assertNWBFileStructure(self, nwbfile_path: FilePath, expected_session_start_ assert "ElectricalSeriesLFImec11" in nwbfile.acquisition assert len(nwbfile.acquisition) == 16 - assert "NeuropixelImec0" in nwbfile.devices - assert "NeuropixelImec1" in nwbfile.devices + assert "NeuropixelsImec0" in nwbfile.devices + assert "NeuropixelsImec1" in nwbfile.devices assert len(nwbfile.devices) == 2 - assert "Imec0" in nwbfile.electrode_groups - assert "Imec1" in nwbfile.electrode_groups + assert "NeuropixelsImec0" in nwbfile.electrode_groups + assert "NeuropixelsImec1" in nwbfile.electrode_groups assert len(nwbfile.electrode_groups) == 2 def test_multi_probe_spikeglx_converter(self): @@ -160,8 +160,16 @@ def test_multi_probe_spikeglx_converter(self): device_metadata = test_ecephys_metadata.pop("Device") expected_device_metadata = expected_ecephys_metadata.pop("Device") - assert device_metadata == expected_device_metadata + + assert test_ecephys_metadata["ElectrodeGroup"] == expected_ecephys_metadata["ElectrodeGroup"] + assert test_ecephys_metadata["Electrodes"] == expected_ecephys_metadata["Electrodes"] + assert test_ecephys_metadata["ElectricalSeriesAPImec0"] == expected_ecephys_metadata["ElectricalSeriesAPImec0"] + assert test_ecephys_metadata["ElectricalSeriesAPImec1"] == expected_ecephys_metadata["ElectricalSeriesAPImec1"] + assert test_ecephys_metadata["ElectricalSeriesLFImec0"] == expected_ecephys_metadata["ElectricalSeriesLFImec0"] + assert test_ecephys_metadata["ElectricalSeriesLFImec1"] == expected_ecephys_metadata["ElectricalSeriesLFImec1"] + + # Test all the dictionary assert test_ecephys_metadata == expected_ecephys_metadata nwbfile_path = self.tmpdir / "test_multi_probe_spikeglx_converter.nwb" diff --git a/tests/test_on_data/ecephys/test_spikeglx_metadata.py b/tests/test_on_data/ecephys/test_spikeglx_metadata.py index 9d57f049d..462d67725 100644 --- a/tests/test_on_data/ecephys/test_spikeglx_metadata.py +++ b/tests/test_on_data/ecephys/test_spikeglx_metadata.py @@ -32,7 +32,8 @@ def test_spikelgx_recording_property_addition(): probe_name = "Imec0" expected_shank_ids = probe.shank_ids - expected_group_name = [f"{probe_name}{shank_id}" for shank_id in expected_shank_ids] + expected_group_name = [f"Neuropixels{probe_name}Shank{shank_id}" for shank_id in expected_shank_ids] + expected_contact_shapes = ["square"] * n_channels expected_contact_ids = probe.contact_ids From aa060de05c12b7d2b9542aa7697d8c44825990b4 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 16 Jan 2025 14:25:40 -0600 Subject: [PATCH 111/118] Improve error message on `get_json_schema_from_method_signature` when passing parameters without type hints (#1157) --- CHANGELOG.md | 6 ++++-- .../ecephys/basesortingextractorinterface.py | 2 +- src/neuroconv/utils/json_schema.py | 10 +++++++--- ...est_get_json_schema_from_method_signature.py | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b932e6d..3391f828c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ ## Bug Fixes ## Features -* Source validation is no longer performed when initializing interfaces or converters [PR #1168](https://github.com/catalystneuro/neuroconv/pull/1168) ## Improvements @@ -21,9 +20,11 @@ Small fixes should be here. * Add description to inter-sample-shift for `SpikeGLXRecordingInterface` [PR #1177](https://github.com/catalystneuro/neuroconv/pull/1177) ## Improvements +* `get_json_schema_from_method_signature` now throws a more informative error when an untyped parameter is passed [#1157](https://github.com/catalystneuro/neuroconv/pull/1157) * Improve the naming of ElectrodeGroups in the `SpikeGLXRecordingInterface` when multi probes are present [PR #1177](https://github.com/catalystneuro/neuroconv/pull/1177) * Detect mismatch errors between group and group names when writing ElectrodeGroups [PR #1165](https://github.com/catalystneuro/neuroconv/pull/1165) * Fix metadata bug in `IntanRecordingInterface` where extra devices were added incorrectly if the recording contained multiple electrode groups or names [#1166](https://github.com/catalystneuro/neuroconv/pull/1166) +* Source validation is no longer performed when initializing interfaces or converters [PR #1168](https://github.com/catalystneuro/neuroconv/pull/1168) # v0.6.6 (December 20, 2024) @@ -42,6 +43,7 @@ Small fixes should be here. * `SpikeGLXNIDQInterface` is no longer written as an ElectricalSeries [#1152](https://github.com/catalystneuro/neuroconv/pull/1152) * Fix a bug on ecephys interfaces where extra electrode group and devices were written if the property of the "group_name" was set in the recording extractor [#1164](https://github.com/catalystneuro/neuroconv/pull/1164) + ## Features * Propagate the `unit_electrode_indices` argument from the spikeinterface tools to `BaseSortingExtractorInterface`. This allows users to map units to the electrode table when adding sorting data [PR #1124](https://github.com/catalystneuro/neuroconv/pull/1124) * Imaging interfaces have a new conversion option `always_write_timestamps` that can be used to force writing timestamps even if neuroconv's heuristics indicates regular sampling rate [PR #1125](https://github.com/catalystneuro/neuroconv/pull/1125) @@ -56,7 +58,7 @@ Small fixes should be here. * Use mixing tests for ecephy's mocks [PR #1136](https://github.com/catalystneuro/neuroconv/pull/1136) * Use pytest format for dandi tests to avoid window permission error on teardown [PR #1151](https://github.com/catalystneuro/neuroconv/pull/1151) * Added many docstrings for public functions [PR #1063](https://github.com/catalystneuro/neuroconv/pull/1063) -* Clean up warnings and deprecations in the testing framework [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) +* Clean up warnings and deprecations in the testing framework for the ecephys pipeline [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) * Enhance the typing of the signature on the `NWBConverter` by adding zarr as a literal option on the backend and backend configuration [PR #1160](https://github.com/catalystneuro/neuroconv/pull/1160) diff --git a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py index 8eeb59324..9dfd8c82e 100644 --- a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py @@ -18,7 +18,7 @@ class BaseSortingExtractorInterface(BaseExtractorInterface): ExtractorModuleName = "spikeinterface.extractors" - def __init__(self, verbose=True, **source_data): + def __init__(self, verbose: bool = True, **source_data): super().__init__(**source_data) self.sorting_extractor = self.get_extractor()(**source_data) self.verbose = verbose diff --git a/src/neuroconv/utils/json_schema.py b/src/neuroconv/utils/json_schema.py index 6aa7a75d0..b05019fe4 100644 --- a/src/neuroconv/utils/json_schema.py +++ b/src/neuroconv/utils/json_schema.py @@ -140,12 +140,16 @@ def get_json_schema_from_method_signature(method: Callable, exclude: Optional[li additional_properties = True continue - annotation = parameter.annotation + # Raise error if the type annotation is missing as a json schema cannot be generated in that case + if parameter.annotation is inspect._empty: + raise TypeError( + f"Parameter '{argument_name}' in method '{method_display}' is missing a type annotation. " + f"Either add a type annotation for '{argument_name}' or add it to the exclude list." + ) # Pydantic uses ellipsis for required pydantic_default = ... if parameter.default is inspect._empty else parameter.default - - arguments_to_annotations.update({argument_name: (annotation, pydantic_default)}) + arguments_to_annotations.update({argument_name: (parameter.annotation, pydantic_default)}) # The ConfigDict is required to support custom types like NumPy arrays model = pydantic.create_model( diff --git a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py index e6fe27e16..111053797 100644 --- a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py +++ b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py @@ -424,3 +424,20 @@ def test_class_method(self, integer: int): } assert test_json_schema == expected_json_schema + + +def test_json_schema_raises_error_for_missing_type_annotations(): + """Test that attempting to generate a JSON schema for a method with missing type annotations raises a TypeError.""" + # https://github.com/catalystneuro/neuroconv/pull/1157 + + def test_method(param_with_type: int, param_without_type, param_with_default="default_value"): + pass + + with pytest.raises( + TypeError, + match=( + "Parameter 'param_without_type' in method 'test_method' is missing a type annotation. " + "Either add a type annotation for 'param_without_type' or add it to the exclude list." + ), + ): + get_json_schema_from_method_signature(method=test_method) From 78349074d27d508df693407f95b07f58eb5d60a9 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 20 Jan 2025 15:22:26 -0600 Subject: [PATCH 112/118] Release 0.67. (#1181) --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 3 +-- pyproject.toml | 10 ++++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae61219c1..32eccd1c4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: exclude: ^docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.3 + rev: v0.9.1 hooks: - id: ruff args: [ --fix ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 3391f828c..b10cd158c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,7 @@ ## Improvements -# v0.6.9 (Upcoming) -Small fixes should be here. +# v0.6.9 (January 20, 2024) ## Deprecations diff --git a/pyproject.toml b/pyproject.toml index 9debeb5e9..4eab293ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -160,7 +160,7 @@ biocam = [ "spikeinterface>=0.101.0", ] blackrock = [ - "neo>=0.13.3", + "neo>=0.14", "spikeinterface>=0.101.0", ] cellexplorer = [ @@ -168,6 +168,7 @@ cellexplorer = [ "neo>=0.13.3", "pymatreader>=0.0.32", "spikeinterface>=0.101.0", + "setuptools; python_version >= '3.12'" ] edf = [ "neo>=0.13.3", @@ -175,7 +176,7 @@ edf = [ "spikeinterface>=0.101.0", ] intan = [ - "neo>=0.13.3", + "neo>=0.14", "spikeinterface>=0.101.0", ] kilosort = [ @@ -195,6 +196,7 @@ mearec = [ "MEArec>=1.8.0", "neo>=0.13.3", "spikeinterface>=0.101.0", + "setuptools; python_version >= '3.12'" ] neuralynx = [ "natsort>=7.1.1", @@ -208,7 +210,7 @@ neuroscope = [ ] openephys = [ "lxml>=4.9.4", - "neo>=0.13.3", + "neo>=0.14", "spikeinterface>=0.101.0", ] phy = [ @@ -230,7 +232,7 @@ spikegadgets = [ "spikeinterface>=0.101.0", ] spikeglx = [ - "neo>=0.13.3", + "neo>=0.14", "spikeinterface>=0.101.0", ] tdt = [ From 2b4112d415d32b953a6bb08cbe6653efe5232fcd Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 20 Jan 2025 15:29:47 -0600 Subject: [PATCH 113/118] version bump to 0.7 --- CHANGELOG.md | 2 +- docs/developer_guide/making_a_release.rst | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b10cd158c..901c92514 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ## Improvements -# v0.6.9 (January 20, 2024) +# v0.6.7 (January 20, 2024) ## Deprecations diff --git a/docs/developer_guide/making_a_release.rst b/docs/developer_guide/making_a_release.rst index 412a7ad9a..ba5576abd 100644 --- a/docs/developer_guide/making_a_release.rst +++ b/docs/developer_guide/making_a_release.rst @@ -17,7 +17,7 @@ A simple to-do list for the Neuroconv release process: 3. **Perform Checks**: - - Ensure that no requirement files include pointers to `git`-based dependencies (including specific branches or commit hashes). All dependencies for a PyPI release should point to the released package versions that are available on conda-forge or PyPI. This can be done efficiently by searching for `@ git` in an IDE. + - Ensure that no requirement files include pointers to `git`-based dependencies (including specific branches or commit hashes). All dependencies for a PyPI release should point to the released package versions that are available on conda-forge or PyPI. This can be done efficiently by searching for `@ git` in the pyproject.toml on an IDE. 4. **Tag on GitHub**: diff --git a/pyproject.toml b/pyproject.toml index 4eab293ad..f7a250651 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "neuroconv" -version = "0.6.7" +version = "0.7.0" description = "Convert data from proprietary formats to NWB format." readme = "README.md" authors = [ From 41f4b9a8873d978d481d5639c8a49f6156fe6f02 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Tue, 21 Jan 2025 08:32:12 +1100 Subject: [PATCH 114/118] Improve temporally_align_data_interfaces (#1162) --- CHANGELOG.md | 1 + src/neuroconv/nwbconverter.py | 6 ++++-- .../test_temporal_alignment_methods.py | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 901c92514..86c51a2c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bug Fixes ## Features +* Added `metadata` and `conversion_options` as arguments to `NWBConverter.temporally_align_data_interfaces` [PR #1162](https://github.com/catalystneuro/neuroconv/pull/1162) ## Improvements diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index 5b024f126..5e8950079 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -255,7 +255,7 @@ def run_conversion( self.validate_metadata(metadata=metadata, append_mode=append_mode) self.validate_conversion_options(conversion_options=conversion_options) - self.temporally_align_data_interfaces() + self.temporally_align_data_interfaces(metadata=metadata, conversion_options=conversion_options) with make_or_load_nwbfile( nwbfile_path=nwbfile_path, @@ -273,7 +273,9 @@ def run_conversion( configure_backend(nwbfile=nwbfile_out, backend_configuration=backend_configuration) - def temporally_align_data_interfaces(self): + def temporally_align_data_interfaces( + self, metadata: Optional[dict] = None, conversion_options: Optional[dict] = None + ): """Override this method to implement custom alignment.""" pass diff --git a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py index 081cd172d..724749574 100644 --- a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py +++ b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py @@ -2,7 +2,7 @@ from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, Union +from typing import Dict, Optional, Union import numpy as np from hdmf.testing import TestCase @@ -186,7 +186,9 @@ class TestAlignmentConverter(NWBConverter): NIDQ=MockSpikeGLXNIDQInterface, Trials=CsvTimeIntervalsInterface, Behavior=MockBehaviorEventInterface ) - def temporally_align_data_interfaces(self): + def temporally_align_data_interfaces( + self, metadata: Optional[dict] = None, conversion_options: Optional[dict] = None + ): unaligned_trial_start_times = self.data_interface_objects["Trials"].get_original_timestamps( column="start_time" ) From 9290b679bd71749ef172c59fdbfd42bf5c76f27f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 21 Jan 2025 15:36:20 -0600 Subject: [PATCH 115/118] Set `verbose = False` as default on all the interfaces (#1153) Co-authored-by: Paul Adkisson Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 5 ++++- .../recording/openephys.rst | 1 - .../behavior/audio/audiointerface.py | 4 ---- .../deeplabcut/deeplabcutdatainterface.py | 2 +- .../behavior/fictrac/fictracdatainterface.py | 4 ++-- .../lightningpose/lightningposeconverter.py | 4 ++-- .../lightningposedatainterface.py | 4 ++-- .../behavior/medpc/medpcdatainterface.py | 2 +- .../neuralynx/neuralynx_nvt_interface.py | 4 ++-- .../behavior/sleap/sleapdatainterface.py | 4 ++-- .../alphaomega/alphaomegadatainterface.py | 4 ++-- .../ecephys/axona/axonadatainterface.py | 2 +- .../ecephys/baselfpextractorinterface.py | 2 +- .../baserecordingextractorinterface.py | 4 ++-- .../ecephys/basesortingextractorinterface.py | 3 ++- .../ecephys/biocam/biocamdatainterface.py | 4 ++-- .../blackrock/blackrockdatainterface.py | 6 +++--- .../cellexplorer/cellexplorerdatainterface.py | 8 ++++---- .../ecephys/edf/edfdatainterface.py | 4 ++-- .../ecephys/intan/intandatainterface.py | 4 ++-- .../ecephys/kilosort/kilosortdatainterface.py | 2 +- .../ecephys/maxwell/maxonedatainterface.py | 2 +- .../ecephys/mcsraw/mcsrawdatainterface.py | 2 +- .../ecephys/mearec/mearecdatainterface.py | 4 ++-- .../neuralynx/neuralynxdatainterface.py | 4 ++-- .../neuroscope/neuroscopedatainterface.py | 4 ++-- .../openephys/openephysbinarydatainterface.py | 4 ++-- .../openephys/openephysdatainterface.py | 4 ++-- .../openephys/openephyslegacydatainterface.py | 4 ++-- .../ecephys/phy/phydatainterface.py | 4 ++-- .../ecephys/plexon/plexondatainterface.py | 10 +++++----- .../ecephys/spike2/spike2datainterface.py | 4 ++-- .../spikegadgets/spikegadgetsdatainterface.py | 2 +- .../ecephys/spikeglx/spikeglxdatainterface.py | 4 ++-- .../ecephys/spikeglx/spikeglxnidqinterface.py | 4 ++-- .../ecephys/tdt/tdtdatainterface.py | 4 ++-- .../ophys/baseimagingextractorinterface.py | 2 +- .../ophys/brukertiff/brukertiffconverter.py | 4 ++-- .../brukertiff/brukertiffdatainterface.py | 8 ++++---- .../ophys/caiman/caimandatainterface.py | 2 +- .../ophys/cnmfe/cnmfedatainterface.py | 2 +- .../ophys/extract/extractdatainterface.py | 2 +- .../ophys/hdf5/hdf5datainterface.py | 4 ++-- .../micromanagertiffdatainterface.py | 4 ++-- .../ophys/miniscope/miniscopeconverter.py | 4 ++-- .../ophys/sbx/sbxdatainterface.py | 4 ++-- .../scanimage/scanimageimaginginterfaces.py | 14 ++++++------- .../ophys/suite2p/suite2pdatainterface.py | 2 +- .../tdt_fp/tdtfiberphotometrydatainterface.py | 2 +- .../ophys/tiff/tiffdatainterface.py | 4 ++-- .../text/excel/exceltimeintervalsinterface.py | 4 ++-- .../text/timeintervalsinterface.py | 5 ++--- src/neuroconv/nwbconverter.py | 8 ++++---- .../nwb_helpers/_metadata_and_file_helpers.py | 2 +- .../tools/roiextractors/roiextractors.py | 8 ++++---- .../tools/spikeinterface/spikeinterface.py | 20 +++++++++---------- .../tools/testing/mock_interfaces.py | 4 ++-- ...t_get_json_schema_from_method_signature.py | 2 +- .../test_temporal_alignment_methods.py | 4 ++-- 59 files changed, 125 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c51a2c7..3cab36eab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ * Added `metadata` and `conversion_options` as arguments to `NWBConverter.temporally_align_data_interfaces` [PR #1162](https://github.com/catalystneuro/neuroconv/pull/1162) ## Improvements +* Interfaces and converters now have `verbose=False` by default [PR #1153](https://github.com/catalystneuro/neuroconv/pull/1153) + # v0.6.7 (January 20, 2024) @@ -29,8 +31,9 @@ # v0.6.6 (December 20, 2024) -## Deprecations +## Deprecations and Changes * Removed use of `jsonschema.RefResolver` as it will be deprecated from the jsonschema library [PR #1133](https://github.com/catalystneuro/neuroconv/pull/1133) +* Completely removed compression settings from most places[PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) * Completely removed compression settings from most places [PR #1126](https://github.com/catalystneuro/neuroconv/pull/1126) * Soft deprecation for `file_path` as an argument of `SpikeGLXNIDQInterface` and `SpikeGLXRecordingInterface` [PR #1155](https://github.com/catalystneuro/neuroconv/pull/1155) * `starting_time` in RecordingInterfaces has given a soft deprecation in favor of time alignment methods [PR #1158](https://github.com/catalystneuro/neuroconv/pull/1158) diff --git a/docs/conversion_examples_gallery/recording/openephys.rst b/docs/conversion_examples_gallery/recording/openephys.rst index a5d4e960a..bced73b7f 100644 --- a/docs/conversion_examples_gallery/recording/openephys.rst +++ b/docs/conversion_examples_gallery/recording/openephys.rst @@ -30,4 +30,3 @@ Convert OpenEphys data to NWB using :py:class:`~neuroconv.datainterfaces.ecephys >>> # Choose a path for saving the nwb file and run the conversion >>> nwbfile_path = f"{path_to_save_nwbfile}" # This should be something like: "./saved_file.nwb" >>> interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata) - NWB file saved at ... diff --git a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py index 6561096b5..ec2920432 100644 --- a/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py +++ b/src/neuroconv/datainterfaces/behavior/audio/audiointerface.py @@ -170,8 +170,6 @@ def add_to_nwbfile( stub_frames: int = 1000, write_as: Literal["stimulus", "acquisition"] = "stimulus", iterator_options: Optional[dict] = None, - overwrite: bool = False, - verbose: bool = True, ): """ Parameters @@ -186,8 +184,6 @@ def add_to_nwbfile( "stimulus" or as "acquisition". iterator_options : dict, optional Dictionary of options for the SliceableDataChunkIterator. - overwrite : bool, default: False - verbose : bool, default: True Returns ------- diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index 147fcf6ea..8c81a0264 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -31,7 +31,7 @@ def __init__( file_path: FilePath, config_file_path: Optional[FilePath] = None, subject_name: str = "ind1", - verbose: bool = True, + verbose: bool = False, ): """ Interface for writing DLC's output files to nwb using dlc2nwb. diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index 1d822f919..75e1415dd 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -159,7 +159,7 @@ def __init__( file_path: FilePath, radius: Optional[float] = None, configuration_file_path: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, ): """ Interface for writing FicTrac files to nwb. @@ -173,7 +173,7 @@ def __init__( and the units are set to meters. If not provided the units are set to radians. configuration_file_path : FilePath, optional Path to the .txt file with the configuration metadata. Usually called config.txt - verbose : bool, default: True + verbose : bool, default: False controls verbosity. ``True`` by default. """ self.file_path = Path(file_path) diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py index 62edaf140..07a06227b 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py @@ -34,7 +34,7 @@ def __init__( labeled_video_file_path: Optional[FilePath] = None, image_series_original_video_name: Optional[str] = None, image_series_labeled_video_name: Optional[str] = None, - verbose: bool = True, + verbose: bool = False, ): """ The converter for Lightning Pose format to convert the pose estimation data @@ -52,7 +52,7 @@ def __init__( The name of the ImageSeries to add for the original video. image_series_labeled_video_name: string, optional The name of the ImageSeries to add for the labeled video. - verbose : bool, default: True + verbose : bool, default: False controls verbosity. ``True`` by default. """ self.verbose = verbose diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py index dbd425b5b..1211c31d1 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py @@ -64,7 +64,7 @@ def __init__( file_path: FilePath, original_video_file_path: FilePath, labeled_video_file_path: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, ): """ Interface for writing pose estimation data from the Lightning Pose algorithm. @@ -77,7 +77,7 @@ def __init__( Path to the original video file (.mp4). labeled_video_file_path : a string or a path, optional Path to the labeled video file (.mp4). - verbose : bool, default: True + verbose : bool, default: False controls verbosity. ``True`` by default. """ # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created diff --git a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py index d519dc71a..17c2cfc03 100644 --- a/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py @@ -49,7 +49,7 @@ def __init__( start_variable: str, metadata_medpc_name_to_info_dict: dict, aligned_timestamp_names: Optional[list[str]] = None, - verbose: bool = True, + verbose: bool = False, ): """ Initialize MedpcInterface. diff --git a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py index 213adf731..e118850ae 100644 --- a/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py +++ b/src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py @@ -22,7 +22,7 @@ class NeuralynxNvtInterface(BaseTemporalAlignmentInterface): info = "Interface for writing Neuralynx position tracking .nvt files to NWB." @validate_call - def __init__(self, file_path: FilePath, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = False): """ Interface for writing Neuralynx .nvt files to nwb. @@ -30,7 +30,7 @@ def __init__(self, file_path: FilePath, verbose: bool = True): ---------- file_path : FilePath Path to the .nvt file - verbose : bool, default: True + verbose : bool, default: Falsee controls verbosity. """ diff --git a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py index 713b21c98..90b9b91e1 100644 --- a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py @@ -32,7 +32,7 @@ def __init__( self, file_path: FilePath, video_file_path: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, frames_per_second: Optional[float] = None, ): """ @@ -42,7 +42,7 @@ def __init__( ---------- file_path : FilePath Path to the .slp file (the output of sleap) - verbose : bool, default: True + verbose : bool, default: Falsee controls verbosity. ``True`` by default. video_file_path : FilePath, optional The file path of the video for extracting timestamps. diff --git a/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py b/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py index 97f89abca..0a9bd80c8 100644 --- a/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/alphaomega/alphaomegadatainterface.py @@ -26,7 +26,7 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: extractor_kwargs["stream_id"] = self.stream_id return extractor_kwargs - def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, folder_path: DirectoryPath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Load and prepare data for AlphaOmega. @@ -36,7 +36,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str Path to the folder of .mpx files. verbose: boolean Allows verbose. - Default is True. + Default is False. es_key: str, default: "ElectricalSeries" """ super().__init__(folder_path=folder_path, verbose=verbose, es_key=es_key) diff --git a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py index 3c8a1067c..d592f4ee1 100644 --- a/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py @@ -36,7 +36,7 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: return extractor_kwargs - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Parameters diff --git a/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py index af16601bb..79e006c3b 100644 --- a/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baselfpextractorinterface.py @@ -15,7 +15,7 @@ class BaseLFPExtractorInterface(BaseRecordingExtractorInterface): "LF", ) - def __init__(self, verbose: bool = True, es_key: str = "ElectricalSeriesLFP", **source_data): + def __init__(self, verbose: bool = False, es_key: str = "ElectricalSeriesLFP", **source_data): super().__init__(verbose=verbose, es_key=es_key, **source_data) def add_to_nwbfile( diff --git a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py index c9df3ba52..1b7b18119 100644 --- a/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/baserecordingextractorinterface.py @@ -20,11 +20,11 @@ class BaseRecordingExtractorInterface(BaseExtractorInterface): ExtractorModuleName = "spikeinterface.extractors" - def __init__(self, verbose: bool = True, es_key: str = "ElectricalSeries", **source_data): + def __init__(self, verbose: bool = False, es_key: str = "ElectricalSeries", **source_data): """ Parameters ---------- - verbose : bool, default: True + verbose : bool, default: False If True, will print out additional information. es_key : str, default: "ElectricalSeries" The key of this ElectricalSeries in the metadata dictionary. diff --git a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py index 9dfd8c82e..872927cd7 100644 --- a/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/basesortingextractorinterface.py @@ -18,7 +18,8 @@ class BaseSortingExtractorInterface(BaseExtractorInterface): ExtractorModuleName = "spikeinterface.extractors" - def __init__(self, verbose: bool = True, **source_data): + def __init__(self, verbose: bool = False, **source_data): + super().__init__(**source_data) self.sorting_extractor = self.get_extractor()(**source_data) self.verbose = verbose diff --git a/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py b/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py index f12f3a93d..056a8d31f 100644 --- a/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/biocam/biocamdatainterface.py @@ -20,7 +20,7 @@ def get_source_schema(cls) -> dict: schema["properties"]["file_path"]["description"] = "Path to the .bwr file." return schema - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Load and prepare data for Biocam. @@ -28,7 +28,7 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele ---------- file_path : string or Path Path to the .bwr file. - verbose : bool, default: True + verbose : bool, default: False Allows verbose. es_key: str, default: "ElectricalSeries" """ diff --git a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py index bc9f9b51b..811aab81a 100644 --- a/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/blackrock/blackrockdatainterface.py @@ -35,7 +35,7 @@ def __init__( self, file_path: FilePath, nsx_override: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): """ @@ -90,7 +90,7 @@ def get_source_schema(cls) -> dict: metadata_schema["properties"]["file_path"].update(description="Path to Blackrock .nev file.") return metadata_schema - def __init__(self, file_path: FilePath, sampling_frequency: Optional[float] = None, verbose: bool = True): + def __init__(self, file_path: FilePath, sampling_frequency: Optional[float] = None, verbose: bool = False): """ Parameters ---------- @@ -100,7 +100,7 @@ def __init__(self, file_path: FilePath, sampling_frequency: Optional[float] = No The sampling frequency for the sorting extractor. When the signal data is available (.ncs) those files will be used to extract the frequency automatically. Otherwise, the sampling frequency needs to be specified for this extractor to be initialized. - verbose : bool, default: True + verbose : bool, default: False Enables verbosity """ super().__init__(file_path=file_path, sampling_frequency=sampling_frequency, verbose=verbose) diff --git a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py index 9ffeb78a2..ebd921117 100644 --- a/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/cellexplorer/cellexplorerdatainterface.py @@ -254,7 +254,7 @@ class CellExplorerRecordingInterface(BaseRecordingExtractorInterface): The folder where the session data is located. It should contain a `{folder.name}.session.mat` file and the binary files `{folder.name}.dat` or `{folder.name}.lfp` for the LFP interface. - verbose : bool, default: True + verbose : bool, default: Falsee Whether to output verbose text. es_key : str, default: "ElectricalSeries" and "ElectricalSeriesLFP" for the LFP interface @@ -294,7 +294,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["folder_path"]["description"] = "Folder containing the .session.mat file" return source_schema - def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, folder_path: DirectoryPath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Parameters @@ -382,7 +382,7 @@ class CellExplorerLFPInterface(CellExplorerRecordingInterface): sampling_frequency_key = "srLfp" binary_file_extension = "lfp" - def __init__(self, folder_path: DirectoryPath, verbose: bool = True, es_key: str = "ElectricalSeriesLFP"): + def __init__(self, folder_path: DirectoryPath, verbose: bool = False, es_key: str = "ElectricalSeriesLFP"): super().__init__(folder_path, verbose, es_key) def add_to_nwbfile( @@ -425,7 +425,7 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: return extractor_kwargs - def __init__(self, file_path: FilePath, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = False): """ Initialize read of Cell Explorer file. diff --git a/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py b/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py index ef169f66f..150aa574b 100644 --- a/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/edf/edfdatainterface.py @@ -37,7 +37,7 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: def __init__( self, file_path: FilePath, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", channels_to_skip: Optional[list] = None, ): @@ -50,7 +50,7 @@ def __init__( ---------- file_path : str or Path Path to the edf file - verbose : bool, default: True + verbose : bool, default: Falseeeeee Allows verbose. es_key : str, default: "ElectricalSeries" Key for the ElectricalSeries metadata diff --git a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py index 126bea3b4..73c430c22 100644 --- a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py @@ -35,7 +35,7 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: def __init__( self, file_path: FilePath, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ignore_integrity_checks: bool = False, ): @@ -47,7 +47,7 @@ def __init__( file_path : FilePathType Path to either a rhd or a rhs file - verbose : bool, default: True + verbose : bool, default: False Verbose es_key : str, default: "ElectricalSeries" ignore_integrity_checks, bool, default: False. diff --git a/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py b/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py index aafde42f0..58fa80169 100644 --- a/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/kilosort/kilosortdatainterface.py @@ -22,7 +22,7 @@ def __init__( self, folder_path: DirectoryPath, keep_good_only: bool = False, - verbose: bool = True, + verbose: bool = False, ): """ Load and prepare sorting data for kilosort diff --git a/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py b/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py index 11902f81b..6351e02b2 100644 --- a/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/maxwell/maxonedatainterface.py @@ -47,7 +47,7 @@ def __init__( file_path: FilePath, hdf5_plugin_path: Optional[DirectoryPath] = None, download_plugin: bool = True, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ) -> None: """ diff --git a/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py b/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py index ff8e82139..d646a4b35 100644 --- a/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/mcsraw/mcsrawdatainterface.py @@ -20,7 +20,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the .raw file." return source_schema - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Load and prepare data for MCSRaw. diff --git a/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py b/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py index 7a82025ca..96a56c130 100644 --- a/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/mearec/mearecdatainterface.py @@ -23,7 +23,7 @@ def get_source_schema(cls) -> dict: source_schema["properties"]["file_path"]["description"] = "Path to the MEArec .h5 file." return source_schema - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Load and prepare data for MEArec. @@ -31,7 +31,7 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele ---------- folder_path : str or Path Path to the MEArec .h5 file. - verbose : bool, default: True + verbose : bool, default: Falsee Allows verbose. es_key : str, default: "ElectricalSeries" """ diff --git a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py index d87c8f0bd..cdb80c28d 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuralynx/neuralynxdatainterface.py @@ -116,7 +116,7 @@ def __init__( self, folder_path: DirectoryPath, sampling_frequency: Optional[float] = None, - verbose: bool = True, + verbose: bool = False, stream_id: Optional[str] = None, ): """_summary_ @@ -127,7 +127,7 @@ def __init__( The path to the folder/directory containing the data files for the session (nse, ntt, nse, nev) sampling_frequency : float, optional If a specific sampling_frequency is desired it can be set with this argument. - verbose : bool, default: True + verbose : bool, default: False Enables verbosity stream_id: str, optional Used by Spikeinterface and neo to calculate the t_start, if not provided and the stream is unique diff --git a/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py b/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py index d68532a94..9ed1201ab 100644 --- a/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/neuroscope/neuroscopedatainterface.py @@ -128,7 +128,7 @@ def __init__( file_path: FilePath, gain: Optional[float] = None, xml_file_path: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): """ @@ -271,7 +271,7 @@ def __init__( keep_mua_units: bool = True, exclude_shanks: Optional[list[int]] = None, xml_file_path: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, ): """ Load and prepare spike sorted data and corresponding metadata from the Neuroscope format (.res/.clu files). diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py index ef67fde40..503faa025 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysbinarydatainterface.py @@ -43,7 +43,7 @@ def __init__( stream_name: Optional[str] = None, block_index: Optional[int] = None, stub_test: bool = False, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): """ @@ -59,7 +59,7 @@ def __init__( block_index : int, optional, default: None The index of the block to extract from the data. stub_test : bool, default: False - verbose : bool, default: True + verbose : bool, default: Falsee es_key : str, default: "ElectricalSeries" """ from ._openephys_utils import _read_settings_xml diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py index 81b84c36c..c25cc63ca 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephysdatainterface.py @@ -39,7 +39,7 @@ def __new__( folder_path: DirectoryPath, stream_name: Optional[str] = None, block_index: Optional[int] = None, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): """ @@ -58,7 +58,7 @@ def __new__( When channel stream is not available the name of the stream must be specified. block_index : int, optional, default: None The index of the block to extract from the data. - verbose : bool, default: True + verbose : bool, default: False es_key : str, default: "ElectricalSeries" """ super().__new__(cls) diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py index b3392d2db..e6a8e0d79 100644 --- a/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py @@ -41,7 +41,7 @@ def __init__( folder_path: DirectoryPath, stream_name: Optional[str] = None, block_index: Optional[int] = None, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): """ @@ -57,7 +57,7 @@ def __init__( The name of the recording stream. block_index : int, optional, default: None The index of the block to extract from the data. - verbose : bool, default: True + verbose : bool, default: Falseee es_key : str, default: "ElectricalSeries" """ available_streams = self.get_stream_names(folder_path=folder_path) diff --git a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py index 157fef2e5..367463f75 100644 --- a/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py @@ -29,7 +29,7 @@ def __init__( self, folder_path: DirectoryPath, exclude_cluster_groups: Optional[list[str]] = None, - verbose: bool = True, + verbose: bool = False, ): """ Initialize a PhySortingInterface. @@ -40,7 +40,7 @@ def __init__( Path to the output Phy folder (containing the params.py). exclude_cluster_groups : str or list of str, optional Cluster groups to exclude (e.g. "noise" or ["noise", "mua"]). - verbose : bool, default: True + verbose : bool, default: Falsee """ super().__init__(folder_path=folder_path, exclude_cluster_groups=exclude_cluster_groups, verbose=verbose) diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py index d1dcc4d45..4d41f5854 100644 --- a/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py @@ -28,7 +28,7 @@ def get_source_schema(cls) -> dict: def __init__( self, file_path: FilePath, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", stream_name: str = "WB-Wideband", ): @@ -39,7 +39,7 @@ def __init__( ---------- file_path : str or Path Path to the .plx file. - verbose : bool, default: True + verbose : bool, default: Falsee Allows verbosity. es_key : str, default: "ElectricalSeries" stream_name: str, optional @@ -90,7 +90,7 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: return extractor_kwargs @validate_call - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Load and prepare data for Plexon. @@ -98,7 +98,7 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele ---------- file_path : str or Path Path to the .plx file. - verbose : bool, default: True + verbose : bool, default: False Allows verbosity. es_key : str, default: "ElectricalSeries" """ @@ -148,7 +148,7 @@ def get_source_schema(cls) -> dict: return source_schema @validate_call - def __init__(self, file_path: FilePath, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = False): """ Load and prepare data for Plexon. diff --git a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py index bf0ddc860..4fdf4239b 100644 --- a/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py @@ -41,7 +41,7 @@ def get_all_channels_info(cls, file_path: FilePath): return cls.get_extractor().get_all_channels_info(file_path=file_path) @validate_call - def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"): + def __init__(self, file_path: FilePath, verbose: bool = False, es_key: str = "ElectricalSeries"): """ Initialize reading of Spike2 file. @@ -49,7 +49,7 @@ def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "Ele ---------- file_path : FilePathType Path to .smr or .smrx file. - verbose : bool, default: True + verbose : bool, default: False es_key : str, default: "ElectricalSeries" """ _test_sonpy_installation() diff --git a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py index b8b483dd0..475572e7b 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py @@ -28,7 +28,7 @@ def __init__( file_path: FilePath, stream_id: str = "trodes", gains: Optional[ArrayType] = None, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): """ diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py index 9ebe8ac48..65dbc2b2e 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py @@ -47,7 +47,7 @@ def _source_data_to_extractor_kwargs(self, source_data: dict) -> dict: def __init__( self, file_path: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, es_key: Optional[str] = None, folder_path: Optional[DirectoryPath] = None, stream_id: Optional[str] = None, @@ -62,7 +62,7 @@ def __init__( Examples are 'imec0.ap', 'imec0.lf', 'imec1.ap', 'imec1.lf', etc. file_path : FilePathType Path to .bin file. Point to .ap.bin for SpikeGLXRecordingInterface and .lf.bin for SpikeGLXLFPInterface. - verbose : bool, default: True + verbose : bool, default: False Whether to output verbose text. es_key : str, the key to access the metadata of the ElectricalSeries. """ diff --git a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py index 4c7e5a6d9..eae58421f 100644 --- a/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py +++ b/src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py @@ -35,7 +35,7 @@ def get_source_schema(cls) -> dict: def __init__( self, file_path: Optional[FilePath] = None, - verbose: bool = True, + verbose: bool = False, load_sync_channel: Optional[bool] = None, es_key: str = "ElectricalSeriesNIDQ", folder_path: Optional[DirectoryPath] = None, @@ -51,7 +51,7 @@ def __init__( Path to the folder containing the .nidq.bin file. file_path : FilePathType Path to .nidq.bin file. - verbose : bool, default: True + verbose : bool, default: False Whether to output verbose text. es_key : str, default: "ElectricalSeriesNIDQ" """ diff --git a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py index a46f3e0f8..1dacb355a 100644 --- a/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py @@ -23,7 +23,7 @@ def __init__( folder_path: DirectoryPath, gain: float, stream_id: str = "0", - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): """ @@ -37,7 +37,7 @@ def __init__( Select from multiple streams. gain : float The conversion factor from int16 to microvolts. - verbose : bool, default: True + verbose : bool, default: Falsee Allows verbose. es_key : str, optional diff --git a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py index 04407a3d4..08cd31125 100644 --- a/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py +++ b/src/neuroconv/datainterfaces/ophys/baseimagingextractorinterface.py @@ -35,7 +35,7 @@ class BaseImagingExtractorInterface(BaseExtractorInterface): def __init__( self, - verbose: bool = True, + verbose: bool = False, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", **source_data, ): diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py index 1fc854b81..81c17474d 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffconverter.py @@ -51,7 +51,7 @@ def __init__( plane_separation_type: {'contiguous', 'disjoint'} Defines how to write volumetric imaging data. Use 'contiguous' to create the volumetric two photon series, and 'disjoint' to create separate imaging plane and two photon series for each plane. - verbose : bool, default: True + verbose : bool, default: False Controls verbosity. """ self.verbose = verbose @@ -190,7 +190,7 @@ def __init__( ---------- folder_path : DirectoryPath The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). - verbose : bool, default: True + verbose : bool, default: False Controls verbosity. """ from roiextractors.extractors.tiffimagingextractors.brukertiffimagingextractor import ( diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py index f7e7bee1b..ea1d1e4ba 100644 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/brukertiff/brukertiffdatainterface.py @@ -60,7 +60,7 @@ def __init__( self, folder_path: DirectoryPath, stream_name: Optional[str] = None, - verbose: bool = True, + verbose: bool = False, ): """ Initialize reading of TIFF files. @@ -71,7 +71,7 @@ def __init__( The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). stream_name : str, optional The name of the recording stream (e.g. 'Ch2'). - verbose : bool, default: True + verbose : bool, default: False """ self.folder_path = folder_path super().__init__( @@ -233,7 +233,7 @@ def __init__( self, folder_path: DirectoryPath, stream_name: Optional[str] = None, - verbose: bool = True, + verbose: bool = False, ): """ Initialize reading of TIFF files. @@ -244,7 +244,7 @@ def __init__( The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). stream_name : str, optional The name of the recording stream (e.g. 'Ch2'). - verbose : bool, default: True + verbose : bool, default: False """ super().__init__( folder_path=folder_path, diff --git a/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py b/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py index 802645139..9ab1f460f 100644 --- a/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/caiman/caimandatainterface.py @@ -17,7 +17,7 @@ def get_source_schema(cls) -> dict: source_metadata["properties"]["file_path"]["description"] = "Path to .hdf5 file." return source_metadata - def __init__(self, file_path: FilePath, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = False): """ Parameters diff --git a/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py b/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py index 4ea60c892..5e84aa282 100644 --- a/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/cnmfe/cnmfedatainterface.py @@ -10,6 +10,6 @@ class CnmfeSegmentationInterface(BaseSegmentationExtractorInterface): associated_suffixes = (".mat",) info = "Interface for constrained non-negative matrix factorization (CNMFE) segmentation." - def __init__(self, file_path: FilePath, verbose: bool = True): + def __init__(self, file_path: FilePath, verbose: bool = False): super().__init__(file_path=file_path) self.verbose = verbose diff --git a/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py b/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py index 39b3d0a79..3263edbc1 100644 --- a/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/extract/extractdatainterface.py @@ -17,7 +17,7 @@ def __init__( file_path: FilePath, sampling_frequency: float, output_struct_name: Optional[str] = None, - verbose: bool = True, + verbose: bool = False, ): """ diff --git a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py index 4bb13f86b..c1cfefc95 100644 --- a/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py +++ b/src/neuroconv/datainterfaces/ophys/hdf5/hdf5datainterface.py @@ -22,7 +22,7 @@ def __init__( start_time: Optional[float] = None, metadata: Optional[dict] = None, channel_names: Optional[ArrayType] = None, - verbose: bool = True, + verbose: bool = False, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", ): """ @@ -36,7 +36,7 @@ def __init__( start_time : float, optional metadata : dict, optional channel_names : list of str, optional - verbose : bool, default: True + verbose : bool, default: False """ super().__init__( file_path=file_path, diff --git a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py index 5373b7004..8484920cd 100644 --- a/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/micromanagertiff/micromanagertiffdatainterface.py @@ -20,7 +20,7 @@ def get_source_schema(cls) -> dict: return source_schema @validate_call - def __init__(self, folder_path: DirectoryPath, verbose: bool = True): + def __init__(self, folder_path: DirectoryPath, verbose: bool = False): """ Data Interface for MicroManagerTiffImagingExtractor. @@ -29,7 +29,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True): folder_path : FolderPathType The folder path that contains the OME-TIF image files (.ome.tif files) and the 'DisplaySettings' JSON file. - verbose : bool, default: True + verbose : bool, default: False """ super().__init__(folder_path=folder_path) self.verbose = verbose diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py index 59424d7a5..75b72d06a 100644 --- a/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py +++ b/src/neuroconv/datainterfaces/ophys/miniscope/miniscopeconverter.py @@ -24,7 +24,7 @@ def get_source_schema(cls): return source_schema @validate_call - def __init__(self, folder_path: DirectoryPath, verbose: bool = True): + def __init__(self, folder_path: DirectoryPath, verbose: bool = False): """ Initializes the data interfaces for the Miniscope recording and behavioral data stream. @@ -51,7 +51,7 @@ def __init__(self, folder_path: DirectoryPath, verbose: bool = True): ---------- folder_path : FolderPathType The path to the main Miniscope folder. - verbose : bool, default: True + verbose : bool, default: False Controls verbosity. """ self.verbose = verbose diff --git a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py index 49e556d06..fc259cad2 100644 --- a/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/sbx/sbxdatainterface.py @@ -17,7 +17,7 @@ def __init__( self, file_path: FilePath, sampling_frequency: Optional[float] = None, - verbose: bool = True, + verbose: bool = False, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", ): """ @@ -26,7 +26,7 @@ def __init__( file_path : FilePathType Path to .sbx file. sampling_frequency : float, optional - verbose : bool, default: True + verbose : bool, default: False """ super().__init__( diff --git a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py index 7d9d7003b..b004b8ee0 100644 --- a/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py +++ b/src/neuroconv/datainterfaces/ophys/scanimage/scanimageimaginginterfaces.py @@ -40,7 +40,7 @@ def __new__( channel_name: Optional[str] = None, plane_name: Optional[str] = None, fallback_sampling_frequency: Optional[float] = None, - verbose: bool = True, + verbose: bool = False, ): from roiextractors.extractors.tiffimagingextractors.scanimagetiff_utils import ( extract_extra_metadata, @@ -104,7 +104,7 @@ def __init__( self, file_path: FilePath, fallback_sampling_frequency: Optional[float] = None, - verbose: bool = True, + verbose: bool = False, ): """ DataInterface for reading Tiff files that are generated by ScanImage v3.8. This interface extracts the metadata @@ -189,7 +189,7 @@ def __new__( channel_name: Optional[str] = None, plane_name: Optional[str] = None, extract_all_metadata: bool = False, - verbose: bool = True, + verbose: bool = False, ): from natsort import natsorted from roiextractors.extractors.tiffimagingextractors.scanimagetiff_utils import ( @@ -254,7 +254,7 @@ def __init__( channel_name: Optional[str] = None, image_metadata: Optional[dict] = None, parsed_metadata: Optional[dict] = None, - verbose: bool = True, + verbose: bool = False, ): """ DataInterface for reading multi-file (buffered) TIFF files that are generated by ScanImage. @@ -361,7 +361,7 @@ def __init__( extract_all_metadata: bool = False, image_metadata: Optional[dict] = None, parsed_metadata: Optional[dict] = None, - verbose: bool = True, + verbose: bool = False, ): """ DataInterface for reading multi-file (buffered) TIFF files that are generated by ScanImage. @@ -485,7 +485,7 @@ def __init__( plane_name: Optional[str] = None, image_metadata: Optional[dict] = None, parsed_metadata: Optional[dict] = None, - verbose: bool = True, + verbose: bool = False, ): """ DataInterface for reading multi-file (buffered) TIFF files that are generated by ScanImage. @@ -608,7 +608,7 @@ def __init__( image_metadata: Optional[dict] = None, parsed_metadata: Optional[dict] = None, extract_all_metadata: bool = False, - verbose: bool = True, + verbose: bool = False, ): """ DataInterface for reading multi-file (buffered) TIFF files that are generated by ScanImage. diff --git a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py index 8a3f876c2..a2a8e876c 100644 --- a/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/suite2p/suite2pdatainterface.py @@ -80,7 +80,7 @@ def __init__( channel_name: Optional[str] = None, plane_name: Optional[str] = None, plane_segmentation_name: Optional[str] = None, - verbose: bool = True, + verbose: bool = False, ): """ diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py index 0c6b90aea..bef262cab 100644 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tdt_fp/tdtfiberphotometrydatainterface.py @@ -29,7 +29,7 @@ class TDTFiberPhotometryInterface(BaseTemporalAlignmentInterface): associated_suffixes = ("Tbk", "Tdx", "tev", "tin", "tsq") @validate_call - def __init__(self, folder_path: DirectoryPath, verbose: bool = True): + def __init__(self, folder_path: DirectoryPath, verbose: bool = False): """Initialize the TDTFiberPhotometryInterface. Parameters diff --git a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py index ce98561de..5c83f0c72 100644 --- a/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py +++ b/src/neuroconv/datainterfaces/ophys/tiff/tiffdatainterface.py @@ -24,7 +24,7 @@ def __init__( self, file_path: FilePath, sampling_frequency: float, - verbose: bool = True, + verbose: bool = False, photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", ): """ @@ -34,7 +34,7 @@ def __init__( ---------- file_path : FilePathType sampling_frequency : float - verbose : bool, default: True + verbose : bool, default: False photon_series_type : {'OnePhotonSeries', 'TwoPhotonSeries'}, default: "TwoPhotonSeries" """ super().__init__( diff --git a/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py b/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py index 92fbbbaa7..313029b9b 100644 --- a/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/excel/exceltimeintervalsinterface.py @@ -18,7 +18,7 @@ def __init__( self, file_path: FilePath, read_kwargs: Optional[dict] = None, - verbose: bool = True, + verbose: bool = False, ): """ Parameters @@ -26,7 +26,7 @@ def __init__( file_path : FilePath read_kwargs : dict, optional Passed to pandas.read_excel() - verbose : bool, default: True + verbose : bool, default: False """ super().__init__(file_path=file_path, read_kwargs=read_kwargs, verbose=verbose) diff --git a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py index a1de63a07..2520a1236 100644 --- a/src/neuroconv/datainterfaces/text/timeintervalsinterface.py +++ b/src/neuroconv/datainterfaces/text/timeintervalsinterface.py @@ -21,7 +21,7 @@ def __init__( self, file_path: FilePath, read_kwargs: Optional[dict] = None, - verbose: bool = True, + verbose: bool = False, ): """ Initialize the TimeIntervalsInterface. @@ -32,8 +32,7 @@ def __init__( The path to the file containing time intervals data. read_kwargs : dict, optional Additional arguments for reading the file, by default None. - verbose : bool, optional - If True, provides verbose output, by default True. + verbose : bool, default: False """ read_kwargs = read_kwargs or dict() super().__init__(file_path=file_path) diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index 5e8950079..20df647d6 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -61,11 +61,11 @@ def get_source_schema(cls) -> dict: return source_schema @classmethod - def validate_source(cls, source_data: dict[str, dict], verbose: bool = True): + def validate_source(cls, source_data: dict[str, dict], verbose: bool = False): """Validate source_data against Converter source_schema.""" cls._validate_source_data(source_data=source_data, verbose=verbose) - def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = True): + def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = False): # We do this to ensure that python objects are in string format for the JSON schema encoder = _NWBSourceDataEncoder() @@ -77,7 +77,7 @@ def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = Tr print("Source data is valid!") @validate_call - def __init__(self, source_data: dict[str, dict], verbose: bool = True): + def __init__(self, source_data: dict[str, dict], verbose: bool = False): """Validate source_data against source_schema and initialize all data interfaces.""" self.verbose = verbose self.data_interface_objects = { @@ -313,7 +313,7 @@ def get_source_schema(cls) -> dict: def validate_source(cls): raise NotImplementedError("Source data not available with previously initialized classes.") - def __init__(self, data_interfaces: Union[list[BaseDataInterface], dict[str, BaseDataInterface]], verbose=True): + def __init__(self, data_interfaces: Union[list[BaseDataInterface], dict[str, BaseDataInterface]], verbose=False): self.verbose = verbose if isinstance(data_interfaces, list): # Create unique names for each interface diff --git a/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py b/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py index 355c86510..e7473f228 100644 --- a/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py +++ b/src/neuroconv/tools/nwb_helpers/_metadata_and_file_helpers.py @@ -166,7 +166,7 @@ def make_or_load_nwbfile( metadata: Optional[dict] = None, overwrite: bool = False, backend: Literal["hdf5", "zarr"] = "hdf5", - verbose: bool = True, + verbose: bool = False, ): """ Context for automatically handling decision of write vs. append for writing an NWBFile. diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 27d3b5f9c..afe58ede6 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -753,7 +753,7 @@ def write_imaging( nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, iterator_type: str = "v2", iterator_options: Optional[dict] = None, photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", @@ -792,7 +792,7 @@ def write_imaging_to_nwbfile( nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, iterator_type: str = "v2", iterator_options: Optional[dict] = None, photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", @@ -1904,7 +1904,7 @@ def write_segmentation( nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, include_background_segmentation: bool = False, include_roi_centroids: bool = True, include_roi_acceptance: bool = True, @@ -1949,7 +1949,7 @@ def write_segmentation_to_nwbfile( nwbfile: Optional[NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, include_background_segmentation: bool = False, include_roi_centroids: bool = True, include_roi_acceptance: bool = True, diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 7df25c703..e88e7d480 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -1187,7 +1187,7 @@ def write_recording( nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, starting_time: Optional[float] = None, write_as: Optional[str] = "raw", es_key: Optional[str] = None, @@ -1228,7 +1228,7 @@ def write_recording_to_nwbfile( nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, starting_time: Optional[float] = None, write_as: Optional[str] = "raw", es_key: Optional[str] = None, @@ -1291,7 +1291,7 @@ def write_recording_to_nwbfile( properties in the RecordingExtractor object. overwrite : bool, default: False Whether to overwrite the NWBFile if one exists at the nwbfile_path. - verbose : bool, default: True + verbose : bool, default: False If 'nwbfile_path' is specified, informs user after a successful write operation. starting_time : float, optional Sets the starting time of the ElectricalSeries to a manually set value. @@ -1784,7 +1784,7 @@ def write_sorting( nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, unit_ids: Optional[list[Union[str, int]]] = None, property_descriptions: Optional[dict] = None, skip_properties: Optional[list[str]] = None, @@ -1831,7 +1831,7 @@ def write_sorting_to_nwbfile( nwbfile: Optional[pynwb.NWBFile] = None, metadata: Optional[dict] = None, overwrite: bool = False, - verbose: bool = True, + verbose: bool = False, unit_ids: Optional[list[Union[str, int]]] = None, property_descriptions: Optional[dict] = None, skip_properties: Optional[list[str]] = None, @@ -1866,7 +1866,7 @@ def write_sorting_to_nwbfile( overwrite : bool, default: False Whether to overwrite the NWBFile if one exists at the nwbfile_path. The default is False (append mode). - verbose : bool, default: True + verbose : bool, default: False If 'nwbfile_path' is specified, informs user after a successful write operation. unit_ids : list, optional Controls the unit_ids that will be written to the nwb file. If None (default), all @@ -2066,7 +2066,7 @@ def write_sorting_analyzer( metadata: Optional[dict] = None, overwrite: bool = False, recording: Optional[BaseRecording] = None, - verbose: bool = True, + verbose: bool = False, unit_ids: Optional[Union[list[str], list[int]]] = None, write_electrical_series: bool = False, add_electrical_series_kwargs: Optional[dict] = None, @@ -2111,7 +2111,7 @@ def write_sorting_analyzer_to_nwbfile( metadata: Optional[dict] = None, overwrite: bool = False, recording: Optional[BaseRecording] = None, - verbose: bool = True, + verbose: bool = False, unit_ids: Optional[Union[list[str], list[int]]] = None, write_electrical_series: bool = False, add_electrical_series_kwargs: Optional[dict] = None, @@ -2153,7 +2153,7 @@ def write_sorting_analyzer_to_nwbfile( recording : BaseRecording, optional If the sorting_analyzer is 'recordingless', this argument needs to be passed to save electrode info. Otherwise, electrodes info is not added to the nwb file. - verbose : bool, default: True + verbose : bool, default: False If 'nwbfile_path' is specified, informs user after a successful write operation. unit_ids : list, optional Controls the unit_ids that will be written to the nwb file. If None (default), all @@ -2218,7 +2218,7 @@ def write_waveforms( metadata: Optional[dict] = None, overwrite: bool = False, recording: Optional[BaseRecording] = None, - verbose: bool = True, + verbose: bool = False, unit_ids: Optional[list[Union[str, int]]] = None, write_electrical_series: bool = False, add_electrical_series_kwargs: Optional[dict] = None, diff --git a/src/neuroconv/tools/testing/mock_interfaces.py b/src/neuroconv/tools/testing/mock_interfaces.py index 42bf699df..128b9365f 100644 --- a/src/neuroconv/tools/testing/mock_interfaces.py +++ b/src/neuroconv/tools/testing/mock_interfaces.py @@ -208,7 +208,7 @@ def __init__( sampling_frequency: float = 30_000.0, durations: tuple[float, ...] = (1.0,), seed: int = 0, - verbose: bool = True, + verbose: bool = False, es_key: str = "ElectricalSeries", ): super().__init__( @@ -251,7 +251,7 @@ def __init__( sampling_frequency: float = 30_000.0, durations: tuple[float, ...] = (1.0,), seed: int = 0, - verbose: bool = True, + verbose: bool = False, ): """ Parameters diff --git a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py index 111053797..09609b9f7 100644 --- a/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py +++ b/tests/test_minimal/test_utils/test_get_json_schema_from_method_signature.py @@ -213,7 +213,7 @@ def test_get_json_schema_from_example_data_interface(): "type": "string", "description": "Path to the folder of .mpx files.", }, - "verbose": {"default": True, "type": "boolean", "description": "Allows verbose.\nDefault is True."}, + "verbose": {"default": False, "type": "boolean", "description": "Allows verbose.\nDefault is False."}, "es_key": {"default": "ElectricalSeries", "type": "string"}, }, "required": ["folder_path"], diff --git a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py index 724749574..9ed728b16 100644 --- a/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py +++ b/tests/test_on_data/test_temporal_alignment/test_temporal_alignment_methods.py @@ -357,7 +357,7 @@ def mimic_reading_externally_aligned_timestamps(): class TestAlignmentConverter(NWBConverter): data_interface_classes = dict(Trials=CsvTimeIntervalsInterface, Behavior=MockBehaviorEventInterface) - def __init__(self, source_data: Dict[str, dict], verbose: bool = True): + def __init__(self, source_data: Dict[str, dict], verbose: bool = False): super().__init__(source_data=source_data, verbose=verbose) unaligned_trial_start_timestamps = self.data_interface_objects["Trials"].get_timestamps( @@ -505,7 +505,7 @@ class TestAlignmentConverter(NWBConverter): NIDQ=MockSpikeGLXNIDQInterface, Trials=CsvTimeIntervalsInterface, Behavior=MockBehaviorEventInterface ) - def __init__(self, source_data: Dict[str, dict], verbose: bool = True): + def __init__(self, source_data: Dict[str, dict], verbose: bool = False): super().__init__(source_data=source_data, verbose=verbose) inferred_aligned_trial_start_time = self.data_interface_objects["NIDQ"].get_event_times_from_ttl( From b3c6d91748dbb9d761f54a47d5f8ceb6ace120e8 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Fri, 24 Jan 2025 06:41:19 +1100 Subject: [PATCH 116/118] updated ndx-events to 0.2.1 (#1176) Co-authored-by: Heberto Mayorquin --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7a250651..8a33fb965 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ neuroconv = "neuroconv.tools.yaml_conversion_specification._yaml_conversion_spec test = [ "pytest", "pytest-cov", - "ndx-events>=0.2.0", # for special tests to ensure load_namespaces is set to allow NWBFile load at all times + "ndx-events==0.2.1", # for special tests to ensure load_namespaces is set to allow NWBFile load at all times "parameterized>=0.8.1", "ndx-miniscope", "spikeinterface[qualitymetrics]>=0.101.0", @@ -132,7 +132,7 @@ lightningpose = [ "neuroconv[video]", ] medpc = [ - "ndx-events==0.2.0", + "ndx-events==0.2.1", ] behavior = [ "neuroconv[sleap]", From b78770806635a7fe76a979fcca8151fa3515446a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 23 Jan 2025 16:56:16 -0600 Subject: [PATCH 117/118] Use `ndx-pose>=0.2` for `DeepLabCutInterface` and `LightningPoseDataInterface` (#1128) Co-authored-by: Paul Adkisson --- .github/workflows/testing.yml | 6 + CHANGELOG.md | 4 +- docs/conversion_examples_gallery/conftest.py | 9 +- pyproject.toml | 4 +- src/neuroconv/basedatainterface.py | 4 +- .../behavior/deeplabcut/_dlc_utils.py | 72 ++- .../deeplabcut/deeplabcutdatainterface.py | 28 +- .../lightningposedatainterface.py | 57 +- .../behavior/sleap/sleapdatainterface.py | 18 + .../behavior/test_behavior_interfaces.py | 521 ---------------- .../behavior/test_lightningpose_converter.py | 1 + .../test_pose_estimation_interfaces.py | 566 ++++++++++++++++++ 12 files changed, 722 insertions(+), 568 deletions(-) create mode 100644 tests/test_on_data/behavior/test_pose_estimation_interfaces.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d8c5bb9fd..644329cbd 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -98,6 +98,12 @@ jobs: s3-gin-bucket: ${{ secrets.S3_GIN_BUCKET }} os: ${{ matrix.os }} + # TODO: remove this setp after this is merged https://github.com/talmolab/sleap-io/pull/143 + - name: Run Sleap Tests until sleap.io adds support for ndx-pose > 2.0 + run : | + pip install ndx-pose==0.1.1 + pytest tests/test_on_data/behavior/test_pose_estimation_interfaces.py + - name: Install full requirements run: pip install .[full] diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cab36eab..6df9fd842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,14 @@ ## Features * Added `metadata` and `conversion_options` as arguments to `NWBConverter.temporally_align_data_interfaces` [PR #1162](https://github.com/catalystneuro/neuroconv/pull/1162) +* Use the latest version of ndx-pose for `DeepLabCutInterface` and `LightningPoseDataInterface` [PR #1128](https://github.com/catalystneuro/neuroconv/pull/1128) ## Improvements * Interfaces and converters now have `verbose=False` by default [PR #1153](https://github.com/catalystneuro/neuroconv/pull/1153) -# v0.6.7 (January 20, 2024) + +# v0.6.7 (January 20, 2025) ## Deprecations diff --git a/docs/conversion_examples_gallery/conftest.py b/docs/conversion_examples_gallery/conftest.py index 6618d6d52..775eb4f9d 100644 --- a/docs/conversion_examples_gallery/conftest.py +++ b/docs/conversion_examples_gallery/conftest.py @@ -1,4 +1,5 @@ import platform +from importlib.metadata import version as importlib_version from pathlib import Path import pytest @@ -29,9 +30,15 @@ def add_data_space(doctest_namespace, tmp_path): # Hook to conditionally skip doctests in deeplabcut.rst for Python 3.9 on macOS (Darwin) def pytest_runtest_setup(item): if isinstance(item, pytest.DoctestItem): - # Check if we are running the doctest from deeplabcut.rst test_file = Path(item.fspath) + # Check if we are running the doctest from deeplabcut.rst if test_file.name == "deeplabcut.rst": # Check if Python version is 3.9 and platform is Darwin (macOS) if version.parse(python_version) < version.parse("3.10") and os == "Darwin": pytest.skip("Skipping doctests for deeplabcut.rst on Python 3.9 and macOS") + # Check if we are running the doctest from sleap.rst + # TODO: remove after this is merged https://github.com/talmolab/sleap-io/pull/143 and released + elif test_file.name in ["ecephys_pose_estimation.rst", "sleap.rst"]: + ndx_pose_version = version.parse(importlib_version("ndx-pose")) + if ndx_pose_version >= version.parse("0.2.0"): + pytest.skip("Skipping doctests because sleeps only run when ndx-pose version < 0.2.0") diff --git a/pyproject.toml b/pyproject.toml index 8a33fb965..5915a2b54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,7 +118,7 @@ sleap = [ "sleap-io>=0.0.2; python_version>='3.9'", ] deeplabcut = [ - "ndx-pose==0.1.1", + "ndx-pose>=0.2", "tables; platform_system != 'Darwin'", "tables>=3.10.1; platform_system == 'Darwin' and python_version >= '3.10'", ] @@ -128,7 +128,7 @@ video = [ "opencv-python-headless>=4.8.1.78", ] lightningpose = [ - "ndx-pose==0.1.1", + "ndx-pose>=0.2", "neuroconv[video]", ] medpc = [ diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 64af908e3..95b80f6d7 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -212,9 +212,7 @@ def run_conversion( @staticmethod def get_default_backend_configuration( nwbfile: NWBFile, - # TODO: when all H5DataIO prewraps are gone, introduce Zarr safely - # backend: Union[Literal["hdf5", "zarr"]], - backend: Literal["hdf5"] = "hdf5", + backend: Literal["hdf5", "zarr"] = "hdf5", ) -> Union[HDF5BackendConfiguration, ZarrBackendConfiguration]: """ Fill and return a default backend configuration to serve as a starting point for further customization. diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py index 14866510d..1c64c59a9 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/_dlc_utils.py @@ -1,4 +1,3 @@ -import importlib import pickle import warnings from pathlib import Path @@ -11,6 +10,8 @@ from pynwb import NWBFile from ruamel.yaml import YAML +from ....tools import get_module + def _read_config(config_file_path: FilePath) -> dict: """ @@ -93,7 +94,7 @@ def _get_cv2_timestamps(file_path: Union[Path, str]): return timestamps -def _get_movie_timestamps(movie_file, VARIABILITYBOUND=1000, infer_timestamps=True): +def _get_video_timestamps(movie_file, VARIABILITYBOUND=1000, infer_timestamps=True): """ Return numpy array of the timestamps for a video. @@ -263,13 +264,44 @@ def _write_pes_to_nwbfile( exclude_nans, pose_estimation_container_kwargs: Optional[dict] = None, ): - - from ndx_pose import PoseEstimation, PoseEstimationSeries + """ + Updated version of _write_pes_to_nwbfile to work with ndx-pose v0.2.0+ + """ + from ndx_pose import PoseEstimation, PoseEstimationSeries, Skeleton, Skeletons + from pynwb.file import Subject pose_estimation_container_kwargs = pose_estimation_container_kwargs or dict() + pose_estimation_name = pose_estimation_container_kwargs.get("name", "PoseEstimationDeepLabCut") + + # Create a subject if it doesn't exist + if nwbfile.subject is None: + subject = Subject(subject_id=animal) + nwbfile.subject = subject + else: + subject = nwbfile.subject + + # Create skeleton from the keypoints + keypoints = df_animal.columns.get_level_values("bodyparts").unique() + animal = animal if animal else "" + subject = subject if animal == subject.subject_id else None + skeleton_name = f"Skeleton{pose_estimation_name}_{animal.capitalize()}" + skeleton = Skeleton( + name=skeleton_name, + nodes=list(keypoints), + edges=np.array(paf_graph) if paf_graph else None, # Convert paf_graph to numpy array + subject=subject, + ) + + behavior_processing_module = get_module(nwbfile=nwbfile, name="behavior", description="processed behavioral data") + if "Skeletons" not in behavior_processing_module.data_interfaces: + skeletons = Skeletons(skeletons=[skeleton]) + behavior_processing_module.add(skeletons) + else: + skeletons = behavior_processing_module["Skeletons"] + skeletons.add_skeletons(skeleton) pose_estimation_series = [] - for keypoint in df_animal.columns.get_level_values("bodyparts").unique(): + for keypoint in keypoints: data = df_animal.xs(keypoint, level="bodyparts", axis=1).to_numpy() if exclude_nans: @@ -292,35 +324,31 @@ def _write_pes_to_nwbfile( ) pose_estimation_series.append(pes) - deeplabcut_version = None - is_deeplabcut_installed = importlib.util.find_spec(name="deeplabcut") is not None - if is_deeplabcut_installed: - deeplabcut_version = importlib.metadata.version(distribution_name="deeplabcut") + camera_name = pose_estimation_name + if camera_name not in nwbfile.devices: + camera = nwbfile.create_device( + name=camera_name, + description="Camera used for behavioral recording and pose estimation.", + ) + else: + camera = nwbfile.devices[camera_name] - # TODO, taken from the original implementation, improve it if the video is passed + # Create PoseEstimation container with updated arguments dimensions = [list(map(int, image_shape.split(",")))[1::2]] dimensions = np.array(dimensions, dtype="uint32") pose_estimation_default_kwargs = dict( pose_estimation_series=pose_estimation_series, description="2D keypoint coordinates estimated using DeepLabCut.", - original_videos=[video_file_path], + original_videos=[video_file_path] if video_file_path else None, dimensions=dimensions, + devices=[camera], scorer=scorer, source_software="DeepLabCut", - source_software_version=deeplabcut_version, - nodes=[pes.name for pes in pose_estimation_series], - edges=paf_graph if paf_graph else None, - **pose_estimation_container_kwargs, + skeleton=skeleton, ) pose_estimation_default_kwargs.update(pose_estimation_container_kwargs) pose_estimation_container = PoseEstimation(**pose_estimation_default_kwargs) - if "behavior" in nwbfile.processing: # TODO: replace with get_module - behavior_processing_module = nwbfile.processing["behavior"] - else: - behavior_processing_module = nwbfile.create_processing_module( - name="behavior", description="processed behavioral data" - ) behavior_processing_module.add(pose_estimation_container) return nwbfile @@ -387,7 +415,7 @@ def _add_subject_to_nwbfile( if video_file_path is None: timestamps = df.index.tolist() # setting timestamps to dummy else: - timestamps = _get_movie_timestamps(video_file_path, infer_timestamps=True) + timestamps = _get_video_timestamps(video_file_path, infer_timestamps=True) # Fetch the corresponding metadata pickle file, we extract the edges graph from here # TODO: This is the original implementation way to extract the file name but looks very brittle. Improve it diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py index 8c81a0264..c73551e59 100644 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py @@ -12,7 +12,7 @@ class DeepLabCutInterface(BaseTemporalAlignmentInterface): """Data interface for DeepLabCut datasets.""" display_name = "DeepLabCut" - keywords = ("DLC",) + keywords = ("DLC", "DeepLabCut", "pose estimation", "behavior") associated_suffixes = (".h5", ".csv") info = "Interface for handling data from DeepLabCut." @@ -48,7 +48,18 @@ def __init__( Controls verbosity. """ # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created - from ndx_pose import PoseEstimation, PoseEstimationSeries # noqa: F401 + from importlib.metadata import version + + import ndx_pose # noqa: F401 + from packaging import version as version_parse + + ndx_pose_version = version("ndx-pose") + if version_parse.parse(ndx_pose_version) < version_parse.parse("0.2.0"): + raise ImportError( + "DeepLabCut interface requires ndx-pose version 0.2.0 or later. " + f"Found version {ndx_pose_version}. Please upgrade: " + "pip install 'ndx-pose>=0.2.0'" + ) from ._dlc_utils import _read_config @@ -62,6 +73,8 @@ def __init__( self.config_dict = _read_config(config_file_path=config_file_path) self.subject_name = subject_name self.verbose = verbose + self.pose_estimation_container_kwargs = dict() + super().__init__(file_path=file_path, config_file_path=config_file_path) def get_metadata(self): @@ -101,7 +114,7 @@ def add_to_nwbfile( self, nwbfile: NWBFile, metadata: Optional[dict] = None, - container_name: str = "PoseEstimation", + container_name: str = "PoseEstimationDeepLabCut", ): """ Conversion from DLC output files to nwb. Derived from dlc2nwb library. @@ -112,16 +125,19 @@ def add_to_nwbfile( nwb file to which the recording information is to be added metadata: dict metadata info for constructing the nwb file (optional). - container_name: str, default: "PoseEstimation" - Name of the container to store the pose estimation. + container_name: str, default: "PoseEstimationDeepLabCut" + name of the PoseEstimation container in the nwb + """ from ._dlc_utils import _add_subject_to_nwbfile + self.pose_estimation_container_kwargs["name"] = container_name + _add_subject_to_nwbfile( nwbfile=nwbfile, file_path=str(self.source_data["file_path"]), individual_name=self.subject_name, config_file=self.source_data["config_file_path"], timestamps=self._timestamps, - pose_estimation_container_kwargs=dict(name=container_name), + pose_estimation_container_kwargs=self.pose_estimation_container_kwargs, ) diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py index 1211c31d1..05f569a52 100644 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py @@ -40,9 +40,10 @@ def get_metadata_schema(self) -> dict: description=dict(type="string"), scorer=dict(type="string"), source_software=dict(type="string", default="LightningPose"), + camera_name=dict(type="string", default="CameraPoseEstimation"), ), patternProperties={ - "^(?!(name|description|scorer|source_software)$)[a-zA-Z0-9_]+$": dict( + "^(?!(name|description|scorer|source_software|camera_name)$)[a-zA-Z0-9_]+$": dict( title="PoseEstimationSeries", type="object", properties=dict(name=dict(type="string"), description=dict(type="string")), @@ -80,9 +81,21 @@ def __init__( verbose : bool, default: False controls verbosity. ``True`` by default. """ + # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created # For more detail, see https://github.com/rly/ndx-pose/issues/36 + from importlib.metadata import version + import ndx_pose # noqa: F401 + from packaging import version as version_parse + + ndx_pose_version = version("ndx-pose") + if version_parse.parse(ndx_pose_version) < version_parse.parse("0.2.0"): + raise ImportError( + "LightningPose interface requires ndx-pose version 0.2.0 or later. " + f"Found version {ndx_pose_version}. Please upgrade: " + "pip install 'ndx-pose>=0.2.0'" + ) from neuroconv.datainterfaces.behavior.video.video_utils import ( VideoCaptureContext, @@ -162,6 +175,7 @@ def get_metadata(self) -> DeepDict: description="Contains the pose estimation series for each keypoint.", scorer=self.scorer_name, source_software="LightningPose", + camera_name="CameraPoseEstimation", ) for keypoint_name in self.keypoint_names: keypoint_name_without_spaces = keypoint_name.replace(" ", "") @@ -198,7 +212,7 @@ def add_to_nwbfile( The description of how the confidence was computed, e.g., 'Softmax output of the deep neural network'. stub_test : bool, default: False """ - from ndx_pose import PoseEstimation, PoseEstimationSeries + from ndx_pose import PoseEstimation, PoseEstimationSeries, Skeleton, Skeletons metadata_copy = deepcopy(metadata) @@ -215,15 +229,14 @@ def add_to_nwbfile( original_video_name = str(self.original_video_file_path) else: original_video_name = metadata_copy["Behavior"]["Videos"][0]["name"] - - pose_estimation_kwargs = dict( - name=pose_estimation_metadata["name"], - description=pose_estimation_metadata["description"], - source_software=pose_estimation_metadata["source_software"], - scorer=pose_estimation_metadata["scorer"], - original_videos=[original_video_name], - dimensions=[self.dimension], - ) + camera_name = pose_estimation_metadata["camera_name"] + if camera_name in nwbfile.devices: + camera = nwbfile.devices[camera_name] + else: + camera = nwbfile.create_device( + name=camera_name, + description="Camera used for behavioral recording and pose estimation.", + ) pose_estimation_data = self.pose_estimation_data if not stub_test else self.pose_estimation_data.head(n=10) timestamps = self.get_timestamps(stub_test=stub_test) @@ -255,8 +268,28 @@ def add_to_nwbfile( pose_estimation_series.append(PoseEstimationSeries(**pose_estimation_series_kwargs)) - pose_estimation_kwargs.update( + # Add Skeleton(s) + nodes = [keypoint_name.replace(" ", "") for keypoint_name in self.keypoint_names] + subject = nwbfile.subject if nwbfile.subject is not None else None + name = f"Skeleton{pose_estimation_name}" + skeleton = Skeleton(name=name, nodes=nodes, subject=subject) + if "Skeletons" in behavior.data_interfaces: + skeletons = behavior.data_interfaces["Skeletons"] + skeletons.add_skeletons(skeleton) + else: + skeletons = Skeletons(skeletons=[skeleton]) + behavior.add(skeletons) + + pose_estimation_kwargs = dict( + name=pose_estimation_metadata["name"], + description=pose_estimation_metadata["description"], + source_software=pose_estimation_metadata["source_software"], + scorer=pose_estimation_metadata["scorer"], + original_videos=[original_video_name], + dimensions=[self.dimension], pose_estimation_series=pose_estimation_series, + devices=[camera], + skeleton=skeleton, ) if self.source_data["labeled_video_file_path"]: diff --git a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py index 90b9b91e1..c84820270 100644 --- a/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py @@ -49,6 +49,24 @@ def __init__( frames_per_second : float, optional The frames per second (fps) or sampling rate of the video. """ + + # This import is to assure that the ndx_pose is in the global namespace when an pynwb.io object is created + # For more detail, see https://github.com/rly/ndx-pose/issues/36 + from importlib.metadata import version + + import ndx_pose # noqa: F401 + from packaging import version as version_parse + + ndx_pose_version = version("ndx-pose") + + # TODO: remove after this is merged https://github.com/talmolab/sleap-io/pull/143 and released + if version_parse.parse(ndx_pose_version) != version_parse.parse("0.1.1"): + raise ImportError( + "SLEAP interface requires ndx-pose version 0.1.1. " + f"Found version {ndx_pose_version}. Please install the required version: " + "pip install 'ndx-pose==0.1.1'" + ) + self.file_path = Path(file_path) self.sleap_io = get_package(package_name="sleap_io") self.video_file_path = video_file_path diff --git a/tests/test_on_data/behavior/test_behavior_interfaces.py b/tests/test_on_data/behavior/test_behavior_interfaces.py index 0b5c63376..2115e6dd4 100644 --- a/tests/test_on_data/behavior/test_behavior_interfaces.py +++ b/tests/test_on_data/behavior/test_behavior_interfaces.py @@ -1,30 +1,23 @@ -import sys import unittest from datetime import datetime, timezone from pathlib import Path import numpy as np -import pandas as pd import pytest -import sleap_io from hdmf.testing import TestCase from natsort import natsorted from ndx_miniscope import Miniscope from ndx_miniscope.utils import get_timestamps from numpy.testing import assert_array_equal -from parameterized import param, parameterized from pynwb import NWBHDF5IO from pynwb.behavior import Position, SpatialSeries from neuroconv import NWBConverter from neuroconv.datainterfaces import ( - DeepLabCutInterface, FicTracDataInterface, - LightningPoseDataInterface, MedPCInterface, MiniscopeBehaviorInterface, NeuralynxNvtInterface, - SLEAPInterface, VideoInterface, ) from neuroconv.tools.testing.data_interface_mixins import ( @@ -33,7 +26,6 @@ TemporalAlignmentMixin, VideoInterfaceMixin, ) -from neuroconv.utils import DeepDict try: from ..setup_paths import BEHAVIOR_DATA_PATH, OPHYS_DATA_PATH, OUTPUT_PATH @@ -41,142 +33,6 @@ from setup_paths import BEHAVIOR_DATA_PATH, OUTPUT_PATH -class TestLightningPoseDataInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): - data_interface_cls = LightningPoseDataInterface - interface_kwargs = dict( - file_path=str(BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.csv"), - original_video_file_path=str( - BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.mp4" - ), - ) - conversion_options = dict(reference_frame="(0,0) corresponds to the top left corner of the video.") - save_directory = OUTPUT_PATH - - @pytest.fixture(scope="class", autouse=True) - def setup_metadata(self, request): - - cls = request.cls - - cls.pose_estimation_name = "PoseEstimation" - cls.original_video_height = 406 - cls.original_video_width = 396 - cls.expected_keypoint_names = [ - "paw1LH_top", - "paw2LF_top", - "paw3RF_top", - "paw4RH_top", - "tailBase_top", - "tailMid_top", - "nose_top", - "obs_top", - "paw1LH_bot", - "paw2LF_bot", - "paw3RF_bot", - "paw4RH_bot", - "tailBase_bot", - "tailMid_bot", - "nose_bot", - "obsHigh_bot", - "obsLow_bot", - ] - cls.expected_metadata = DeepDict( - PoseEstimation=dict( - name=cls.pose_estimation_name, - description="Contains the pose estimation series for each keypoint.", - scorer="heatmap_tracker", - source_software="LightningPose", - ) - ) - cls.expected_metadata[cls.pose_estimation_name].update( - { - keypoint_name: dict( - name=f"PoseEstimationSeries{keypoint_name}", - description=f"The estimated position (x, y) of {keypoint_name} over time.", - ) - for keypoint_name in cls.expected_keypoint_names - } - ) - - cls.test_data = pd.read_csv(cls.interface_kwargs["file_path"], header=[0, 1, 2])["heatmap_tracker"] - - def check_extracted_metadata(self, metadata: dict): - assert metadata["NWBFile"]["session_start_time"] == datetime(2023, 11, 9, 10, 14, 37, 0) - assert self.pose_estimation_name in metadata["Behavior"] - assert metadata["Behavior"][self.pose_estimation_name] == self.expected_metadata[self.pose_estimation_name] - - def check_read_nwb(self, nwbfile_path: str): - from ndx_pose import PoseEstimation, PoseEstimationSeries - - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - - # Replacing assertIn with pytest-style assert - assert "behavior" in nwbfile.processing - assert self.pose_estimation_name in nwbfile.processing["behavior"].data_interfaces - - pose_estimation_container = nwbfile.processing["behavior"].data_interfaces[self.pose_estimation_name] - - # Replacing assertIsInstance with pytest-style assert - assert isinstance(pose_estimation_container, PoseEstimation) - - pose_estimation_metadata = self.expected_metadata[self.pose_estimation_name] - - # Replacing assertEqual with pytest-style assert - assert pose_estimation_container.description == pose_estimation_metadata["description"] - assert pose_estimation_container.scorer == pose_estimation_metadata["scorer"] - assert pose_estimation_container.source_software == pose_estimation_metadata["source_software"] - - # Using numpy's assert_array_equal - assert_array_equal( - pose_estimation_container.dimensions[:], [[self.original_video_height, self.original_video_width]] - ) - - # Replacing assertEqual with pytest-style assert - assert len(pose_estimation_container.pose_estimation_series) == len(self.expected_keypoint_names) - - for keypoint_name in self.expected_keypoint_names: - series_metadata = pose_estimation_metadata[keypoint_name] - - # Replacing assertIn with pytest-style assert - assert series_metadata["name"] in pose_estimation_container.pose_estimation_series - - pose_estimation_series = pose_estimation_container.pose_estimation_series[series_metadata["name"]] - - # Replacing assertIsInstance with pytest-style assert - assert isinstance(pose_estimation_series, PoseEstimationSeries) - - # Replacing assertEqual with pytest-style assert - assert pose_estimation_series.unit == "px" - assert pose_estimation_series.description == series_metadata["description"] - assert pose_estimation_series.reference_frame == self.conversion_options["reference_frame"] - - test_data = self.test_data[keypoint_name] - - # Using numpy's assert_array_equal - assert_array_equal(pose_estimation_series.data[:], test_data[["x", "y"]].values) - - -class TestLightningPoseDataInterfaceWithStubTest(DataInterfaceTestMixin, TemporalAlignmentMixin): - data_interface_cls = LightningPoseDataInterface - interface_kwargs = dict( - file_path=str(BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.csv"), - original_video_file_path=str( - BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.mp4" - ), - ) - - conversion_options = dict(stub_test=True) - save_directory = OUTPUT_PATH - - def check_read_nwb(self, nwbfile_path: str): - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - pose_estimation_container = nwbfile.processing["behavior"].data_interfaces["PoseEstimation"] - for pose_estimation_series in pose_estimation_container.pose_estimation_series.values(): - assert pose_estimation_series.data.shape[0] == 10 - assert pose_estimation_series.confidence.shape[0] == 10 - - class TestFicTracDataInterface(DataInterfaceTestMixin): data_interface_cls = FicTracDataInterface interface_kwargs = dict( @@ -321,268 +177,6 @@ class TestFicTracDataInterfaceTiming(TemporalAlignmentMixin): save_directory = OUTPUT_PATH -from platform import python_version - -from packaging import version - -python_version = version.parse(python_version()) -from sys import platform - - -@pytest.mark.skipif( - platform == "darwin" and python_version < version.parse("3.10"), - reason="interface not supported on macOS with Python < 3.10", -) -class TestDeepLabCutInterface(DataInterfaceTestMixin): - data_interface_cls = DeepLabCutInterface - interface_kwargs = dict( - file_path=str( - BEHAVIOR_DATA_PATH - / "DLC" - / "open_field_without_video" - / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" - ), - config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), - subject_name="ind1", - ) - save_directory = OUTPUT_PATH - - def run_custom_checks(self): - self.check_renaming_instance(nwbfile_path=self.nwbfile_path) - - def check_renaming_instance(self, nwbfile_path: str): - custom_container_name = "TestPoseEstimation" - - metadata = self.interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - self.interface.run_conversion( - nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata, container_name=custom_container_name - ) - - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - assert "behavior" in nwbfile.processing - assert "PoseEstimation" not in nwbfile.processing["behavior"].data_interfaces - assert custom_container_name in nwbfile.processing["behavior"].data_interfaces - - def check_read_nwb(self, nwbfile_path: str): - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - assert "behavior" in nwbfile.processing - processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces - assert "PoseEstimation" in processing_module_interfaces - - pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series - expected_pose_estimation_series = ["ind1_leftear", "ind1_rightear", "ind1_snout", "ind1_tailbase"] - - expected_pose_estimation_series_are_in_nwb_file = [ - pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series - ] - - assert all(expected_pose_estimation_series_are_in_nwb_file) - - -@pytest.fixture -def clean_pose_extension_import(): - modules_to_remove = [m for m in sys.modules if m.startswith("ndx_pose")] - for module in modules_to_remove: - del sys.modules[module] - - -@pytest.mark.skipif( - platform == "darwin" and python_version < version.parse("3.10"), - reason="interface not supported on macOS with Python < 3.10", -) -def test_deep_lab_cut_import_pose_extension_bug(clean_pose_extension_import, tmp_path): - """ - Test that the DeepLabCutInterface writes correctly without importing the ndx-pose extension. - See issues: - https://github.com/catalystneuro/neuroconv/issues/1114 - https://github.com/rly/ndx-pose/issues/36 - - """ - - interface_kwargs = dict( - file_path=str( - BEHAVIOR_DATA_PATH - / "DLC" - / "open_field_without_video" - / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" - ), - config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), - ) - - interface = DeepLabCutInterface(**interface_kwargs) - metadata = interface.get_metadata() - metadata["NWBFile"]["session_start_time"] = datetime(2023, 7, 24, 9, 30, 55, 440600, tzinfo=timezone.utc) - - nwbfile_path = tmp_path / "test.nwb" - interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) - with NWBHDF5IO(path=nwbfile_path, mode="r") as io: - read_nwbfile = io.read() - pose_estimation_container = read_nwbfile.processing["behavior"]["PoseEstimation"] - - assert len(pose_estimation_container.fields) > 0 - - -@pytest.mark.skipif( - platform == "darwin" and python_version < version.parse("3.10"), - reason="interface not supported on macOS with Python < 3.10", -) -class TestDeepLabCutInterfaceNoConfigFile(DataInterfaceTestMixin): - data_interface_cls = DeepLabCutInterface - interface_kwargs = dict( - file_path=str( - BEHAVIOR_DATA_PATH - / "DLC" - / "open_field_without_video" - / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" - ), - config_file_path=None, - subject_name="ind1", - ) - save_directory = OUTPUT_PATH - - def check_read_nwb(self, nwbfile_path: str): - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - assert "behavior" in nwbfile.processing - processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces - assert "PoseEstimation" in processing_module_interfaces - - pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series - expected_pose_estimation_series = ["ind1_leftear", "ind1_rightear", "ind1_snout", "ind1_tailbase"] - - expected_pose_estimation_series_are_in_nwb_file = [ - pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series - ] - - assert all(expected_pose_estimation_series_are_in_nwb_file) - - -@pytest.mark.skipif( - platform == "darwin" and python_version < version.parse("3.10"), - reason="interface not supported on macOS with Python < 3.10", -) -class TestDeepLabCutInterfaceSetTimestamps(DataInterfaceTestMixin): - data_interface_cls = DeepLabCutInterface - interface_kwargs = dict( - file_path=str( - BEHAVIOR_DATA_PATH - / "DLC" - / "open_field_without_video" - / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" - ), - config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), - subject_name="ind1", - ) - - save_directory = OUTPUT_PATH - - def run_custom_checks(self): - self.check_custom_timestamps(nwbfile_path=self.nwbfile_path) - - def check_custom_timestamps(self, nwbfile_path: str): - custom_timestamps = np.concatenate( - (np.linspace(10, 110, 1000), np.linspace(150, 250, 1000), np.linspace(300, 400, 330)) - ) - - metadata = self.interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - - self.interface.set_aligned_timestamps(custom_timestamps) - assert len(self.interface._timestamps) == 2330 - - self.interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) - - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - assert "behavior" in nwbfile.processing - processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces - assert "PoseEstimation" in processing_module_interfaces - - pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series - - for pose_estimation in pose_estimation_series_in_nwb.values(): - pose_timestamps = pose_estimation.timestamps - np.testing.assert_array_equal(pose_timestamps, custom_timestamps) - - # This was tested in the other test - def check_read_nwb(self, nwbfile_path: str): - pass - - -@pytest.mark.skipif( - platform == "darwin" and python_version < version.parse("3.10"), - reason="interface not supported on macOS with Python < 3.10", -) -class TestDeepLabCutInterfaceFromCSV(DataInterfaceTestMixin): - data_interface_cls = DeepLabCutInterface - interface_kwargs = dict( - file_path=str( - BEHAVIOR_DATA_PATH - / "DLC" - / "SL18_csv" - / "SL18_D19_S01_F01_BOX_SLP_20230503_112642.1DLC_resnet50_SubLearnSleepBoxRedLightJun26shuffle1_100000_stubbed.csv" - ), - config_file_path=None, - subject_name="SL18", - ) - save_directory = OUTPUT_PATH - - def check_read_nwb(self, nwbfile_path: str): - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - assert "behavior" in nwbfile.processing - processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces - assert "PoseEstimation" in processing_module_interfaces - - pose_estimation_series_in_nwb = processing_module_interfaces["PoseEstimation"].pose_estimation_series - expected_pose_estimation_series = ["SL18_redled", "SL18_shoulder", "SL18_haunch", "SL18_baseoftail"] - - expected_pose_estimation_series_are_in_nwb_file = [ - pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series - ] - - assert all(expected_pose_estimation_series_are_in_nwb_file) - - -class TestSLEAPInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): - data_interface_cls = SLEAPInterface - interface_kwargs = dict( - file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "predictions_1.2.7_provenance_and_tracking.slp"), - video_file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "melanogaster_courtship.mp4"), - ) - save_directory = OUTPUT_PATH - - def check_read_nwb(self, nwbfile_path: str): # This is currently structured to be file-specific - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - assert "SLEAP_VIDEO_000_20190128_113421" in nwbfile.processing - processing_module_interfaces = nwbfile.processing["SLEAP_VIDEO_000_20190128_113421"].data_interfaces - assert "track=track_0" in processing_module_interfaces - - pose_estimation_series_in_nwb = processing_module_interfaces["track=track_0"].pose_estimation_series - expected_pose_estimation_series = [ - "abdomen", - "eyeL", - "eyeR", - "forelegL4", - "forelegR4", - "head", - "hindlegL4", - "hindlegR4", - "midlegL4", - "midlegR4", - "thorax", - "wingL", - "wingR", - ] - - assert set(pose_estimation_series_in_nwb) == set(expected_pose_estimation_series) - - class TestMiniscopeInterface(DataInterfaceTestMixin): data_interface_cls = MiniscopeBehaviorInterface interface_kwargs = dict(folder_path=str(OPHYS_DATA_PATH / "imaging_datasets" / "Miniscope" / "C6-J588_Disc5")) @@ -662,121 +256,6 @@ def check_metadata(self): assert metadata["NWBFile"]["session_start_time"] == datetime(2023, 5, 15, 10, 35, 29) -class CustomTestSLEAPInterface(TestCase): - savedir = OUTPUT_PATH - - @parameterized.expand( - [ - param( - data_interface=SLEAPInterface, - interface_kwargs=dict( - file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "predictions_1.2.7_provenance_and_tracking.slp"), - ), - ) - ] - ) - def test_sleap_to_nwb_interface(self, data_interface, interface_kwargs): - nwbfile_path = str(self.savedir / f"{data_interface.__name__}.nwb") - - interface = SLEAPInterface(**interface_kwargs) - metadata = interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata) - - slp_predictions_path = interface_kwargs["file_path"] - labels = sleap_io.load_slp(slp_predictions_path) - - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - # Test matching number of processing modules - number_of_videos = len(labels.videos) - assert len(nwbfile.processing) == number_of_videos - - # Test processing module naming as video - processing_module_name = "SLEAP_VIDEO_000_20190128_113421" - assert processing_module_name in nwbfile.processing - - # For this case we have as many containers as tracks - # Each track usually represents a subject - processing_module = nwbfile.processing[processing_module_name] - processing_module_interfaces = processing_module.data_interfaces - assert len(processing_module_interfaces) == len(labels.tracks) - - # Test name of PoseEstimation containers - extracted_container_names = processing_module_interfaces.keys() - for track in labels.tracks: - expected_track_name = f"track={track.name}" - assert expected_track_name in extracted_container_names - - # Test one PoseEstimation container - container_name = f"track={track.name}" - pose_estimation_container = processing_module_interfaces[container_name] - # Test that the skeleton nodes are store as nodes in containers - expected_node_names = [node.name for node in labels.skeletons[0]] - assert expected_node_names == list(pose_estimation_container.nodes[:]) - - # Test that each PoseEstimationSeries is named as a node - for node_name in pose_estimation_container.nodes[:]: - assert node_name in pose_estimation_container.pose_estimation_series - - @parameterized.expand( - [ - param( - data_interface=SLEAPInterface, - interface_kwargs=dict( - file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "melanogaster_courtship.slp"), - video_file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "melanogaster_courtship.mp4"), - ), - ) - ] - ) - def test_sleap_interface_timestamps_propagation(self, data_interface, interface_kwargs): - nwbfile_path = str(self.savedir / f"{data_interface.__name__}.nwb") - - interface = SLEAPInterface(**interface_kwargs) - metadata = interface.get_metadata() - metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) - interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata) - - slp_predictions_path = interface_kwargs["file_path"] - labels = sleap_io.load_slp(slp_predictions_path) - - from neuroconv.datainterfaces.behavior.sleap.sleap_utils import ( - extract_timestamps, - ) - - expected_timestamps = set(extract_timestamps(interface_kwargs["video_file_path"])) - - with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: - nwbfile = io.read() - # Test matching number of processing modules - number_of_videos = len(labels.videos) - assert len(nwbfile.processing) == number_of_videos - - # Test processing module naming as video - video_name = Path(labels.videos[0].filename).stem - processing_module_name = f"SLEAP_VIDEO_000_{video_name}" - - # For this case we have as many containers as tracks - processing_module_interfaces = nwbfile.processing[processing_module_name].data_interfaces - - extracted_container_names = processing_module_interfaces.keys() - for track in labels.tracks: - expected_track_name = f"track={track.name}" - assert expected_track_name in extracted_container_names - - container_name = f"track={track.name}" - pose_estimation_container = processing_module_interfaces[container_name] - - # Test that each PoseEstimationSeries is named as a node - for node_name in pose_estimation_container.nodes[:]: - pose_estimation_series = pose_estimation_container.pose_estimation_series[node_name] - extracted_timestamps = pose_estimation_series.timestamps[:] - - # Some frames do not have predictions associated with them, so we test for sub-set - assert set(extracted_timestamps).issubset(expected_timestamps) - - class TestVideoInterface(VideoInterfaceMixin): data_interface_cls = VideoInterface save_directory = OUTPUT_PATH diff --git a/tests/test_on_data/behavior/test_lightningpose_converter.py b/tests/test_on_data/behavior/test_lightningpose_converter.py index dd93632a4..e72a3f687 100644 --- a/tests/test_on_data/behavior/test_lightningpose_converter.py +++ b/tests/test_on_data/behavior/test_lightningpose_converter.py @@ -64,6 +64,7 @@ def setUpClass(cls) -> None: description="Contains the pose estimation series for each keypoint.", scorer="heatmap_tracker", source_software="LightningPose", + camera_name="CameraPoseEstimation", ) cls.pose_estimation_metadata.update( diff --git a/tests/test_on_data/behavior/test_pose_estimation_interfaces.py b/tests/test_on_data/behavior/test_pose_estimation_interfaces.py new file mode 100644 index 000000000..5dbaa4633 --- /dev/null +++ b/tests/test_on_data/behavior/test_pose_estimation_interfaces.py @@ -0,0 +1,566 @@ +import sys +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import sleap_io +from hdmf.testing import TestCase +from numpy.testing import assert_array_equal +from parameterized import param, parameterized +from pynwb import NWBHDF5IO + +from neuroconv.datainterfaces import ( + DeepLabCutInterface, + LightningPoseDataInterface, + SLEAPInterface, +) +from neuroconv.tools.testing.data_interface_mixins import ( + DataInterfaceTestMixin, + TemporalAlignmentMixin, +) +from neuroconv.utils import DeepDict + +try: + from ..setup_paths import BEHAVIOR_DATA_PATH, OUTPUT_PATH +except ImportError: + from setup_paths import BEHAVIOR_DATA_PATH, OUTPUT_PATH + +from importlib.metadata import version as importlib_version +from platform import python_version +from sys import platform + +from packaging import version + +python_version = version.parse(python_version()) +# TODO: remove after this is merged https://github.com/talmolab/sleap-io/pull/143 and released +ndx_pose_version = version.parse(importlib_version("ndx-pose")) + + +@pytest.mark.skipif(ndx_pose_version < version.parse("0.2.0"), reason="Interface requires ndx-pose version >= 0.2.0") +class TestLightningPoseDataInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): + data_interface_cls = LightningPoseDataInterface + interface_kwargs = dict( + file_path=str(BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.csv"), + original_video_file_path=str( + BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.mp4" + ), + ) + conversion_options = dict(reference_frame="(0,0) corresponds to the top left corner of the video.") + save_directory = OUTPUT_PATH + + @pytest.fixture(scope="class", autouse=True) + def setup_metadata(self, request): + + cls = request.cls + + cls.pose_estimation_name = "PoseEstimation" + cls.original_video_height = 406 + cls.original_video_width = 396 + cls.expected_keypoint_names = [ + "paw1LH_top", + "paw2LF_top", + "paw3RF_top", + "paw4RH_top", + "tailBase_top", + "tailMid_top", + "nose_top", + "obs_top", + "paw1LH_bot", + "paw2LF_bot", + "paw3RF_bot", + "paw4RH_bot", + "tailBase_bot", + "tailMid_bot", + "nose_bot", + "obsHigh_bot", + "obsLow_bot", + ] + cls.expected_metadata = DeepDict( + PoseEstimation=dict( + name=cls.pose_estimation_name, + description="Contains the pose estimation series for each keypoint.", + scorer="heatmap_tracker", + source_software="LightningPose", + camera_name="CameraPoseEstimation", + ) + ) + cls.expected_metadata[cls.pose_estimation_name].update( + { + keypoint_name: dict( + name=f"PoseEstimationSeries{keypoint_name}", + description=f"The estimated position (x, y) of {keypoint_name} over time.", + ) + for keypoint_name in cls.expected_keypoint_names + } + ) + + cls.test_data = pd.read_csv(cls.interface_kwargs["file_path"], header=[0, 1, 2])["heatmap_tracker"] + + def check_extracted_metadata(self, metadata: dict): + assert metadata["NWBFile"]["session_start_time"] == datetime(2023, 11, 9, 10, 14, 37, 0) + assert self.pose_estimation_name in metadata["Behavior"] + assert metadata["Behavior"][self.pose_estimation_name] == self.expected_metadata[self.pose_estimation_name] + + def check_read_nwb(self, nwbfile_path: str): + from ndx_pose import PoseEstimation, PoseEstimationSeries + + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + + # Replacing assertIn with pytest-style assert + assert "behavior" in nwbfile.processing + assert self.pose_estimation_name in nwbfile.processing["behavior"].data_interfaces + assert "Skeletons" in nwbfile.processing["behavior"].data_interfaces + + pose_estimation_container = nwbfile.processing["behavior"].data_interfaces[self.pose_estimation_name] + + # Replacing assertIsInstance with pytest-style assert + assert isinstance(pose_estimation_container, PoseEstimation) + + pose_estimation_metadata = self.expected_metadata[self.pose_estimation_name] + + # Replacing assertEqual with pytest-style assert + assert pose_estimation_container.description == pose_estimation_metadata["description"] + assert pose_estimation_container.scorer == pose_estimation_metadata["scorer"] + assert pose_estimation_container.source_software == pose_estimation_metadata["source_software"] + + # Using numpy's assert_array_equal + assert_array_equal( + pose_estimation_container.dimensions[:], [[self.original_video_height, self.original_video_width]] + ) + + # Replacing assertEqual with pytest-style assert + assert len(pose_estimation_container.pose_estimation_series) == len(self.expected_keypoint_names) + + assert pose_estimation_container.skeleton.nodes[:].tolist() == self.expected_keypoint_names + + for keypoint_name in self.expected_keypoint_names: + series_metadata = pose_estimation_metadata[keypoint_name] + + # Replacing assertIn with pytest-style assert + assert series_metadata["name"] in pose_estimation_container.pose_estimation_series + + pose_estimation_series = pose_estimation_container.pose_estimation_series[series_metadata["name"]] + + # Replacing assertIsInstance with pytest-style assert + assert isinstance(pose_estimation_series, PoseEstimationSeries) + + # Replacing assertEqual with pytest-style assert + assert pose_estimation_series.unit == "px" + assert pose_estimation_series.description == series_metadata["description"] + assert pose_estimation_series.reference_frame == self.conversion_options["reference_frame"] + + test_data = self.test_data[keypoint_name] + + # Using numpy's assert_array_equal + assert_array_equal(pose_estimation_series.data[:], test_data[["x", "y"]].values) + + +@pytest.mark.skipif(ndx_pose_version < version.parse("0.2.0"), reason="Interface requires ndx-pose version >= 0.2.0") +class TestLightningPoseDataInterfaceWithStubTest(DataInterfaceTestMixin, TemporalAlignmentMixin): + data_interface_cls = LightningPoseDataInterface + interface_kwargs = dict( + file_path=str(BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.csv"), + original_video_file_path=str( + BEHAVIOR_DATA_PATH / "lightningpose" / "outputs/2023-11-09/10-14-37/video_preds/test_vid.mp4" + ), + ) + + conversion_options = dict(stub_test=True) + save_directory = OUTPUT_PATH + + def check_read_nwb(self, nwbfile_path: str): + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + pose_estimation_container = nwbfile.processing["behavior"].data_interfaces["PoseEstimation"] + for pose_estimation_series in pose_estimation_container.pose_estimation_series.values(): + assert pose_estimation_series.data.shape[0] == 10 + assert pose_estimation_series.confidence.shape[0] == 10 + + +@pytest.mark.skipif( + ndx_pose_version >= version.parse("0.2.0"), reason="SLEAPInterface requires ndx-pose version < 0.2.0" +) +class TestSLEAPInterface(DataInterfaceTestMixin, TemporalAlignmentMixin): + + data_interface_cls = SLEAPInterface + interface_kwargs = dict( + file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "predictions_1.2.7_provenance_and_tracking.slp"), + video_file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "melanogaster_courtship.mp4"), + ) + save_directory = OUTPUT_PATH + + def check_read_nwb(self, nwbfile_path: str): # This is currently structured to be file-specific + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "SLEAP_VIDEO_000_20190128_113421" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["SLEAP_VIDEO_000_20190128_113421"].data_interfaces + assert "track=track_0" in processing_module_interfaces + + pose_estimation_series_in_nwb = processing_module_interfaces["track=track_0"].pose_estimation_series + expected_pose_estimation_series = [ + "abdomen", + "eyeL", + "eyeR", + "forelegL4", + "forelegR4", + "head", + "hindlegL4", + "hindlegR4", + "midlegL4", + "midlegR4", + "thorax", + "wingL", + "wingR", + ] + + assert set(pose_estimation_series_in_nwb) == set(expected_pose_estimation_series) + + +@pytest.mark.skipif( + ndx_pose_version >= version.parse("0.2.0"), reason="SLEAPInterface requires ndx-pose version < 0.2.0" +) +class CustomTestSLEAPInterface(TestCase): + savedir = OUTPUT_PATH + + @parameterized.expand( + [ + param( + data_interface=SLEAPInterface, + interface_kwargs=dict( + file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "predictions_1.2.7_provenance_and_tracking.slp"), + ), + ) + ] + ) + def test_sleap_to_nwb_interface(self, data_interface, interface_kwargs): + nwbfile_path = str(self.savedir / f"{data_interface.__name__}.nwb") + + interface = SLEAPInterface(**interface_kwargs) + metadata = interface.get_metadata() + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata) + + slp_predictions_path = interface_kwargs["file_path"] + labels = sleap_io.load_slp(slp_predictions_path) + + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + # Test matching number of processing modules + number_of_videos = len(labels.videos) + assert len(nwbfile.processing) == number_of_videos + + # Test processing module naming as video + processing_module_name = "SLEAP_VIDEO_000_20190128_113421" + assert processing_module_name in nwbfile.processing + + # For this case we have as many containers as tracks + # Each track usually represents a subject + processing_module = nwbfile.processing[processing_module_name] + processing_module_interfaces = processing_module.data_interfaces + assert len(processing_module_interfaces) == len(labels.tracks) + + # Test name of PoseEstimation containers + extracted_container_names = processing_module_interfaces.keys() + for track in labels.tracks: + expected_track_name = f"track={track.name}" + assert expected_track_name in extracted_container_names + + # Test one PoseEstimation container + container_name = f"track={track.name}" + pose_estimation_container = processing_module_interfaces[container_name] + # Test that the skeleton nodes are store as nodes in containers + expected_node_names = [node.name for node in labels.skeletons[0]] + assert expected_node_names == list(pose_estimation_container.nodes[:]) + + # Test that each PoseEstimationSeries is named as a node + for node_name in pose_estimation_container.nodes[:]: + assert node_name in pose_estimation_container.pose_estimation_series + + @parameterized.expand( + [ + param( + data_interface=SLEAPInterface, + interface_kwargs=dict( + file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "melanogaster_courtship.slp"), + video_file_path=str(BEHAVIOR_DATA_PATH / "sleap" / "melanogaster_courtship.mp4"), + ), + ) + ] + ) + def test_sleap_interface_timestamps_propagation(self, data_interface, interface_kwargs): + nwbfile_path = str(self.savedir / f"{data_interface.__name__}.nwb") + + interface = SLEAPInterface(**interface_kwargs) + metadata = interface.get_metadata() + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + interface.run_conversion(nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata) + + slp_predictions_path = interface_kwargs["file_path"] + labels = sleap_io.load_slp(slp_predictions_path) + + from neuroconv.datainterfaces.behavior.sleap.sleap_utils import ( + extract_timestamps, + ) + + expected_timestamps = set(extract_timestamps(interface_kwargs["video_file_path"])) + + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + # Test matching number of processing modules + number_of_videos = len(labels.videos) + assert len(nwbfile.processing) == number_of_videos + + # Test processing module naming as video + video_name = Path(labels.videos[0].filename).stem + processing_module_name = f"SLEAP_VIDEO_000_{video_name}" + + # For this case we have as many containers as tracks + processing_module_interfaces = nwbfile.processing[processing_module_name].data_interfaces + + extracted_container_names = processing_module_interfaces.keys() + for track in labels.tracks: + expected_track_name = f"track={track.name}" + assert expected_track_name in extracted_container_names + + container_name = f"track={track.name}" + pose_estimation_container = processing_module_interfaces[container_name] + + # Test that each PoseEstimationSeries is named as a node + for node_name in pose_estimation_container.nodes[:]: + pose_estimation_series = pose_estimation_container.pose_estimation_series[node_name] + extracted_timestamps = pose_estimation_series.timestamps[:] + + # Some frames do not have predictions associated with them, so we test for sub-set + assert set(extracted_timestamps).issubset(expected_timestamps) + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10") or ndx_pose_version < version.parse("0.2.0"), + reason="Interface requires ndx-pose version >= 0.2.0 and not supported on macOS with Python < 3.10", +) +class TestDeepLabCutInterface(DataInterfaceTestMixin): + data_interface_cls = DeepLabCutInterface + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), + subject_name="ind1", + ) + save_directory = OUTPUT_PATH + + def run_custom_checks(self): + self.check_renaming_instance(nwbfile_path=self.nwbfile_path) + + def check_renaming_instance(self, nwbfile_path: str): + custom_container_name = "TestPoseEstimation" + + metadata = self.interface.get_metadata() + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + self.interface.run_conversion( + nwbfile_path=nwbfile_path, overwrite=True, metadata=metadata, container_name=custom_container_name + ) + + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + assert custom_container_name in nwbfile.processing["behavior"].data_interfaces + + def check_read_nwb(self, nwbfile_path: str): + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces + assert "PoseEstimationDeepLabCut" in processing_module_interfaces + assert "Skeletons" in processing_module_interfaces + + pose_estimation_container = processing_module_interfaces["PoseEstimationDeepLabCut"] + pose_estimation_series_in_nwb = pose_estimation_container.pose_estimation_series + expected_pose_estimation_series = ["ind1_leftear", "ind1_rightear", "ind1_snout", "ind1_tailbase"] + + expected_pose_estimation_series_are_in_nwb_file = [ + pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series + ] + + assert all(expected_pose_estimation_series_are_in_nwb_file) + + skeleton = pose_estimation_container.skeleton + assert skeleton.nodes[:].tolist() == ["snout", "leftear", "rightear", "tailbase"] + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10") or ndx_pose_version < version.parse("0.2.0"), + reason="Interface requires ndx-pose version >= 0.2.0 and not supported on macOS with Python < 3.10", +) +class TestDeepLabCutInterfaceNoConfigFile(DataInterfaceTestMixin): + data_interface_cls = DeepLabCutInterface + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=None, + subject_name="ind1", + ) + save_directory = OUTPUT_PATH + + def check_read_nwb(self, nwbfile_path: str): + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces + assert "PoseEstimationDeepLabCut" in processing_module_interfaces + + pose_estimation_series_in_nwb = processing_module_interfaces[ + "PoseEstimationDeepLabCut" + ].pose_estimation_series + expected_pose_estimation_series = ["ind1_leftear", "ind1_rightear", "ind1_snout", "ind1_tailbase"] + + expected_pose_estimation_series_are_in_nwb_file = [ + pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series + ] + + assert all(expected_pose_estimation_series_are_in_nwb_file) + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10") or ndx_pose_version < version.parse("0.2.0"), + reason="Interface requires ndx-pose version >= 0.2.0 and not supported on macOS with Python < 3.10", +) +class TestDeepLabCutInterfaceSetTimestamps(DataInterfaceTestMixin): + data_interface_cls = DeepLabCutInterface + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), + subject_name="ind1", + ) + + save_directory = OUTPUT_PATH + + def run_custom_checks(self): + self.check_custom_timestamps(nwbfile_path=self.nwbfile_path) + + def check_custom_timestamps(self, nwbfile_path: str): + custom_timestamps = np.concatenate( + (np.linspace(10, 110, 1000), np.linspace(150, 250, 1000), np.linspace(300, 400, 330)) + ) + + metadata = self.interface.get_metadata() + metadata["NWBFile"].update(session_start_time=datetime.now().astimezone()) + + self.interface.set_aligned_timestamps(custom_timestamps) + assert len(self.interface._timestamps) == 2330 + + self.interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) + + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces + assert "PoseEstimationDeepLabCut" in processing_module_interfaces + + pose_estimation_series_in_nwb = processing_module_interfaces[ + "PoseEstimationDeepLabCut" + ].pose_estimation_series + + for pose_estimation in pose_estimation_series_in_nwb.values(): + pose_timestamps = pose_estimation.timestamps + np.testing.assert_array_equal(pose_timestamps, custom_timestamps) + + # This was tested in the other test + def check_read_nwb(self, nwbfile_path: str): + pass + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10") or ndx_pose_version < version.parse("0.2.0"), + reason="Interface requires ndx-pose version >= 0.2.0 and not supported on macOS with Python < 3.10", +) +class TestDeepLabCutInterfaceFromCSV(DataInterfaceTestMixin): + data_interface_cls = DeepLabCutInterface + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "SL18_csv" + / "SL18_D19_S01_F01_BOX_SLP_20230503_112642.1DLC_resnet50_SubLearnSleepBoxRedLightJun26shuffle1_100000_stubbed.csv" + ), + config_file_path=None, + subject_name="SL18", + ) + save_directory = OUTPUT_PATH + + def check_read_nwb(self, nwbfile_path: str): + with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + assert "behavior" in nwbfile.processing + processing_module_interfaces = nwbfile.processing["behavior"].data_interfaces + assert "PoseEstimationDeepLabCut" in processing_module_interfaces + + pose_estimation_series_in_nwb = processing_module_interfaces[ + "PoseEstimationDeepLabCut" + ].pose_estimation_series + expected_pose_estimation_series = ["SL18_redled", "SL18_shoulder", "SL18_haunch", "SL18_baseoftail"] + + expected_pose_estimation_series_are_in_nwb_file = [ + pose_estimation in pose_estimation_series_in_nwb for pose_estimation in expected_pose_estimation_series + ] + + assert all(expected_pose_estimation_series_are_in_nwb_file) + + +@pytest.fixture +def clean_pose_extension_import(): + modules_to_remove = [m for m in sys.modules if m.startswith("ndx_pose")] + for module in modules_to_remove: + del sys.modules[module] + + +@pytest.mark.skipif( + platform == "darwin" and python_version < version.parse("3.10") or ndx_pose_version < version.parse("0.2.0"), + reason="Interface requires ndx-pose version >= 0.2.0 and not supported on macOS with Python < 3.10", +) +def test_deep_lab_cut_import_pose_extension_bug(clean_pose_extension_import, tmp_path): + """ + Test that the DeepLabCutInterface writes correctly without importing the ndx-pose extension. + See issues: + https://github.com/catalystneuro/neuroconv/issues/1114 + https://github.com/rly/ndx-pose/issues/36 + + """ + + interface_kwargs = dict( + file_path=str( + BEHAVIOR_DATA_PATH + / "DLC" + / "open_field_without_video" + / "m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5" + ), + config_file_path=str(BEHAVIOR_DATA_PATH / "DLC" / "open_field_without_video" / "config.yaml"), + ) + + interface = DeepLabCutInterface(**interface_kwargs) + metadata = interface.get_metadata() + metadata["NWBFile"]["session_start_time"] = datetime(2023, 7, 24, 9, 30, 55, 440600, tzinfo=timezone.utc) + + nwbfile_path = tmp_path / "test.nwb" + interface.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata, overwrite=True) + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + read_nwbfile = io.read() + pose_estimation_container = read_nwbfile.processing["behavior"]["PoseEstimationDeepLabCut"] + + assert len(pose_estimation_container.fields) > 0 From ce63b7c4d6d47eb42dfe966ce3243ed7f0f73b0b Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Thu, 23 Jan 2025 20:50:13 -0600 Subject: [PATCH 118/118] run conversion: Use context manager only in append mode (#1180) --- CHANGELOG.md | 11 ++-- src/neuroconv/basedatainterface.py | 64 +++++++++++++------ src/neuroconv/nwbconverter.py | 56 ++++++++++------ .../tools/testing/data_interface_mixins.py | 7 +- tests/test_ecephys/test_ecephys_interfaces.py | 19 ++++++ 5 files changed, 107 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df9fd842..b8b887fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,22 @@ # v0.7.0 (Upcoming) -## Deprecations +## Deprecations and Changes +* Interfaces and converters now have `verbose=False` by default [PR #1153](https://github.com/catalystneuro/neuroconv/pull/1153) +* Added `metadata` and `conversion_options` as arguments to `NWBConverter.temporally_align_data_interfaces` [PR #1162](https://github.com/catalystneuro/neuroconv/pull/1162) ## Bug Fixes +* `run_conversion` does not longer trigger append mode an index error when `nwbfile_path` points to a faulty file [PR #1180](https://github.com/catalystneuro/neuroconv/pull/1180) ## Features -* Added `metadata` and `conversion_options` as arguments to `NWBConverter.temporally_align_data_interfaces` [PR #1162](https://github.com/catalystneuro/neuroconv/pull/1162) * Use the latest version of ndx-pose for `DeepLabCutInterface` and `LightningPoseDataInterface` [PR #1128](https://github.com/catalystneuro/neuroconv/pull/1128) ## Improvements -* Interfaces and converters now have `verbose=False` by default [PR #1153](https://github.com/catalystneuro/neuroconv/pull/1153) - +* Simple writing no longer uses a context manager [PR #1180](https://github.com/catalystneuro/neuroconv/pull/1180) # v0.6.7 (January 20, 2025) -## Deprecations +## Deprecations and Changes ## Bug Fixes * Temporary set a ceiling for hdmf to avoid a chunking bug [PR #1175](https://github.com/catalystneuro/neuroconv/pull/1175) diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index 95b80f6d7..9a2f25844 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -17,7 +17,10 @@ make_nwbfile_from_metadata, make_or_load_nwbfile, ) -from .tools.nwb_helpers._metadata_and_file_helpers import _resolve_backend +from .tools.nwb_helpers._metadata_and_file_helpers import ( + _resolve_backend, + configure_and_write_nwbfile, +) from .utils import ( get_json_schema_from_method_signature, load_dict_from_file, @@ -163,7 +166,7 @@ def run_conversion( Parameters ---------- nwbfile_path : FilePathType - Path for where the data will be written or appended. + Path for where to write or load (if overwrite=False) the NWBFile. nwbfile : NWBFile, optional An in-memory NWBFile object to write to the location. metadata : dict, optional @@ -182,32 +185,51 @@ def run_conversion( Otherwise, all datasets will use default configuration settings. """ - backend = _resolve_backend(backend, backend_configuration) - no_nwbfile_provided = nwbfile is None # Otherwise, variable reference may mutate later on inside the context + appending_to_in_memory_nwbfile = nwbfile is not None + file_initially_exists = Path(nwbfile_path).exists() if nwbfile_path is not None else False + appending_to_in_disk_nwbfile = file_initially_exists and not overwrite + + if appending_to_in_disk_nwbfile and appending_to_in_memory_nwbfile: + raise ValueError( + "Cannot append to an existing file while also providing an in-memory NWBFile. " + "Either set overwrite=True to replace the existing file, or remove the nwbfile parameter to append to the existing file on disk." + ) if metadata is None: metadata = self.get_metadata() + self.validate_metadata(metadata=metadata, append_mode=appending_to_in_disk_nwbfile) + + if not appending_to_in_disk_nwbfile: + if appending_to_in_memory_nwbfile: + self.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, **conversion_options) + else: + nwbfile = self.create_nwbfile(metadata=metadata, **conversion_options) + + configure_and_write_nwbfile( + nwbfile=nwbfile, + output_filepath=nwbfile_path, + backend=backend, + backend_configuration=backend_configuration, + ) + + else: # We are only using the context in append mode, see issue #1143 + + backend = _resolve_backend(backend, backend_configuration) + with make_or_load_nwbfile( + nwbfile_path=nwbfile_path, + nwbfile=nwbfile, + metadata=metadata, + overwrite=overwrite, + backend=backend, + verbose=getattr(self, "verbose", False), + ) as nwbfile_out: - file_initially_exists = Path(nwbfile_path).exists() if nwbfile_path is not None else False - append_mode = file_initially_exists and not overwrite - - self.validate_metadata(metadata=metadata, append_mode=append_mode) - - with make_or_load_nwbfile( - nwbfile_path=nwbfile_path, - nwbfile=nwbfile, - metadata=metadata, - overwrite=overwrite, - backend=backend, - verbose=getattr(self, "verbose", False), - ) as nwbfile_out: - if no_nwbfile_provided: self.add_to_nwbfile(nwbfile=nwbfile_out, metadata=metadata, **conversion_options) - if backend_configuration is None: - backend_configuration = self.get_default_backend_configuration(nwbfile=nwbfile_out, backend=backend) + if backend_configuration is None: + backend_configuration = self.get_default_backend_configuration(nwbfile=nwbfile_out, backend=backend) - configure_backend(nwbfile=nwbfile_out, backend_configuration=backend_configuration) + configure_backend(nwbfile=nwbfile_out, backend_configuration=backend_configuration) @staticmethod def get_default_backend_configuration( diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index 20df647d6..05e26e866 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -14,6 +14,7 @@ from .tools.nwb_helpers import ( HDF5BackendConfiguration, ZarrBackendConfiguration, + configure_and_write_nwbfile, configure_backend, get_default_backend_configuration, get_default_nwbfile_metadata, @@ -243,35 +244,54 @@ def run_conversion( " use Converter.add_to_nwbfile." ) - backend = _resolve_backend(backend, backend_configuration) - no_nwbfile_provided = nwbfile is None # Otherwise, variable reference may mutate later on inside the context - + appending_to_in_memory_nwbfile = nwbfile is not None file_initially_exists = Path(nwbfile_path).exists() if nwbfile_path is not None else False - append_mode = file_initially_exists and not overwrite + appending_to_in_disk_nwbfile = file_initially_exists and not overwrite + + if appending_to_in_disk_nwbfile and appending_to_in_memory_nwbfile: + raise ValueError( + "Cannot append to an existing file while also providing an in-memory NWBFile. " + "Either set overwrite=True to replace the existing file, or remove the nwbfile parameter to append to the existing file on disk." + ) if metadata is None: metadata = self.get_metadata() - self.validate_metadata(metadata=metadata, append_mode=append_mode) + self.validate_metadata(metadata=metadata, append_mode=appending_to_in_disk_nwbfile) self.validate_conversion_options(conversion_options=conversion_options) - self.temporally_align_data_interfaces(metadata=metadata, conversion_options=conversion_options) - with make_or_load_nwbfile( - nwbfile_path=nwbfile_path, - nwbfile=nwbfile, - metadata=metadata, - overwrite=overwrite, - backend=backend, - verbose=getattr(self, "verbose", False), - ) as nwbfile_out: - if no_nwbfile_provided: + if not appending_to_in_disk_nwbfile: + + if appending_to_in_memory_nwbfile: + self.add_to_nwbfile(nwbfile=nwbfile, metadata=metadata, conversion_options=conversion_options) + else: + nwbfile = self.create_nwbfile(metadata=metadata, conversion_options=conversion_options) + + configure_and_write_nwbfile( + nwbfile=nwbfile, + output_filepath=nwbfile_path, + backend=backend, + backend_configuration=backend_configuration, + ) + + else: # We are only using the context in append mode, see issue #1143 + + backend = _resolve_backend(backend, backend_configuration) + with make_or_load_nwbfile( + nwbfile_path=nwbfile_path, + nwbfile=nwbfile, + metadata=metadata, + overwrite=overwrite, + backend=backend, + verbose=getattr(self, "verbose", False), + ) as nwbfile_out: self.add_to_nwbfile(nwbfile=nwbfile_out, metadata=metadata, conversion_options=conversion_options) - if backend_configuration is None: - backend_configuration = self.get_default_backend_configuration(nwbfile=nwbfile_out, backend=backend) + if backend_configuration is None: + backend_configuration = self.get_default_backend_configuration(nwbfile=nwbfile_out, backend=backend) - configure_backend(nwbfile=nwbfile_out, backend_configuration=backend_configuration) + configure_backend(nwbfile=nwbfile_out, backend_configuration=backend_configuration) def temporally_align_data_interfaces( self, metadata: Optional[dict] = None, conversion_options: Optional[dict] = None diff --git a/src/neuroconv/tools/testing/data_interface_mixins.py b/src/neuroconv/tools/testing/data_interface_mixins.py index d8586f44f..bfba59a45 100644 --- a/src/neuroconv/tools/testing/data_interface_mixins.py +++ b/src/neuroconv/tools/testing/data_interface_mixins.py @@ -142,7 +142,6 @@ def test_run_conversion_with_backend_configuration(self, setup_interface, tmp_pa backend_configuration = self.interface.get_default_backend_configuration(nwbfile=nwbfile, backend=backend) self.interface.run_conversion( nwbfile_path=nwbfile_path, - nwbfile=nwbfile, overwrite=True, metadata=metadata, backend_configuration=backend_configuration, @@ -227,7 +226,6 @@ class TestNWBConverter(NWBConverter): backend_configuration = converter.get_default_backend_configuration(nwbfile=nwbfile, backend=backend) converter.run_conversion( nwbfile_path=nwbfile_path, - nwbfile=nwbfile, overwrite=True, metadata=metadata, backend_configuration=backend_configuration, @@ -889,7 +887,6 @@ def check_run_conversion_with_backend_configuration( backend_configuration = self.interface.get_default_backend_configuration(nwbfile=nwbfile, backend=backend) self.interface.run_conversion( nwbfile_path=nwbfile_path, - nwbfile=nwbfile, overwrite=True, metadata=metadata, backend_configuration=backend_configuration, @@ -931,7 +928,6 @@ class TestNWBConverter(NWBConverter): backend_configuration = converter.get_default_backend_configuration(nwbfile=nwbfile, backend=backend) converter.run_conversion( nwbfile_path=nwbfile_path, - nwbfile=nwbfile, overwrite=True, metadata=metadata, backend_configuration=backend_configuration, @@ -1262,7 +1258,7 @@ def check_run_conversion_with_backend_configuration( backend_configuration = self.interface.get_default_backend_configuration(nwbfile=nwbfile, backend=backend) self.interface.run_conversion( nwbfile_path=nwbfile_path, - nwbfile=nwbfile, + metadata=metadata, overwrite=True, backend_configuration=backend_configuration, **self.conversion_options, @@ -1303,7 +1299,6 @@ class TestNWBConverter(NWBConverter): backend_configuration = converter.get_default_backend_configuration(nwbfile=nwbfile, backend=backend) converter.run_conversion( nwbfile_path=nwbfile_path, - nwbfile=nwbfile, overwrite=True, metadata=metadata, backend_configuration=backend_configuration, diff --git a/tests/test_ecephys/test_ecephys_interfaces.py b/tests/test_ecephys/test_ecephys_interfaces.py index d2fdd68ba..8a32abcac 100644 --- a/tests/test_ecephys/test_ecephys_interfaces.py +++ b/tests/test_ecephys/test_ecephys_interfaces.py @@ -6,6 +6,7 @@ from hdmf.testing import TestCase from packaging.version import Version +from neuroconv import ConverterPipe from neuroconv.datainterfaces import Spike2RecordingInterface from neuroconv.tools.nwb_helpers import get_module from neuroconv.tools.testing.mock_interfaces import ( @@ -158,6 +159,24 @@ def test_group_naming_not_adding_extra_devices(self, setup_interface): assert len(nwbfile.devices) == 1 assert len(nwbfile.electrode_groups) == 4 + def test_error_for_append_with_in_memory_file(self, setup_interface, tmp_path): + + nwbfile_path = tmp_path / "test.nwb" + self.interface.run_conversion(nwbfile_path=nwbfile_path) + + nwbfile = self.interface.create_nwbfile() + + expected_error_message = ( + "Cannot append to an existing file while also providing an in-memory NWBFile. " + "Either set overwrite=True to replace the existing file, or remove the nwbfile parameter to append to the existing file on disk." + ) + with pytest.raises(ValueError, match=expected_error_message): + self.interface.run_conversion(nwbfile=nwbfile, nwbfile_path=nwbfile_path, overwrite=False) + + converter = ConverterPipe(data_interfaces=[self.interface]) + with pytest.raises(ValueError, match=expected_error_message): + converter.run_conversion(nwbfile=nwbfile, nwbfile_path=nwbfile_path, overwrite=False) + class TestAssertions(TestCase): @pytest.mark.skipif(python_version.minor != 10, reason="Only testing with Python 3.10!")