Adds support for video and intrinsic signal optical imaging to convert_all_sessions and convert_session

pyproject.toml

@@ -15,9 +15,9 @@ classifiers = [
 ]
 
 dependencies = [
-  "neuroconv",
-  "nwbinspector",
-  "ndx-events==0.2.0"
+  "neuroconv==0.6.6",
+  "ndx-events==0.2.0",
+  "spikeinterface==0.101.2"
 ]
 
 [project.urls]

src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_all_sessions.py

@@ -106,21 +106,32 @@ def get_session_to_nwb_kwargs_per_session(*, data_dir_path: DirectoryPath):
     """
     a1_ephys_path = data_dir_path / "A1_EphysFiles"
     a1_ephys_behavior_path = data_dir_path / "A1_EphysBehavioralFiles"
+    a1_ephys_video_path = data_dir_path / "Videos" / "A1EphysVideos"
     a1_opto_path = data_dir_path / "A1_OptoBehavioralFiles"
+    a1_opto_video_path = data_dir_path / "Videos" / "A1OptoVideos"
     m2_ephys_path = data_dir_path / "M2_EphysFiles"
     m2_ephys_behavior_path = data_dir_path / "M2_EphysBehavioralFiles"
+    m2_ephys_video_path = data_dir_path / "Videos" / "M2EphysVideos"
     m2_opto_path = data_dir_path / "M2_OptoBehavioralFiles"
+    m2_opto_video_path = data_dir_path / "Videos" / "M2OptoVideos"
+    intrinsic_signal_optical_imaging_path = data_dir_path / "Intrinsic Imaging Data"
 
     a1_kwargs = get_brain_region_kwargs(
         ephys_path=a1_ephys_path,
         ephys_behavior_path=a1_ephys_behavior_path,
+        ephys_video_path=a1_ephys_video_path,
         opto_path=a1_opto_path,
+        opto_video_path=a1_opto_video_path,
+        intrinsic_signal_optical_imaging_path=intrinsic_signal_optical_imaging_path,
         brain_region="A1",
     )
     m2_kwargs = get_brain_region_kwargs(
         ephys_path=m2_ephys_path,
         ephys_behavior_path=m2_ephys_behavior_path,
+        ephys_video_path=m2_ephys_video_path,
         opto_path=m2_opto_path,
+        opto_video_path=m2_opto_video_path,
+        intrinsic_signal_optical_imaging_path=intrinsic_signal_optical_imaging_path,
         brain_region="M2",
     )
     session_to_nwb_kwargs_per_session = a1_kwargs + m2_kwargs
@@ -129,7 +140,13 @@ def get_session_to_nwb_kwargs_per_session(*, data_dir_path: DirectoryPath):
 
 
 def get_brain_region_kwargs(
-    ephys_path: DirectoryPath, ephys_behavior_path: DirectoryPath, opto_path: DirectoryPath, brain_region: str
+    ephys_path: DirectoryPath,
+    ephys_behavior_path: DirectoryPath,
+    opto_path: DirectoryPath,
+    ephys_video_path: DirectoryPath,
+    opto_video_path: DirectoryPath,
+    intrinsic_signal_optical_imaging_path: DirectoryPath,
+    brain_region: str,
 ):
     """Get the session_to_nwb kwargs for each session in the dataset for a given brain region.
 
@@ -139,8 +156,14 @@ def get_brain_region_kwargs(
         Path to the directory containing electrophysiology data for subjects.
     ephys_behavior_path : DirectoryPath
         Path to the directory containing electrophysiology behavior data files.
+    ephys_video_path : DirectoryPath
+        Path to the directory containing electrophysiology video data files.
     opto_path : DirectoryPath
         Path to the directory containing optogenetics behavior data files.
+    opto_video_path : DirectoryPath
+        Path to the directory containing optogenetics video data files.
+    intrinsic_signal_optical_imaging_path : DirectoryPath
+        Path to the directory containing intrinsic signal optical imaging data files.
     brain_region : str
         The brain region associated with the sessions.
 
@@ -153,23 +176,33 @@ def get_brain_region_kwargs(
     for subject_dir in ephys_path.iterdir():
         subject_id = subject_dir.name
         matched_behavior_paths = sorted(ephys_behavior_path.glob(f"raw_{subject_id}_*.mat"))
+        video_subject_path = ephys_video_path / subject_id
+        matched_video_paths = sorted(list(video_subject_path.iterdir()))
+        matched_isoi_path = intrinsic_signal_optical_imaging_path / subject_id
         sorted_session_dirs = sorted(subject_dir.iterdir())
-        for ephys_folder_path, behavior_file_path in zip(sorted_session_dirs, matched_behavior_paths):
+        for ephys_folder_path, behavior_file_path, video_folder_path in zip(
+            sorted_session_dirs, matched_behavior_paths, matched_video_paths
+        ):
             session_to_nwb_kwargs = dict(
                 ephys_folder_path=ephys_folder_path,
                 behavior_file_path=behavior_file_path,
                 brain_region=brain_region,
-                intrinsic_signal_optical_imaging_folder_path="",  # TODO: Add intrinsic signal optical imaging folder path
-                video_folder_path="",  # TODO: Add video folder path
+                intrinsic_signal_optical_imaging_folder_path=matched_isoi_path,
+                video_folder_path=video_folder_path,
             )
             session_to_nwb_kwargs_per_session.append(session_to_nwb_kwargs)
     for behavior_file_path in opto_path.iterdir():
+        split_behavior_file_path = behavior_file_path.name.split("_")
+        subject_id = split_behavior_file_path[1]
+        date = split_behavior_file_path[2]
+        video_folder_path = opto_video_path / subject_id / date
+        matched_isoi_path = intrinsic_signal_optical_imaging_path / subject_id
         session_to_nwb_kwargs = dict(
             behavior_file_path=behavior_file_path,
             brain_region=brain_region,
             has_opto=True,
-            intrinsic_signal_optical_imaging_folder_path="",  # TODO: Add intrinsic signal optical imaging folder path
-            video_folder_path="",  # TODO: Add video folder path
+            intrinsic_signal_optical_imaging_folder_path=matched_isoi_path,
+            video_folder_path=video_folder_path,
         )
         session_to_nwb_kwargs_per_session.append(session_to_nwb_kwargs)
     return session_to_nwb_kwargs_per_session

src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_convert_session.py

@@ -80,10 +80,10 @@ def session_to_nwb(
     conversion_options.update(dict(Behavior=dict()))
 
     # Add Video(s)
-    # for i, video_file_path in enumerate(video_file_paths):
-    #     metadata_key_name = f"VideoCamera{i+1}"
-    #     source_data.update({metadata_key_name: dict(file_paths=[video_file_path], metadata_key_name=metadata_key_name)})
-    #     conversion_options.update({metadata_key_name: dict()})
+    for i, video_file_path in enumerate(video_file_paths):
+        metadata_key_name = f"VideoCamera{i+1}"
+        source_data.update({metadata_key_name: dict(file_paths=[video_file_path], metadata_key_name=metadata_key_name)})
+        conversion_options.update({metadata_key_name: dict()})
 
     # Add Optogenetic
     if has_opto:
@@ -92,8 +92,10 @@ def session_to_nwb(
         conversion_options["Behavior"]["normalize_timestamps"] = True
 
     # Add Intrinsic Signal Optical Imaging
-    # source_data.update(dict(ISOI=dict(folder_path=intrinsic_signal_optical_imaging_folder_path)))
-    # conversion_options.update(dict(ISOI=dict()))
+    overlaid_image_path = intrinsic_signal_optical_imaging_folder_path / "Overlaid.jpg"
+    target_image_path = intrinsic_signal_optical_imaging_folder_path / "Target.jpg"
+    source_data.update(dict(ISOI=dict(overlaid_image_path=overlaid_image_path, target_image_path=target_image_path)))
+    conversion_options.update(dict(ISOI=dict()))
 
     converter = Zempolich2024NWBConverter(source_data=source_data, verbose=verbose)
     metadata = converter.get_metadata()
@@ -110,10 +112,10 @@ def session_to_nwb(
     if has_ephys:
         conversion_options["Sorting"]["units_description"] = metadata["Sorting"]["units_description"]
 
-    # # Overwrite video metadata
-    # for i, video_file_path in enumerate(video_file_paths):
-    #     metadata_key_name = f"VideoCamera{i+1}"
-    #     metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name]
+    # Overwrite video metadata
+    for i, video_file_path in enumerate(video_file_paths):
+        metadata_key_name = f"VideoCamera{i+1}"
+        metadata["Behavior"][metadata_key_name] = editable_metadata["Behavior"][metadata_key_name]
 
     subject_id = behavior_file_path.name.split("_")[1]
     session_id = behavior_file_path.name.split("_")[2]
@@ -168,8 +170,8 @@ def main():
     # Example Session A1 Ephys + Behavior
     ephys_folder_path = data_dir_path / "A1_EphysFiles" / "m53" / "Day1_A1"
     behavior_file_path = data_dir_path / "A1_EphysBehavioralFiles" / "raw_m53_231029_001.mat"
-    video_folder_path = Path("")
-    intrinsic_signal_optical_imaging_folder_path = Path("")
+    video_folder_path = data_dir_path / "Videos" / "A1EphysVideos" / "m53" / "231029"
+    intrinsic_signal_optical_imaging_folder_path = data_dir_path / "Intrinsic Imaging Data" / "m53"
     session_to_nwb(
         ephys_folder_path=ephys_folder_path,
         behavior_file_path=behavior_file_path,
@@ -182,8 +184,8 @@ def main():
 
     # Example Session A1 Ogen + Behavior
     behavior_file_path = data_dir_path / "A1_OptoBehavioralFiles" / "raw_m53_231013_001.mat"
-    video_folder_path = Path("")
-    intrinsic_signal_optical_imaging_folder_path = Path("")
+    video_folder_path = data_dir_path / "Videos" / "A1OptoVideos" / "m53" / "231013"
+    intrinsic_signal_optical_imaging_folder_path = data_dir_path / "Intrinsic Imaging Data" / "m53"
     session_to_nwb(
         behavior_file_path=behavior_file_path,
         video_folder_path=video_folder_path,
@@ -197,8 +199,8 @@ def main():
     # Example Session M2 Ephys + Behavior
     ephys_folder_path = data_dir_path / "M2_EphysFiles" / "m74" / "M2_Day1"
     behavior_file_path = data_dir_path / "M2_EphysBehavioralFiles" / "raw_m74_240815_001.mat"
-    video_folder_path = Path("")
-    intrinsic_signal_optical_imaging_folder_path = Path("")
+    video_folder_path = data_dir_path / "Videos" / "M2EphysVideos" / "m74" / "240815"
+    intrinsic_signal_optical_imaging_folder_path = data_dir_path / "Intrinsic Imaging Data" / "m74"
     session_to_nwb(
         ephys_folder_path=ephys_folder_path,
         behavior_file_path=behavior_file_path,
@@ -212,8 +214,8 @@ def main():
 
     # Example Session M2 Opto + Behavior
     behavior_file_path = data_dir_path / "M2_OptoBehavioralFiles" / "raw_m74_240809_001.mat"
-    video_folder_path = Path("")
-    intrinsic_signal_optical_imaging_folder_path = Path("")
+    video_folder_path = data_dir_path / "Videos" / "M2OptoVideos" / "m74" / "240809"
+    intrinsic_signal_optical_imaging_folder_path = data_dir_path / "Intrinsic Imaging Data" / "m74"
     session_to_nwb(
         behavior_file_path=behavior_file_path,
         video_folder_path=video_folder_path,

src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_intrinsic_signal_imaging_interface.py

@@ -1,11 +1,12 @@
 """Primary class for converting intrinsic signal optical imaging."""
 from pynwb.file import NWBFile
 from pynwb.base import Images
-from pynwb.image import GrayscaleImage, RGBImage
+from pynwb.image import RGBImage
 from pynwb.device import Device
-from pydantic import DirectoryPath
+from pydantic import FilePath
 import numpy as np
 from PIL import Image
+from pathlib import Path
 
 from neuroconv.basedatainterface import BaseDataInterface
 from neuroconv.tools import nwb_helpers
@@ -16,15 +17,17 @@ class Zempolich2024IntrinsicSignalOpticalImagingInterface(BaseDataInterface):
 
     keywords = ("intrinsic signal optical imaging",)
 
-    def __init__(self, folder_path: DirectoryPath):
+    def __init__(self, overlaid_image_path: FilePath, target_image_path: FilePath):
         """Initialize the intrinsic signal optical imaging interface.
 
         Parameters
         ----------
-        folder_path : DirectoryPath
-            Path to the folder containing the intrinsic signal optical imaging files.
+        overlaid_image_path : FilePath
+            Path to the intrinsic signal optical imaging overlaid image file.
+        target_image_path : FilePath
+            Path to the intrinsic signal optical imaging target image file.
         """
-        super().__init__(folder_path=folder_path)
+        super().__init__(overlaid_image_path=overlaid_image_path, target_image_path=target_image_path)
 
     def get_metadata_schema(self) -> dict:
         metadata_schema = super().get_metadata_schema()
@@ -59,13 +62,12 @@ def get_metadata_schema(self) -> dict:
 
     def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict):
         # Read Data
-        folder_path = self.source_data["folder_path"]
-        raw_image_path = folder_path / "BloodvesselPattern.tiff"
-        processed_image_path = folder_path / "IOS_imageOverlaidFinal.jpg"
-        with Image.open(raw_image_path) as image:
-            raw_image_array = np.array(image)
-        with Image.open(processed_image_path) as image:
-            processed_image_array = np.array(image)
+        overlaid_image_path = Path(self.source_data["overlaid_image_path"])
+        target_image_path = Path(self.source_data["target_image_path"])
+        with Image.open(overlaid_image_path) as image:
+            overlaid_image_array = np.array(image)
+        with Image.open(target_image_path) as image:
+            target_image_array = np.array(image)
 
         # Add Data to NWBFile
         isoi_metadata = metadata["IntrinsicSignalOpticalImaging"]
@@ -74,20 +76,20 @@ def add_to_nwbfile(self, nwbfile: NWBFile, metadata: dict):
             name=isoi_metadata["Module"]["name"],
             description=isoi_metadata["Module"]["description"],
         )
-        raw_image = GrayscaleImage(
-            name=isoi_metadata["RawImage"]["name"],
-            data=raw_image_array,
-            description=isoi_metadata["RawImage"]["description"],
+        overlaid_image = RGBImage(
+            name=isoi_metadata["OverlaidImage"]["name"],
+            data=overlaid_image_array,
+            description=isoi_metadata["OverlaidImage"]["description"],
         )
-        processed_image = RGBImage(
-            name=isoi_metadata["ProcessedImage"]["name"],
-            data=processed_image_array,
-            description=isoi_metadata["ProcessedImage"]["description"],
+        target_image = RGBImage(
+            name=isoi_metadata["TargetImage"]["name"],
+            data=target_image_array,
+            description=isoi_metadata["TargetImage"]["description"],
         )
         images = Images(
             name=isoi_metadata["Images"]["name"],
             description=isoi_metadata["Images"]["description"],
-            images=[raw_image, processed_image],
+            images=[overlaid_image, target_image],
         )
         isoi_module.add(images)
 

src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_metadata.yaml

@@ -101,14 +101,14 @@ Behavior:
     - name: endZone_ThresholdVector
       description: Position of ending/exit position of target zone on a given trial (in raw encoder values corresponding to value read out by quadrature encoder).
       dtype: float64
-  # VideoCamera1:
-  #   - name: video_camera_1
-  #     description: Two IR video cameras (AAK CA20 600TVL 2.8MM) were used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data as necessary. Camera 1 is a side angle view of the mouse.
-  #     unit: Frames
-  # VideoCamera2:
-  #   - name: video_camera_2
-  #     description: Two IR video cameras (AAK CA20 600TVL 2.8MM) were used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data as necessary. Camera 2 is a zoomed-in view of the pupil of the mouse.
-  #     unit: Frames
+  VideoCamera1:
+    - name: video_camera_1
+      description: Two IR video cameras (AAK CA20 600TVL 2.8MM) were used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data as necessary. Camera 1 is a side angle view of the mouse.
+      unit: Frames
+  VideoCamera2:
+    - name: video_camera_2
+      description: Two IR video cameras (AAK CA20 600TVL 2.8MM) were used to monitor the experiments from different angles of interest, allowing for offline analysis of body movements, pupillometry, and other behavioral data as necessary. Camera 2 is a zoomed-in view of the pupil of the mouse.
+      unit: Frames
 
 Sorting:
   units_description: Neural spikes were sorted offline using Kilosort 2.5 and Phy2 software and manually curated to ensure precise spike time acquisition.
@@ -136,16 +136,16 @@ IntrinsicSignalOpticalImaging:
   Images:
     name: images
     description: Intrinsic signal optical images.
-  RawImage:
-    name: raw_image
-    description: Original image capture of ROI (blood vessel pattern) for intrinsic imaging.
-  ProcessedImage:
-    name: processed_image
+  OverlaidImage:
+    name: overlaid_image
     description: The primary auditory cortex was identified through tonotopic mapping using a temporally periodic acoustic stimulation consisting of sequences of single pure tones of ascending or descending frequencies (20 tones per sequence, 2 to 40kHz exponentially separated, 75dB SPL, 50ms tone duration, 500ms onset-to-onset interval). Tone sequences were delivered from a free-field speaker 10cm above the mouse and repeated 40-60 times (0.1Hz). To compute tonotopic maps, the time course of each pixel was first highpass filtered using a moving average (10s window). Next, a Fourier transform was computed to extract the phase and the power of the frequency component at the frequency of acoustic stimulation (0.1 Hz). The phase indicates the sound frequency driving the response of a pixel, and the power indicates the strength of its response. To account for the hemodynamic delay and compute maps of absolute tonotopy, the response time to ascending sequence of tones was subtracted from the response time to the descending sequence. From these maps of absolute tonotopy, equally spaced iso-frequency contour lines were extracted, color-coded for sound frequency, and overlaid on top of the image of the blood vessel pattern.
+  TargetImage:
+    name: target_image
+    description: The ephys probe target location is indicated by an ellipse overlaid on top of the blood vessel pattern image.
   Devices:
     - name: two_photon_microscope
-      description: <microscope model> (Neurolabware)
+      description: Standard Microscope by Neurolabware.
       manufacturer: Neurolabware
     - name: intrinsic_signal_optical_imaging_laser
-      description: <laser model> (ThorLabs)
+      description: ThorLabs 700nm fiber coupled LED (M700F3) driven by their basic LED driver (LEDD1B).
       manufacturer: ThorLabs

src/schneider_lab_to_nwb/zempolich_2024/zempolich_2024_notes.md

@@ -3,6 +3,7 @@
 ## Behavior
 
 ## Video
+- M2EphysVideos/m80/240819 is missing video for Cam2
 
 ## Optogenetics
 - is_opto_trial in the trials table is `np.logical_not(np.isnan(onset_times))` rather than reading from the .mat file
@@ -12,9 +13,7 @@
 ## Intrinsic Signal Optical Imaging
 - Just including raw blood vessel image and processed overlay + pixel locations bc including the isoi roi response series would really require an extension for context, but seems like it has limited reuse potential.
 - Used the Audette paper for description of overlay image.
-- Need pixel locs for ephys
-- Need device info for 2p microscope and red light laser
-- Why is the overlaid image flipped left/right compared to the original?
+- m73 is missing the target image
 
 ## Temporal Alignment
 - For session A1/m53/Day1 (raw_m53_231029_001.mat),
@@ -32,12 +31,10 @@
 
 ## Active Requests
 - Mice sexes
-- Video for each session and ISOI data for each animal
-- pixel locs for ephys
-- ISOI device info for 2p microscope and red light laser
+- target image for m73
+- Cam2 Video for M2EphysVideos/m80/240819
 
 ## Questions for Midway Meeting
 - injection vs stimulation location(s) for A1 vs M2???
-- Why is the overlaid image flipped left/right compared to the original?
 - Want to split data into epochs: Active Behavior, Passive Listening, ??? What is happening post-2164? Before 1187s?
 - Double Check: Is it ok to normalize opto sessions to first encoder timestamp?