From 0fb854be85d760721f391d2cc5b388370e412063 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 27 Jul 2023 12:38:29 -0500 Subject: [PATCH 01/33] First commit for DataJoint to NWB export --- element_calcium_imaging/export/__init__.py | 0 .../export/nwb/__init__.py | 1 + element_calcium_imaging/export/nwb/nwb.py | 123 ++++++++++++++++++ .../export/nwb/requirements.txt | 2 + 4 files changed, 126 insertions(+) create mode 100644 element_calcium_imaging/export/__init__.py create mode 100644 element_calcium_imaging/export/nwb/__init__.py create mode 100644 element_calcium_imaging/export/nwb/nwb.py create mode 100644 element_calcium_imaging/export/nwb/requirements.txt diff --git a/element_calcium_imaging/export/__init__.py b/element_calcium_imaging/export/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/element_calcium_imaging/export/nwb/__init__.py b/element_calcium_imaging/export/nwb/__init__.py new file mode 100644 index 00000000..55785b00 --- /dev/null +++ b/element_calcium_imaging/export/nwb/__init__.py @@ -0,0 +1 @@ +from .nwb import imaging_session_to_nwb diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py new file mode 100644 index 00000000..612ac2d2 --- /dev/null +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -0,0 +1,123 @@ +from datetime import datetime +from uuid import uuid4 + +import datajoint as dj +import matplotlib.pyplot as plt +import numpy as np +from dateutil.tz import tzlocal + +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from pynwb.image import ImageSeries +from pynwb.ophys import ( + CorrectedImageStack, + Fluorescence, + ImageSegmentation, + MotionCorrection, + OnePhotonSeries, + OpticalChannel, + RoiResponseSeries, + TwoPhotonSeries, +) + +from ... import imaging, scan + + +def add_scan_to_nwb(session_key, nwbfile, nwbfile_kwargs=None): + from math import nan + + scan_keys = (scan.Scan & session_key).fetch("KEY") + + for scan_key in scan_keys: + scan_data = (scan.Scan & scan_key).fetch1("scanner", "scan_notes") + device = nwbfile.create_device( + name=scan_data["scanner"] + if scan_data["scanner"] is not None + else "TwoPhotonMicroscope", + description="Two photon microscope", + manufacturer="Microscope manufacturer", + ) + + no_channels, frame_rate = (scan.ScanInfo & scan_key).fetch1("nchannels", "fps") + + field_keys = (scan.ScanInfo.Field & scan_key).fetch("KEY") + + for channel in range(no_channels): + optical_channel = OpticalChannel( + name=f"OpticalChannel{channel+1}", + description=f"Optical channel number {channel+1}", + emission_lambda=nan, + ) + + for field_key in field_keys: + field_no = (scan.ScanInfo.Field & field_key).fetch1("field_idx") + imaging_plane = nwbfile.create_imaging_plane( + name=f"ImagingPlane{field_no+1}", + optical_channel=optical_channel, + imaging_rate=frame_rate, + description=scan_data["scan_notes"] + if session_info["scan_notes"] != "" + else f"Imaging plane for channel {channel+1}", + device=device, + excitation_lambda=nan, + indicator="unknown", + location="unknown", + grid_spacing=(0.01, 0.01), + grid_spacing_unit="meters", + origin_coords=[1.0, 2.0, 3.0], + origin_coords_units="meters", + ) + + +def add_data_to_nwb(session_key, nwbfile, raw_data): + if raw_data: + imaging_data = get_image_files(session_key) + frame_rate = (scan.ScanInfo & session_key).fetch1("fps") + two_p_series = TwoPhotonSeries( + name="TwoPhotonSeries", + data=imaging_data, + imaging_plane=imaging_plane, + rate=frame_rate, + unit="raw fluorescence", + ) + else: + imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") + + +def add_motion_correction_to_nwb(session_key, nwbfile): + pass + + +def imaging_session_to_nwb( + session_key, + raw=True, + spikes=True, + lfp="source", + end_frame=None, + lab_key=None, + project_key=None, + protocol_key=None, + nwbfile_kwargs=None, +): + session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) + + if session_to_nwb: + nwbfile = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, + ) + else: + nwbfile = NWBFile(**nwbfile_kwargs) + + add_scan_to_nwb(session_key, nwbfile, **nwbfile_kwargs) + add_data_to_nwb(session_key, nwbfile, raw_data=raw) + + try: + (imaging.MotionCorrection & "scan_id='1'").fetch1("KEY") + add_motion_correction_to_nwb(session_key) + except DataJointError: + raise DataJointError(f"No motion correction data found for key {session_key}") + + return nwbfile diff --git a/element_calcium_imaging/export/nwb/requirements.txt b/element_calcium_imaging/export/nwb/requirements.txt new file mode 100644 index 00000000..fad8e1bf --- /dev/null +++ b/element_calcium_imaging/export/nwb/requirements.txt @@ -0,0 +1,2 @@ +dandi +pynwb \ No newline at end of file From 2254bef50f5272fb2917d710c0c5c7336264f0be Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 27 Jul 2023 16:22:18 -0500 Subject: [PATCH 02/33] First functioning version --- element_calcium_imaging/export/nwb/nwb.py | 162 ++++++++++++++-------- 1 file changed, 108 insertions(+), 54 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 612ac2d2..ce946a1f 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,8 +1,7 @@ +import datajoint as dj from datetime import datetime from uuid import uuid4 -import datajoint as dj -import matplotlib.pyplot as plt import numpy as np from dateutil.tz import tzlocal @@ -22,77 +21,137 @@ from ... import imaging, scan -def add_scan_to_nwb(session_key, nwbfile, nwbfile_kwargs=None): +def add_scan_to_nwb(session_key, nwbfile): from math import nan - scan_keys = (scan.Scan & session_key).fetch("KEY") - - for scan_key in scan_keys: - scan_data = (scan.Scan & scan_key).fetch1("scanner", "scan_notes") - device = nwbfile.create_device( - name=scan_data["scanner"] - if scan_data["scanner"] is not None - else "TwoPhotonMicroscope", - description="Two photon microscope", - manufacturer="Microscope manufacturer", + try: + scan_key = (scan.Scan & session_key).fetch1("KEY") + except DataJointError: + raise NotImplementedError( + "Exporting more than one scan per session to NWB is not supported yet." ) - no_channels, frame_rate = (scan.ScanInfo & scan_key).fetch1("nchannels", "fps") + scanner_name, scan_notes = (scan.Scan & scan_key).fetch1("scanner", "scan_notes") + device = nwbfile.create_device( + name=scanner_name if scanner_name is not None else "TwoPhotonMicroscope", + description="Two photon microscope", + manufacturer="Microscope manufacturer", + ) - field_keys = (scan.ScanInfo.Field & scan_key).fetch("KEY") + no_channels, frame_rate = (scan.ScanInfo & scan_key).fetch1("nchannels", "fps") - for channel in range(no_channels): - optical_channel = OpticalChannel( - name=f"OpticalChannel{channel+1}", - description=f"Optical channel number {channel+1}", - emission_lambda=nan, - ) + field_keys = (scan.ScanInfo.Field & scan_key).fetch("KEY") - for field_key in field_keys: - field_no = (scan.ScanInfo.Field & field_key).fetch1("field_idx") - imaging_plane = nwbfile.create_imaging_plane( - name=f"ImagingPlane{field_no+1}", - optical_channel=optical_channel, - imaging_rate=frame_rate, - description=scan_data["scan_notes"] - if session_info["scan_notes"] != "" - else f"Imaging plane for channel {channel+1}", - device=device, - excitation_lambda=nan, - indicator="unknown", - location="unknown", - grid_spacing=(0.01, 0.01), - grid_spacing_unit="meters", - origin_coords=[1.0, 2.0, 3.0], - origin_coords_units="meters", - ) + for channel in range(no_channels): + optical_channel = OpticalChannel( + name=f"OpticalChannel{channel+1}", + description=f"Optical channel number {channel+1}", + emission_lambda=nan, + ) + for field_key in field_keys: + field_no = (scan.ScanInfo.Field & field_key).fetch1("field_idx") + imaging_plane = nwbfile.create_imaging_plane( + name=f"ImagingPlane{field_no+1}", + optical_channel=optical_channel, + imaging_rate=frame_rate, + description=scan_data["scan_notes"] + if scan_notes != "" + else f"Imaging plane for channel {channel+1}", + device=device, + excitation_lambda=nan, + indicator="unknown", + location="unknown", + grid_spacing=(0.01, 0.01), + grid_spacing_unit="meters", + origin_coords=[1.0, 2.0, 3.0], + origin_coords_unit="meters", + ) + return imaging_plane -def add_data_to_nwb(session_key, nwbfile, raw_data): + +def add_data_to_nwb(session_key, nwbfile, raw_data, plane): if raw_data: imaging_data = get_image_files(session_key) frame_rate = (scan.ScanInfo & session_key).fetch1("fps") two_p_series = TwoPhotonSeries( name="TwoPhotonSeries", data=imaging_data, - imaging_plane=imaging_plane, + imaging_plane=plane, rate=frame_rate, unit="raw fluorescence", ) else: imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") + two_p_series = TwoPhotonSeries( + name="TwoPhotonSeries", + dimension=(scan.ScanInfo.Field & session_key).fetch1( + "px_height", "px_width" + ), + external_file=imaging_files, + imaging_plane=plane, + starting_frame=[0], + format="external", + starting_time=0.0, + rate=(scan.ScanInfo & session_key).fetch1("fps"), + ) def add_motion_correction_to_nwb(session_key, nwbfile): - pass + raise NotImplementedError( + "Motion Correction data cannot be packaged into NWB at this time." + ) + + +def add_segmentation_data_to_nwb(session_key, nwbfile, plane): + ophys_module = nwbfile.create_processing_module( + name="ophys", description="optical physiology processed data" + ) + img_seg = ImageSegmentation() + ps = img_seg.create_plane_segmentation( + name="PlaneSegmentation", + description="output from segmenting", + imaging_plane=plane, + ) + ophys_module.add(img_seg) + + mask_keys = (imaging.Segmentation.Mask & session_key).fetch("KEY") + for mask_key in mask_keys: + ps.add_roi( + image_mask=np.asarray( + (imaging.Segmentation.Mask() & mask_key).fetch1( + "mask_xpix", "mask_ypix", "mask_weights" + ) + ) + ) + + rt_region = ps.create_roi_table_region( + region=((imaging.Segmentation.Mask & session_key).fetch("mask")).tolist(), + description="All ROIs from database.", + ) + + channels = (scan.ScanInfo & session_key).fetch1("nchannels") + for channel in range(channels): + roi_resp_series = RoiResponseSeries( + name=f"RoiResponseSeries{channel}", + data=np.stack( + ( + imaging.Fluorescence.Trace + & session_key + & f"fluo_channel='{channel}'" + ).fetch("fluorescence") + ).T, + rois=rt_region, + unit="a.u.", + rate=(scan.ScanInfo & session_key).fetch1("fps"), + ) + fl = Fluorescence(roi_response_series=roi_resp_series) + ophys_module.add(fl) def imaging_session_to_nwb( session_key, - raw=True, - spikes=True, - lfp="source", - end_frame=None, + raw=False, lab_key=None, project_key=None, protocol_key=None, @@ -111,13 +170,8 @@ def imaging_session_to_nwb( else: nwbfile = NWBFile(**nwbfile_kwargs) - add_scan_to_nwb(session_key, nwbfile, **nwbfile_kwargs) - add_data_to_nwb(session_key, nwbfile, raw_data=raw) - - try: - (imaging.MotionCorrection & "scan_id='1'").fetch1("KEY") - add_motion_correction_to_nwb(session_key) - except DataJointError: - raise DataJointError(f"No motion correction data found for key {session_key}") + imaging_plane = add_scan_to_nwb(session_key, nwbfile) + add_data_to_nwb(session_key, nwbfile, raw_data=raw, plane=imaging_plane) + add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane) return nwbfile From 2ff92e0f94d70679cd3966cb79ca20fca480586f Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 27 Jul 2023 16:37:11 -0500 Subject: [PATCH 03/33] Minor fixes and updates --- element_calcium_imaging/export/nwb/nwb.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index ce946a1f..e8a1e847 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,10 +1,7 @@ import datajoint as dj -from datetime import datetime -from uuid import uuid4 - import numpy as np -from dateutil.tz import tzlocal +from datajoint import DataJointError from pynwb import NWBHDF5IO, NWBFile, TimeSeries from pynwb.image import ImageSeries from pynwb.ophys import ( @@ -19,6 +16,7 @@ ) from ... import imaging, scan +from ...scan import get_image_files def add_scan_to_nwb(session_key, nwbfile): @@ -55,7 +53,7 @@ def add_scan_to_nwb(session_key, nwbfile): name=f"ImagingPlane{field_no+1}", optical_channel=optical_channel, imaging_rate=frame_rate, - description=scan_data["scan_notes"] + description=scan_notes if scan_notes != "" else f"Imaging plane for channel {channel+1}", device=device, From 142179b3137f4f7ecd2acc48d14dd7ee06aa9cbd Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Tue, 15 Aug 2023 09:49:36 -0500 Subject: [PATCH 04/33] Add neuroconv for raw data to NWB --- element_calcium_imaging/export/nwb/nwb.py | 71 ++++++++++++++++--- .../export/nwb/requirements.txt | 3 +- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index e8a1e847..348d2e8f 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,8 +1,9 @@ +import pathlib import datajoint as dj import numpy as np from datajoint import DataJointError -from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from pynwb import NWBHDF5IO, NWBFile from pynwb.image import ImageSeries from pynwb.ophys import ( CorrectedImageStack, @@ -16,7 +17,7 @@ ) from ... import imaging, scan -from ...scan import get_image_files +from ...scan import get_imaging_root_data_dir, get_image_files def add_scan_to_nwb(session_key, nwbfile): @@ -70,15 +71,63 @@ def add_scan_to_nwb(session_key, nwbfile): def add_data_to_nwb(session_key, nwbfile, raw_data, plane): if raw_data: - imaging_data = get_image_files(session_key) - frame_rate = (scan.ScanInfo & session_key).fetch1("fps") - two_p_series = TwoPhotonSeries( - name="TwoPhotonSeries", - data=imaging_data, - imaging_plane=plane, - rate=frame_rate, - unit="raw fluorescence", + from element_interface.utils import find_full_path + + acquisition_software = (scan.Scan & session_key).fetch1("acq_software") + output_dir = (imaging.ProcessingTask & session_key).fetch1( + "processing_output_dir" ) + + save_path = get_imaging_root_data_dir() / pathlib.Path(output_dir) + + if acquisition_software == "ScanImage": + from neuroconv.datainterfaces import ScanImageImagingInterface + + imaging_data_location = get_image_files(session_key) + scanimage_interface = ScanImageImagingInterface( + file_path=imaging_data_location[0], + fallback_sampling_frequency=(scan.ScanInfo & session_key).fetch1("fps"), + ) + scanimage_interface.run_conversion(nwbfile_path=(save_path / "nwbfile")) + + with NWBHDF5IO((save_path / "nwbfile"), "r+") as io: + nwbfile = io.read() + + # frame_rate = (scan.ScanInfo & session_key).fetch1("fps") + # two_p_series = TwoPhotonSeries( + # name="TwoPhotonSeries", + # data=imaging_data, + # imaging_plane=plane, + # rate=frame_rate, + # unit="raw fluorescence", + # ) + elif acquisition_software == "PrairieView": + from neuroconv.datainterfaces import BrukerTiffImagingInterface + + imaging_data_location = get_image_files(session_key) + for file in imaging_data_location: + bruker_tiff_interface = BrukerTiffImagingInterface( + file_path=file, + fallback_sampling_frequency=(scan.ScanInfo & session_key).fetch1( + "fps" + ), + ) + bruker_tiff_interface.run_conversion(nwbfile_path=(save_path / "nwbfile")) + + elif acquisition_software == "Scanbox": + from neuroconv.datainterfaces import SbxImagingInterface + + imaging_data_location = get_image_files(session_key) + sbx_interface = SbxImagingInterface( + file_path=imaging_data_location[0], + fallback_sampling_frequency=(scan.ScanInfo & session_key).fetch1("fps"), + ) + sbx_interface.run_conversion(nwbfile_path=(save_path / "nwbfile")) + elif acquisition_software == "NIS": + raise NotImplementedError( + "Exporting raw data from .nd2 files to NWB files is not supported yet. Please set `raw=False` and try again." + ) + else: imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") two_p_series = TwoPhotonSeries( @@ -131,7 +180,7 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, plane): channels = (scan.ScanInfo & session_key).fetch1("nchannels") for channel in range(channels): roi_resp_series = RoiResponseSeries( - name=f"RoiResponseSeries{channel}", + name=f"RoiFluorescenceSeries{channel}", data=np.stack( ( imaging.Fluorescence.Trace diff --git a/element_calcium_imaging/export/nwb/requirements.txt b/element_calcium_imaging/export/nwb/requirements.txt index fad8e1bf..4dccc67c 100644 --- a/element_calcium_imaging/export/nwb/requirements.txt +++ b/element_calcium_imaging/export/nwb/requirements.txt @@ -1,2 +1,3 @@ dandi -pynwb \ No newline at end of file +pynwb +neuroconv[scanimage, brukertiff, scanbox] \ No newline at end of file From 42e4c8b71d18623ad49f8324d62321cac4682aac Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 24 Aug 2023 09:56:10 -0500 Subject: [PATCH 05/33] MVP - exports for raw data and processed data only --- element_calcium_imaging/export/nwb/nwb.py | 199 +++++++++++------- .../export/nwb/requirements.txt | 2 +- 2 files changed, 124 insertions(+), 77 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 348d2e8f..501bad29 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -3,6 +3,7 @@ import numpy as np from datajoint import DataJointError +from element_interface.utils import find_full_path from pynwb import NWBHDF5IO, NWBFile from pynwb.image import ImageSeries from pynwb.ophys import ( @@ -15,6 +16,7 @@ RoiResponseSeries, TwoPhotonSeries, ) +from neuroconv import ConverterPipe from ... import imaging, scan from ...scan import get_imaging_root_data_dir, get_image_files @@ -69,80 +71,88 @@ def add_scan_to_nwb(session_key, nwbfile): return imaging_plane -def add_data_to_nwb(session_key, nwbfile, raw_data, plane): - if raw_data: - from element_interface.utils import find_full_path +def rawdata_to_nwb(session_key, output_directory, nwb_path): - acquisition_software = (scan.Scan & session_key).fetch1("acq_software") - output_dir = (imaging.ProcessingTask & session_key).fetch1( - "processing_output_dir" + acquisition_software = (scan.Scan & session_key).fetch1("acq_software") + session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") + processing_method = ( + imaging.ProcessingParamSet & f"paramset_idx='{session_paramset_key}'" + ).fetch1("processing_method") + + if acquisition_software == "ScanImage" and processing_method == "suite2p": + from neuroconv.datainterfaces import ( + ScanImageImagingInterface, + Suite2pSegmentationInterface, ) - save_path = get_imaging_root_data_dir() / pathlib.Path(output_dir) + processing_folder_location = pathlib.Path(output_directory).as_posix() + raw_data_files_location = get_image_files(session_key, "*.tif") + scan_interface = ScanImageImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + ) + s2p_interface = Suite2pSegmentationInterface( + folder_path=processing_folder_location + ) + converter = ConverterPipe(data_interfaces=[scan_interface, s2p_interface]) - if acquisition_software == "ScanImage": - from neuroconv.datainterfaces import ScanImageImagingInterface + elif acquisition_software == "ScanImage" and processing_method == "CaImAn": + from neuroconv.datainterfaces import ( + ScanImageImagingInterface, + CaimanSegmentationInterface, + ) - imaging_data_location = get_image_files(session_key) - scanimage_interface = ScanImageImagingInterface( - file_path=imaging_data_location[0], - fallback_sampling_frequency=(scan.ScanInfo & session_key).fetch1("fps"), - ) - scanimage_interface.run_conversion(nwbfile_path=(save_path / "nwbfile")) - - with NWBHDF5IO((save_path / "nwbfile"), "r+") as io: - nwbfile = io.read() - - # frame_rate = (scan.ScanInfo & session_key).fetch1("fps") - # two_p_series = TwoPhotonSeries( - # name="TwoPhotonSeries", - # data=imaging_data, - # imaging_plane=plane, - # rate=frame_rate, - # unit="raw fluorescence", - # ) - elif acquisition_software == "PrairieView": - from neuroconv.datainterfaces import BrukerTiffImagingInterface - - imaging_data_location = get_image_files(session_key) - for file in imaging_data_location: - bruker_tiff_interface = BrukerTiffImagingInterface( - file_path=file, - fallback_sampling_frequency=(scan.ScanInfo & session_key).fetch1( - "fps" - ), - ) - bruker_tiff_interface.run_conversion(nwbfile_path=(save_path / "nwbfile")) + caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) + scan_interface = ScanImageImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + ) + caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) + converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) - elif acquisition_software == "Scanbox": - from neuroconv.datainterfaces import SbxImagingInterface + elif acquisition_software == "Scanbox" and processing_method == "suite2p": + from neuroconv.datainterfaces import ( + SbxImagingInterface, + Suite2pSegmentationInterface, + ) - imaging_data_location = get_image_files(session_key) - sbx_interface = SbxImagingInterface( - file_path=imaging_data_location[0], - fallback_sampling_frequency=(scan.ScanInfo & session_key).fetch1("fps"), - ) - sbx_interface.run_conversion(nwbfile_path=(save_path / "nwbfile")) - elif acquisition_software == "NIS": - raise NotImplementedError( - "Exporting raw data from .nd2 files to NWB files is not supported yet. Please set `raw=False` and try again." - ) + scan_interface = SbxImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + ) + s2p_interface = Suite2pSegmentationInterface( + folder_path=processing_folder_location + ) + converter = ConverterPipe(data_interfaces=[scan_interface, s2p_interface]) - else: - imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") - two_p_series = TwoPhotonSeries( - name="TwoPhotonSeries", - dimension=(scan.ScanInfo.Field & session_key).fetch1( - "px_height", "px_width" - ), - external_file=imaging_files, - imaging_plane=plane, - starting_frame=[0], - format="external", - starting_time=0.0, - rate=(scan.ScanInfo & session_key).fetch1("fps"), + elif acquisition_software == "Scanbox" and processing_method == "CaImAn": + from neuroconv.datainterfaces import ( + SbxImagingInterface, + CaimanSegmentationInterface, ) + caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) + scan_interface = SbxImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + ) + caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) + converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) + + metadata = converter.get_metadata() + metadata["NWBFile"].update(session_description="DataJoint Session") + converter.run_conversion(nwbfile_path=(nwb_path / "nwbfile"), metadata=metadata) + + +def add_image_series_to_nwb(session_key, plane): + imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") + two_p_series = TwoPhotonSeries( + name="TwoPhotonSeries", + dimension=(scan.ScanInfo.Field & session_key).fetch1("px_height", "px_width"), + external_file=imaging_files, + imaging_plane=plane, + starting_frame=[0], + format="external", + starting_time=0.0, + rate=(scan.ScanInfo & session_key).fetch1("fps"), + ) + def add_motion_correction_to_nwb(session_key, nwbfile): raise NotImplementedError( @@ -198,7 +208,9 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, plane): def imaging_session_to_nwb( session_key, + save_path=None, raw=False, + processed_data_source="database" or "filesystem", lab_key=None, project_key=None, protocol_key=None, @@ -206,19 +218,54 @@ def imaging_session_to_nwb( ): session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) - if session_to_nwb: - nwbfile = session_to_nwb( - session_key, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - additional_nwbfile_kwargs=nwbfile_kwargs, + output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( + "processing_output_dir" + ) + output_dir = find_full_path(get_imaging_root_data_dir(), output_relative_dir) + + if not save_path: + output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( + "processing_output_dir" ) + save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) + + if raw: + rawdata_to_nwb(session_key, output_directory=output_dir, nwb_path=save_path) + with NWBHDF5IO((save_path / "nwbfile"), mode="r+") as io: + nwb_file = io.read() + if session_to_nwb: + nwb_file = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, + ) + io.write(nwb_file) + else: + if "Subject" in nwbfile_kwargs: + from pynwb.file import Subject + nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) + else: + nwb_file = NWBFile(**nwbfile_kwargs) + io.write(nwb_file) else: - nwbfile = NWBFile(**nwbfile_kwargs) + if session_to_nwb: + nwb_file = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, + ) + else: + nwb_file = NWBFile(**nwbfile_kwargs) + + imaging_plane = add_scan_to_nwb(session_key, nwbfile=nwb_file) + add_image_series_to_nwb(session_key, plane=imaging_plane) + add_segmentation_data_to_nwb(session_key, nwbfile=nwb_file, plane=imaging_plane) + + return nwb_file - imaging_plane = add_scan_to_nwb(session_key, nwbfile) - add_data_to_nwb(session_key, nwbfile, raw_data=raw, plane=imaging_plane) - add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane) - return nwbfile +## TODO: Add a `from_source` flag as with ephys NWB with options for \ No newline at end of file diff --git a/element_calcium_imaging/export/nwb/requirements.txt b/element_calcium_imaging/export/nwb/requirements.txt index 4dccc67c..102856b7 100644 --- a/element_calcium_imaging/export/nwb/requirements.txt +++ b/element_calcium_imaging/export/nwb/requirements.txt @@ -1,3 +1,3 @@ dandi pynwb -neuroconv[scanimage, brukertiff, scanbox] \ No newline at end of file +neuroconv[scanimage, brukertiff, scanbox, caiman, suite2p] \ No newline at end of file From a82f54b9d4199f47fcde16ded2feb4db9b7b4ed9 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 24 Aug 2023 12:47:25 -0500 Subject: [PATCH 06/33] Add Neuropil and Deconvolve traces to nwb --- element_calcium_imaging/export/nwb/nwb.py | 152 +++++++++++++--------- 1 file changed, 89 insertions(+), 63 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 501bad29..5fa74717 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -22,56 +22,7 @@ from ...scan import get_imaging_root_data_dir, get_image_files -def add_scan_to_nwb(session_key, nwbfile): - from math import nan - - try: - scan_key = (scan.Scan & session_key).fetch1("KEY") - except DataJointError: - raise NotImplementedError( - "Exporting more than one scan per session to NWB is not supported yet." - ) - - scanner_name, scan_notes = (scan.Scan & scan_key).fetch1("scanner", "scan_notes") - device = nwbfile.create_device( - name=scanner_name if scanner_name is not None else "TwoPhotonMicroscope", - description="Two photon microscope", - manufacturer="Microscope manufacturer", - ) - - no_channels, frame_rate = (scan.ScanInfo & scan_key).fetch1("nchannels", "fps") - - field_keys = (scan.ScanInfo.Field & scan_key).fetch("KEY") - - for channel in range(no_channels): - optical_channel = OpticalChannel( - name=f"OpticalChannel{channel+1}", - description=f"Optical channel number {channel+1}", - emission_lambda=nan, - ) - - for field_key in field_keys: - field_no = (scan.ScanInfo.Field & field_key).fetch1("field_idx") - imaging_plane = nwbfile.create_imaging_plane( - name=f"ImagingPlane{field_no+1}", - optical_channel=optical_channel, - imaging_rate=frame_rate, - description=scan_notes - if scan_notes != "" - else f"Imaging plane for channel {channel+1}", - device=device, - excitation_lambda=nan, - indicator="unknown", - location="unknown", - grid_spacing=(0.01, 0.01), - grid_spacing_unit="meters", - origin_coords=[1.0, 2.0, 3.0], - origin_coords_unit="meters", - ) - return imaging_plane - - -def rawdata_to_nwb(session_key, output_directory, nwb_path): +def create_raw_data_nwbfile(session_key, output_directory, nwb_path): acquisition_software = (scan.Scan & session_key).fetch1("acq_software") session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") @@ -137,16 +88,65 @@ def rawdata_to_nwb(session_key, output_directory, nwb_path): metadata = converter.get_metadata() metadata["NWBFile"].update(session_description="DataJoint Session") - converter.run_conversion(nwbfile_path=(nwb_path / "nwbfile"), metadata=metadata) + converter.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) + + +def add_scan_to_nwb(session_key, nwbfile): + from math import nan + try: + scan_key = (scan.Scan & session_key).fetch1("KEY") + except DataJointError: + raise NotImplementedError( + "Exporting more than one scan per session to NWB is not supported yet." + ) -def add_image_series_to_nwb(session_key, plane): + scanner_name, scan_notes = (scan.Scan & scan_key).fetch1("scanner", "scan_notes") + device = nwbfile.create_device( + name=scanner_name if scanner_name is not None else "TwoPhotonMicroscope", + description="Two photon microscope", + manufacturer="Microscope manufacturer", + ) + + no_channels, frame_rate = (scan.ScanInfo & scan_key).fetch1("nchannels", "fps") + + field_keys = (scan.ScanInfo.Field & scan_key).fetch("KEY") + + for channel in range(no_channels): + optical_channel = OpticalChannel( + name=f"OpticalChannel{channel+1}", + description=f"Optical channel number {channel+1}", + emission_lambda=nan, + ) + + for field_key in field_keys: + field_no = (scan.ScanInfo.Field & field_key).fetch1("field_idx") + imaging_plane = nwbfile.create_imaging_plane( + name=f"ImagingPlane{field_no+1}", + optical_channel=optical_channel, + imaging_rate=frame_rate, + description=scan_notes + if scan_notes != "" + else f"Imaging plane for channel {channel+1}", + device=device, + excitation_lambda=nan, + indicator="unknown", + location="unknown", + grid_spacing=(0.01, 0.01), + grid_spacing_unit="meters", + origin_coords=[1.0, 2.0, 3.0], + origin_coords_unit="meters", + ) + return imaging_plane + + +def add_image_series_to_nwb(session_key, imaging_plane): imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") two_p_series = TwoPhotonSeries( name="TwoPhotonSeries", dimension=(scan.ScanInfo.Field & session_key).fetch1("px_height", "px_width"), external_file=imaging_files, - imaging_plane=plane, + imaging_plane=imaging_plane, starting_frame=[0], format="external", starting_time=0.0, @@ -160,7 +160,7 @@ def add_motion_correction_to_nwb(session_key, nwbfile): ) -def add_segmentation_data_to_nwb(session_key, nwbfile, plane): +def add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): ophys_module = nwbfile.create_processing_module( name="ophys", description="optical physiology processed data" ) @@ -168,7 +168,7 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, plane): ps = img_seg.create_plane_segmentation( name="PlaneSegmentation", description="output from segmenting", - imaging_plane=plane, + imaging_plane=imaging_plane, ) ophys_module.add(img_seg) @@ -190,7 +190,7 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, plane): channels = (scan.ScanInfo & session_key).fetch1("nchannels") for channel in range(channels): roi_resp_series = RoiResponseSeries( - name=f"RoiFluorescenceSeries{channel}", + name=f"Fluorescence_{channel}", data=np.stack( ( imaging.Fluorescence.Trace @@ -202,14 +202,40 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, plane): unit="a.u.", rate=(scan.ScanInfo & session_key).fetch1("fps"), ) - fl = Fluorescence(roi_response_series=roi_resp_series) + neuorpil_series = RoiResponseSeries( + name=f"Neuropil_{channel}", + data=np.stack( + ( + imaging.Fluorescence.Trace + & session_key + & f"fluo_channel='{channel}'" + ).fetch("neuropil_fluorescence") + ).T, + rois=rt_region, + unit="a.u.", + rate=(scan.ScanInfo & session_key).fetch1("fps"), + ) + deconvolved_series = RoiResponseSeries( + name=f"Deconvolved_{channel}", + data=np.stack( + ( + imaging.Activity.Trace + & session_key + & f"fluo_channel='{channel}'" + ).fetch("activity_trace") + ).T, + rois=rt_region, + unit="a.u.", + rate=(scan.ScanInfo & session_key).fetch1("fps"), + ) + fl = Fluorescence(roi_response_series=[roi_resp_series, neuorpil_series, deconvolved_series]) ophys_module.add(fl) def imaging_session_to_nwb( session_key, save_path=None, - raw=False, + include_raw_data=False, processed_data_source="database" or "filesystem", lab_key=None, project_key=None, @@ -229,9 +255,9 @@ def imaging_session_to_nwb( ) save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) - if raw: - rawdata_to_nwb(session_key, output_directory=output_dir, nwb_path=save_path) - with NWBHDF5IO((save_path / "nwbfile"), mode="r+") as io: + if include_raw_data: + create_raw_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) + with NWBHDF5IO((save_path / f'{session_key["subject"]}_nwbfile'), mode="r+") as io: nwb_file = io.read() if session_to_nwb: nwb_file = session_to_nwb( @@ -262,8 +288,8 @@ def imaging_session_to_nwb( nwb_file = NWBFile(**nwbfile_kwargs) imaging_plane = add_scan_to_nwb(session_key, nwbfile=nwb_file) - add_image_series_to_nwb(session_key, plane=imaging_plane) - add_segmentation_data_to_nwb(session_key, nwbfile=nwb_file, plane=imaging_plane) + add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) + add_segmentation_data_to_nwb(session_key, nwbfile=nwb_file, imaging_plane=imaging_plane) return nwb_file From 2cee4d6ffa922f3b6df36b48d8d141c20d8ad99d Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 25 Aug 2023 12:56:47 -0500 Subject: [PATCH 07/33] Add extract segementation interface --- element_calcium_imaging/export/nwb/nwb.py | 16 ++++++++++++++++ .../export/nwb/requirements.txt | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 5fa74717..21356592 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -59,6 +59,22 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) + elif acquisition_software == "ScanImage" and processing_method == "extract": + from neuroconv.datainterfaces import ( + ScanImageImagingInterface, + ExtractSegmentationInterface, + ) + + processing_file_location = pathlib.Path(output_directory).as_posix() + raw_data_files_location = get_image_files(session_key, "*.tif") + scan_interface = ScanImageImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + ) + extract_interface = Suite2pSegmentationInterface( + file_path=processing_file_location + ) + converter = ConverterPipe(data_interfaces=[scan_interface, extract_interface]) + elif acquisition_software == "Scanbox" and processing_method == "suite2p": from neuroconv.datainterfaces import ( SbxImagingInterface, diff --git a/element_calcium_imaging/export/nwb/requirements.txt b/element_calcium_imaging/export/nwb/requirements.txt index 102856b7..2375658c 100644 --- a/element_calcium_imaging/export/nwb/requirements.txt +++ b/element_calcium_imaging/export/nwb/requirements.txt @@ -1,3 +1,3 @@ dandi pynwb -neuroconv[scanimage, brukertiff, scanbox, caiman, suite2p] \ No newline at end of file +neuroconv[scanimage, brukertiff, scanbox, caiman, suite2p, extract] \ No newline at end of file From b2e26d1b88969374a5aefd6b533c511076f849a3 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 25 Aug 2023 13:26:17 -0500 Subject: [PATCH 08/33] Add bruker tiff NWB data interface --- element_calcium_imaging/export/nwb/nwb.py | 36 ++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 21356592..0e4d4c8f 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -70,7 +70,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): scan_interface = ScanImageImagingInterface( file_path=raw_data_files_location[0], fallback_sampling_frequency=30 ) - extract_interface = Suite2pSegmentationInterface( + extract_interface = ExtractSegmentationInterface( file_path=processing_file_location ) converter = ConverterPipe(data_interfaces=[scan_interface, extract_interface]) @@ -101,6 +101,40 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) + + elif acquisition_software == "Scanbox" and processing_method == "extract": + from neuroconv.datainterfaces import ( + SbxImagingInterface, + ExtractSegmentationInterface, + ) + + processing_file_location = pathlib.Path(output_directory).as_posix() + raw_data_files_location = get_image_files(session_key, "*.tif") + scan_interface = SbxImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + ) + extract_interface = ExtractSegmentationInterface( + file_path=processing_file_location + ) + converter = ConverterPipe(data_interfaces=[scan_interface, extract_interface]) + + elif acquisition_software == "PrairieView" and processing_method == "suite2p": + n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") + if n_planes > 1: + from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter + else: + from neuroconv.datainterfaces import BrukerTiffSinglePlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import Suite2pSegmentationInterface + + processing_folder_location = pathlib.Path(output_directory).as_posix() + raw_data_files_location = get_image_files(session_key, "*.tif") + bruker_interface = BrukerTiffConverter( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + ) + s2p_interface = Suite2pSegmentationInterface( + folder_path=processing_folder_location + ) + converter = ConverterPipe(data_interfaces=[scan_interface, s2p_interface]) metadata = converter.get_metadata() metadata["NWBFile"].update(session_description="DataJoint Session") From a0d091976c9d1fd5e72788f48587f63bfa5e6eb5 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 25 Aug 2023 13:54:49 -0500 Subject: [PATCH 09/33] Introduce `processed_data_source` flag --- element_calcium_imaging/export/nwb/nwb.py | 79 +++++++++++++++++------ 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 0e4d4c8f..5a8b2a56 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -129,12 +129,47 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): processing_folder_location = pathlib.Path(output_directory).as_posix() raw_data_files_location = get_image_files(session_key, "*.tif") bruker_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + file_path=raw_data_files_location[0], fallback_sampling_frequency=30, **bruker_kwargs ) s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location ) - converter = ConverterPipe(data_interfaces=[scan_interface, s2p_interface]) + converter = ConverterPipe(data_interfaces=[bruker_interface, s2p_interface]) + + elif acquisition_software == "PrairieView" and processing_method == "caiman": + n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") + if n_planes > 1: + from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter + else: + from neuroconv.datainterfaces import BrukerTiffSinglePlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import CaimanSegmentationInterface + + processing_folder_location = pathlib.Path(output_directory).as_posix() + raw_data_files_location = get_image_files(session_key, "*.tif") + bruker_interface = BrukerTiffConverter( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30, **bruker_kwargs + ) + caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) + caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) + converter = ConverterPipe(data_interfaces=[bruker_interface, caiman_interface]) + + elif acquisition_software == "PrairieView" and processing_method == "extract": + n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") + if n_planes > 1: + from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter + else: + from neuroconv.datainterfaces import BrukerTiffSinglePlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import ExtractSegmentationInterface + + processing_folder_location = pathlib.Path(output_directory).as_posix() + raw_data_files_location = get_image_files(session_key, "*.tif") + bruker_interface = BrukerTiffConverter( + file_path=raw_data_files_location[0], fallback_sampling_frequency=30, **bruker_kwargs + ) + extract_interface = ExtractSegmentationInterface( + file_path=processing_file_location + ) + converter = ConverterPipe(data_interfaces=[bruker_interface, extract_interface]) metadata = converter.get_metadata() metadata["NWBFile"].update(session_description="DataJoint Session") @@ -286,12 +321,16 @@ def imaging_session_to_nwb( session_key, save_path=None, include_raw_data=False, - processed_data_source="database" or "filesystem", + processed_data_source="database", lab_key=None, project_key=None, protocol_key=None, nwbfile_kwargs=None, -): + ): + + if processed_data_source not in ["database", "filesystem"]: + raise ValueError("Invalid processed data source. Expected one of 'database', 'filesystem'") + session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( @@ -306,6 +345,7 @@ def imaging_session_to_nwb( save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) if include_raw_data: + processed_data_source="filesystem" create_raw_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) with NWBHDF5IO((save_path / f'{session_key["subject"]}_nwbfile'), mode="r+") as io: nwb_file = io.read() @@ -326,22 +366,25 @@ def imaging_session_to_nwb( nwb_file = NWBFile(**nwbfile_kwargs) io.write(nwb_file) else: - if session_to_nwb: - nwb_file = session_to_nwb( - session_key, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - additional_nwbfile_kwargs=nwbfile_kwargs, - ) - else: - nwb_file = NWBFile(**nwbfile_kwargs) + if processed_data_source == "database": + if session_to_nwb: + nwb_file = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, + ) + else: + nwb_file = NWBFile(**nwbfile_kwargs) - imaging_plane = add_scan_to_nwb(session_key, nwbfile=nwb_file) - add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) - add_segmentation_data_to_nwb(session_key, nwbfile=nwb_file, imaging_plane=imaging_plane) + imaging_plane = add_scan_to_nwb(session_key, nwbfile=nwb_file) + add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) + add_segmentation_data_to_nwb(session_key, nwbfile=nwb_file, imaging_plane=imaging_plane) - return nwb_file + return nwb_file + elif processed_data_source == "filesystem": + create_processed_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) ## TODO: Add a `from_source` flag as with ephys NWB with options for \ No newline at end of file From 8c7bb5c8f267f6c46d25357bc65c24ab195f80ae Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 25 Aug 2023 14:24:38 -0500 Subject: [PATCH 10/33] Add `create_processed_data_nwbfile()` --- element_calcium_imaging/export/nwb/nwb.py | 42 ++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 5a8b2a56..b7e78acb 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -136,7 +136,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) converter = ConverterPipe(data_interfaces=[bruker_interface, s2p_interface]) - elif acquisition_software == "PrairieView" and processing_method == "caiman": + elif acquisition_software == "PrairieView" and processing_method == "caiman": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter @@ -153,7 +153,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) converter = ConverterPipe(data_interfaces=[bruker_interface, caiman_interface]) - elif acquisition_software == "PrairieView" and processing_method == "extract": + elif acquisition_software == "PrairieView" and processing_method == "extract": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter @@ -161,7 +161,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): from neuroconv.datainterfaces import BrukerTiffSinglePlaneConverter as BrukerTiffConverter from neuroconv.datainterfaces import ExtractSegmentationInterface - processing_folder_location = pathlib.Path(output_directory).as_posix() + processing_file_location = pathlib.Path(output_directory).as_posix() raw_data_files_location = get_image_files(session_key, "*.tif") bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=30, **bruker_kwargs @@ -175,6 +175,40 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): metadata["NWBFile"].update(session_description="DataJoint Session") converter.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) +def create_processed_data_nwbfile(session_key, output_directory, nwb_path): + session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") + processing_method = ( + imaging.ProcessingParamSet & f"paramset_idx='{session_paramset_key}'" + ).fetch1("processing_method") + + if processing_method == "suite2p": + from neuroconv.datainterfaces import Suite2pSegmentationInterface + + s2p_interface = Suite2pSegmentationInterface( + folder_path=processing_folder_location + ) + metadata = interface.get_metadata() + s2p_interface.run_conversion(nwbfile_path=nwb_path, metadata=metadata) + + elif processing_method == "caiman": + from neuroconv.datainterfaces import CaimanSegmentationInterface + + processing_folder_location = pathlib.Path(output_directory).as_posix() + caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) + caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) + metadata = caiman_interface.get_metadata() + caiman_interface.run_conversion(nwbfile_path=nwb_path, metadata=metadata) + + elif processing_method == "extract": + from neuroconv.datainterfaces import ExtractSegmentationInterface + + processing_file_location = pathlib.Path(output_directory).as_posix() + extract_interface = ExtractSegmentationInterface( + file_path=processing_file_location + ) + metadata = extract_interface.get_metadata() + extract_interface.run_conversion(nwbfile_path=nwb_path, metadata=metadata) + def add_scan_to_nwb(session_key, nwbfile): from math import nan @@ -383,8 +417,8 @@ def imaging_session_to_nwb( add_segmentation_data_to_nwb(session_key, nwbfile=nwb_file, imaging_plane=imaging_plane) return nwb_file + elif processed_data_source == "filesystem": create_processed_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) - ## TODO: Add a `from_source` flag as with ephys NWB with options for \ No newline at end of file From 65a293f090357ea951265e394d1bea62ff97f8c6 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 25 Aug 2023 14:57:36 -0500 Subject: [PATCH 11/33] Add subject for create_processed_data_nwbfile --- element_calcium_imaging/export/nwb/nwb.py | 29 +++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index b7e78acb..b7e666eb 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -25,6 +25,9 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): acquisition_software = (scan.Scan & session_key).fetch1("acq_software") + if acquisition_software == "NIS": + raise NotImplementedError("Packaging raw data from Nikon NIS acquisition software (.nd2 file format) is not currently supported.") + session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") processing_method = ( imaging.ProcessingParamSet & f"paramset_idx='{session_paramset_key}'" @@ -188,7 +191,7 @@ def create_processed_data_nwbfile(session_key, output_directory, nwb_path): folder_path=processing_folder_location ) metadata = interface.get_metadata() - s2p_interface.run_conversion(nwbfile_path=nwb_path, metadata=metadata) + s2p_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) elif processing_method == "caiman": from neuroconv.datainterfaces import CaimanSegmentationInterface @@ -197,7 +200,7 @@ def create_processed_data_nwbfile(session_key, output_directory, nwb_path): caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) metadata = caiman_interface.get_metadata() - caiman_interface.run_conversion(nwbfile_path=nwb_path, metadata=metadata) + caiman_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) elif processing_method == "extract": from neuroconv.datainterfaces import ExtractSegmentationInterface @@ -207,7 +210,7 @@ def create_processed_data_nwbfile(session_key, output_directory, nwb_path): file_path=processing_file_location ) metadata = extract_interface.get_metadata() - extract_interface.run_conversion(nwbfile_path=nwb_path, metadata=metadata) + extract_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) def add_scan_to_nwb(session_key, nwbfile): @@ -420,5 +423,21 @@ def imaging_session_to_nwb( elif processed_data_source == "filesystem": create_processed_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) - -## TODO: Add a `from_source` flag as with ephys NWB with options for \ No newline at end of file + with NWBHDF5IO((save_path / f'{session_key["subject"]}_nwbfile'), mode="r+") as io: + nwb_file = io.read() + if session_to_nwb: + nwb_file = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, + ) + io.write(nwb_file) + else: + if "Subject" in nwbfile_kwargs: + from pynwb.file import Subject + nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) + else: + nwb_file = NWBFile(**nwbfile_kwargs) + io.write(nwb_file) From c92e81601c908c76bd5751ba72463eaa6b856ce8 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 25 Aug 2023 16:04:56 -0500 Subject: [PATCH 12/33] Update formatting and syntax errors --- element_calcium_imaging/export/nwb/nwb.py | 159 ++++++++++++++-------- 1 file changed, 104 insertions(+), 55 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index b7e666eb..02ea0c68 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,9 +1,10 @@ import pathlib + import datajoint as dj import numpy as np - from datajoint import DataJointError from element_interface.utils import find_full_path +from neuroconv import ConverterPipe from pynwb import NWBHDF5IO, NWBFile from pynwb.image import ImageSeries from pynwb.ophys import ( @@ -16,17 +17,18 @@ RoiResponseSeries, TwoPhotonSeries, ) -from neuroconv import ConverterPipe from ... import imaging, scan -from ...scan import get_imaging_root_data_dir, get_image_files +from ...scan import get_image_files, get_imaging_root_data_dir def create_raw_data_nwbfile(session_key, output_directory, nwb_path): acquisition_software = (scan.Scan & session_key).fetch1("acq_software") if acquisition_software == "NIS": - raise NotImplementedError("Packaging raw data from Nikon NIS acquisition software (.nd2 file format) is not currently supported.") + raise NotImplementedError( + "Packaging raw data from Nikon NIS acquisition software (.nd2 file format) is not currently supported." + ) session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") processing_method = ( @@ -51,8 +53,8 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "ScanImage" and processing_method == "CaImAn": from neuroconv.datainterfaces import ( - ScanImageImagingInterface, CaimanSegmentationInterface, + ScanImageImagingInterface, ) caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) @@ -64,8 +66,8 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "ScanImage" and processing_method == "extract": from neuroconv.datainterfaces import ( - ScanImageImagingInterface, ExtractSegmentationInterface, + ScanImageImagingInterface, ) processing_file_location = pathlib.Path(output_directory).as_posix() @@ -94,8 +96,8 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "Scanbox" and processing_method == "CaImAn": from neuroconv.datainterfaces import ( - SbxImagingInterface, CaimanSegmentationInterface, + SbxImagingInterface, ) caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) @@ -104,11 +106,11 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) - + elif acquisition_software == "Scanbox" and processing_method == "extract": from neuroconv.datainterfaces import ( - SbxImagingInterface, ExtractSegmentationInterface, + SbxImagingInterface, ) processing_file_location = pathlib.Path(output_directory).as_posix() @@ -120,19 +122,25 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): file_path=processing_file_location ) converter = ConverterPipe(data_interfaces=[scan_interface, extract_interface]) - + elif acquisition_software == "PrairieView" and processing_method == "suite2p": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: - from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import ( + BrukerTiffMultiPlaneConverter as BrukerTiffConverter, + ) else: - from neuroconv.datainterfaces import BrukerTiffSinglePlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import ( + BrukerTiffSinglePlaneConverter as BrukerTiffConverter, + ) from neuroconv.datainterfaces import Suite2pSegmentationInterface processing_folder_location = pathlib.Path(output_directory).as_posix() raw_data_files_location = get_image_files(session_key, "*.tif") bruker_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30, **bruker_kwargs + file_path=raw_data_files_location[0], + fallback_sampling_frequency=30, + **bruker_kwargs, ) s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location @@ -142,15 +150,21 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "PrairieView" and processing_method == "caiman": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: - from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import ( + BrukerTiffMultiPlaneConverter as BrukerTiffConverter, + ) else: - from neuroconv.datainterfaces import BrukerTiffSinglePlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import ( + BrukerTiffSinglePlaneConverter as BrukerTiffConverter, + ) from neuroconv.datainterfaces import CaimanSegmentationInterface processing_folder_location = pathlib.Path(output_directory).as_posix() raw_data_files_location = get_image_files(session_key, "*.tif") bruker_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30, **bruker_kwargs + file_path=raw_data_files_location[0], + fallback_sampling_frequency=30, + **bruker_kwargs, ) caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) @@ -159,15 +173,21 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "PrairieView" and processing_method == "extract": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: - from neuroconv.datainterfaces import BrukerTiffMultiPlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import ( + BrukerTiffMultiPlaneConverter as BrukerTiffConverter, + ) else: - from neuroconv.datainterfaces import BrukerTiffSinglePlaneConverter as BrukerTiffConverter + from neuroconv.datainterfaces import ( + BrukerTiffSinglePlaneConverter as BrukerTiffConverter, + ) from neuroconv.datainterfaces import ExtractSegmentationInterface processing_file_location = pathlib.Path(output_directory).as_posix() raw_data_files_location = get_image_files(session_key, "*.tif") bruker_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30, **bruker_kwargs + file_path=raw_data_files_location[0], + fallback_sampling_frequency=30, + **bruker_kwargs, ) extract_interface = ExtractSegmentationInterface( file_path=processing_file_location @@ -176,7 +196,10 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): metadata = converter.get_metadata() metadata["NWBFile"].update(session_description="DataJoint Session") - converter.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) + converter.run_conversion( + nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata + ) + def create_processed_data_nwbfile(session_key, output_directory, nwb_path): session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") @@ -186,13 +209,16 @@ def create_processed_data_nwbfile(session_key, output_directory, nwb_path): if processing_method == "suite2p": from neuroconv.datainterfaces import Suite2pSegmentationInterface - + s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location ) metadata = interface.get_metadata() - s2p_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) - + s2p_interface.run_conversion( + nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + metadata=metadata, + ) + elif processing_method == "caiman": from neuroconv.datainterfaces import CaimanSegmentationInterface @@ -200,7 +226,10 @@ def create_processed_data_nwbfile(session_key, output_directory, nwb_path): caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) metadata = caiman_interface.get_metadata() - caiman_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) + caiman_interface.run_conversion( + nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + metadata=metadata, + ) elif processing_method == "extract": from neuroconv.datainterfaces import ExtractSegmentationInterface @@ -210,7 +239,10 @@ def create_processed_data_nwbfile(session_key, output_directory, nwb_path): file_path=processing_file_location ) metadata = extract_interface.get_metadata() - extract_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) + extract_interface.run_conversion( + nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + metadata=metadata, + ) def add_scan_to_nwb(session_key, nwbfile): @@ -341,16 +373,16 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): name=f"Deconvolved_{channel}", data=np.stack( ( - imaging.Activity.Trace - & session_key - & f"fluo_channel='{channel}'" + imaging.Activity.Trace & session_key & f"fluo_channel='{channel}'" ).fetch("activity_trace") ).T, rois=rt_region, unit="a.u.", rate=(scan.ScanInfo & session_key).fetch1("fps"), ) - fl = Fluorescence(roi_response_series=[roi_resp_series, neuorpil_series, deconvolved_series]) + fl = Fluorescence( + roi_response_series=[roi_resp_series, neuorpil_series, deconvolved_series] + ) ophys_module.add(fl) @@ -363,10 +395,12 @@ def imaging_session_to_nwb( project_key=None, protocol_key=None, nwbfile_kwargs=None, - ): +): if processed_data_source not in ["database", "filesystem"]: - raise ValueError("Invalid processed data source. Expected one of 'database', 'filesystem'") + raise ValueError( + "Invalid processed data source. Expected one of 'database', 'filesystem'" + ) session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) @@ -382,9 +416,13 @@ def imaging_session_to_nwb( save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) if include_raw_data: - processed_data_source="filesystem" - create_raw_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) - with NWBHDF5IO((save_path / f'{session_key["subject"]}_nwbfile'), mode="r+") as io: + processed_data_source = "filesystem" + create_raw_data_nwbfile( + session_key, output_directory=output_dir, nwb_path=save_path + ) + with NWBHDF5IO( + (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" + ) as io: nwb_file = io.read() if session_to_nwb: nwb_file = session_to_nwb( @@ -398,6 +436,7 @@ def imaging_session_to_nwb( else: if "Subject" in nwbfile_kwargs: from pynwb.file import Subject + nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) else: nwb_file = NWBFile(**nwbfile_kwargs) @@ -417,27 +456,37 @@ def imaging_session_to_nwb( imaging_plane = add_scan_to_nwb(session_key, nwbfile=nwb_file) add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) - add_segmentation_data_to_nwb(session_key, nwbfile=nwb_file, imaging_plane=imaging_plane) + add_segmentation_data_to_nwb( + session_key, nwbfile=nwb_file, imaging_plane=imaging_plane + ) - return nwb_file - - elif processed_data_source == "filesystem": - create_processed_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) - with NWBHDF5IO((save_path / f'{session_key["subject"]}_nwbfile'), mode="r+") as io: - nwb_file = io.read() - if session_to_nwb: - nwb_file = session_to_nwb( - session_key, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - additional_nwbfile_kwargs=nwbfile_kwargs, - ) + with NWBHDF5IO( + (save_path / f'{session_key["subject"]}_nwbfile'), mode="w" + ) as io: io.write(nwb_file) - else: - if "Subject" in nwbfile_kwargs: - from pynwb.file import Subject - nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) + + elif processed_data_source == "filesystem": + create_processed_data_nwbfile( + session_key, output_directory=output_dir, nwb_path=save_path + ) + with NWBHDF5IO( + (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" + ) as io: + nwb_file = io.read() + if session_to_nwb: + nwb_file = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, + ) + io.write(nwb_file) else: - nwb_file = NWBFile(**nwbfile_kwargs) - io.write(nwb_file) + if "Subject" in nwbfile_kwargs: + from pynwb.file import Subject + + nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) + else: + nwb_file = NWBFile(**nwbfile_kwargs) + io.write(nwb_file) From 63ea11106ca88c8fbede9328cdce589e7e769816 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 25 Aug 2023 16:10:55 -0500 Subject: [PATCH 13/33] Updates to pass style checks --- element_calcium_imaging/export/nwb/nwb.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 02ea0c68..3ba1b707 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,18 +1,13 @@ import pathlib -import datajoint as dj import numpy as np from datajoint import DataJointError from element_interface.utils import find_full_path from neuroconv import ConverterPipe from pynwb import NWBHDF5IO, NWBFile -from pynwb.image import ImageSeries from pynwb.ophys import ( - CorrectedImageStack, Fluorescence, ImageSegmentation, - MotionCorrection, - OnePhotonSeries, OpticalChannel, RoiResponseSeries, TwoPhotonSeries, @@ -140,7 +135,6 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=30, - **bruker_kwargs, ) s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location @@ -164,7 +158,6 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=30, - **bruker_kwargs, ) caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) @@ -187,7 +180,6 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=30, - **bruker_kwargs, ) extract_interface = ExtractSegmentationInterface( file_path=processing_file_location @@ -210,10 +202,11 @@ def create_processed_data_nwbfile(session_key, output_directory, nwb_path): if processing_method == "suite2p": from neuroconv.datainterfaces import Suite2pSegmentationInterface + processing_folder_location = pathlib.Path(output_directory).as_posix() s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location ) - metadata = interface.get_metadata() + metadata = s2p_interface.get_metadata() s2p_interface.run_conversion( nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata, @@ -306,6 +299,7 @@ def add_image_series_to_nwb(session_key, imaging_plane): starting_time=0.0, rate=(scan.ScanInfo & session_key).fetch1("fps"), ) + return two_p_series def add_motion_correction_to_nwb(session_key, nwbfile): From c3ca1f6fb553f3c60fd7427829273e146700d56f Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Tue, 29 Aug 2023 10:03:46 -0500 Subject: [PATCH 14/33] Debug export --- element_calcium_imaging/export/nwb/nwb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 3ba1b707..bf0056f0 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -475,7 +475,6 @@ def imaging_session_to_nwb( protocol_key=protocol_key, additional_nwbfile_kwargs=nwbfile_kwargs, ) - io.write(nwb_file) else: if "Subject" in nwbfile_kwargs: from pynwb.file import Subject @@ -483,4 +482,4 @@ def imaging_session_to_nwb( nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) else: nwb_file = NWBFile(**nwbfile_kwargs) - io.write(nwb_file) + io.write(nwb_file) From 2aab1da6ad34b64f1e2c5781fad880e054dba5fb Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 8 Sep 2023 16:09:32 -0500 Subject: [PATCH 15/33] Successful export of database only NWB file --- element_calcium_imaging/export/nwb/nwb.py | 37 ++++++++++++++--------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index bf0056f0..08a4491b 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -13,12 +13,12 @@ TwoPhotonSeries, ) -from ... import imaging, scan -from ...scan import get_image_files, get_imaging_root_data_dir +from ... import scan +from ... import imaging_no_curation as imaging +from ...scan import get_calcium_imaging_files, get_imaging_root_data_dir def create_raw_data_nwbfile(session_key, output_directory, nwb_path): - acquisition_software = (scan.Scan & session_key).fetch1("acq_software") if acquisition_software == "NIS": raise NotImplementedError( @@ -37,7 +37,9 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) processing_folder_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_image_files(session_key, "*.tif") + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software + ) scan_interface = ScanImageImagingInterface( file_path=raw_data_files_location[0], fallback_sampling_frequency=30 ) @@ -66,7 +68,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_image_files(session_key, "*.tif") + raw_data_files_location = get_calcium_imaging_files(session_key, "*.tif") scan_interface = ScanImageImagingInterface( file_path=raw_data_files_location[0], fallback_sampling_frequency=30 ) @@ -109,7 +111,9 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_image_files(session_key, "*.tif") + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software + ) scan_interface = SbxImagingInterface( file_path=raw_data_files_location[0], fallback_sampling_frequency=30 ) @@ -131,7 +135,9 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): from neuroconv.datainterfaces import Suite2pSegmentationInterface processing_folder_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_image_files(session_key, "*.tif") + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software + ) bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=30, @@ -154,7 +160,9 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): from neuroconv.datainterfaces import CaimanSegmentationInterface processing_folder_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_image_files(session_key, "*.tif") + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software + ) bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=30, @@ -176,7 +184,9 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): from neuroconv.datainterfaces import ExtractSegmentationInterface processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_image_files(session_key, "*.tif") + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software + ) bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=30, @@ -323,11 +333,11 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): mask_keys = (imaging.Segmentation.Mask & session_key).fetch("KEY") for mask_key in mask_keys: ps.add_roi( - image_mask=np.asarray( + pixel_mask=np.asarray( (imaging.Segmentation.Mask() & mask_key).fetch1( "mask_xpix", "mask_ypix", "mask_weights" ) - ) + ).T ) rt_region = ps.create_roi_table_region( @@ -350,7 +360,7 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): unit="a.u.", rate=(scan.ScanInfo & session_key).fetch1("fps"), ) - neuorpil_series = RoiResponseSeries( + neuropil_series = RoiResponseSeries( name=f"Neuropil_{channel}", data=np.stack( ( @@ -375,7 +385,7 @@ def add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): rate=(scan.ScanInfo & session_key).fetch1("fps"), ) fl = Fluorescence( - roi_response_series=[roi_resp_series, neuorpil_series, deconvolved_series] + roi_response_series=[roi_resp_series, neuropil_series, deconvolved_series] ) ophys_module.add(fl) @@ -390,7 +400,6 @@ def imaging_session_to_nwb( protocol_key=None, nwbfile_kwargs=None, ): - if processed_data_source not in ["database", "filesystem"]: raise ValueError( "Invalid processed data source. Expected one of 'database', 'filesystem'" From 6df88e17c998b3b642cef493ddd70ffb2ab35751 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 8 Sep 2023 16:13:33 -0500 Subject: [PATCH 16/33] Fetch frame rate from ScanInfo for fallback frequency --- element_calcium_imaging/export/nwb/nwb.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 08a4491b..ea1d7da4 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -30,6 +30,8 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): imaging.ProcessingParamSet & f"paramset_idx='{session_paramset_key}'" ).fetch1("processing_method") + frame_rate = (scan.ScanInfo & session_key).fetch1("fps") + if acquisition_software == "ScanImage" and processing_method == "suite2p": from neuroconv.datainterfaces import ( ScanImageImagingInterface, @@ -41,7 +43,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): session_key, acquisition_software ) scan_interface = ScanImageImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location @@ -56,7 +58,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) scan_interface = ScanImageImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) @@ -70,7 +72,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): processing_file_location = pathlib.Path(output_directory).as_posix() raw_data_files_location = get_calcium_imaging_files(session_key, "*.tif") scan_interface = ScanImageImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) extract_interface = ExtractSegmentationInterface( file_path=processing_file_location @@ -84,7 +86,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) scan_interface = SbxImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location @@ -99,7 +101,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) scan_interface = SbxImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) @@ -115,7 +117,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): session_key, acquisition_software ) scan_interface = SbxImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=30 + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) extract_interface = ExtractSegmentationInterface( file_path=processing_file_location @@ -140,7 +142,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], - fallback_sampling_frequency=30, + fallback_sampling_frequency=frame_rate, ) s2p_interface = Suite2pSegmentationInterface( folder_path=processing_folder_location @@ -165,7 +167,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], - fallback_sampling_frequency=30, + fallback_sampling_frequency=frame_rate, ) caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) @@ -189,7 +191,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) bruker_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], - fallback_sampling_frequency=30, + fallback_sampling_frequency=frame_rate, ) extract_interface = ExtractSegmentationInterface( file_path=processing_file_location From cfe9bf6588050eb4899c2f29f1cd2107fc4c7eca Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 14 Sep 2023 16:54:44 -0500 Subject: [PATCH 17/33] Confirmed export for all three combinations --- element_calcium_imaging/export/nwb/nwb.py | 181 +++++++++++----------- 1 file changed, 87 insertions(+), 94 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index ea1d7da4..ac6af8cf 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,6 +1,7 @@ import pathlib import numpy as np +import datajoint as dj from datajoint import DataJointError from element_interface.utils import find_full_path from neuroconv import ConverterPipe @@ -17,8 +18,10 @@ from ... import imaging_no_curation as imaging from ...scan import get_calcium_imaging_files, get_imaging_root_data_dir +logger = dj.logger -def create_raw_data_nwbfile(session_key, output_directory, nwb_path): + +def _create_full_nwbfile(session_key, output_directory, nwb_path): acquisition_software = (scan.Scan & session_key).fetch1("acq_software") if acquisition_software == "NIS": raise NotImplementedError( @@ -70,7 +73,7 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_calcium_imaging_files(session_key, "*.tif") + raw_data_files_location = get_calcium_imaging_files(session_key, acquisition_software) scan_interface = ScanImageImagingInterface( file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) @@ -127,11 +130,11 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "PrairieView" and processing_method == "suite2p": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: - from neuroconv.datainterfaces import ( + from neuroconv.converters import ( BrukerTiffMultiPlaneConverter as BrukerTiffConverter, ) else: - from neuroconv.datainterfaces import ( + from neuroconv.converters import ( BrukerTiffSinglePlaneConverter as BrukerTiffConverter, ) from neuroconv.datainterfaces import Suite2pSegmentationInterface @@ -152,11 +155,11 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "PrairieView" and processing_method == "caiman": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: - from neuroconv.datainterfaces import ( + from neuroconv.converters import ( BrukerTiffMultiPlaneConverter as BrukerTiffConverter, ) else: - from neuroconv.datainterfaces import ( + from neuroconv.converters import ( BrukerTiffSinglePlaneConverter as BrukerTiffConverter, ) from neuroconv.datainterfaces import CaimanSegmentationInterface @@ -176,11 +179,11 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): elif acquisition_software == "PrairieView" and processing_method == "extract": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: - from neuroconv.datainterfaces import ( + from neuroconv.converters import ( BrukerTiffMultiPlaneConverter as BrukerTiffConverter, ) else: - from neuroconv.datainterfaces import ( + from neuroconv.converters import ( BrukerTiffSinglePlaneConverter as BrukerTiffConverter, ) from neuroconv.datainterfaces import ExtractSegmentationInterface @@ -205,52 +208,54 @@ def create_raw_data_nwbfile(session_key, output_directory, nwb_path): ) -def create_processed_data_nwbfile(session_key, output_directory, nwb_path): - session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") - processing_method = ( - imaging.ProcessingParamSet & f"paramset_idx='{session_paramset_key}'" - ).fetch1("processing_method") - - if processing_method == "suite2p": - from neuroconv.datainterfaces import Suite2pSegmentationInterface +def _create_raw_data_nwbfile(session_key, output_directory, nwb_path): + acquisition_software = (scan.Scan & session_key).fetch1("acq_software") + frame_rate = (scan.ScanInfo & session_key).fetch1("fps") + if acquisition_software == "ScanImage": + from neuroconv.datainterfaces import ScanImageImagingInterface - processing_folder_location = pathlib.Path(output_directory).as_posix() - s2p_interface = Suite2pSegmentationInterface( - folder_path=processing_folder_location - ) - metadata = s2p_interface.get_metadata() - s2p_interface.run_conversion( - nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), - metadata=metadata, + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software ) - elif processing_method == "caiman": - from neuroconv.datainterfaces import CaimanSegmentationInterface - - processing_folder_location = pathlib.Path(output_directory).as_posix() - caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) - caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) - metadata = caiman_interface.get_metadata() - caiman_interface.run_conversion( - nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), - metadata=metadata, + imaging_interface = ScanImageImagingInterface(file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate) + metadata = imaging_interface.get_metadata() + imaging_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) + + elif acquisition_software == "Scanbox": + from neuroconv.datainterfaces import SbxImagingInterface + + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software ) - elif processing_method == "extract": - from neuroconv.datainterfaces import ExtractSegmentationInterface + imaging_interface = SbxImagingInterface(file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate) + metadata = imaging_interface.get_metadata() + imaging_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) - processing_file_location = pathlib.Path(output_directory).as_posix() - extract_interface = ExtractSegmentationInterface( - file_path=processing_file_location + elif acquisition_software == "PrairieView": + n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") + if n_planes > 1: + from neuroconv.converters import ( + BrukerTiffMultiPlaneConverter as BrukerTiffConverter, + ) + else: + from neuroconv.converters import ( + BrukerTiffSinglePlaneConverter as BrukerTiffConverter, + ) + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software ) - metadata = extract_interface.get_metadata() - extract_interface.run_conversion( - nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), - metadata=metadata, + + imaging_interface = BrukerTiffConverter( + file_path=raw_data_files_location[0], + fallback_sampling_frequency=frame_rate, ) + metadata = imaging_interface.get_metadata() + imaging_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) -def add_scan_to_nwb(session_key, nwbfile): +def _add_scan_to_nwb(session_key, nwbfile): from math import nan try: @@ -299,7 +304,7 @@ def add_scan_to_nwb(session_key, nwbfile): return imaging_plane -def add_image_series_to_nwb(session_key, imaging_plane): +def _add_image_series_to_nwb(session_key, imaging_plane): imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") two_p_series = TwoPhotonSeries( name="TwoPhotonSeries", @@ -314,13 +319,13 @@ def add_image_series_to_nwb(session_key, imaging_plane): return two_p_series -def add_motion_correction_to_nwb(session_key, nwbfile): +def _add_motion_correction_to_nwb(session_key, nwbfile): raise NotImplementedError( "Motion Correction data cannot be packaged into NWB at this time." ) -def add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): +def _add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): ophys_module = nwbfile.create_processing_module( name="ophys", description="optical physiology processed data" ) @@ -408,7 +413,6 @@ def imaging_session_to_nwb( ) session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) - output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( "processing_output_dir" ) @@ -420,9 +424,8 @@ def imaging_session_to_nwb( ) save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) - if include_raw_data: - processed_data_source = "filesystem" - create_raw_data_nwbfile( + if include_raw_data and processed_data_source == "filesystem": + _create_full_nwbfile( session_key, output_directory=output_dir, nwb_path=save_path ) with NWBHDF5IO( @@ -437,17 +440,15 @@ def imaging_session_to_nwb( protocol_key=protocol_key, additional_nwbfile_kwargs=nwbfile_kwargs, ) - io.write(nwb_file) else: - if "Subject" in nwbfile_kwargs: - from pynwb.file import Subject - - nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) - else: - nwb_file = NWBFile(**nwbfile_kwargs) - io.write(nwb_file) - else: - if processed_data_source == "database": + nwb_file = NWBFile(**nwbfile_kwargs) + io.write(nwb_file) + elif include_raw_data and processed_data_source == "database": + _create_raw_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) + with NWBHDF5IO( + (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" + ) as io: + nwb_file = io.read() if session_to_nwb: nwb_file = session_to_nwb( session_key, @@ -458,39 +459,31 @@ def imaging_session_to_nwb( ) else: nwb_file = NWBFile(**nwbfile_kwargs) - - imaging_plane = add_scan_to_nwb(session_key, nwbfile=nwb_file) - add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) - add_segmentation_data_to_nwb( - session_key, nwbfile=nwb_file, imaging_plane=imaging_plane + try: + io.write(nwb_file) + except ValueError: + logger.warn("Group already exists in NWB file. Unable to update values.") + elif not include_raw_data and processed_data_source == "database": + if session_to_nwb: + nwb_file = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, ) + else: + nwb_file = NWBFile(**nwbfile_kwargs) + imaging_plane = _add_scan_to_nwb(session_key, nwbfile=nwb_file) + _add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) + _add_segmentation_data_to_nwb( + session_key, nwbfile=nwb_file, imaging_plane=imaging_plane + ) - with NWBHDF5IO( - (save_path / f'{session_key["subject"]}_nwbfile'), mode="w" - ) as io: - io.write(nwb_file) + with NWBHDF5IO( + (save_path / f'{session_key["subject"]}_nwbfile'), mode="w" + ) as io: + io.write(nwb_file) - elif processed_data_source == "filesystem": - create_processed_data_nwbfile( - session_key, output_directory=output_dir, nwb_path=save_path - ) - with NWBHDF5IO( - (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" - ) as io: - nwb_file = io.read() - if session_to_nwb: - nwb_file = session_to_nwb( - session_key, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - additional_nwbfile_kwargs=nwbfile_kwargs, - ) - else: - if "Subject" in nwbfile_kwargs: - from pynwb.file import Subject - - nwb_file.subject = Subject(**nwbfile_kwargs["Subject"]) - else: - nwb_file = NWBFile(**nwbfile_kwargs) - io.write(nwb_file) + elif not include_raw_data and processed_data_source == "filesystem": + raise NotImplementedError("Creating NWB files without raw data from the filesystem is not supported. Please set `include_raw_data=True` or `processed_data_source='filesystem'`.") From b2cd4346f0863f743e73107aa529c678ef5ae8d8 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 14 Sep 2023 17:27:10 -0500 Subject: [PATCH 18/33] Black formatting --- element_calcium_imaging/export/nwb/nwb.py | 43 +++++++++++++++++------ 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index ac6af8cf..bb4b464b 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -73,7 +73,9 @@ def _create_full_nwbfile(session_key, output_directory, nwb_path): ) processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_calcium_imaging_files(session_key, acquisition_software) + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software + ) scan_interface = ScanImageImagingInterface( file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) @@ -218,20 +220,30 @@ def _create_raw_data_nwbfile(session_key, output_directory, nwb_path): session_key, acquisition_software ) - imaging_interface = ScanImageImagingInterface(file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate) + imaging_interface = ScanImageImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate + ) metadata = imaging_interface.get_metadata() - imaging_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) - + imaging_interface.run_conversion( + nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + metadata=metadata, + ) + elif acquisition_software == "Scanbox": from neuroconv.datainterfaces import SbxImagingInterface - + raw_data_files_location = get_calcium_imaging_files( session_key, acquisition_software ) - imaging_interface = SbxImagingInterface(file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate) + imaging_interface = SbxImagingInterface( + file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate + ) metadata = imaging_interface.get_metadata() - imaging_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) + imaging_interface.run_conversion( + nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + metadata=metadata, + ) elif acquisition_software == "PrairieView": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") @@ -252,7 +264,10 @@ def _create_raw_data_nwbfile(session_key, output_directory, nwb_path): fallback_sampling_frequency=frame_rate, ) metadata = imaging_interface.get_metadata() - imaging_interface.run_conversion(nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata) + imaging_interface.run_conversion( + nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + metadata=metadata, + ) def _add_scan_to_nwb(session_key, nwbfile): @@ -444,7 +459,9 @@ def imaging_session_to_nwb( nwb_file = NWBFile(**nwbfile_kwargs) io.write(nwb_file) elif include_raw_data and processed_data_source == "database": - _create_raw_data_nwbfile(session_key, output_directory=output_dir, nwb_path=save_path) + _create_raw_data_nwbfile( + session_key, output_directory=output_dir, nwb_path=save_path + ) with NWBHDF5IO( (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" ) as io: @@ -462,7 +479,9 @@ def imaging_session_to_nwb( try: io.write(nwb_file) except ValueError: - logger.warn("Group already exists in NWB file. Unable to update values.") + logger.warn( + "Group already exists in NWB file. Unable to update values." + ) elif not include_raw_data and processed_data_source == "database": if session_to_nwb: nwb_file = session_to_nwb( @@ -486,4 +505,6 @@ def imaging_session_to_nwb( io.write(nwb_file) elif not include_raw_data and processed_data_source == "filesystem": - raise NotImplementedError("Creating NWB files without raw data from the filesystem is not supported. Please set `include_raw_data=True` or `processed_data_source='filesystem'`.") + raise NotImplementedError( + "Creating NWB files without raw data from the filesystem is not supported. Please set `include_raw_data=True` or `processed_data_source='filesystem'`." + ) From 7d9523effa54ea6daf17313898cce59e66695e91 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Tue, 19 Sep 2023 15:50:59 -0500 Subject: [PATCH 19/33] Check for activated module --- element_calcium_imaging/export/nwb/nwb.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index bb4b464b..77f73375 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -15,11 +15,18 @@ ) from ... import scan -from ... import imaging_no_curation as imaging +from ... import imaging_no_curation, imaging, imaging_preprocess from ...scan import get_calcium_imaging_files, get_imaging_root_data_dir logger = dj.logger +if imaging_no_curation.schema.is_activated(): + imaging = imaging_no_curation +else: + raise DataJointError( + "This export function is designed for the `imaging_no_curation` module." + ) + def _create_full_nwbfile(session_key, output_directory, nwb_path): acquisition_software = (scan.Scan & session_key).fetch1("acq_software") @@ -422,17 +429,12 @@ def imaging_session_to_nwb( protocol_key=None, nwbfile_kwargs=None, ): + session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) if processed_data_source not in ["database", "filesystem"]: raise ValueError( "Invalid processed data source. Expected one of 'database', 'filesystem'" ) - session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) - output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( - "processing_output_dir" - ) - output_dir = find_full_path(get_imaging_root_data_dir(), output_relative_dir) - if not save_path: output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( "processing_output_dir" @@ -441,7 +443,7 @@ def imaging_session_to_nwb( if include_raw_data and processed_data_source == "filesystem": _create_full_nwbfile( - session_key, output_directory=output_dir, nwb_path=save_path + session_key, output_directory=save_path, nwb_path=save_path ) with NWBHDF5IO( (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" @@ -460,7 +462,7 @@ def imaging_session_to_nwb( io.write(nwb_file) elif include_raw_data and processed_data_source == "database": _create_raw_data_nwbfile( - session_key, output_directory=output_dir, nwb_path=save_path + session_key, output_directory=save_path, nwb_path=save_path ) with NWBHDF5IO( (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" From ef64152553b21673d83dda6d1cdfb945ccd50a1a Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 21 Sep 2023 13:12:48 -0500 Subject: [PATCH 20/33] Remove filesystem flag --- element_calcium_imaging/export/nwb/nwb.py | 357 ++++------------------ 1 file changed, 59 insertions(+), 298 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 77f73375..0e103588 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -15,7 +15,7 @@ ) from ... import scan -from ... import imaging_no_curation, imaging, imaging_preprocess +from ... import imaging_no_curation from ...scan import get_calcium_imaging_files, get_imaging_root_data_dir logger = dj.logger @@ -28,140 +28,55 @@ ) -def _create_full_nwbfile(session_key, output_directory, nwb_path): - acquisition_software = (scan.Scan & session_key).fetch1("acq_software") - if acquisition_software == "NIS": - raise NotImplementedError( - "Packaging raw data from Nikon NIS acquisition software (.nd2 file format) is not currently supported." - ) - - session_paramset_key = (imaging.ProcessingTask & session_key).fetch1("paramset_idx") - processing_method = ( - imaging.ProcessingParamSet & f"paramset_idx='{session_paramset_key}'" - ).fetch1("processing_method") - - frame_rate = (scan.ScanInfo & session_key).fetch1("fps") - - if acquisition_software == "ScanImage" and processing_method == "suite2p": - from neuroconv.datainterfaces import ( - ScanImageImagingInterface, - Suite2pSegmentationInterface, - ) - - processing_folder_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - scan_interface = ScanImageImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate - ) - s2p_interface = Suite2pSegmentationInterface( - folder_path=processing_folder_location - ) - converter = ConverterPipe(data_interfaces=[scan_interface, s2p_interface]) - - elif acquisition_software == "ScanImage" and processing_method == "CaImAn": - from neuroconv.datainterfaces import ( - CaimanSegmentationInterface, - ScanImageImagingInterface, - ) - - caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) - scan_interface = ScanImageImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate - ) - caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) - converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) - - elif acquisition_software == "ScanImage" and processing_method == "extract": - from neuroconv.datainterfaces import ( - ExtractSegmentationInterface, - ScanImageImagingInterface, - ) - - processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - scan_interface = ScanImageImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate - ) - extract_interface = ExtractSegmentationInterface( - file_path=processing_file_location - ) - converter = ConverterPipe(data_interfaces=[scan_interface, extract_interface]) - - elif acquisition_software == "Scanbox" and processing_method == "suite2p": - from neuroconv.datainterfaces import ( - SbxImagingInterface, - Suite2pSegmentationInterface, - ) - - scan_interface = SbxImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate - ) - s2p_interface = Suite2pSegmentationInterface( - folder_path=processing_folder_location - ) - converter = ConverterPipe(data_interfaces=[scan_interface, s2p_interface]) +def imaging_session_to_nwb( + session_key, + save_path=None, + include_raw_data=False, + lab_key=None, + project_key=None, + protocol_key=None, + nwbfile_kwargs=None, +): + session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) - elif acquisition_software == "Scanbox" and processing_method == "CaImAn": - from neuroconv.datainterfaces import ( - CaimanSegmentationInterface, - SbxImagingInterface, + if not save_path: + output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( + "processing_output_dir" ) + save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) - caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) - scan_interface = SbxImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate + if session_to_nwb: + nwb_file = session_to_nwb( + session_key, + lab_key=lab_key, + project_key=project_key, + protocol_key=protocol_key, + additional_nwbfile_kwargs=nwbfile_kwargs, + ) + else: + nwb_file = NWBFile(**nwbfile_kwargs) + if include_raw_data: + _create_raw_data_nwbfile(session_key, linked_nwb_file=nwb_file) + else: + imaging_plane = _add_scan_to_nwb(session_key, nwbfile=nwb_file) + _add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) + _add_segmentation_data_to_nwb( + session_key, nwbfile=nwb_file, imaging_plane=imaging_plane ) - caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) - converter = ConverterPipe(data_interfaces=[scan_interface, caiman_interface]) - elif acquisition_software == "Scanbox" and processing_method == "extract": - from neuroconv.datainterfaces import ( - ExtractSegmentationInterface, - SbxImagingInterface, - ) + return nwb_file - processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - scan_interface = SbxImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate - ) - extract_interface = ExtractSegmentationInterface( - file_path=processing_file_location - ) - converter = ConverterPipe(data_interfaces=[scan_interface, extract_interface]) - elif acquisition_software == "PrairieView" and processing_method == "suite2p": - n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") - if n_planes > 1: - from neuroconv.converters import ( - BrukerTiffMultiPlaneConverter as BrukerTiffConverter, - ) - else: - from neuroconv.converters import ( - BrukerTiffSinglePlaneConverter as BrukerTiffConverter, - ) - from neuroconv.datainterfaces import Suite2pSegmentationInterface +def _create_raw_data_nwbfile(session_key, linked_nwb_file): + acquisition_software = (scan.Scan & session_key).fetch1("acq_software") + frame_rate = (scan.ScanInfo & session_key).fetch1("fps") - processing_folder_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - bruker_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], - fallback_sampling_frequency=frame_rate, - ) - s2p_interface = Suite2pSegmentationInterface( - folder_path=processing_folder_location + if acquisition_software == "NIS": + raise NotImplementedError( + "Packaging raw data acquired from `Nikon NIS Elements` software is not supported at this time." ) - converter = ConverterPipe(data_interfaces=[bruker_interface, s2p_interface]) - elif acquisition_software == "PrairieView" and processing_method == "caiman": + elif acquisition_software == "PrairieView": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") if n_planes > 1: from neuroconv.converters import ( @@ -171,108 +86,37 @@ def _create_full_nwbfile(session_key, output_directory, nwb_path): from neuroconv.converters import ( BrukerTiffSinglePlaneConverter as BrukerTiffConverter, ) - from neuroconv.datainterfaces import CaimanSegmentationInterface - - processing_folder_location = pathlib.Path(output_directory).as_posix() raw_data_files_location = get_calcium_imaging_files( session_key, acquisition_software ) - bruker_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], - fallback_sampling_frequency=frame_rate, - ) - caiman_hdf5 = list(processing_folder_location.rglob("caiman_analysis.hdf5")) - caiman_interface = CaimanSegmentationInterface(file_path=caiman_hdf5[0]) - converter = ConverterPipe(data_interfaces=[bruker_interface, caiman_interface]) - elif acquisition_software == "PrairieView" and processing_method == "extract": - n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") - if n_planes > 1: - from neuroconv.converters import ( - BrukerTiffMultiPlaneConverter as BrukerTiffConverter, - ) - else: - from neuroconv.converters import ( - BrukerTiffSinglePlaneConverter as BrukerTiffConverter, - ) - from neuroconv.datainterfaces import ExtractSegmentationInterface - - processing_file_location = pathlib.Path(output_directory).as_posix() - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - bruker_interface = BrukerTiffConverter( + imaging_interface = BrukerTiffConverter( file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate, ) - extract_interface = ExtractSegmentationInterface( - file_path=processing_file_location - ) - converter = ConverterPipe(data_interfaces=[bruker_interface, extract_interface]) - - metadata = converter.get_metadata() - metadata["NWBFile"].update(session_description="DataJoint Session") - converter.run_conversion( - nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), metadata=metadata - ) - - -def _create_raw_data_nwbfile(session_key, output_directory, nwb_path): - acquisition_software = (scan.Scan & session_key).fetch1("acq_software") - frame_rate = (scan.ScanInfo & session_key).fetch1("fps") - if acquisition_software == "ScanImage": - from neuroconv.datainterfaces import ScanImageImagingInterface - - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - - imaging_interface = ScanImageImagingInterface( - file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate - ) metadata = imaging_interface.get_metadata() imaging_interface.run_conversion( - nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + nwbfile=linked_nwb_file, metadata=metadata, ) - - elif acquisition_software == "Scanbox": - from neuroconv.datainterfaces import SbxImagingInterface + else: + if acquisition_software == "ScanImage": + from neuroconv.datainterfaces import ( + ScanImageImagingInterface as ImagingInterface, + ) + elif acquisition_software == "Scanbox": + from neuroconv.datainterfaces import SbxImagingInterface as ImagingInterface raw_data_files_location = get_calcium_imaging_files( session_key, acquisition_software ) - imaging_interface = SbxImagingInterface( + imaging_interface = ImagingInterface( file_path=raw_data_files_location[0], fallback_sampling_frequency=frame_rate ) metadata = imaging_interface.get_metadata() - imaging_interface.run_conversion( - nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), - metadata=metadata, - ) - - elif acquisition_software == "PrairieView": - n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") - if n_planes > 1: - from neuroconv.converters import ( - BrukerTiffMultiPlaneConverter as BrukerTiffConverter, - ) - else: - from neuroconv.converters import ( - BrukerTiffSinglePlaneConverter as BrukerTiffConverter, - ) - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - - imaging_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], - fallback_sampling_frequency=frame_rate, - ) - metadata = imaging_interface.get_metadata() - imaging_interface.run_conversion( - nwbfile_path=(nwb_path / f'{session_key["subject"]}_nwbfile'), + imaging_interface.add_to_nwbfile( + nwbfile=linked_nwb_file, metadata=metadata, ) @@ -419,94 +263,11 @@ def _add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): ophys_module.add(fl) -def imaging_session_to_nwb( - session_key, - save_path=None, - include_raw_data=False, - processed_data_source="database", - lab_key=None, - project_key=None, - protocol_key=None, - nwbfile_kwargs=None, -): - session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) - if processed_data_source not in ["database", "filesystem"]: - raise ValueError( - "Invalid processed data source. Expected one of 'database', 'filesystem'" - ) +def write_nwb(nwbfile, fname, check_read=True): + with pynwb.NWBHDF5IO(fname, "w") as io: + io.write(nwbfile) - if not save_path: - output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( - "processing_output_dir" - ) - save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) - - if include_raw_data and processed_data_source == "filesystem": - _create_full_nwbfile( - session_key, output_directory=save_path, nwb_path=save_path - ) - with NWBHDF5IO( - (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" - ) as io: - nwb_file = io.read() - if session_to_nwb: - nwb_file = session_to_nwb( - session_key, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - additional_nwbfile_kwargs=nwbfile_kwargs, - ) - else: - nwb_file = NWBFile(**nwbfile_kwargs) - io.write(nwb_file) - elif include_raw_data and processed_data_source == "database": - _create_raw_data_nwbfile( - session_key, output_directory=save_path, nwb_path=save_path - ) - with NWBHDF5IO( - (save_path / f'{session_key["subject"]}_nwbfile'), mode="r+" - ) as io: - nwb_file = io.read() - if session_to_nwb: - nwb_file = session_to_nwb( - session_key, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - additional_nwbfile_kwargs=nwbfile_kwargs, - ) - else: - nwb_file = NWBFile(**nwbfile_kwargs) - try: - io.write(nwb_file) - except ValueError: - logger.warn( - "Group already exists in NWB file. Unable to update values." - ) - elif not include_raw_data and processed_data_source == "database": - if session_to_nwb: - nwb_file = session_to_nwb( - session_key, - lab_key=lab_key, - project_key=project_key, - protocol_key=protocol_key, - additional_nwbfile_kwargs=nwbfile_kwargs, - ) - else: - nwb_file = NWBFile(**nwbfile_kwargs) - imaging_plane = _add_scan_to_nwb(session_key, nwbfile=nwb_file) - _add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) - _add_segmentation_data_to_nwb( - session_key, nwbfile=nwb_file, imaging_plane=imaging_plane - ) - - with NWBHDF5IO( - (save_path / f'{session_key["subject"]}_nwbfile'), mode="w" - ) as io: - io.write(nwb_file) - - elif not include_raw_data and processed_data_source == "filesystem": - raise NotImplementedError( - "Creating NWB files without raw data from the filesystem is not supported. Please set `include_raw_data=True` or `processed_data_source='filesystem'`." - ) + if check_read: + with pynwb.NWBHDF5IO(fname, "r") as io: + io.read() + logger.info("File saved successfully") From ceb6c6b97251c7fc64d24508527665c5230eb29e Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 21 Sep 2023 16:31:31 -0500 Subject: [PATCH 21/33] Add docstrings + fix minor bugs --- .../export/nwb/__init__.py | 2 +- element_calcium_imaging/export/nwb/nwb.py | 66 +++++++++++++++---- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/element_calcium_imaging/export/nwb/__init__.py b/element_calcium_imaging/export/nwb/__init__.py index 55785b00..a37b7999 100644 --- a/element_calcium_imaging/export/nwb/__init__.py +++ b/element_calcium_imaging/export/nwb/__init__.py @@ -1 +1 @@ -from .nwb import imaging_session_to_nwb +from .nwb import imaging_session_to_nwb, write_nwb diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 0e103588..8ee39354 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,5 +1,4 @@ import pathlib - import numpy as np import datajoint as dj from datajoint import DataJointError @@ -14,10 +13,10 @@ TwoPhotonSeries, ) -from ... import scan -from ... import imaging_no_curation +from ... import scan, imaging_no_curation from ...scan import get_calcium_imaging_files, get_imaging_root_data_dir + logger = dj.logger if imaging_no_curation.schema.is_activated(): @@ -30,20 +29,32 @@ def imaging_session_to_nwb( session_key, - save_path=None, include_raw_data=False, lab_key=None, project_key=None, protocol_key=None, nwbfile_kwargs=None, ): - session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) + """Main function for converting calcium imaging data to NWB. + + Args: + session_key (dict): key from Session table. + include_raw_data (bool): Optional. Default False. Include the raw data from + source. `ScanImage`, `Scanbox`, and `PrairieView` are supported. + lab_key (dict): Optional key to add metadata from Element Lab. + project_key (dict): Optional key to add metadata from Element Lab. + protocol_key (dict): Optional key to add metadata from Element Lab. + nwbfile_kwargs (dict): Optional. If Element Session is not used, this argument + is required and must be a dictionary containing 'session_description' (str), + 'identifier' (str), and 'session_start_time' (datetime), the required + minimal data for instantiating an NWBFile object. If element-session is + being used, this argument can optionally be used to overwrite NWBFile + fields. + Returns: + nwbfile (NWBFile): nwb file + """ - if not save_path: - output_relative_dir = (imaging.ProcessingTask & session_key).fetch1( - "processing_output_dir" - ) - save_path = find_full_path(get_imaging_root_data_dir(), output_relative_dir) + session_to_nwb = getattr(imaging._linking_module, "session_to_nwb", False) if session_to_nwb: nwb_file = session_to_nwb( @@ -55,6 +66,7 @@ def imaging_session_to_nwb( ) else: nwb_file = NWBFile(**nwbfile_kwargs) + if include_raw_data: _create_raw_data_nwbfile(session_key, linked_nwb_file=nwb_file) else: @@ -68,6 +80,13 @@ def imaging_session_to_nwb( def _create_raw_data_nwbfile(session_key, linked_nwb_file): + """Adds raw data to NWB file. + + Args: + session_key (dict): key from Session table + linked_nwb_file (NWBFile): nwb file + """ + acquisition_software = (scan.Scan & session_key).fetch1("acq_software") frame_rate = (scan.ScanInfo & session_key).fetch1("fps") @@ -122,6 +141,13 @@ def _create_raw_data_nwbfile(session_key, linked_nwb_file): def _add_scan_to_nwb(session_key, nwbfile): + """Adds metadata for a scan from database. + + Args: + session_key (dict): key from Session table + nwbfile (NWBFile): nwb file + """ + from math import nan try: @@ -192,6 +218,14 @@ def _add_motion_correction_to_nwb(session_key, nwbfile): def _add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): + """Adds segmentation data from database. + + Args: + session_key (dict): key from Session table + nwbfile (NWBFile): nwb file + imaging_plane (NWBFile Imaging Plane): nwb file imaging plane object + """ + ophys_module = nwbfile.create_processing_module( name="ophys", description="optical physiology processed data" ) @@ -264,10 +298,18 @@ def _add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): def write_nwb(nwbfile, fname, check_read=True): - with pynwb.NWBHDF5IO(fname, "w") as io: + """Export NWBFile + + Args: + nwbfile (NWBFile): nwb file + fname (str): Absolute path including `*.nwb` extension. + check_read (bool): If True, PyNWB will try to read the produced NWB file and + ensure that it can be read. + """ + with NWBHDF5IO(fname, "w") as io: io.write(nwbfile) if check_read: - with pynwb.NWBHDF5IO(fname, "r") as io: + with NWBHDF5IO(fname, "r") as io: io.read() logger.info("File saved successfully") From dc2b82c0175dba3e9fbac240a8c03eb2a59a4f23 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 13 Oct 2023 15:05:27 -0500 Subject: [PATCH 22/33] Update export function, add notebook, black formatting --- element_calcium_imaging/export/nwb/nwb.py | 3 - notebooks/nwb_export.ipynb | 181 ++++++++++++++++++++++ tests/__init__.py | 1 - 3 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 notebooks/nwb_export.ipynb diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 8ee39354..d786ea86 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -1,9 +1,6 @@ -import pathlib import numpy as np import datajoint as dj from datajoint import DataJointError -from element_interface.utils import find_full_path -from neuroconv import ConverterPipe from pynwb import NWBHDF5IO, NWBFile from pynwb.ophys import ( Fluorescence, diff --git a/notebooks/nwb_export.ipynb b/notebooks/nwb_export.ipynb new file mode 100644 index 00000000..695b34a6 --- /dev/null +++ b/notebooks/nwb_export.ipynb @@ -0,0 +1,181 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Export workflow to Neurodata Without Borders file and upload to DANDI\n", + "\n", + "In this tutorial, we will walk through exporting two-photon calcium imaging data\n", + "stored in a DataJoint pipeline to [Neurodata Without Borders (NWB)](https://www.nwb.org/) file format and\n", + "uploading the NWB file to the [DANDI archive](https://dandiarchive.org/).\n", + "\n", + "Please ensure the following before running this notebook for the best experience:\n", + "1. Set up an account with the [DANDI archive](https://dandiarchive.org/).\n", + "2. Obtain your `DANDI api key` and `Dandiset ID`.\n", + "\n", + "*Before running this notebook, please see the calcium imaging [tutorial](./tutorial.ipynb) notebook to\n", + "set up and insert data into the calcium imaging data pipeline.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if os.path.basename(os.getcwd()) == \"notebooks\":\n", + " os.chdir(\"..\")\n", + "\n", + "import datajoint as dj\n", + "from datetime import datetime\n", + "import uuid\n", + "from workflow.pipeline import *\n", + "from element_lab.export import element_lab_to_nwb_dict\n", + "from element_animal.export.nwb import subject_to_nwb\n", + "from element_session.export.nwb import session_to_nwb\n", + "from element_calcium_imaging.export.nwb import imaging_session_to_nwb, write_nwb\n", + "from element_interface.dandi import upload_to_dandi" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Fetch the `session_key` for the session you want to convert to NWB. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session_key = (session.Session & \"subject='subject1'\").fetch1(\"KEY\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define any optional keyword arguments for the NWB file and call the\n", + "`imaging_session_to_nwb` function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nwb_kwargs = {\n", + " \"session_start_time\": datetime.now(),\n", + " \"session_description\": \"DataJoint Session\",\n", + " \"identifier\": uuid(\"DJNWBFile\"),\n", + "}\n", + "test_file = imaging_session_to_nwb(session_key=session_key,include_raw_data=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Uploading to the DANDI archive requires information in the NWB file that may not be\n", + "present when calling `imaging_session_to_nwb` such as the subject's date of birth or\n", + "species. We can check for this information within the NWB file and add the information\n", + "if it's missing. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_file.subject" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dob, subject_sex = (subject.Subject & session_key).fetch1(\"subject_birth_date\", \"sex\")\n", + "species = \"Mus musculus\"\n", + "test_file.subject.sex = subject_sex\n", + "test_file.subject.date_of_birth = dob\n", + "test_file.subject.species = species" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Save the NWB file to a local directory prior to uploading to DANDI. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "write_nwb(test_file, \"subject1_nwb\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "upload_to_dandi(\n", + " data_directory=\"./example_data/000683/\",\n", + " working_directory=\"./example_data/\",\n", + " dandiset_id=dj.config['custom']['dandiset_id'],\n", + " staging=False,\n", + " api_key=dj.config['custom']['dandi.api'],\n", + " sync=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nwb", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/__init__.py b/tests/__init__.py index b1c1a71d..5fe3536d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -76,4 +76,3 @@ def test_data(pipeline): _ = [find_full_path(root_dirs(), p) for p in sessions_dirs] except FileNotFoundError as e: print(e) - From c4c6a1f287bcb89d1ba2a7c52f9f7f635ab25a7b Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 13 Oct 2023 15:09:12 -0500 Subject: [PATCH 23/33] Add function docstrings --- element_calcium_imaging/export/nwb/nwb.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index d786ea86..fac5c3b7 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -194,6 +194,13 @@ def _add_scan_to_nwb(session_key, nwbfile): def _add_image_series_to_nwb(session_key, imaging_plane): + """Adds TwoPhotonSeries to NWB file. + + Args: + session_key (dict): key from Session table + imaging_plane (NWBFile Imaging Plane): nwb file imaging plane object + """ + imaging_files = (scan.ScanInfo.ScanFile & session_key).fetch("file_path") two_p_series = TwoPhotonSeries( name="TwoPhotonSeries", From 1e53a2a84461f2eca37e35d9f390ae6e93f17703 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 13 Oct 2023 15:13:09 -0500 Subject: [PATCH 24/33] Update version and CHANGELOG --- CHANGELOG.md | 4 ++++ element_calcium_imaging/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a35a03c3..75f9b38b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.9.0] - 2023-10-13 + ++ Add - Export to NWB and upload to DANDI + ## [0.8.1] - 2023-08-31 + Fix - Rename `get_image_files` to `get_calcium_imaging_files` where missed diff --git a/element_calcium_imaging/version.py b/element_calcium_imaging/version.py index f53466cb..d3830931 100644 --- a/element_calcium_imaging/version.py +++ b/element_calcium_imaging/version.py @@ -1,2 +1,2 @@ """Package metadata.""" -__version__ = "0.8.1" +__version__ = "0.9.0" From d2341f9848cb6a1a5d6e659dd346edd64db692e7 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 13 Oct 2023 15:14:09 -0500 Subject: [PATCH 25/33] Update documentation --- docs/src/roadmap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/roadmap.md b/docs/src/roadmap.md index f9864931..41715c9e 100644 --- a/docs/src/roadmap.md +++ b/docs/src/roadmap.md @@ -16,8 +16,8 @@ the common motifs to create Element Calcium Imaging. Major features include: - [x] Quality metrics - [ ] Data compression - [ ] Deepinterpolation -- [ ] Data export to NWB -- [ ] Data publishing to DANDI +- [x] Data export to NWB +- [x] Data publishing to DANDI Further development of this Element is community driven. Upon user requests and based on guidance from the Scientific Steering Group we will continue adding features to this From 416b8835caf1946e0e6630c3c98f4b4c65535581 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 13 Oct 2023 15:16:08 -0500 Subject: [PATCH 26/33] Black formatting to tutorial notebook --- notebooks/nwb_export.ipynb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/notebooks/nwb_export.ipynb b/notebooks/nwb_export.ipynb index 695b34a6..b8f536f6 100644 --- a/notebooks/nwb_export.ipynb +++ b/notebooks/nwb_export.ipynb @@ -75,7 +75,7 @@ " \"session_description\": \"DataJoint Session\",\n", " \"identifier\": uuid(\"DJNWBFile\"),\n", "}\n", - "test_file = imaging_session_to_nwb(session_key=session_key,include_raw_data=False)" + "test_file = imaging_session_to_nwb(session_key=session_key, include_raw_data=False)" ] }, { @@ -135,10 +135,11 @@ "upload_to_dandi(\n", " data_directory=\"./example_data/000683/\",\n", " working_directory=\"./example_data/\",\n", - " dandiset_id=dj.config['custom']['dandiset_id'],\n", + " dandiset_id=dj.config[\"custom\"][\"dandiset_id\"],\n", " staging=False,\n", - " api_key=dj.config['custom']['dandi.api'],\n", - " sync=False)" + " api_key=dj.config[\"custom\"][\"dandi.api\"],\n", + " sync=False,\n", + ")" ] }, { From 16bb13aaae0d6a48cddad1802a1627817d069fbd Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 18 Oct 2023 17:13:47 -0500 Subject: [PATCH 27/33] Update docs to include nwb export --- docs/src/concepts.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/src/concepts.md b/docs/src/concepts.md index 8aea519d..abfc763f 100644 --- a/docs/src/concepts.md +++ b/docs/src/concepts.md @@ -60,3 +60,18 @@ segmented. - Scanbox - Nikon NIS-Elements - Bruker Prairie View + +## Data Export and Publishing + +Element Calcium Imaging supports exporting of all data into standard Neurodata +Without Borders (NWB) files. This makes it easy to share files with collaborators and +publish results on [DANDI Archive](https://dandiarchive.org/). +[NWB](https://www.nwb.org/), as an organization, is dedicated to standardizing data +formats and maximizing interoperability across tools for neurophysiology. + +To use the export functionality with additional related dependencies, install the +Element with the `nwb` option as follows: + +```console +pip install element-calcium-imaging[nwb] +``` \ No newline at end of file From f39d20148600ae514f00af23a324ef790dfbc20f Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 19 Oct 2023 11:51:42 -0500 Subject: [PATCH 28/33] Remove tutorial notebook for now --- notebooks/nwb_export.ipynb | 182 ------------------------------------- 1 file changed, 182 deletions(-) delete mode 100644 notebooks/nwb_export.ipynb diff --git a/notebooks/nwb_export.ipynb b/notebooks/nwb_export.ipynb deleted file mode 100644 index b8f536f6..00000000 --- a/notebooks/nwb_export.ipynb +++ /dev/null @@ -1,182 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Export workflow to Neurodata Without Borders file and upload to DANDI\n", - "\n", - "In this tutorial, we will walk through exporting two-photon calcium imaging data\n", - "stored in a DataJoint pipeline to [Neurodata Without Borders (NWB)](https://www.nwb.org/) file format and\n", - "uploading the NWB file to the [DANDI archive](https://dandiarchive.org/).\n", - "\n", - "Please ensure the following before running this notebook for the best experience:\n", - "1. Set up an account with the [DANDI archive](https://dandiarchive.org/).\n", - "2. Obtain your `DANDI api key` and `Dandiset ID`.\n", - "\n", - "*Before running this notebook, please see the calcium imaging [tutorial](./tutorial.ipynb) notebook to\n", - "set up and insert data into the calcium imaging data pipeline.*" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "if os.path.basename(os.getcwd()) == \"notebooks\":\n", - " os.chdir(\"..\")\n", - "\n", - "import datajoint as dj\n", - "from datetime import datetime\n", - "import uuid\n", - "from workflow.pipeline import *\n", - "from element_lab.export import element_lab_to_nwb_dict\n", - "from element_animal.export.nwb import subject_to_nwb\n", - "from element_session.export.nwb import session_to_nwb\n", - "from element_calcium_imaging.export.nwb import imaging_session_to_nwb, write_nwb\n", - "from element_interface.dandi import upload_to_dandi" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Fetch the `session_key` for the session you want to convert to NWB. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "session_key = (session.Session & \"subject='subject1'\").fetch1(\"KEY\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Define any optional keyword arguments for the NWB file and call the\n", - "`imaging_session_to_nwb` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nwb_kwargs = {\n", - " \"session_start_time\": datetime.now(),\n", - " \"session_description\": \"DataJoint Session\",\n", - " \"identifier\": uuid(\"DJNWBFile\"),\n", - "}\n", - "test_file = imaging_session_to_nwb(session_key=session_key, include_raw_data=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Uploading to the DANDI archive requires information in the NWB file that may not be\n", - "present when calling `imaging_session_to_nwb` such as the subject's date of birth or\n", - "species. We can check for this information within the NWB file and add the information\n", - "if it's missing. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_file.subject" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dob, subject_sex = (subject.Subject & session_key).fetch1(\"subject_birth_date\", \"sex\")\n", - "species = \"Mus musculus\"\n", - "test_file.subject.sex = subject_sex\n", - "test_file.subject.date_of_birth = dob\n", - "test_file.subject.species = species" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Save the NWB file to a local directory prior to uploading to DANDI. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "write_nwb(test_file, \"subject1_nwb\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "upload_to_dandi(\n", - " data_directory=\"./example_data/000683/\",\n", - " working_directory=\"./example_data/\",\n", - " dandiset_id=dj.config[\"custom\"][\"dandiset_id\"],\n", - " staging=False,\n", - " api_key=dj.config[\"custom\"][\"dandi.api\"],\n", - " sync=False,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nwb", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.17" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} From b9ac549e8c0c0650be2d8562dedb651d5c1ebb15 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 19 Oct 2023 15:06:43 -0500 Subject: [PATCH 29/33] Improvements to raw data and Bruker packaging --- element_calcium_imaging/export/nwb/nwb.py | 34 ++++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index fac5c3b7..3adfc4ac 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -66,6 +66,19 @@ def imaging_session_to_nwb( if include_raw_data: _create_raw_data_nwbfile(session_key, linked_nwb_file=nwb_file) + + if not nwb_file.imaging_planes: + imaging_plane = _add_scan_to_nwb(session_key, nwbfile=nwb_file) + + _add_image_series_to_nwb( + session_key, imaging_plane=nwb_file.imaging_planes["ImagingPlane"] + ) + _add_segmentation_data_to_nwb( + session_key, + nwbfile=nwb_file, + imaging_plane=nwb_file.imaging_planes["ImagingPlane"], + ) + else: imaging_plane = _add_scan_to_nwb(session_key, nwbfile=nwb_file) _add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) @@ -94,22 +107,29 @@ def _create_raw_data_nwbfile(session_key, linked_nwb_file): elif acquisition_software == "PrairieView": n_planes = (scan.ScanInfo & session_key).fetch1("ndepths") + raw_data_files_location = get_calcium_imaging_files( + session_key, acquisition_software + ) + if n_planes > 1: from neuroconv.converters import ( BrukerTiffMultiPlaneConverter as BrukerTiffConverter, ) + + imaging_interface = BrukerTiffConverter( + file_path=raw_data_files_location[0], + fallback_sampling_frequency=frame_rate, + plane_separation_type="disjoint", + ) else: from neuroconv.converters import ( BrukerTiffSinglePlaneConverter as BrukerTiffConverter, ) - raw_data_files_location = get_calcium_imaging_files( - session_key, acquisition_software - ) - imaging_interface = BrukerTiffConverter( - file_path=raw_data_files_location[0], - fallback_sampling_frequency=frame_rate, - ) + imaging_interface = BrukerTiffConverter( + file_path=raw_data_files_location[0], + fallback_sampling_frequency=frame_rate, + ) metadata = imaging_interface.get_metadata() imaging_interface.run_conversion( nwbfile=linked_nwb_file, From 53446de9c0679d1dee4224593ec727cf9f4a79d5 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 23 Oct 2023 17:04:38 -0500 Subject: [PATCH 30/33] Updates from code review --- element_calcium_imaging/export/nwb/nwb.py | 58 ++++++++++------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index 3adfc4ac..c312d6fd 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -66,25 +66,19 @@ def imaging_session_to_nwb( if include_raw_data: _create_raw_data_nwbfile(session_key, linked_nwb_file=nwb_file) - if not nwb_file.imaging_planes: - imaging_plane = _add_scan_to_nwb(session_key, nwbfile=nwb_file) - - _add_image_series_to_nwb( - session_key, imaging_plane=nwb_file.imaging_planes["ImagingPlane"] - ) - _add_segmentation_data_to_nwb( - session_key, - nwbfile=nwb_file, - imaging_plane=nwb_file.imaging_planes["ImagingPlane"], - ) + _add_scan_to_nwb(session_key, nwbfile=nwb_file) else: - imaging_plane = _add_scan_to_nwb(session_key, nwbfile=nwb_file) - _add_image_series_to_nwb(session_key, imaging_plane=imaging_plane) - _add_segmentation_data_to_nwb( - session_key, nwbfile=nwb_file, imaging_plane=imaging_plane - ) + _add_scan_to_nwb(session_key, nwbfile=nwb_file) + _add_image_series_to_nwb( + session_key, imaging_plane=nwb_file.imaging_planes["ImagingPlane"] + ) + _add_segmentation_data_to_nwb( + session_key, + nwbfile=nwb_file, + imaging_plane=nwb_file.imaging_planes["ImagingPlane"], + ) return nwb_file @@ -131,7 +125,7 @@ def _create_raw_data_nwbfile(session_key, linked_nwb_file): fallback_sampling_frequency=frame_rate, ) metadata = imaging_interface.get_metadata() - imaging_interface.run_conversion( + imaging_interface.add_to_nwbfile( nwbfile=linked_nwb_file, metadata=metadata, ) @@ -195,12 +189,12 @@ def _add_scan_to_nwb(session_key, nwbfile): for field_key in field_keys: field_no = (scan.ScanInfo.Field & field_key).fetch1("field_idx") imaging_plane = nwbfile.create_imaging_plane( - name=f"ImagingPlane{field_no+1}", + name="ImagingPlane", optical_channel=optical_channel, imaging_rate=frame_rate, description=scan_notes if scan_notes != "" - else f"Imaging plane for channel {channel+1}", + else f"Imaging plane for field {field_no+1}, channel {channel+1}", device=device, excitation_lambda=nan, indicator="unknown", @@ -304,20 +298,18 @@ def _add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): unit="a.u.", rate=(scan.ScanInfo & session_key).fetch1("fps"), ) - deconvolved_series = RoiResponseSeries( - name=f"Deconvolved_{channel}", - data=np.stack( - ( - imaging.Activity.Trace & session_key & f"fluo_channel='{channel}'" - ).fetch("activity_trace") - ).T, - rois=rt_region, - unit="a.u.", - rate=(scan.ScanInfo & session_key).fetch1("fps"), - ) - fl = Fluorescence( - roi_response_series=[roi_resp_series, neuropil_series, deconvolved_series] - ) + # deconvolved_series = RoiResponseSeries( + # name=f"Deconvolved_{channel}", + # data=np.stack( + # ( + # imaging.Activity.Trace & session_key & f"fluo_channel='{channel}'" + # ).fetch("activity_trace") + # ).T, + # rois=rt_region, + # unit="a.u.", + # rate=(scan.ScanInfo & session_key).fetch1("fps"), + # ) + fl = Fluorescence(roi_response_series=[roi_resp_series, neuropil_series]) ophys_module.add(fl) From e2f17d07857e7790a7e59771d32ccdecbec5742d Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 25 Oct 2023 10:58:40 -0500 Subject: [PATCH 31/33] Update link to contribution page --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e04d1708..2bd0f498 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ # Contribution Guidelines This project follows the -[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/). +[DataJoint Contribution Guidelines](https://datajoint.com/docs/about/contribute/). Please reference the link for more full details. From c96ceb8d165a467a3ce53dea97e1743fec395bc5 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 25 Oct 2023 13:20:27 -0500 Subject: [PATCH 32/33] Revert commented code --- element_calcium_imaging/export/nwb/nwb.py | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index c312d6fd..b34fbf02 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -298,18 +298,18 @@ def _add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): unit="a.u.", rate=(scan.ScanInfo & session_key).fetch1("fps"), ) - # deconvolved_series = RoiResponseSeries( - # name=f"Deconvolved_{channel}", - # data=np.stack( - # ( - # imaging.Activity.Trace & session_key & f"fluo_channel='{channel}'" - # ).fetch("activity_trace") - # ).T, - # rois=rt_region, - # unit="a.u.", - # rate=(scan.ScanInfo & session_key).fetch1("fps"), - # ) - fl = Fluorescence(roi_response_series=[roi_resp_series, neuropil_series]) + deconvolved_series = RoiResponseSeries( + name=f"Deconvolved_{channel}", + data=np.stack( + ( + imaging.Activity.Trace & session_key & f"fluo_channel='{channel}'" + ).fetch("activity_trace") + ).T, + rois=rt_region, + unit="a.u.", + rate=(scan.ScanInfo & session_key).fetch1("fps"), + ) + fl = Fluorescence(roi_response_series=[roi_resp_series, neuropil_series, deconvolved_series]) ophys_module.add(fl) From cbf572ce9e75023d58feb7d46807715977648e58 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 25 Oct 2023 13:22:20 -0500 Subject: [PATCH 33/33] Black formatting --- element_calcium_imaging/export/nwb/nwb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/element_calcium_imaging/export/nwb/nwb.py b/element_calcium_imaging/export/nwb/nwb.py index b34fbf02..63fc005c 100644 --- a/element_calcium_imaging/export/nwb/nwb.py +++ b/element_calcium_imaging/export/nwb/nwb.py @@ -309,7 +309,9 @@ def _add_segmentation_data_to_nwb(session_key, nwbfile, imaging_plane): unit="a.u.", rate=(scan.ScanInfo & session_key).fetch1("fps"), ) - fl = Fluorescence(roi_response_series=[roi_resp_series, neuropil_series, deconvolved_series]) + fl = Fluorescence( + roi_response_series=[roi_resp_series, neuropil_series, deconvolved_series] + ) ophys_module.add(fl)