Interval list types (#742)

* add interval_type secondary key to IntervalList

* add interval_type to IntervalList inserts

* document interval types in tutorial

* update changelog

* change interval_type to pipeline

* Update src/spyglass/common/common_interval.py

Co-authored-by: Chris Brozdowski <[email protected]>

---------

Co-authored-by: Eric Denovellis <[email protected]>
Co-authored-by: Chris Brozdowski <[email protected]>

CHANGELOG.md

@@ -11,6 +11,7 @@
 - Add `cautious_delete` to Mixin class, initial implementation. #711
 - Add `deprecation_factory` to facilitate table migration. #717
 - Add Spyglass logger. #730
+- IntervalList: Add secondary key `pipeline` #742
 
 ### Pipelines
 
@@ -30,6 +31,7 @@
     - Allow multiple spike waveform features for clusterelss decoding #731
     - Reorder notebooks #731
 
+
 ## [0.4.3] (November 7, 2023)
 
 - Migrate `config` helper scripts to Spyglass codebase. #662

notebooks/01_Insert_Data.ipynb

@@ -2048,6 +2048,28 @@
     ").fetch(\"interval_list_name\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`IntervalList` has an additional secondary key `pipeline` which can describe the origin of the data.\n",
+    "Because it is a _secondary_ key, it is not required to uniquely identify an entry. \n",
+    "Current values for this key from spyglass pipelines are:\n",
+    "\n",
+    "| pipeline | Source|\n",
+    "| ---   | --- |\n",
+    "| position | sg.common.PositionSource |\n",
+    "| lfp_v0 | sg.common.LFP |\n",
+    "| lfp_v1 | sg.lfp.v1.LFPV1 |\n",
+    "| lfp_band | sg.common.LFPBand,<br>  sg.lfp.analysis.v1.LFPBandV1 |\n",
+    "| lfp_artifact | sg.lfp.v1.LFPArtifactDetection |\n",
+    "| spikesorting_artifact_v0 | sg.spikesorting.ArtifactDetection |\n",
+    "| spikesorting_artifact_v1 | sg.spikesorting.v1.ArtifactDetection |\n",
+    "| spikesorting_recording_v0 | sg.spikesorting.SpikeSortingRecording |\n",
+    "| spikesorting_recording_v1 | sg.spikesorting.v1.SpikeSortingRecording |\n",
+    "\n"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

notebooks/py_scripts/01_Insert_Data.py

@@ -302,6 +302,24 @@
     - {"interval_list_name": "pos 2 valid times"}
 ).fetch("interval_list_name")
 
+# `IntervalList` has an additional secondary key `pipeline` which can describe the origin of the data.
+# Because it is a _secondary_ key, it is not required to uniquely identify an entry.
+# Current values for this key from spyglass pipelines are:
+#
+# | pipeline | Source|
+# | ---   | --- |
+# | position | sg.common.PositionSource |
+# | lfp_v0 | sg.common.LFP |
+# | lfp_v1 | sg.lfp.v1.LFPV1 |
+# | lfp_band | sg.common.LFPBand,<br>  sg.lfp.analysis.v1.LFPBandV1 |
+# | lfp_artifact | sg.lfp.v1.LFPArtifactDetection |
+# | spikesorting_artifact_v0 | sg.spikesorting.ArtifactDetection |
+# | spikesorting_artifact_v1 | sg.spikesorting.v1.ArtifactDetection |
+# | spikesorting_recording_v0 | sg.spikesorting.SpikeSortingRecording |
+# | spikesorting_recording_v1 | sg.spikesorting.v1.SpikeSortingRecording |
+#
+#
+
 # ## Deleting data
 #
 

src/spyglass/common/common_behav.py

@@ -76,6 +76,7 @@ def insert_from_nwbfile(cls, nwb_file_name):
                     **sess_key,
                     **ind_key,
                     valid_times=epoch_list[0]["valid_times"],
+                    pipeline="position",
                 )
             )
 

src/spyglass/common/common_ephys.py

@@ -495,6 +495,7 @@ def make(self, key):
                 "nwb_file_name": key["nwb_file_name"],
                 "interval_list_name": key["interval_list_name"],
                 "valid_times": lfp_valid_times,
+                "pipeline": "lfp_v0",
             },
             replace=True,
         )
@@ -838,6 +839,7 @@ def make(self, key):
                     "nwb_file_name": key["nwb_file_name"],
                     "interval_list_name": key["interval_list_name"],
                     "valid_times": lfp_band_valid_times,
+                    "pipeline": "lfp_band",
                 }
             )
         else:

src/spyglass/common/common_interval.py

@@ -23,6 +23,7 @@ class IntervalList(SpyglassMixin, dj.Manual):
     interval_list_name: varchar(170)  # descriptive name of this interval list
     ---
     valid_times: longblob  # numpy array with start/end times for each interval
+    pipeline = "": varchar(64)  # type of interval list (e.g. 'position', 'spikesorting_recording_v1')
     """
 
     # See #630, #664. Excessive key length.

src/spyglass/lfp/analysis/v1/lfp_band.py

@@ -353,6 +353,7 @@ def make(self, key):
                     "nwb_file_name": key["nwb_file_name"],
                     "interval_list_name": key["interval_list_name"],
                     "valid_times": lfp_band_valid_times,
+                    "pipeline": "lfp band",
                 }
             )
         else:

src/spyglass/lfp/v1/lfp.py

@@ -166,6 +166,7 @@ def make(self, key):
                 "nwb_file_name": key["nwb_file_name"],
                 "interval_list_name": key["interval_list_name"],
                 "valid_times": lfp_valid_times,
+                "pipeline": "lfp_v1",
             },
             replace=True,
         )

src/spyglass/lfp/v1/lfp_artifact.py

@@ -192,6 +192,7 @@ def make(self, key):
             "nwb_file_name": key["nwb_file_name"],
             "interval_list_name": key["artifact_removed_interval_list_name"],
             "valid_times": key["artifact_removed_valid_times"],
+            "pipeline": "lfp_artifact",
         }
 
         LFPArtifactRemovedIntervalList.insert1(key, replace=True)

src/spyglass/spikesorting/spikesorting_artifact.py

@@ -130,6 +130,7 @@ def make(self, key):
                 "artifact_removed_interval_list_name"
             ]
             tmp_key["valid_times"] = key["artifact_removed_valid_times"]
+            tmp_key["pipeline"] = "spikesorting_artifact_v0"
             IntervalList.insert1(tmp_key, replace=True)
 
             # insert into computed table

src/spyglass/spikesorting/spikesorting_recording.py

@@ -400,6 +400,7 @@ def make(self, key):
                 "nwb_file_name": key["nwb_file_name"],
                 "interval_list_name": recording_name,
                 "valid_times": sort_interval_valid_times,
+                "pipeline": "spikesorting_recording_v0",
             },
             replace=True,
         )

src/spyglass/spikesorting/v1/artifact.py

@@ -156,6 +156,7 @@ def make(self, key):
                 ).fetch1("nwb_file_name"),
                 interval_list_name=str(key["artifact_id"]),
                 valid_times=artifact_removed_valid_times,
+                pipeline="spikesorting_artifact_v1",
             ),
             skip_duplicates=True,
         )

src/spyglass/spikesorting/v1/recording.py

@@ -275,6 +275,7 @@ def make(self, key):
                 ),
                 "interval_list_name": key["recording_id"],
                 "valid_times": sort_interval_valid_times,
+                "pipeline": "spikesorting_recording_v1",
             }
         )
         AnalysisNwbfile().add(