From 9fa3d87eee9405d61d84787ab82f2f5fde8f0db6 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 22 Nov 2023 11:39:53 +0100 Subject: [PATCH 1/3] Fix typo _serializablility -> _serializability --- src/spikeinterface/comparison/hybrid.py | 4 ++-- .../comparison/multicomparisons.py | 6 ++--- src/spikeinterface/core/base.py | 22 +++++++++---------- src/spikeinterface/core/generate.py | 2 +- src/spikeinterface/core/numpyextractors.py | 22 +++++++++---------- src/spikeinterface/core/old_api_utils.py | 12 +++++----- src/spikeinterface/core/tests/test_base.py | 10 ++++----- .../core/tests/test_jsonification.py | 6 ++--- src/spikeinterface/core/waveform_extractor.py | 20 ++++++++--------- src/spikeinterface/preprocessing/motion.py | 2 +- src/spikeinterface/sorters/basesorter.py | 14 +++++++++--- 11 files changed, 64 insertions(+), 56 deletions(-) diff --git a/src/spikeinterface/comparison/hybrid.py b/src/spikeinterface/comparison/hybrid.py index e0c98cd772..8af64536ff 100644 --- a/src/spikeinterface/comparison/hybrid.py +++ b/src/spikeinterface/comparison/hybrid.py @@ -84,7 +84,7 @@ def __init__( ) # save injected sorting if necessary self.injected_sorting = injected_sorting - if not self.injected_sorting.check_serializablility("json"): + if not self.injected_sorting.check_serializability("json"): # TODO later : also use pickle assert injected_sorting_folder is not None, "Provide injected_sorting_folder to injected sorting object" self.injected_sorting = self.injected_sorting.save(folder=injected_sorting_folder) @@ -181,7 +181,7 @@ def __init__( self.injected_sorting = injected_sorting # save injected sorting if necessary - if not self.injected_sorting.check_serializablility("json"): + if not self.injected_sorting.check_serializability("json"): # TODO later : also use pickle assert injected_sorting_folder is not None, "Provide injected_sorting_folder to injected sorting object" self.injected_sorting = self.injected_sorting.save(folder=injected_sorting_folder) diff --git a/src/spikeinterface/comparison/multicomparisons.py b/src/spikeinterface/comparison/multicomparisons.py index bc7d76ea5a..56e2c5c0c2 100644 --- a/src/spikeinterface/comparison/multicomparisons.py +++ b/src/spikeinterface/comparison/multicomparisons.py @@ -189,7 +189,7 @@ def save_to_folder(self, save_folder): stacklevel=2, ) for sorting in self.object_list: - assert sorting.check_serializablility( + assert sorting.check_serializability( "json" ), "MultiSortingComparison.save_to_folder() need json serializable sortings" @@ -259,8 +259,8 @@ def __init__( BaseSorting.__init__(self, sampling_frequency=sampling_frequency, unit_ids=unit_ids) - self._serializablility["json"] = False - self._serializablility["pickle"] = True + self._serializability["json"] = False + self._serializability["pickle"] = True if len(unit_ids) > 0: for k in ("agreement_number", "avg_agreement", "unit_ids"): diff --git a/src/spikeinterface/core/base.py b/src/spikeinterface/core/base.py index 7cde209a8d..33a7ca41f5 100644 --- a/src/spikeinterface/core/base.py +++ b/src/spikeinterface/core/base.py @@ -62,7 +62,7 @@ def __init__(self, main_ids: Sequence) -> None: # * number of units for sorting self._properties = {} - self._serializablility = {"memory": True, "json": True, "pickle": True} + self._serializability = {"memory": True, "json": True, "pickle": True} # extractor specific list of pip extra requirements self.extra_requirements = [] @@ -507,22 +507,22 @@ def clone(self) -> "BaseExtractor": clone = BaseExtractor.from_dict(d) return clone - def check_serializablility(self, type): + def check_serializability(self, type): kwargs = self._kwargs for value in kwargs.values(): # here we check if the value is a BaseExtractor, a list of BaseExtractors, or a dict of BaseExtractors if isinstance(value, BaseExtractor): - if not value.check_serializablility(type=type): + if not value.check_serializability(type=type): return False elif isinstance(value, list): for v in value: - if isinstance(v, BaseExtractor) and not v.check_serializablility(type=type): + if isinstance(v, BaseExtractor) and not v.check_serializability(type=type): return False elif isinstance(value, dict): for v in value.values(): - if isinstance(v, BaseExtractor) and not v.check_serializablility(type=type): + if isinstance(v, BaseExtractor) and not v.check_serializability(type=type): return False - return self._serializablility[type] + return self._serializability[type] def check_if_memory_serializable(self): """ @@ -533,7 +533,7 @@ def check_if_memory_serializable(self): bool True if the object is memory serializable, False otherwise. """ - return self.check_serializablility("memory") + return self.check_serializability("memory") def check_if_json_serializable(self): """ @@ -546,11 +546,11 @@ def check_if_json_serializable(self): """ # we keep this for backward compatilibity or not ???? # is this needed ??? I think no. - return self.check_serializablility("json") + return self.check_serializability("json") def check_if_pickle_serializable(self): # is this needed ??? I think no. - return self.check_serializablility("pickle") + return self.check_serializability("pickle") @staticmethod def _get_file_path(file_path: Union[str, Path], extensions: Sequence) -> Path: @@ -623,7 +623,7 @@ def dump_to_json( folder_metadata: str, Path, or None Folder with files containing additional information (e.g. probe in BaseRecording) and properties """ - assert self.check_serializablility("json"), "The extractor is not json serializable" + assert self.check_serializability("json"), "The extractor is not json serializable" # Writing paths as relative_to requires recursively expanding the dict if relative_to: @@ -882,7 +882,7 @@ def save_to_folder(self, name=None, folder=None, overwrite=False, verbose=True, # dump provenance provenance_file = folder / f"provenance.json" - if self.check_serializablility("json"): + if self.check_serializability("json"): self.dump(provenance_file) else: provenance_file.write_text( diff --git a/src/spikeinterface/core/generate.py b/src/spikeinterface/core/generate.py index 1c8661d12d..2adf08c6ec 100644 --- a/src/spikeinterface/core/generate.py +++ b/src/spikeinterface/core/generate.py @@ -1118,7 +1118,7 @@ def __init__( dtype = parent_recording.dtype if parent_recording is not None else templates.dtype BaseRecording.__init__(self, sorting.get_sampling_frequency(), channel_ids, dtype) - # Important : self._serializablility is not change here because it will depend on the sorting parents itself. + # Important : self._serializability is not change here because it will depend on the sorting parents itself. n_units = len(sorting.unit_ids) assert len(templates) == n_units diff --git a/src/spikeinterface/core/numpyextractors.py b/src/spikeinterface/core/numpyextractors.py index 82075e638c..7cced7a2ae 100644 --- a/src/spikeinterface/core/numpyextractors.py +++ b/src/spikeinterface/core/numpyextractors.py @@ -66,8 +66,8 @@ def __init__(self, traces_list, sampling_frequency, t_starts=None, channel_ids=N assert len(t_starts) == len(traces_list), "t_starts must be a list of same size than traces_list" t_starts = [float(t_start) for t_start in t_starts] - self._serializablility["json"] = False - self._serializablility["pickle"] = False + self._serializability["json"] = False + self._serializability["pickle"] = False for i, traces in enumerate(traces_list): if t_starts is None: @@ -129,10 +129,10 @@ def __init__(self, spikes, sampling_frequency, unit_ids): """ """ BaseSorting.__init__(self, sampling_frequency, unit_ids) - self._serializablility["memory"] = True - self._serializablility["json"] = False + self._serializability["memory"] = True + self._serializability["json"] = False # theorically this should be False but for simplicity make generators simples we still need this. - self._serializablility["pickle"] = True + self._serializability["pickle"] = True if spikes.size == 0: nseg = 1 @@ -364,9 +364,9 @@ def __init__(self, shm_name, shape, sampling_frequency, unit_ids, dtype=minimum_ BaseSorting.__init__(self, sampling_frequency, unit_ids) - self._serializablility["memory"] = True - self._serializablility["json"] = False - self._serializablility["pickle"] = False + self._serializability["memory"] = True + self._serializability["json"] = False + self._serializability["pickle"] = False self.shm = SharedMemory(shm_name, create=False) self.shm_spikes = np.ndarray(shape=shape, dtype=dtype, buffer=self.shm.buf) @@ -524,9 +524,9 @@ def __init__(self, snippets_list, spikesframes_list, sampling_frequency, nbefore dtype=dtype, ) - self._serializablility["memory"] = False - self._serializablility["json"] = False - self._serializablility["pickle"] = False + self._serializability["memory"] = False + self._serializability["json"] = False + self._serializability["pickle"] = False for snippets, spikesframes in zip(snippets_list, spikesframes_list): snp_segment = NumpySnippetsSegment(snippets, spikesframes) diff --git a/src/spikeinterface/core/old_api_utils.py b/src/spikeinterface/core/old_api_utils.py index 879700cc15..46418450a3 100644 --- a/src/spikeinterface/core/old_api_utils.py +++ b/src/spikeinterface/core/old_api_utils.py @@ -182,9 +182,9 @@ def __init__(self, oldapi_recording_extractor): ) # set to False to use dumping mechanism of old extractor - self._serializablility["memory"] = False - self._serializablility["json"] = False - self._serializablility["pickle"] = False + self._serializability["memory"] = False + self._serializability["json"] = False + self._serializability["pickle"] = False self.annotate(is_filtered=oldapi_recording_extractor.is_filtered) @@ -269,9 +269,9 @@ def __init__(self, oldapi_sorting_extractor): sorting_segment = OldToNewSortingSegment(oldapi_sorting_extractor) self.add_sorting_segment(sorting_segment) - self._serializablility["memory"] = False - self._serializablility["json"] = False - self._serializablility["pickle"] = False + self._serializability["memory"] = False + self._serializability["json"] = False + self._serializability["pickle"] = False # add old properties copy_properties(oldapi_extractor=oldapi_sorting_extractor, new_extractor=self) diff --git a/src/spikeinterface/core/tests/test_base.py b/src/spikeinterface/core/tests/test_base.py index a944be3da0..9e2a3b22ab 100644 --- a/src/spikeinterface/core/tests/test_base.py +++ b/src/spikeinterface/core/tests/test_base.py @@ -40,7 +40,7 @@ def test_check_if_memory_serializable(): assert extractor.check_if_memory_serializable() # make not not memory serilizable - test_extractor._serializablility["memory"] = False + test_extractor._serializability["memory"] = False extractors_not_mem_serializable = make_nested_extractors(test_extractor) for extractor in extractors_not_mem_serializable: assert not extractor.check_if_memory_serializable() @@ -50,18 +50,18 @@ def test_check_if_serializable(): test_extractor = generate_recording(seed=0, durations=[2]) # make a list of json serializable objects - test_extractor._serializablility["json"] = True + test_extractor._serializability["json"] = True extractors_json_serializable = make_nested_extractors(test_extractor) for extractor in extractors_json_serializable: print(extractor) - assert extractor.check_serializablility("json") + assert extractor.check_serializability("json") # make of not json serializable objects - test_extractor._serializablility["json"] = False + test_extractor._serializability["json"] = False extractors_not_json_serializable = make_nested_extractors(test_extractor) for extractor in extractors_not_json_serializable: print(extractor) - assert not extractor.check_serializablility("json") + assert not extractor.check_serializability("json") if __name__ == "__main__": diff --git a/src/spikeinterface/core/tests/test_jsonification.py b/src/spikeinterface/core/tests/test_jsonification.py index 1c491bd7a6..f63cfb16d8 100644 --- a/src/spikeinterface/core/tests/test_jsonification.py +++ b/src/spikeinterface/core/tests/test_jsonification.py @@ -144,9 +144,9 @@ def __init__(self, attribute, other_extractor=None, extractor_list=None, extract BaseExtractor.__init__(self, main_ids=["1", "2"]) # this already the case by default - self._serializablility["memory"] = True - self._serializablility["json"] = True - self._serializablility["pickle"] = True + self._serializability["memory"] = True + self._serializability["json"] = True + self._serializability["pickle"] = True self._kwargs = { "attribute": attribute, diff --git a/src/spikeinterface/core/waveform_extractor.py b/src/spikeinterface/core/waveform_extractor.py index a81d36139d..84fa895948 100644 --- a/src/spikeinterface/core/waveform_extractor.py +++ b/src/spikeinterface/core/waveform_extractor.py @@ -284,14 +284,14 @@ def create( else: relative_to = None - if recording.check_serializablility("json"): + if recording.check_serializability("json"): recording.dump(folder / "recording.json", relative_to=relative_to) - elif recording.check_serializablility("pickle"): + elif recording.check_serializability("pickle"): recording.dump(folder / "recording.pickle", relative_to=relative_to) - if sorting.check_serializablility("json"): + if sorting.check_serializability("json"): sorting.dump(folder / "sorting.json", relative_to=relative_to) - elif sorting.check_serializablility("pickle"): + elif sorting.check_serializability("pickle"): sorting.dump(folder / "sorting.pickle", relative_to=relative_to) else: warn( @@ -920,14 +920,14 @@ def save( (folder / "params.json").write_text(json.dumps(check_json(self._params), indent=4), encoding="utf8") if self.has_recording(): - if self.recording.check_serializablility("json"): + if self.recording.check_serializability("json"): self.recording.dump(folder / "recording.json", relative_to=relative_to) - elif self.recording.check_serializablility("pickle"): + elif self.recording.check_serializability("pickle"): self.recording.dump(folder / "recording.pickle", relative_to=relative_to) - if self.sorting.check_serializablility("json"): + if self.sorting.check_serializability("json"): self.sorting.dump(folder / "sorting.json", relative_to=relative_to) - elif self.sorting.check_serializablility("pickle"): + elif self.sorting.check_serializability("pickle"): self.sorting.dump(folder / "sorting.pickle", relative_to=relative_to) else: warn( @@ -977,10 +977,10 @@ def save( # write metadata zarr_root.attrs["params"] = check_json(self._params) if self.has_recording(): - if self.recording.check_serializablility("json"): + if self.recording.check_serializability("json"): rec_dict = self.recording.to_dict(relative_to=relative_to, recursive=True) zarr_root.attrs["recording"] = check_json(rec_dict) - if self.sorting.check_serializablility("json"): + if self.sorting.check_serializability("json"): sort_dict = self.sorting.to_dict(relative_to=relative_to, recursive=True) zarr_root.attrs["sorting"] = check_json(sort_dict) else: diff --git a/src/spikeinterface/preprocessing/motion.py b/src/spikeinterface/preprocessing/motion.py index 8672b48340..3992a4c8c6 100644 --- a/src/spikeinterface/preprocessing/motion.py +++ b/src/spikeinterface/preprocessing/motion.py @@ -333,7 +333,7 @@ def correct_motion( ) (folder / "parameters.json").write_text(json.dumps(parameters, indent=4, cls=SIJsonEncoder), encoding="utf8") (folder / "run_times.json").write_text(json.dumps(run_times, indent=4), encoding="utf8") - if recording.check_serializablility("json"): + if recording.check_serializability("json"): recording.dump_to_json(folder / "recording.json") np.save(folder / "peaks.npy", peaks) diff --git a/src/spikeinterface/sorters/basesorter.py b/src/spikeinterface/sorters/basesorter.py index 894918cbc4..87d9a2febc 100644 --- a/src/spikeinterface/sorters/basesorter.py +++ b/src/spikeinterface/sorters/basesorter.py @@ -136,10 +136,18 @@ def initialize_folder(cls, recording, output_folder, verbose, remove_existing_fo f"This sorter {cls.sorter_name} does not handle multi-segment recordings, use si.concatenate_recordings(...)" ) + if are_paths_relative_possible(recording, output_folder): + relative_to = output_folder + else: + relative_to = None + rec_file = output_folder / "spikeinterface_recording.json" - if recording.check_serializablility("json"): - recording.dump(rec_file, relative_to=output_folder) - elif recording.check_serializablility("pickle"): + if recording.check_serializability("json"): + if are_paths_relative_possible(recording, output_folder): + recording.dump(rec_file, relative_to=output_folder) + else: + recording.dump(rec_file) + elif recording.check_serializability("pickle"): recording.dump(output_folder / "spikeinterface_recording.pickle", relative_to=output_folder) else: # TODO: deprecate and finally remove this after 0.100 From 11727454a31b6081a93a7958162eb40cbef451d7 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 22 Nov 2023 13:20:31 +0100 Subject: [PATCH 2/3] oups --- src/spikeinterface/sorters/basesorter.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/spikeinterface/sorters/basesorter.py b/src/spikeinterface/sorters/basesorter.py index 87d9a2febc..8116587044 100644 --- a/src/spikeinterface/sorters/basesorter.py +++ b/src/spikeinterface/sorters/basesorter.py @@ -136,17 +136,9 @@ def initialize_folder(cls, recording, output_folder, verbose, remove_existing_fo f"This sorter {cls.sorter_name} does not handle multi-segment recordings, use si.concatenate_recordings(...)" ) - if are_paths_relative_possible(recording, output_folder): - relative_to = output_folder - else: - relative_to = None - rec_file = output_folder / "spikeinterface_recording.json" if recording.check_serializability("json"): - if are_paths_relative_possible(recording, output_folder): - recording.dump(rec_file, relative_to=output_folder) - else: - recording.dump(rec_file) + recording.dump(rec_file) elif recording.check_serializability("pickle"): recording.dump(output_folder / "spikeinterface_recording.pickle", relative_to=output_folder) else: From 09d9259cd4ac72932b290a3c06e833ed5f4d1542 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 22 Nov 2023 14:33:40 +0100 Subject: [PATCH 3/3] Update src/spikeinterface/comparison/multicomparisons.py Co-authored-by: Zach McKenzie <92116279+zm711@users.noreply.github.com> --- src/spikeinterface/comparison/multicomparisons.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/comparison/multicomparisons.py b/src/spikeinterface/comparison/multicomparisons.py index 56e2c5c0c2..fc5a216895 100644 --- a/src/spikeinterface/comparison/multicomparisons.py +++ b/src/spikeinterface/comparison/multicomparisons.py @@ -191,7 +191,7 @@ def save_to_folder(self, save_folder): for sorting in self.object_list: assert sorting.check_serializability( "json" - ), "MultiSortingComparison.save_to_folder() need json serializable sortings" + ), "MultiSortingComparison.save_to_folder() needs json serializable sortings" save_folder = Path(save_folder) save_folder.mkdir(parents=True, exist_ok=True)