From 84d96a39c911cfe24ba76eba3223d95f4d6f877b Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 21 Nov 2023 16:23:35 -0500 Subject: [PATCH 01/15] cnmf/params.py: Toards json serialisation --- caiman/source_extraction/cnmf/params.py | 29 +++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 712834521..d62801c33 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import json import logging import numpy as np import os @@ -1029,7 +1030,7 @@ def __eq__(self, other): return True - def to_dict(self): + def to_dict(self) -> dict: """Returns the params class as a dictionary with subdictionaries for each category.""" return {'data': self.data, 'spatial_params': self.spatial, 'temporal_params': self.temporal, @@ -1038,8 +1039,28 @@ def to_dict(self): 'merging': self.merging, 'motion': self.motion, 'ring_CNN': self.ring_CNN } - def __repr__(self): - + def to_json(self) -> str: + """ Reversibly serialise CNMFParams to json """ + dictdata = self.to_dict() + # now we need to tweak dictdata to convert ndarrays to something json can represent + class NumpyEncoder(json.JSONEncoder): # Custom json encoder that handles ndarrays better + def default(self, obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + elif isinstance(obj, np.integer): + return int(obj) + elif isinstance(obj, slice): + return list([obj.start, obj.stop, obj.step]) + return json.JSONEncoder.default(self, obj) + return json.dumps(dictdata, cls=NumpyEncoder) + + + def to_jsonfile(self, targfn:str) -> None: + """ Reversibly serialise CNMFParams to a json file """ + with open(targfn, 'w') as targfh: + targfh.write(self.to_json()) + + def __repr__(self) -> str: formatted_outputs = [ f'{group_name}:\n\n{pformat(group_dict)}' for group_name, group_dict in self.to_dict().items() @@ -1064,7 +1085,7 @@ def change_params(self, params_dict, verbose=False): d = getattr(self, gr) if k in d: flag = False - if flag: + if flag and verbose: logging.warning(f'No parameter {k} found!') self.check_consistency() return self From 8ce73e149c035898b8a6800b88b4099432382df5 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 27 Nov 2023 11:35:24 -0500 Subject: [PATCH 02/15] CNMFParams:to_dict() - key names should match attributes so saves/loads are symmetric --- caiman/source_extraction/cnmf/params.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index d62801c33..a2db822b0 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -1033,11 +1033,19 @@ def __eq__(self, other): def to_dict(self) -> dict: """Returns the params class as a dictionary with subdictionaries for each category.""" - return {'data': self.data, 'spatial_params': self.spatial, 'temporal_params': self.temporal, - 'init_params': self.init, 'preprocess_params': self.preprocess, - 'patch_params': self.patch, 'online': self.online, 'quality': self.quality, - 'merging': self.merging, 'motion': self.motion, 'ring_CNN': self.ring_CNN - } + return { + 'data': self.data, + 'init': self.init, + 'merging': self.merging, + 'motion': self.motion, + 'online': self.online, + 'patch': self.patch, + 'preprocess': self.preprocess, + 'ring_CNN': self.ring_CNN, + 'quality': self.quality, + 'spatial': self.spatial, + 'temporal': self.temporal + } def to_json(self) -> str: """ Reversibly serialise CNMFParams to json """ From 855b45af8dfbc6610025f0745e0edd81044148a1 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 27 Nov 2023 11:59:09 -0500 Subject: [PATCH 03/15] CNMFParams.change_params() - dict follows struct of object, avoids resetting unintended fields. No longer returns another reference to CNMFParams --- caiman/source_extraction/cnmf/params.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index a2db822b0..bb443301b 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -1076,24 +1076,18 @@ def __repr__(self) -> str: return 'CNMFParams:\n\n' + '\n\n'.join(formatted_outputs) - def change_params(self, params_dict, verbose=False): + def change_params(self, params_dict, verbose=False) -> None: """ Method for updating the params object by providing a single dictionary. - For each key in the provided dictionary the method will search in all - subdictionaries and will update the value if it finds a match. Args: - params_dict: dictionary with parameters to be changed and new values - verbose: bool (False). Print message for all keys + params_dict: dictionary with parameters to be changed + verbose: bool (False). If true, will complain if the params dictionary is not complete """ for gr in list(self.__dict__.keys()): - self.set(gr, params_dict, verbose=verbose) - for k, v in params_dict.items(): - flag = True - for gr in list(self.__dict__.keys()): - d = getattr(self, gr) - if k in d: - flag = False - if flag and verbose: - logging.warning(f'No parameter {k} found!') + if gr in params_dict: + self.set(gr, params_dict[gr], verbose=verbose) + else: + if verbose: + logging.warning(f"No subobject {gr} in parameter dict") self.check_consistency() - return self + From 516b8d8e40141e685a6b7977b7f52df84e39a477 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 27 Nov 2023 17:21:14 -0500 Subject: [PATCH 04/15] CNMFParams: Properly scope dict load/save functions, add json functions, adjust tests --- caiman/source_extraction/cnmf/params.py | 11 ++++ caiman/tests/comparison_humans.py | 76 +++++++++++++++--------- caiman/tests/comparison_humans_online.py | 67 +++++++++++++-------- caiman/tests/test_motion_correction.py | 12 ++-- caiman/tests/test_onacid.py | 55 ++++++++++++----- 5 files changed, 149 insertions(+), 72 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index bb443301b..8ab8ab3e1 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -1091,3 +1091,14 @@ def change_params(self, params_dict, verbose=False) -> None: logging.warning(f"No subobject {gr} in parameter dict") self.check_consistency() + def change_params_from_json(self, jsonstring:str, verbose:bool=False) -> None: + """ Same as change_params, except it takes json as input """ + to_load = json.loads(jsonstring) + self.change_params(to_load, verbose=verbose) + + def change_params_from_jsonfile(self, json_fn:str, verbose:bool=False) -> None: + """ Same as change_params, except it takes a json file as input; pass the filename """ + with open(json_fn, 'r') as json_fh: + to_load = json.load(json_fh) + self.change_params(to_load, verbose=verbose) + diff --git a/caiman/tests/comparison_humans.py b/caiman/tests/comparison_humans.py index 4ab28b3c0..e61664ff4 100644 --- a/caiman/tests/comparison_humans.py +++ b/caiman/tests/comparison_humans.py @@ -313,33 +313,53 @@ c, dview, n_processes = setup_cluster( backend=backend_patch, n_processes=n_processes, single_thread=False) # %% - params_dict = {'fnames': [fname_new], - 'fr': params_movie['fr'], - 'decay_time': params_movie['decay_time'], - 'rf': params_movie['rf'], - 'stride': params_movie['stride_cnmf'], - 'K': params_movie['K'], - 'gSig': params_movie['gSig'], - 'merge_thr': params_movie['merge_thresh'], - 'p': global_params['p'], - 'nb': global_params['gnb'], - 'only_init': global_params['only_init_patch'], - 'dview': dview, - 'method_deconvolution': 'oasis', - 'border_pix': params_movie['crop_pix'], - 'low_rank_background': global_params['low_rank_background'], - 'rolling_sum': True, - 'nb_patch': 1, - 'check_nan': check_nan, - 'block_size_temp': block_size, - 'block_size_spat': block_size, - 'num_blocks_per_run_spat': num_blocks_per_run, - 'num_blocks_per_run_temp': num_blocks_per_run, - 'merge_parallel': True, - 'n_pixels_per_process': n_pixels_per_process, - 'ssub': global_params['ssub'], - 'tsub': global_params['tsub'], - 'thr_method': 'nrg' + params_dict = { + 'data': { + 'decay_time': params_movie['decay_time'], + 'fnames': [fname_new], + 'fr': params_movie['fr'], + }, + 'init': { + 'gSig': params_movie['gSig'], + 'K': params_movie['K'], + 'nb': global_params['gnb'], + 'rolling_sum': True, + 'ssub': global_params['ssub'], + 'tsub': global_params['tsub'], + + }, + 'merging': { + 'merge_parallel': True, + 'merge_thr': params_movie['merge_thresh'], + }, + 'patch': { + 'border_pix': params_movie['crop_pix'], + 'low_rank_background': global_params['low_rank_background'], + 'nb_patch': 1, + 'only_init': global_params['only_init_patch'], + 'rf': params_movie['rf'], + 'stride': params_movie['stride_cnmf'], + + }, + 'preprocess': { + 'check_nan': check_nan, + 'n_pixels_per_process': n_pixels_per_process, + 'p': global_params['p'], + }, + 'spatial': { + 'block_size_spat': block_size, + 'nb': global_params['gnb'], + 'num_blocks_per_run_spat': num_blocks_per_run, + 'n_pixels_per_process': n_pixels_per_process, + 'thr_method': 'nrg' + }, + 'temporal': { + 'block_size_temp': block_size, + 'method_deconvolution': 'oasis', + 'nb': global_params['gnb'], + 'num_blocks_per_run_temp': num_blocks_per_run, + 'p': global_params['p'], + }, } init_method = global_params['init_method'] @@ -415,7 +435,7 @@ f=ld['f_gt'], R=ld['YrA_gt'], dims=(ld['d1'], ld['d2'])) min_size_neuro = 3 * 2 * np.pi - max_size_neuro = (2 * params_dict['gSig'][0]) ** 2 * np.pi + max_size_neuro = (2 * params_dict['init']['gSig'][0]) ** 2 * np.pi gt_estimate.threshold_spatial_components(maxthr=0.2, dview=dview) nrn_size = gt_estimate.remove_small_large_neurons(min_size_neuro, max_size_neuro) nrn_dup = gt_estimate.remove_duplicates(predictions=None, r_values=None, dist_thr=0.1, min_dist=10, diff --git a/caiman/tests/comparison_humans_online.py b/caiman/tests/comparison_humans_online.py index cb16f9e8f..cca1fb1be 100644 --- a/caiman/tests/comparison_humans_online.py +++ b/caiman/tests/comparison_humans_online.py @@ -185,30 +185,49 @@ # %% params_dict = { - 'fnames': fls, - 'fr': fr, - 'decay_time': decay_time, - 'gSig': gSig, - 'p': global_params['p'], - 'min_SNR': global_params['min_SNR'], - 'rval_thr': global_params['rval_thr'], - 'ds_factor': ds_factor, - 'nb': gnb, - 'motion_correct': global_params['mot_corr'], - 'init_batch': init_batch, - 'init_method': 'bare', - 'normalize': True, - 'expected_comps': expected_comps, - 'dist_shape_update': True, - 'K': K, - 'epochs': epochs, - 'show_movie': False, - 'min_num_trial': global_params['min_num_trial'], - 'use_peak_max': True, - 'thresh_CNN_noisy': global_params['thresh_CNN_noisy'], - 'sniper_mode': global_params['sniper_mode'], - 'use_dense': False, - 'update_freq': global_params['update_freq'] + 'data': { + 'decay_time': decay_time, + 'fnames': fls, + 'fr': fr, + }, + 'init': { + 'gSig': gSig, + 'K': K, + 'nb': gnb, + }, + 'online': { + 'dist_shape_update': True, + 'ds_factor': ds_factor, + 'epochs': epochs, + 'expected_comps': expected_comps, + 'init_batch': init_batch, + 'init_method': 'bare', + 'min_num_trial': global_params['min_num_trial'], + 'min_SNR': global_params['min_SNR'], + 'motion_correct': global_params['mot_corr'], + 'normalize': True, + 'rval_thr': global_params['rval_thr'], + 'show_movie': False, + 'sniper_mode': global_params['sniper_mode'], + 'thresh_CNN_noisy': global_params['thresh_CNN_noisy'], + 'update_freq': global_params['update_freq'] + 'use_dense': False, + 'use_peak_max': True, + }, + 'preprocess': { + 'p': global_params['p'], + }, + 'quality': { + 'min_SNR': global_params['min_SNR'], + 'rval_thr': global_params['rval_thr'], + }, + 'spatial': { + 'nb': gnb, + }, + 'temporal': { + 'nb': gnb, + 'p': global_params['p'], + }, } opts = cnmf.params.CNMFParams(params_dict=params_dict) diff --git a/caiman/tests/test_motion_correction.py b/caiman/tests/test_motion_correction.py index 8d458cd24..98a72d5dd 100644 --- a/caiman/tests/test_motion_correction.py +++ b/caiman/tests/test_motion_correction.py @@ -128,10 +128,14 @@ def _test_motion_correct_rigid(D): Y, C, S, A, centers, dims, shifts = gen_data(D) fname = 'testMovie.tif' cm.movie(Y).save(fname) - params_dict = {'max_shifts': (4, 4), # maximum allowed rigid shifts (in pixels) - 'pw_rigid': False, # flag for performing non-rigid motion correction - 'border_nan': True, - 'is3D': D == 3} + params_dict = { + 'motion': { + 'border_nan': True, + 'is3D': D == 3, + 'max_shifts': (4, 4), # maximum allowed rigid shifts (in pixels) + 'pw_rigid': False, # flag for performing non-rigid motion correction + } + } opts = cm.source_extraction.cnmf.params.CNMFParams(params_dict=params_dict) mc = MotionCorrect(fname, dview=None, **opts.get_group('motion')) mc.motion_correct(save_movie=True) diff --git a/caiman/tests/test_onacid.py b/caiman/tests/test_onacid.py index 7c1363319..15248251c 100644 --- a/caiman/tests/test_onacid.py +++ b/caiman/tests/test_onacid.py @@ -1,4 +1,6 @@ #!/usr/bin/env python + +import code import numpy.testing as npt import os from caiman.source_extraction import cnmf @@ -25,25 +27,46 @@ def demo(): K = 4 # max number of components in each patch params_dict = { - 'fr': fr, - 'fnames': fname, - 'decay_time': decay_time, - 'gSig': gSig, - 'p': p, - 'motion_correct': False, - 'min_SNR': min_SNR, - 'nb': gnb, - 'init_batch': init_batch, - 'init_method': init_method, - 'rf': patch_size // 2, - 'stride': stride, - 'sniper_mode': True, - 'thresh_CNN_noisy': thresh_CNN_noisy, - 'K': K + 'data': { + 'decay_time': decay_time, + 'fr': fr, + 'fnames': fname + }, + 'init': { + 'K': K, + 'gSig': gSig, + 'nb': gnb + }, + 'online': { + 'init_batch': init_batch, + 'init_method': init_method, + 'min_SNR': min_SNR, + 'motion_correct': False, + 'sniper_mode': True, + 'thresh_CNN_noisy': thresh_CNN_noisy + }, + 'patch':{ + 'rf': patch_size // 2, + 'stride': stride + }, + 'preprocess': { + 'p': p + }, + 'quality': { + 'min_SNR': min_SNR # FIXME duplicated between online.min_SNR and quality.min_SNR + }, + 'spatial': { + 'nb': gnb # FIXME duplicated between init.nb and spatial.nb and temporal.nb + }, + 'temporal': { + 'nb': gnb, # FIXME duplicated between init.nb and spatial.nb and temporal.nb + 'p': p, # FIXME duplicated between preprocess.p and temporal.p + }, } opts = cnmf.params.CNMFParams(params_dict=params_dict) cnm = cnmf.online_cnmf.OnACID(params=opts) cnm.fit_online() + #code.interact(local=dict(globals(), **locals()) ) cnm.save('test_online.hdf5') cnm2 = cnmf.online_cnmf.load_OnlineCNMF('test_online.hdf5') npt.assert_allclose(cnm.estimates.A.sum(), cnm2.estimates.A.sum()) @@ -52,4 +75,4 @@ def demo(): def test_onacid(): demo() - pass + From be0cd87311594c5445a10632324bb7bd3890abfe Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Wed, 29 Nov 2023 15:27:10 -0500 Subject: [PATCH 05/15] CNMFParams.change_params() - Add allow_legacy and warn_unused flags, internally implement set() --- caiman/source_extraction/cnmf/params.py | 40 +++++++++++++++++++------ 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 8ab8ab3e1..6400f4988 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -1076,19 +1076,41 @@ def __repr__(self) -> str: return 'CNMFParams:\n\n' + '\n\n'.join(formatted_outputs) - def change_params(self, params_dict, verbose=False) -> None: - """ Method for updating the params object by providing a single dictionary. + def change_params(self, params_dict, allow_legacy=True, warn_unused=True, verbose=False) -> None: + """ Method for updating the params object by providing a dictionary. Args: params_dict: dictionary with parameters to be changed - verbose: bool (False). If true, will complain if the params dictionary is not complete + verbose: If true, will complain if the params dictionary is not complete + allow_legacy: If True, throw a deprecation warning and then attempt to + handle unconsumed keys using the older copy-it-everywhere logic. + We will eventually remove this option and the corresponding code. + warn_unused: If True, emit warnings when the params dict has fields in it that + were never used in populating the Params object. You really should not + set this to False. Fix your code. """ - for gr in list(self.__dict__.keys()): - if gr in params_dict: - self.set(gr, params_dict[gr], verbose=verbose) - else: - if verbose: - logging.warning(f"No subobject {gr} in parameter dict") + consumed = {} # Keep track of what parameters in params_dict were used to set something in params + for paramkey in params_dict: + if paramkey in list(self.__dict__.keys()): + self.set(paramkey, params_dict[paramkey], verbose=verbose) + consumed[paramkey] = True + # BEGIN code that we will remove in some future version of caiman + if allow_legacy: + legacy_used = False + for remaining_k in params_dict: # This used to be handled by self.set() + for category in list(self.__dict__.keys()): + cat_handle = getattr(self, category) # Thankfully a read-write handle + if remaining_k in cat_handle: # Is it known? + legacy_used = True + consumed[remaining_k] = True + cat_handle[remaining_k] = params_dict[remaining_k] # Do the update + if legacy_used: + logging.warning(f"In setting CNMFParams, non-pathed parameters were used; this is deprecated. allow_legacy will default to False, and then will be removed in future versions of Caiman") + # END + if warn_unused: + for toplevel_k in params_dict: + if toplevel_k not in consumed: + logging.warning(f"In setting CNMFParams, provided toplevel key {toplevel_k} was unused. This is a bug!") self.check_consistency() def change_params_from_json(self, jsonstring:str, verbose:bool=False) -> None: From 124487fce6c2e44336a4958b9c8d85ebe0646dfd Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 1 Dec 2023 17:15:07 -0500 Subject: [PATCH 06/15] Fix documentation on CNMFParams, document some formerly-undocumented parameters --- caiman/source_extraction/cnmf/params.py | 373 +++++++++++++----------- 1 file changed, 202 insertions(+), 171 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 6400f4988..f76cce3d2 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -42,21 +42,37 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), thresh_fitness_delta=-50, thresh_fitness_raw=None, thresh_overlap=0.5, update_freq=200, update_num_comps=True, use_dense=True, use_peak_max=True, only_init_patch=True, var_name_hdf5='mov', max_merge_area=None, - use_corr_img=False, params_dict={}, + use_corr_img=False, + params_from_file:Optional[str]=None, + params_dict={}, ): """Class for setting the processing parameters. All parameters for CNMF, online-CNMF, quality testing, and motion correction can be set here and then used in the various processing pipeline steps. - The preferred way to set parameters is by using the set function, where a subclass is determined and a - dictionary is passed. The whole dictionary can also be initialized at once by passing a dictionary params_dict - when initializing the CNMFParams object. Direct setting of the positional arguments in CNMFParams is only - present for backwards compatibility reasons and should not be used if possible. + + Params have default values; users can override the defaults in two intended ways: + A) During initialisation of the object, people can pass a nested dictionary through the + params_dict parameter, or the name of a jsonfile containing the same nested dictionary + through the params_from_file parameter + B) If the CNMFParams object already exists, they can call its change_params() method to pass in + a dict or change_params_from_jsonfile() to pass in a filename + With both of these, people only need to name and override values they wish to change; all others keep + their defaults. + + All other means of changing parameters are deprecated (including other constructor arguments) + and will be removed in some future version of Caiman (whether they give a deprecation warning or not). Args: - Any parameter that is not set get a default value specified - by the dictionary default options - DATA PARAMETERS (CNMFParams.data) ##### + params_from_file + name of a json file used to initialise the object + params_dict + a dictionary used to initialise the object + + Any parameter that is not set uses a default value + All other arguments are deprecated and should not be used. - fnames: list[str] + Object Structure: + CNMFParams.data (these represent features of the data and other misc settings): + fnames list of complete paths to files that need to be processed dims: (int, int), default: computed from fnames @@ -75,10 +91,10 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), if loading from hdf5 name of the variable to load caiman_version: str - version of CaImAn being used + version of CaImAn being used. Please do not override this last_commit: str - hash of last commit in the caiman repo + hash of last commit in the caiman repo. Pleaes do not override this. mmap_F: list[str] paths to F-order memory mapped files after motion correction @@ -86,29 +102,30 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), mmap_C: str path to C-order memory mapped file after motion correction - PATCH PARAMS (CNMFParams.patch)###### - - rf: int or list or None, default: None - Half-size of patch in pixels. If None, no patches are constructed and the whole FOV is processed jointly. - If list, it should be a list of two elements corresponding to the height and width of patches - - stride: int or None, default: None - Overlap between neighboring patches in pixels. - - nb_patch: int, default: 1 - Number of (local) background components per patch - + CNMFParams.patch (these control how the data is divided into patches): border_pix: int, default: 0 Number of pixels to exclude around each border. + del_duplicates: bool, default: False + Delete duplicate components in the overlapping regions between neighboring patches. If False, + then merging is used. + + in_memory: bool, default: True + Whether to load patches in memory + low_rank_background: bool, default: True Whether to update the background using a low rank approximation. If False all the nonzero elements of the background components are updated using hals (to be used with one background per patch) - del_duplicates: bool, default: False - Delete duplicate components in the overlapping regions between neighboring patches. If False, - then merging is used. + memory_fact: float, default: 1 + unitless number for increasing the amount of available memory + + n_processes: int + Number of processes used for processing patches in parallel + + nb_patch: int, default: 1 + Number of (local) background components per patch only_init: bool, default: True whether to run only the initialization @@ -116,37 +133,37 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), p_patch: int, default: 0 order of AR dynamics when processing within a patch - skip_refinement: bool, default: False - Whether to skip refinement of components (deprecated?) - remove_very_bad_comps: bool, default: True Whether to remove (very) bad quality components during patch processing + rf: int or list or None, default: None + Half-size of patch in pixels. If None, no patches are constructed and the whole FOV is processed jointly. + If list, it should be a list of two elements corresponding to the height and width of patches + + skip_refinement: bool, default: False + Whether to skip refinement of components (deprecated?) + p_ssub: float, default: 2 Spatial downsampling factor + stride: int or None, default: None + Overlap between neighboring patches in pixels. + p_tsub: float, default: 2 Temporal downsampling factor - memory_fact: float, default: 1 - unitless number for increasing the amount of available memory - - n_processes: int - Number of processes used for processing patches in parallel - - in_memory: bool, default: True - Whether to load patches in memory - - PRE-PROCESS PARAMS (CNMFParams.preprocess) ############# + CNMFParams.preprocess (these control preprocessing steps for the data): + check_nan: bool, default: True + whether to check for NaNs - sn: np.array or None, default: None - noise level for each pixel + compute_g': bool, default: False + whether to estimate global time constant - noise_range: [float, float], default: [.25, .5] - range of normalized frequencies over which to compute the PSD for noise determination + include_noise: bool, default: False + flag for using noise values when estimating g - noise_method: 'mean'|'median'|'logmexp', default: 'mean' - PSD averaging method for computing the noise std + lags: int, default: 5 + number of lags to be considered for time constant estimation max_num_samples_fft: int, default: 3*1024 Chunk size for computing the PSD of the data (for memory considerations) @@ -154,26 +171,22 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), n_pixels_per_process: int, default: 1000 Number of pixels to be allocated to each process - compute_g': bool, default: False - whether to estimate global time constant + noise_method: 'mean'|'median'|'logmexp', default: 'mean' + PSD averaging method for computing the noise std + + noise_range: [float, float], default: [.25, .5] + range of normalized frequencies over which to compute the PSD for noise determination p: int, default: 2 order of AR indicator dynamics - lags: int, default: 5 - number of lags to be considered for time constant estimation - - include_noise: bool, default: False - flag for using noise values when estimating g - pixels: list, default: None pixels to be excluded due to saturation - check_nan: bool, default: True - whether to check for NaNs - - INIT PARAMS (CNMFParams.init)############### + sn: np.array or None, default: None + noise level for each pixel + CNMFParams.init (these control how CNMF should be initialised): K: int, default: 30 number of components to be found (per patch or whole FOV depending on whether rf=None) @@ -195,23 +208,23 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), SC_nnn: int, default: 20 number of nearest neighbors to use + alpha_snmf: float, default: 0.5 + sparse NMF sparsity regularization weight + + center_psf: bool, default: False + whether to use 1p data processing mode. Set to true for 1p + gSig: [int, int], default: [5, 5] radius of average neurons (in pixels) gSiz: [int, int], default: [int(round((x * 2) + 1)) for x in gSig], half-size of bounding box for each neuron - center_psf: bool, default: False - whether to use 1p data processing mode. Set to true for 1p - - ssub: float, default: 2 - spatial downsampling factor - - tsub: float, default: 2 - temporal downsampling factor + init_iter: int, default: 2 + number of iterations during corr_pnr (1p) initialization - nb: int, default: 1 - number of background components + kernel: np.array or None, default: None + user specified template for greedyROI lambda_gnmf: float, default: 1. regularization weight for graph NMF @@ -219,6 +232,9 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), maxIter: int, default: 5 number of HALS iterations during initialization + max_iter_snmf : int, default: 500 + maximum number of iterations for sparse NMF initialization + method_init: 'greedy_roi'|'corr_pnr'|'sparse_NMF'|'local_NMF' default: 'greedy_roi' initialization method. use 'corr_pnr' for 1p processing and 'sparse_NMF' for dendritic processing. @@ -228,55 +244,51 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), min_pnr: float, default: 20 minimum value of psnr image for determining a candidate component during corr_pnr - seed_method: str {'auto', 'manual', 'semi'} - methods for choosing seed pixels during greedy_roi or corr_pnr initialization - 'semi' detects nr components automatically and allows to add more manually - if running as notebook 'semi' and 'manual' require a backend that does not - inline figures, e.g. %matplotlib tk + nIter: int, default: 5 + number of rank-1 refinement iterations during greedy_roi initialization - ring_size_factor: float, default: 1.5 - radius of ring (*gSig) for computing background during corr_pnr + nb: int, default: 1 + number of background components - ssub_B: float, default: 2 - downsampling factor for background during corr_pnr + normalize_init: bool, default: True + whether to equalize the movies during initialization - init_iter: int, default: 2 - number of iterations during corr_pnr (1p) initialization + options_local_NMF: dict + dictionary with parameters to pass to local_NMF initializer - nIter: int, default: 5 - number of rank-1 refinement iterations during greedy_roi initialization + perc_baseline_snmf: float, default: 20 + percentile to be removed from the data in sparse_NMF prior to decomposition - rolling_sum: bool, default: True - use rolling sum (as opposed to full sum) for determining candidate centroids during greedy_roi + ring_size_factor: float, default: 1.5 + radius of ring (*gSig) for computing background during corr_pnr rolling_length: int, default: 100 width of rolling window for rolling sum option - kernel: np.array or None, default: None - user specified template for greedyROI - - max_iter_snmf : int, default: 500 - maximum number of iterations for sparse NMF initialization + rolling_sum: bool, default: True + use rolling sum (as opposed to full sum) for determining candidate centroids during greedy_roi - alpha_snmf: float, default: 0.5 - sparse NMF sparsity regularization weight + seed_method: str {'auto', 'manual', 'semi'} + methods for choosing seed pixels during greedy_roi or corr_pnr initialization + 'semi' detects nr components automatically and allows to add more manually + if running as notebook 'semi' and 'manual' require a backend that does not + inline figures, e.g. %matplotlib tk sigma_smooth_snmf : (float, float, float), default: (.5,.5,.5) std of Gaussian kernel for smoothing data in sparse_NMF - perc_baseline_snmf: float, default: 20 - percentile to be removed from the data in sparse_NMF prior to decomposition - - normalize_init: bool, default: True - whether to equalize the movies during initialization + ssub: float, default: 2 + spatial downsampling factor - options_local_NMF: dict - dictionary with parameters to pass to local_NMF initializer + ssub_B: float, default: 2 + downsampling factor for background during corr_pnr - SPATIAL PARAMS (CNMFParams.spatial) ########## + tsub: float, default: 2 + temporal downsampling factor - method_exp: 'dilate'|'ellipse', default: 'dilate' - method for expanding footprint of spatial components + CNMFParams.spatial (these control how the algorithms handle spatial components): + block_size_spat : int, default: 5000 + Number of pixels to process at the same time for dot product. Reduce if you face memory problems dist: float, default: 3 expansion factor of ellipse @@ -284,75 +296,63 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), expandCore: morphological element, default: None(?) morphological element for expanding footprints under dilate - nb: int, default: 1 - number of global background components - - n_pixels_per_process: int, default: 1000 - number of pixels to be processed by each worker - - thr_method: 'nrg'|'max', default: 'nrg' - thresholding method - - maxthr: float, default: 0.1 - Max threshold - - nrgthr: float, default: 0.9999 - Energy threshold - extract_cc: bool, default: True whether to extract connected components during thresholding (might want to turn to False for dendritic imaging) + maxthr: float, default: 0.1 + Max threshold + medw: (int, int) default: None window of median filter (set to (3,)*len(dims) in cnmf.fit) - se: np.array or None, default: None - Morphological closing structuring element (set to np.ones((3,)*len(dims), dtype=np.uint8) in cnmf.fit) - - ss: np.array or None, default: None - Binary element for determining connectivity (set to np.ones((3,)*len(dims), dtype=np.uint8) in cnmf.fit) - - update_background_components: bool, default: True - whether to update the spatial background components + method_exp: 'dilate'|'ellipse', default: 'dilate' + method for expanding footprint of spatial components method_ls: 'lasso_lars'|'nnls_L0', default: 'lasso_lars' 'nnls_L0'. Nonnegative least square with L0 penalty 'lasso_lars' lasso lars function from scikit learn - block_size : int, default: 5000 - Number of pixels to process at the same time for dot product. Reduce if you face memory problems + n_pixels_per_process: int, default: 1000 + number of pixels to be processed by each worker - num_blocks_per_run: int, default: 20 - Parallelization of A'*Y operation + nb: int, default: 1 + number of global background components normalize_yyt_one: bool, default: True Whether to normalize the C and A matrices so that diag(C*C.T) = 1 during update spatial - TEMPORAL PARAMS (CNMFParams.temporal)########### + nrgthr: float, default: 0.9999 + Energy threshold - ITER: int, default: 2 - block coordinate descent iterations + num_blocks_per_run_spat: int, default: 20 + Parallelization of A'*Y operation - method_deconvolution: 'oasis'|'cvxpy'|'oasis', default: 'oasis' - method for solving the constrained deconvolution problem ('oasis','cvx' or 'cvxpy') - if method cvxpy, primary and secondary (if problem unfeasible for approx solution) + se: np.array or None, default: None + Morphological closing structuring element (set to np.ones((3,)*len(dims), dtype=np.uint8) in cnmf.fit) - solvers: 'ECOS'|'SCS', default: ['ECOS', 'SCS'] - solvers to be used with cvxpy, can be 'ECOS','SCS' or 'CVXOPT' + ss: np.array or None, default: None + Binary element for determining connectivity (set to np.ones((3,)*len(dims), dtype=np.uint8) in cnmf.fit) - p: 0|1|2, default: 2 - order of AR indicator dynamics + thr_method: 'nrg'|'max', default: 'nrg' + thresholding method - memory_efficient: False + update_background_components: bool, default: True + whether to update the spatial background components + + + CNMFParams.temporal (these control how the algorithms handle temporal components): + ITER: int, default: 2 + block coordinate descent iterations bas_nonneg: bool, default: True whether to set a non-negative baseline (otherwise b >= min(y)) - noise_range: [float, float], default: [.25, .5] - range of normalized frequencies over which to compute the PSD for noise determination + block_size_temp : int, default: 5000 + Number of pixels to process at the same time for dot product. Reduce if you face memory problems - noise_method: 'mean'|'median'|'logmexp', default: 'mean' - PSD averaging method for computing the noise std + fudge_factor: float (close but smaller than 1) default: .96 + bias correction factor for discrete time constants lags: int, default: 5 number of autocovariance lags to be considered for time constant estimation @@ -360,29 +360,42 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), optimize_g: bool, default: False flag for optimizing time constants - fudge_factor: float (close but smaller than 1) default: .96 - bias correction factor for discrete time constants + memory_efficient: + (undocumented) + + method_deconvolution: 'oasis'|'cvxpy'|'oasis', default: 'oasis' + method for solving the constrained deconvolution problem ('oasis','cvx' or 'cvxpy') + if method cvxpy, primary and secondary (if problem unfeasible for approx solution) nb: int, default: 1 number of global background components - verbosity: bool, default: False - whether to be verbose + noise_method: 'mean'|'median'|'logmexp', default: 'mean' + PSD averaging method for computing the noise std - block_size : int, default: 5000 - Number of pixels to process at the same time for dot product. Reduce if you face memory problems + noise_range: [float, float], default: [.25, .5] + range of normalized frequencies over which to compute the PSD for noise determination - num_blocks_per_run: int, default: 20 + num_blocks_per_run_temp: int, default: 20 Parallelization of A'*Y operation + p: 0|1|2, default: 2 + order of AR indicator dynamics + s_min: float or None, default: None Minimum spike threshold amplitude (computed in the code if used). - MERGE PARAMS (CNMFParams.merge)##### + solvers: 'ECOS'|'SCS', default: ['ECOS', 'SCS'] + solvers to be used with cvxpy, can be 'ECOS','SCS' or 'CVXOPT' + + verbosity: bool, default: False + whether to be verbose + + CNMFParams.merging (these control how components are merged): do_merge: bool, default: True Whether or not to merge - thr: float, default: 0.8 + merge_thr: float, default: 0.8 Trace correlation threshold for merging two components. merge_parallel: bool, default: False @@ -391,46 +404,50 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), max_merge_area: int or None, default: None maximum area (in pixels) of merged components, used to determine whether to merge components during fitting process - QUALITY EVALUATION PARAMETERS (CNMFParams.quality)########### + CNMFParams.quality (these control how quality of traces are evaluated): + SNR_lowest: float, default: 0.5 + minimum required trace SNR. Traces with SNR below this will get rejected + + cnn_lowest: float, default: 0.1 + minimum required CNN threshold. Components with score lower than this will get rejected. + + gSig_range: list or integers, default: None + gSig scale values for CNN classifier. In not None, multiple values are tested in the CNN classifier. min_SNR: float, default: 2.5 trace SNR threshold. Traces with SNR above this will get accepted - SNR_lowest: float, default: 0.5 - minimum required trace SNR. Traces with SNR below this will get rejected - - rval_thr: float, default: 0.8 - space correlation threshold. Components with correlation higher than this will get accepted + min_cnn_thr: float, default: 0.9 + CNN classifier threshold. Components with score higher than this will get accepted rval_lowest: float, default: -1 minimum required space correlation. Components with correlation below this will get rejected + rval_thr: float, default: 0.8 + space correlation threshold. Components with correlation higher than this will get accepted + use_cnn: bool, default: True flag for using the CNN classifier. - min_cnn_thr: float, default: 0.9 - CNN classifier threshold. Components with score higher than this will get accepted + use_ecc: + (undocumented) - cnn_lowest: float, default: 0.1 - minimum required CNN threshold. Components with score lower than this will get rejected. - - gSig_range: list or integers, default: None - gSig scale values for CNN classifier. In not None, multiple values are tested in the CNN classifier. - - ONLINE CNMF (ONACID) PARAMETERS (CNMFParams.online)##### + max_ecc: + (undocumented) + CNMFParams.online (these control the Online/OnACID mode): N_samples_exceptionality: int, default: np.ceil(decay_time*fr), Number of frames over which trace SNR is computed (usually length of a typical transient) batch_update_suff_stat: bool, default: False Whether to update sufficient statistics in batch mode - ds_factor: int, default: 1, - spatial downsampling factor for faster processing (if > 1) - dist_shape_update: bool, default: False, update shapes in a distributed fashion + ds_factor: int, default: 1, + spatial downsampling factor for faster processing (if > 1) + epochs: int, default: 1, number of times to go over data @@ -484,6 +501,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), Number of additional iterations for computing traces num_times_comp_updated: int, default: np.inf + (undocumented) opencv_codec: str, default: 'H264' FourCC video codec for saving movie. Check http://www.fourcc.org/codecs.php @@ -491,6 +509,9 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), path_to_model: str, default: os.path.join(caiman_datadir(), 'model', 'cnn_model_online.h5') Path to online CNN classifier + ring_CNN: + Whether to use a ring CNN model (XXX due to bugs, this flag may not work and may never have worked) + rval_thr: float, default: 0.8 space correlation threshold for accepting a new component @@ -507,6 +528,9 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), Whether to use the online CNN classifier for screening candidate components (otherwise space correlation is used) + stop_detection: + Stop detecting neurons at the last epoch (XXX what does this mean?) + test_both: bool, default: False Whether to use both the CNN and space correlation for screening new components @@ -528,14 +552,19 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), update_num_comps: bool, default: True Whether to search for new components + use_corr_img: + Use correlation image to detect new components + use_dense: bool, default: True Whether to store and represent A and b as a dense matrix use_peak_max: bool, default: True Whether to find candidate centroids using skimage's find local peaks function - MOTION CORRECTION PARAMETERS (CNMFParams.motion)#### + W_update_factor: + Update W less often than shapes by a given factor (XXX does this work?) + CNMFParams.motion (these control motion-correction): border_nan: bool or str, default: 'copy' flag for allowing NaN in the boundaries. True allows NaN, whereas 'copy' copies the value of the nearest data point. @@ -566,6 +595,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), num_splits_to_process_els, default: [7, None] num_splits_to_process_rig, default: None + (Undocumented, changing this likely to break the code - FIXME why is this a parameter then?) overlaps: (int, int), default: (24, 24) overlap between patches in pixels in pw-rigid motion correction. @@ -594,8 +624,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), indices: tuple(slice), default: (slice(None), slice(None)) Use that to apply motion correction only on a part of the FOV - RING CNN PARAMETERS (CNMFParams.ring_CNN) - + CNMFParams.ring_CNN (these control the ring neural networks): n_channels: int, default: 2 Number of "ring" kernels @@ -885,6 +914,8 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), 'reuse_model': False # reuse an already trained model } + if params_from_file is not None: + self.change_params_from_jsonfile(params_from_file) self.change_params(params_dict) From e3018c9d618965c7ecf4de651f7df96fd6b75cf4 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 1 Dec 2023 19:56:26 -0500 Subject: [PATCH 07/15] params.py: import optional --- caiman/source_extraction/cnmf/params.py | 1 + 1 file changed, 1 insertion(+) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index f76cce3d2..a8a623b58 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -8,6 +8,7 @@ from pprint import pformat import scipy from scipy.ndimage import generate_binary_structure, iterate_structure +from typing import Optional import caiman.utils.utils import caiman.base.movies From 92505b181bb29ae2457779cb3d53f947e697491c Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 4 Dec 2023 17:25:29 -0500 Subject: [PATCH 08/15] CNMFParams.change_params() - Don't use set(), and track unused params everywhere --- caiman/source_extraction/cnmf/params.py | 27 ++++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index a8a623b58..da9dd5e19 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -1123,21 +1123,24 @@ def change_params(self, params_dict, allow_legacy=True, warn_unused=True, verbos """ consumed = {} # Keep track of what parameters in params_dict were used to set something in params for paramkey in params_dict: - if paramkey in list(self.__dict__.keys()): - self.set(paramkey, params_dict[paramkey], verbose=verbose) - consumed[paramkey] = True - # BEGIN code that we will remove in some future version of caiman - if allow_legacy: - legacy_used = False - for remaining_k in params_dict: # This used to be handled by self.set() + if paramkey in list(self.__dict__.keys()): # Proper pathed part + cat_handle = getattr(self, paramkey) + for k, v in params_dict[paramkey].items(): + if k not in cat_handle and warn_unused: + logging.warning(f"In setting CNMFParams, provided key {paramkey}/{k} was not consumed. This is a bug!") + else: + cat_handle[k] = v + # BEGIN code that we will remove in some future version of caiman + elif allow_legacy: for category in list(self.__dict__.keys()): cat_handle = getattr(self, category) # Thankfully a read-write handle - if remaining_k in cat_handle: # Is it known? + if paramkey in cat_handle: # Is it known? legacy_used = True - consumed[remaining_k] = True - cat_handle[remaining_k] = params_dict[remaining_k] # Do the update - if legacy_used: - logging.warning(f"In setting CNMFParams, non-pathed parameters were used; this is deprecated. allow_legacy will default to False, and then will be removed in future versions of Caiman") + consumed[paramkey] = True + cat_handle[paramkey] = params_dict[paramkey] # Do the update + if legacy_used: + logging.warning(f"In setting CNMFParams, non-pathed parameters were used; this is deprecated. allow_legacy will default to False, and then will be removed in future versions of Caiman") + # END if warn_unused: for toplevel_k in params_dict: From 9258508c44e05db2246e3f75098db14a36f3e235 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 4 Dec 2023 17:36:26 -0500 Subject: [PATCH 09/15] Fix a goof in the warning infrastructure --- caiman/source_extraction/cnmf/params.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index da9dd5e19..14779894f 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -1140,11 +1140,10 @@ def change_params(self, params_dict, allow_legacy=True, warn_unused=True, verbos cat_handle[paramkey] = params_dict[paramkey] # Do the update if legacy_used: logging.warning(f"In setting CNMFParams, non-pathed parameters were used; this is deprecated. allow_legacy will default to False, and then will be removed in future versions of Caiman") - # END if warn_unused: for toplevel_k in params_dict: - if toplevel_k not in consumed: + if toplevel_k not in consumed and toplevel_k not in list(self.__dict__.keys()): logging.warning(f"In setting CNMFParams, provided toplevel key {toplevel_k} was unused. This is a bug!") self.check_consistency() From 6e92120d7fedc9603d6a8cab41d836fbfb8d3f6f Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 5 Dec 2023 12:01:42 -0500 Subject: [PATCH 10/15] CNMFParams: more typing, some docs/message tweaks, mark set_if_not_exists in set method as deprecated --- caiman/source_extraction/cnmf/params.py | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 14779894f..90d788093 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -984,16 +984,23 @@ def check_consistency(self): logging.warning(key + "=0, hence setting key update_num_comps " + "in group online automatically to False.") - def set(self, group, val_dict, set_if_not_exists=False, verbose=False): + def set(self, group:str, val_dict:dict, set_if_not_exists:bool=False, verbose=False) -> None: """ Add key-value pairs to a group. Existing key-value pairs will be overwritten if specified in val_dict, but not deleted. Args: - group: The name of the group. - val_dict: A dictionary with key-value pairs to be set for the group. - set_if_not_exists: Whether to set a key-value pair in a group if the key does not currently exist in the group. + group: The name of the group + val_dict: A dictionary with key-value pairs to be set for the group + warn_unused: + set_if_not_exists: Whether to set a key-value pair in a group if the key does not currently exist in the group. (DEPRECATED) """ + if set_if_not_exists: + logging.warning("The set_if_not_exists flag for CNMFParams.set() is deprecated and will be removed in a future version of Caiman") + # can't easily catch if it's passed but set to False, but that wouldn't do anything because of the default, + # and if they get that error it's at least really easy to fix - just remove the flag + # we don't want to support this because it makes the structure of the object unpredictable except at runtime + if not hasattr(self, group): raise KeyError(f'No group in CNMFParams named {group}') @@ -1002,7 +1009,7 @@ def set(self, group, val_dict, set_if_not_exists=False, verbose=False): if k not in d and not set_if_not_exists: if verbose: logging.warning( - f"NOT setting value of key {k} in group {group}, because no prior key existed...") + f"{group}/{k} not set: invalid target in CNMFParams object") else: try: if np.any(d[k] != v): @@ -1108,7 +1115,7 @@ def __repr__(self) -> str: return 'CNMFParams:\n\n' + '\n\n'.join(formatted_outputs) - def change_params(self, params_dict, allow_legacy=True, warn_unused=True, verbose=False) -> None: + def change_params(self, params_dict, allow_legacy:bool=True, warn_unused:bool=True, verbose:bool=False) -> None: """ Method for updating the params object by providing a dictionary. Args: @@ -1121,12 +1128,15 @@ def change_params(self, params_dict, allow_legacy=True, warn_unused=True, verbos were never used in populating the Params object. You really should not set this to False. Fix your code. """ - consumed = {} # Keep track of what parameters in params_dict were used to set something in params + # When we're ready to remove allow_legacy, this code will get a lot simpler + + consumed = {} # Keep track of what parameters in params_dict were used to set something in params (just for legacy API) for paramkey in params_dict: if paramkey in list(self.__dict__.keys()): # Proper pathed part cat_handle = getattr(self, paramkey) for k, v in params_dict[paramkey].items(): if k not in cat_handle and warn_unused: + # For regular/pathed API, we can notice right away if the user gave us something that won't update the object logging.warning(f"In setting CNMFParams, provided key {paramkey}/{k} was not consumed. This is a bug!") else: cat_handle[k] = v @@ -1143,7 +1153,7 @@ def change_params(self, params_dict, allow_legacy=True, warn_unused=True, verbos # END if warn_unused: for toplevel_k in params_dict: - if toplevel_k not in consumed and toplevel_k not in list(self.__dict__.keys()): + if toplevel_k not in consumed and toplevel_k not in list(self.__dict__.keys()): # When we remove legacy behaviour, this logic will simplify and fold into above logging.warning(f"In setting CNMFParams, provided toplevel key {toplevel_k} was unused. This is a bug!") self.check_consistency() From 6894829c8715ee454a74bb23b989ef6eaf3bd3b7 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Wed, 31 Jan 2024 15:44:19 -0500 Subject: [PATCH 11/15] num_splits_to_process_els: note that it does nothing, remove some plumbing around it --- caiman/motion_correction.py | 6 ++---- caiman/source_extraction/cnmf/params.py | 9 ++++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/caiman/motion_correction.py b/caiman/motion_correction.py index aea84698c..9ec1af746 100644 --- a/caiman/motion_correction.py +++ b/caiman/motion_correction.py @@ -131,9 +131,8 @@ def __init__(self, fname, min_mov=None, dview=None, max_shifts=(6, 6), niter_rig splits_els':list for parallelization split the movies in num_splits chunks across time - num_splits_to_process_els: list, - if none all the splits are processed and the movie is saved otherwise at each iteration - num_splits_to_process_els are considered + num_splits_to_process_els: UNUSED + Legacy parameter, does not do anything upsample_factor_grid:int, upsample factor of shifts per patches to avoid smearing when merging patches @@ -192,7 +191,6 @@ def __init__(self, fname, min_mov=None, dview=None, max_shifts=(6, 6), niter_rig self.strides = strides self.overlaps = overlaps self.splits_els = splits_els - self.num_splits_to_process_els = num_splits_to_process_els self.upsample_factor_grid = upsample_factor_grid self.max_deviation_rigid = max_deviation_rigid self.shifts_opencv = bool(shifts_opencv) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 90d788093..9661ec76b 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -594,7 +594,6 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), num_frames_split: int, default: 80 split movie every x frames for parallel processing - num_splits_to_process_els, default: [7, None] num_splits_to_process_rig, default: None (Undocumented, changing this likely to break the code - FIXME why is this a parameter then?) @@ -608,10 +607,10 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), flag for applying shifts using cubic interpolation (otherwise FFT) splits_els: int, default: 14 - number of splits across time for pw-rigid registration + number of splits across time for pw-rigid registration. splits_rig: int, default: 14 - number of splits across time for rigid registration + number of splits across time for rigid registration. strides: (int, int), default: (96, 96) how often to start a new patch in pw-rigid registration. Size of each patch will be strides + overlaps @@ -667,7 +666,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), Flag for reusing an already trained model (saved in path to model) """ - if decay_time == 0 or decay_time == 0.0: + if int(decay_time) == 0: raise Exception("A decay time of 0 is not permitted") self.data = { @@ -886,7 +885,7 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), 'niter_rig': 1, # number of iterations rigid motion correction 'nonneg_movie': True, # flag for producing a non-negative movie 'num_frames_split': 80, # split across time every x frames - 'num_splits_to_process_els': None, # DO NOT MODIFY + 'num_splits_to_process_els': None, # Unused, will be removed in a future version of Caiman 'num_splits_to_process_rig': None, # DO NOT MODIFY 'overlaps': (32, 32), # overlap between patches in pw-rigid motion correction 'pw_rigid': False, # flag for performing pw-rigid motion correction From 0d0ed6cc5e35df4bf027963db841bfc03f0fb56f Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Wed, 31 Jan 2024 15:44:46 -0500 Subject: [PATCH 12/15] Fix typo --- caiman/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index d6aa077af..182827554 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -650,7 +650,7 @@ def get_caiman_version() -> tuple[str, str]: # 'GITW' ) git rev-parse if caiman is built from "pip install -e ." and we are working # out of the checkout directory (the user may have since updated without reinstall) # 'RELF') A release file left in the process to cut a release. Should have a single line - # in it whick looks like "Version:1.4" + # in it which looks like "Version:1.4" # 'FILE') The date of some frequently changing files, which act as a very rough # approximation when no other methods are possible # From cb0f53f51b3e4b82c9a3f981539b216cc794b71f Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Wed, 31 Jan 2024 15:56:09 -0500 Subject: [PATCH 13/15] cnmf/params.py: Improve zero check for decay time --- caiman/source_extraction/cnmf/params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 9661ec76b..771a7a46b 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -666,8 +666,8 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), Flag for reusing an already trained model (saved in path to model) """ - if int(decay_time) == 0: - raise Exception("A decay time of 0 is not permitted") + if float(decay_time) == float(0.0): + raise Exception("A decay time of zero is not permitted") self.data = { 'fnames': fnames, From d61820209f6aef1f7f2072fb5674e55ee14d6ea4 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 27 Feb 2024 15:05:52 -0500 Subject: [PATCH 14/15] CNMFParams: nb parameter isn't actually used outside of the version in init --- caiman/source_extraction/cnmf/params.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index 771a7a46b..c5b8dfac2 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -770,7 +770,6 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), 'method_ls': 'lasso_lars', # number of pixels to be processed by each worker 'n_pixels_per_process': n_pixels_per_process, - 'nb': gnb, # number of background components 'normalize_yyt_one': True, 'nrgthr': 0.9999, # Energy threshold 'num_blocks_per_run_spat': num_blocks_per_run_spat, # number of process to parallelize residual computation ** DECREASE IF MEMORY ISSUES @@ -798,7 +797,6 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), # if method cvxpy, primary and secondary (if problem unfeasible for approx # solution) solvers to be used with cvxpy, can be 'ECOS','SCS' or 'CVXOPT' 'method_deconvolution': method_deconvolution, # 'cvxpy', # 'oasis' - 'nb': gnb, # number of background components 'noise_method': 'mean', # averaging method ('mean','median','logmexp') 'noise_range': [.25, .5], # range of normalized frequencies over which to average 'num_blocks_per_run_temp': num_blocks_per_run_temp, # number of process to parallelize residual computation ** DECREASE IF MEMORY ISSUES From bfc69dcff2d6ba9e9ca3387c573f0fa1d87438e7 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 27 Feb 2024 15:07:31 -0500 Subject: [PATCH 15/15] CNMFParams: Update docs to note nb removed from temporal/spatial --- caiman/source_extraction/cnmf/params.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/caiman/source_extraction/cnmf/params.py b/caiman/source_extraction/cnmf/params.py index c5b8dfac2..50fe3f500 100644 --- a/caiman/source_extraction/cnmf/params.py +++ b/caiman/source_extraction/cnmf/params.py @@ -317,9 +317,6 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), n_pixels_per_process: int, default: 1000 number of pixels to be processed by each worker - nb: int, default: 1 - number of global background components - normalize_yyt_one: bool, default: True Whether to normalize the C and A matrices so that diag(C*C.T) = 1 during update spatial @@ -368,9 +365,6 @@ def __init__(self, fnames=None, dims=None, dxy=(1, 1), method for solving the constrained deconvolution problem ('oasis','cvx' or 'cvxpy') if method cvxpy, primary and secondary (if problem unfeasible for approx solution) - nb: int, default: 1 - number of global background components - noise_method: 'mean'|'median'|'logmexp', default: 'mean' PSD averaging method for computing the noise std