diff --git a/src/spikeinterface/sorters/external/kilosort2_5.py b/src/spikeinterface/sorters/external/kilosort2_5.py index 23359796a1..66d6377c43 100644 --- a/src/spikeinterface/sorters/external/kilosort2_5.py +++ b/src/spikeinterface/sorters/external/kilosort2_5.py @@ -78,7 +78,10 @@ class Kilosort2_5Sorter(KilosortBase, BaseSorter): "ntbuff": "Samples of symmetrical buffer for whitening and spike detection", "nfilt_factor": "Max number of clusters per good channel (even temporary ones) 4", "do_correction": "If True drift registration is applied", - "NT": "Batch size (if None it is automatically computed)", + "NT": "Batch size. If None, 64 * 1024 - ntbuff for Kilosort >= 2.5.2 " + "(64 * 1024 + ntbuff for Kilosort < 2.5.2). " + "Change this value with care: use a multiple of 1024 minus ntbuff (e.g. 32 * 1024 - ntbuff) " + "to avoid spike holes (https://github.com/MouseLand/Kilosort/issues/594).", "AUCsplit": "Threshold on the area under the curve (AUC) criterion for performing a split in the final step", "keep_good_only": "If True only 'good' units are returned", "wave_length": "size of the waveform extracted around each detected peak, (Default 61, maximum 81)", @@ -142,7 +145,7 @@ def set_kilosort2_5_path(kilosort2_5_path: str | Path): def _check_params(cls, recording, output_folder, params): p = params if p["NT"] is None: - p["NT"] = 64 * 1024 + p["ntbuff"] + p["NT"] = cls._get_default_batch_size(p["ntbuff"]) else: p["NT"] = p["NT"] // 32 * 32 # make sure is multiple of 32 if p["wave_length"] % 2 != 1: @@ -213,9 +216,7 @@ def _get_specific_options(cls, ops, params) -> dict: # ops['Nfilt'] = 1024 # max number of clusters ops["nfilt_factor"] = params["nfilt_factor"] # max number of clusters per good channel (even temporary ones) ops["ntbuff"] = params["ntbuff"] # samples of symmetrical buffer for whitening and spike detection - ops["NT"] = params[ - "NT" - ] # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). + ops["NT"] = params["NT"] # batch size (try decreasing if out of memory); poor choices cause "spike holes" ops["whiteningRange"] = params["whiteningRange"] # number of channels to use for whitening each channel ops["nSkipCov"] = 25.0 # compute whitening matrix from every N-th batch ops["nPCs"] = params["nPCs"] # how many PCs to project the spikes into diff --git a/src/spikeinterface/sorters/external/kilosort3.py b/src/spikeinterface/sorters/external/kilosort3.py index be405ae5fc..9424bbe409 100644 --- a/src/spikeinterface/sorters/external/kilosort3.py +++ b/src/spikeinterface/sorters/external/kilosort3.py @@ -74,7 +74,10 @@ class Kilosort3Sorter(KilosortBase, BaseSorter): "ntbuff": "Samples of symmetrical buffer for whitening and spike detection", "nfilt_factor": "Max number of clusters per good channel (even temporary ones) 4", "do_correction": "If True drift registration is applied", - "NT": "Batch size (if None it is automatically computed--recommended Kilosort behavior if ntbuff also not changed)", + "NT": "Batch size. If None, 64 * 1024 - ntbuff for Kilosort >= 3.0.2 " + "(64 * 1024 + ntbuff for Kilosort < 3.0.2). " + "Change this value with care: use a multiple of 1024 minus ntbuff (e.g. 32 * 1024 - ntbuff) " + "to avoid spike holes (https://github.com/MouseLand/Kilosort/issues/594).", "AUCsplit": "Threshold on the area under the curve (AUC) criterion for performing a split in the final step", "wave_length": "size of the waveform extracted around each detected peak, (Default 61, maximum 81)", "keep_good_only": "If True only 'good' units are returned", @@ -138,7 +141,7 @@ def set_kilosort3_path(kilosort3_path: str | Path): def _check_params(cls, recording, output_folder, params): p = params if p["NT"] is None: - p["NT"] = 64 * 1024 + p["ntbuff"] + p["NT"] = cls._get_default_batch_size(p["ntbuff"]) else: p["NT"] = p["NT"] // 32 * 32 # make sure is multiple of 32 if p["wave_length"] % 2 != 1: @@ -206,9 +209,7 @@ def _get_specific_options(cls, ops, params) -> dict: # ops['Nfilt'] = 1024 # max number of clusters ops["nfilt_factor"] = params["nfilt_factor"] # max number of clusters per good channel (even temporary ones) ops["ntbuff"] = params["ntbuff"] # samples of symmetrical buffer for whitening and spike detection - ops["NT"] = params[ - "NT" - ] # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). + ops["NT"] = params["NT"] # batch size (try decreasing if out of memory); poor choices cause "spike holes" ops["whiteningRange"] = params["whiteningRange"] # number of channels to use for whitening each channel ops["nSkipCov"] = 25.0 # compute whitening matrix from every N-th batch ops["scaleproc"] = 200.0 # int16 scaling of whitened data diff --git a/src/spikeinterface/sorters/external/kilosortbase.py b/src/spikeinterface/sorters/external/kilosortbase.py index 68af4fa61e..c53c3fbc2f 100644 --- a/src/spikeinterface/sorters/external/kilosortbase.py +++ b/src/spikeinterface/sorters/external/kilosortbase.py @@ -118,6 +118,27 @@ def _get_specific_options(cls, ops, params): """Specific options should be implemented in subclass""" return ops + @classmethod + def _get_default_batch_size(cls, ntbuff): + """ + Default `NT` for Kilosort 2.5 and 3. + + Since the spike holes fix (https://github.com/MouseLand/Kilosort/issues/594, v2.5.2 and v3.0.2) + Kilosort needs `64 * 1024 - ntbuff`. Installs from before the fix need `64 * 1024 + ntbuff`. + """ + sorter_path = getattr(cls, f"{cls.sorter_name}_path", None) + if sorter_path is not None and not cls.check_compiled(): + track_and_sort = Path(str(sorter_path).strip('"')) / "mainLoop" / "trackAndSort.m" + # the fix added `ntpad` to the batch loading + if track_and_sort.is_file() and "ntpad" not in track_and_sort.read_text(errors="ignore"): + warn( + f"The Kilosort install at {sorter_path} predates the fix for spike holes " + "(https://github.com/MouseLand/Kilosort/issues/594) and misses spikes at the edges of every " + "batch. Update Kilosort to get rid of them." + ) + return 64 * 1024 + ntbuff + return 64 * 1024 - ntbuff + @classmethod def _setup_recording(cls, recording, sorter_output_folder, params, verbose): cls._generate_channel_map_file(recording, sorter_output_folder) diff --git a/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py b/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py index 8f7c7939a5..02515560b4 100644 --- a/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py +++ b/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py @@ -18,6 +18,22 @@ class Kilosort2_5CommonTestSuite(SorterCommonTestSuite, unittest.TestCase): SorterClass = Kilosort2_5Sorter +def test_default_batch_size(tmp_path, monkeypatch): + monkeypatch.setattr(Kilosort2_5Sorter, "check_compiled", classmethod(lambda cls: False)) + monkeypatch.setattr(Kilosort2_5Sorter, "kilosort2_5_path", str(tmp_path)) + track_and_sort = tmp_path / "mainLoop" / "trackAndSort.m" + track_and_sort.parent.mkdir() + + # batch loading from before the spike holes fix + track_and_sort.write_text("batchstart = 0:NT:NT*nBatches;") + with pytest.warns(UserWarning, match="spike holes"): + assert Kilosort2_5Sorter._get_default_batch_size(ntbuff=64) == 64 * 1024 + 64 + + # and after it (v2.5.2) + track_and_sort.write_text("ntpad = rez.ops.ntbuff;\nbatchstart = ntpad:NT:NT*nBatches;") + assert Kilosort2_5Sorter._get_default_batch_size(ntbuff=64) == 64 * 1024 - 64 + + if __name__ == "__main__": test = Kilosort2_5CommonTestSuite() test.setUp()