From 7cb4bd2ba363e3426bce1ef4a6ea5f9d83d7c2ba Mon Sep 17 00:00:00 2001 From: Graham Findlay Date: Tue, 17 Nov 2026 23:06:37 -0600 Subject: [PATCH] Default Kilosort 2.5 / 3 batch size to 64 * 1024 - ntbuff Kilosort 2.5.2 and 3.0.2 expect this value (or another multiple of 1024, minus ntbuff). The old default left a hole of missed spikes at the end of every batch. Kilosort installs from before the fix keep the old value (because it is less harmful than the new one if you don't have the fix) and get a warning. --- .../sorters/external/kilosort2_5.py | 11 +++++----- .../sorters/external/kilosort3.py | 11 +++++----- .../sorters/external/kilosortbase.py | 21 +++++++++++++++++++ .../external/tests/test_kilosort2_5.py | 16 ++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/spikeinterface/sorters/external/kilosort2_5.py b/src/spikeinterface/sorters/external/kilosort2_5.py index 23359796a1..66d6377c43 100644 --- a/src/spikeinterface/sorters/external/kilosort2_5.py +++ b/src/spikeinterface/sorters/external/kilosort2_5.py @@ -78,7 +78,10 @@ class Kilosort2_5Sorter(KilosortBase, BaseSorter): "ntbuff": "Samples of symmetrical buffer for whitening and spike detection", "nfilt_factor": "Max number of clusters per good channel (even temporary ones) 4", "do_correction": "If True drift registration is applied", - "NT": "Batch size (if None it is automatically computed)", + "NT": "Batch size. If None, 64 * 1024 - ntbuff for Kilosort >= 2.5.2 " + "(64 * 1024 + ntbuff for Kilosort < 2.5.2). " + "Change this value with care: use a multiple of 1024 minus ntbuff (e.g. 32 * 1024 - ntbuff) " + "to avoid spike holes (https://github.com/MouseLand/Kilosort/issues/594).", "AUCsplit": "Threshold on the area under the curve (AUC) criterion for performing a split in the final step", "keep_good_only": "If True only 'good' units are returned", "wave_length": "size of the waveform extracted around each detected peak, (Default 61, maximum 81)", @@ -142,7 +145,7 @@ def set_kilosort2_5_path(kilosort2_5_path: str | Path): def _check_params(cls, recording, output_folder, params): p = params if p["NT"] is None: - p["NT"] = 64 * 1024 + p["ntbuff"] + p["NT"] = cls._get_default_batch_size(p["ntbuff"]) else: p["NT"] = p["NT"] // 32 * 32 # make sure is multiple of 32 if p["wave_length"] % 2 != 1: @@ -213,9 +216,7 @@ def _get_specific_options(cls, ops, params) -> dict: # ops['Nfilt'] = 1024 # max number of clusters ops["nfilt_factor"] = params["nfilt_factor"] # max number of clusters per good channel (even temporary ones) ops["ntbuff"] = params["ntbuff"] # samples of symmetrical buffer for whitening and spike detection - ops["NT"] = params[ - "NT" - ] # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). + ops["NT"] = params["NT"] # batch size (try decreasing if out of memory); poor choices cause "spike holes" ops["whiteningRange"] = params["whiteningRange"] # number of channels to use for whitening each channel ops["nSkipCov"] = 25.0 # compute whitening matrix from every N-th batch ops["nPCs"] = params["nPCs"] # how many PCs to project the spikes into diff --git a/src/spikeinterface/sorters/external/kilosort3.py b/src/spikeinterface/sorters/external/kilosort3.py index be405ae5fc..9424bbe409 100644 --- a/src/spikeinterface/sorters/external/kilosort3.py +++ b/src/spikeinterface/sorters/external/kilosort3.py @@ -74,7 +74,10 @@ class Kilosort3Sorter(KilosortBase, BaseSorter): "ntbuff": "Samples of symmetrical buffer for whitening and spike detection", "nfilt_factor": "Max number of clusters per good channel (even temporary ones) 4", "do_correction": "If True drift registration is applied", - "NT": "Batch size (if None it is automatically computed--recommended Kilosort behavior if ntbuff also not changed)", + "NT": "Batch size. If None, 64 * 1024 - ntbuff for Kilosort >= 3.0.2 " + "(64 * 1024 + ntbuff for Kilosort < 3.0.2). " + "Change this value with care: use a multiple of 1024 minus ntbuff (e.g. 32 * 1024 - ntbuff) " + "to avoid spike holes (https://github.com/MouseLand/Kilosort/issues/594).", "AUCsplit": "Threshold on the area under the curve (AUC) criterion for performing a split in the final step", "wave_length": "size of the waveform extracted around each detected peak, (Default 61, maximum 81)", "keep_good_only": "If True only 'good' units are returned", @@ -138,7 +141,7 @@ def set_kilosort3_path(kilosort3_path: str | Path): def _check_params(cls, recording, output_folder, params): p = params if p["NT"] is None: - p["NT"] = 64 * 1024 + p["ntbuff"] + p["NT"] = cls._get_default_batch_size(p["ntbuff"]) else: p["NT"] = p["NT"] // 32 * 32 # make sure is multiple of 32 if p["wave_length"] % 2 != 1: @@ -206,9 +209,7 @@ def _get_specific_options(cls, ops, params) -> dict: # ops['Nfilt'] = 1024 # max number of clusters ops["nfilt_factor"] = params["nfilt_factor"] # max number of clusters per good channel (even temporary ones) ops["ntbuff"] = params["ntbuff"] # samples of symmetrical buffer for whitening and spike detection - ops["NT"] = params[ - "NT" - ] # must be multiple of 32 + ntbuff. This is the batch size (try decreasing if out of memory). + ops["NT"] = params["NT"] # batch size (try decreasing if out of memory); poor choices cause "spike holes" ops["whiteningRange"] = params["whiteningRange"] # number of channels to use for whitening each channel ops["nSkipCov"] = 25.0 # compute whitening matrix from every N-th batch ops["scaleproc"] = 200.0 # int16 scaling of whitened data diff --git a/src/spikeinterface/sorters/external/kilosortbase.py b/src/spikeinterface/sorters/external/kilosortbase.py index 68af4fa61e..c53c3fbc2f 100644 --- a/src/spikeinterface/sorters/external/kilosortbase.py +++ b/src/spikeinterface/sorters/external/kilosortbase.py @@ -118,6 +118,27 @@ def _get_specific_options(cls, ops, params): """Specific options should be implemented in subclass""" return ops + @classmethod + def _get_default_batch_size(cls, ntbuff): + """ + Default `NT` for Kilosort 2.5 and 3. + + Since the spike holes fix (https://github.com/MouseLand/Kilosort/issues/594, v2.5.2 and v3.0.2) + Kilosort needs `64 * 1024 - ntbuff`. Installs from before the fix need `64 * 1024 + ntbuff`. + """ + sorter_path = getattr(cls, f"{cls.sorter_name}_path", None) + if sorter_path is not None and not cls.check_compiled(): + track_and_sort = Path(str(sorter_path).strip('"')) / "mainLoop" / "trackAndSort.m" + # the fix added `ntpad` to the batch loading + if track_and_sort.is_file() and "ntpad" not in track_and_sort.read_text(errors="ignore"): + warn( + f"The Kilosort install at {sorter_path} predates the fix for spike holes " + "(https://github.com/MouseLand/Kilosort/issues/594) and misses spikes at the edges of every " + "batch. Update Kilosort to get rid of them." + ) + return 64 * 1024 + ntbuff + return 64 * 1024 - ntbuff + @classmethod def _setup_recording(cls, recording, sorter_output_folder, params, verbose): cls._generate_channel_map_file(recording, sorter_output_folder) diff --git a/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py b/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py index 8f7c7939a5..02515560b4 100644 --- a/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py +++ b/src/spikeinterface/sorters/external/tests/test_kilosort2_5.py @@ -18,6 +18,22 @@ class Kilosort2_5CommonTestSuite(SorterCommonTestSuite, unittest.TestCase): SorterClass = Kilosort2_5Sorter +def test_default_batch_size(tmp_path, monkeypatch): + monkeypatch.setattr(Kilosort2_5Sorter, "check_compiled", classmethod(lambda cls: False)) + monkeypatch.setattr(Kilosort2_5Sorter, "kilosort2_5_path", str(tmp_path)) + track_and_sort = tmp_path / "mainLoop" / "trackAndSort.m" + track_and_sort.parent.mkdir() + + # batch loading from before the spike holes fix + track_and_sort.write_text("batchstart = 0:NT:NT*nBatches;") + with pytest.warns(UserWarning, match="spike holes"): + assert Kilosort2_5Sorter._get_default_batch_size(ntbuff=64) == 64 * 1024 + 64 + + # and after it (v2.5.2) + track_and_sort.write_text("ntpad = rez.ops.ntbuff;\nbatchstart = ntpad:NT:NT*nBatches;") + assert Kilosort2_5Sorter._get_default_batch_size(ntbuff=64) == 64 * 1024 - 64 + + if __name__ == "__main__": test = Kilosort2_5CommonTestSuite() test.setUp()