Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 85 additions & 77 deletions src/spikeinterface/sortingcomponents/matching/circus.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ class CircusOMPPeeler(BaseTemplateMatching):
is convolved with the templates, and as long as some scalar products
are higher than a given threshold, we use a Cholesky decomposition
to compute the optimal amplitudes needed to reconstruct the signal.
With a vicinity, each selection only re-fits the nearby selected
templates to the residual, as in LocOMP (Mailhé et al., ICASSP 2009,
https://hal.science/hal-00461092).

IMPORTANT NOTE: small chunks are more efficient for such Peeler,
consider using 100ms chunk
Expand All @@ -125,14 +128,12 @@ class CircusOMPPeeler(BaseTemplateMatching):
Stopping criteria of the OMP algorithm, as number of retry while updating amplitudes
rank : int, default: 5
Number of components used internally by the SVD
vicinity : int
Size of the area surrounding a spike to perform modification (expressed in terms
of template temporal width)
vicinity : int, default: 2
Size of the window around a spike, in template widths. A pass selects at most one spike per
window, and each selection re-fits the spikes already selected in its window that share a
channel with it. With 0, each selection re-fits all spikes selected so far
ignore_inds : list
List of template indices to ignore during the matching
vicinity: int
Size of the area surrounding a spike to perform modification (expressed in terms
of template temporal width)
precomputed : dict | None
If not None, a dict with precomputed values for the templates
engine : string in ["numpy", "torch", "auto"]. Default "auto"
Expand Down Expand Up @@ -227,10 +228,10 @@ def __init__(

self.ignore_inds = np.array(ignore_inds)

self.unit_overlaps_tables = {}
# the row of template j in the overlaps of template i, -1 when they share no channel
self.unit_overlaps_tables = np.full((self.num_templates, self.num_templates), -1)
for i in range(self.num_templates):
self.unit_overlaps_tables[i] = np.zeros(self.num_templates, dtype=int)
self.unit_overlaps_tables[i][self.unit_overlaps_indices[i]] = np.arange(len(self.unit_overlaps_indices[i]))
self.unit_overlaps_tables[i, self.unit_overlaps_indices[i]] = np.arange(len(self.unit_overlaps_indices[i]))

self.margin = 2 * self.num_samples
self.is_pushed = False
Expand Down Expand Up @@ -326,8 +327,23 @@ def get_extra_outputs(self):
def get_margin(self):
return self.margin

def _get_gram(self, selection):
"""Gram matrix of the normalized templates (template, peak) in selection, from the overlaps."""
templates, peaks = selection
delta_t = peaks[np.newaxis, :] - peaks[:, np.newaxis]
rows = self.unit_overlaps_tables[templates[:, np.newaxis], templates[np.newaxis, :]]
i, j = np.nonzero((rows >= 0) & (np.abs(delta_t) < self.num_samples))
lags = self.num_samples - 1 + delta_t[i, j]
gram = np.zeros(delta_t.shape)
if self.shared_memory:
gram[i, j] = self.overlaps[templates[i], rows[i, j], lags]
else:
gram[i, j] = [self.overlaps[t][r, lag] for t, r, lag in zip(templates[i], rows[i, j], lags)]
# the overlaps of i with j and of j with i agree up to rounding
return (gram + gram.T) / 2

def compute_matching(self, traces, start_frame, end_frame, segment_index):
from scipy.linalg import get_lapack_funcs, get_blas_funcs, solve_triangular
from scipy.linalg import get_lapack_funcs, get_blas_funcs, solve_triangular, cho_factor, cho_solve
from scipy import ndimage

if not self.is_pushed:
Expand Down Expand Up @@ -381,17 +397,16 @@ def compute_matching(self, traces, start_frame, end_frame, segment_index):

spikes = np.empty(scalar_products.size, dtype=spike_dtype)

M = np.zeros((self.num_templates, self.num_templates), dtype=np.float32)

all_selections = np.empty((2, scalar_products.size), dtype=np.int32)
final_amplitudes = np.zeros(scalar_products.shape, dtype=np.float32)
num_selection = 0
selection = all_selections[:, :0] # empty until the first peak is picked

full_sps = scalar_products.copy()

all_amplitudes = np.zeros(0, dtype=np.float32)
is_in_vicinity = np.zeros(0, dtype=np.int32)
if self.vicinity == 0:
M = np.zeros((self.num_templates, self.num_templates), dtype=np.float32)
full_sps = scalar_products.copy()
else:
is_selected = np.zeros(scalar_products.shape, dtype=bool)

if self.stop_criteria == "omp_min_sps":
stop_criteria = self.omp_min_sps * np.maximum(self.norms, np.sqrt(self.num_channels * self.num_samples))
Expand Down Expand Up @@ -422,31 +437,54 @@ def compute_matching(self, traces, start_frame, end_frame, segment_index):
for peak_index in peak_indices:
best_cluster_ind = best_cluster_inds[0, peak_index]

if num_selection > 0:
delta_t = selection[1] - peak_index
idx = np.flatnonzero((delta_t < self.num_samples) & (delta_t > -self.num_samples))
myline = neighbor_window + delta_t[idx]
myindices = selection[0, idx]

if self.shared_memory:
n_overlaps = len(self.unit_overlaps_indices[best_cluster_ind])
local_overlaps = self.overlaps[best_cluster_ind, :n_overlaps]
else:
local_overlaps = self.overlaps[best_cluster_ind]
if self.vicinity > 0:
# LocOMP: a template is selected at most once at a given time, and each selection re-fits the
# selected templates in its vicinity that share a channel with it to the current residual
if not is_selected[best_cluster_ind, peak_index]:
is_selected[best_cluster_ind, peak_index] = True
all_selections[:, num_selection] = [best_cluster_ind, peak_index]
num_selection += 1

selection = all_selections[:, :num_selection]
nearby = np.flatnonzero(np.abs(selection[1] - peak_index) < self.vicinity)
nearby = nearby[self.units_overlaps[best_cluster_ind][selection[0, nearby]]]
sub_selection = selection[:, nearby]

gram = self._get_gram(sub_selection)
residual_sps = scalar_products[sub_selection[0], sub_selection[1]].astype(np.float64)
try:
factor = cho_factor(gram, lower=True, check_finite=False)
coefficients = cho_solve(factor, residual_sps, check_finite=False)
except np.linalg.LinAlgError: # linearly dependent templates
coefficients = np.linalg.lstsq(gram, residual_sps, rcond=None)[0]

diff_amplitudes = (coefficients / self.norms[sub_selection[0]]).astype(np.float32)
final_amplitudes[sub_selection[0], sub_selection[1]] += diff_amplitudes
else:
if num_selection > 0:
delta_t = selection[1] - peak_index
idx = np.flatnonzero((delta_t < self.num_samples) & (delta_t > -self.num_samples))
myline = neighbor_window + delta_t[idx]
myindices = selection[0, idx]

if self.shared_memory:
n_overlaps = len(self.unit_overlaps_indices[best_cluster_ind])
local_overlaps = self.overlaps[best_cluster_ind, :n_overlaps]
else:
local_overlaps = self.overlaps[best_cluster_ind]

overlapping_templates = self.unit_overlaps_indices[best_cluster_ind]
table = self.unit_overlaps_tables[best_cluster_ind]
overlapping_templates = self.unit_overlaps_indices[best_cluster_ind]
table = self.unit_overlaps_tables[best_cluster_ind]

if num_selection == M.shape[0]:
Z = np.zeros((2 * num_selection, 2 * num_selection), dtype=np.float32)
Z[:num_selection, :num_selection] = M
M = Z
if num_selection == M.shape[0]:
Z = np.zeros((2 * num_selection, 2 * num_selection), dtype=np.float32)
Z[:num_selection, :num_selection] = M
M = Z

mask = np.isin(myindices, overlapping_templates)
a, b = myindices[mask], myline[mask]
M[num_selection, idx[mask]] = local_overlaps[table[a], b]
mask = np.isin(myindices, overlapping_templates)
a, b = myindices[mask], myline[mask]
M[num_selection, idx[mask]] = local_overlaps[table[a], b]

if self.vicinity == 0:
solve_triangular(
M[:num_selection, :num_selection],
M[num_selection, :num_selection],
Expand All @@ -462,61 +500,31 @@ def compute_matching(self, traces, start_frame, end_frame, segment_index):
break
M[num_selection, num_selection] = np.sqrt(Lkk)
else:
is_in_vicinity = np.flatnonzero(np.abs(delta_t) < self.vicinity)

if len(is_in_vicinity) > 0:
L = M[is_in_vicinity, :][:, is_in_vicinity]

M[num_selection, is_in_vicinity] = solve_triangular(
L,
M[num_selection, is_in_vicinity],
trans=0,
lower=1,
overwrite_b=True,
check_finite=False,
)

v = nrm2(M[num_selection, is_in_vicinity]) ** 2
Lkk = 1 - v
if Lkk <= omp_tol: # selected atoms are dependent
break
M[num_selection, num_selection] = np.sqrt(Lkk)
else:
M[num_selection, num_selection] = 1.0
else:
M[0, 0] = 1
M[0, 0] = 1

all_selections[:, num_selection] = [best_cluster_ind, peak_index]
num_selection += 1
all_selections[:, num_selection] = [best_cluster_ind, peak_index]
num_selection += 1

selection = all_selections[:, :num_selection]
res_sps = full_sps[selection[0], selection[1]]

if self.vicinity == 0:
selection = all_selections[:, :num_selection]
res_sps = full_sps[selection[0], selection[1]]
new_amplitudes, _ = potrs(M[:num_selection, :num_selection], res_sps, lower=True, overwrite_b=False)
sub_selection = selection
new_amplitudes /= self.norms[sub_selection[0]]
else:
is_in_vicinity = np.append(is_in_vicinity, num_selection - 1)
all_amplitudes = np.append(all_amplitudes, np.float32(1))
L = M[is_in_vicinity, :][:, is_in_vicinity]
new_amplitudes, _ = potrs(L, res_sps[is_in_vicinity], lower=True, overwrite_b=False)
sub_selection = selection[:, is_in_vicinity]
new_amplitudes /= self.norms[sub_selection[0]]

diff_amplitudes = new_amplitudes - final_amplitudes[sub_selection[0], sub_selection[1]]
diff_amplitudes = new_amplitudes - final_amplitudes[sub_selection[0], sub_selection[1]]
final_amplitudes[sub_selection[0], sub_selection[1]] = new_amplitudes

modified = np.flatnonzero(np.abs(diff_amplitudes) > omp_tol)
final_amplitudes[sub_selection[0], sub_selection[1]] = new_amplitudes

for i in modified:
tmp_best, tmp_peak = sub_selection[:, i]
tmp_best, tmp_peak = sub_selection[:, i].tolist()
diff_amp = diff_amplitudes[i] * self.norms[tmp_best]
if self.shared_memory:
n_overlaps = len(self.unit_overlaps_indices[tmp_best])
local_overlaps = self.overlaps[tmp_best, :n_overlaps]
else:
local_overlaps = self.overlaps[tmp_best]
overlapping_templates = self.units_overlaps[tmp_best]
overlapping_templates = self.unit_overlaps_indices[tmp_best]
tmp = tmp_peak - neighbor_window
idx = [max(0, tmp), min(num_peaks, tmp_peak + self.num_samples)]
tdx = [idx[0] - tmp, idx[1] - tmp]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
import pytest
import numpy as np

from spikeinterface import create_sorting_analyzer, compute_sparsity, get_noise_levels
from spikeinterface import (
NumpyRecording,
Templates,
create_sorting_analyzer,
compute_sparsity,
get_noise_levels,
)

from spikeinterface.sortingcomponents.matching import find_spikes_from_templates
from spikeinterface.sortingcomponents.matching.method_list import matching_methods
Expand Down Expand Up @@ -110,6 +116,34 @@ def test_find_spikes_from_templates(method, sorting_analyzer):
# plt.show()


def test_circus_omp_overlapping_spikes():
# noiseless overlapping spikes, spaced so that re-fits reach spikes whose neighbours lie outside them,
# where selecting a template twice at the same time loses or misfits a spike
sampling_frequency, nbefore, num_samples, num_channels = 30000.0, 20, 60, 4
t = np.arange(num_samples) - nbefore
templates_array = np.zeros((3, num_samples, num_channels), dtype=np.float32)
for unit in range(3):
width = 3 + 1.5 * unit
waveform = -np.exp(-0.5 * (t / width) ** 2) + 0.4 * np.exp(-0.5 * ((t - 2.5 * width) / (2 * width)) ** 2)
profile = np.exp(-0.5 * ((np.arange(num_channels) - 1.5 * unit) / 1.5) ** 2)
# rank 1, so the matcher's SVD keeps the templates exactly
templates_array[unit] = 10 * np.outer(waveform, profile)

units, times = [1, 0, 0, 1, 2], [120, 138, 178, 220, 240]
traces = np.zeros((420, num_channels), dtype=np.float32)
for unit, time in zip(units, times):
traces[time - nbefore : time - nbefore + num_samples] += templates_array[unit]
recording = NumpyRecording([traces], sampling_frequency)
templates = Templates(
templates_array, sampling_frequency, nbefore, channel_ids=recording.channel_ids, unit_ids=np.arange(3)
)

spikes = find_spikes_from_templates(recording, templates, method="circus-omp", job_kwargs=dict(n_jobs=1))
assert spikes["cluster_index"].tolist() == units
assert spikes["sample_index"].tolist() == times
assert np.allclose(spikes["amplitude"], 1, atol=0.01)


if __name__ == "__main__":
sorting_analyzer = get_sorting_analyzer()
# method = "nearest"
Expand Down
Loading