From 5a141b46fb098ac55d31676a8fbc4bbfb6c7c08b Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 1 Sep 2026 17:54:53 +0200 Subject: [PATCH 1/6] feat: extend functionality of processing pipeline --- src/spikeinterface/preprocessing/__init__.py | 4 +- src/spikeinterface/preprocessing/pipeline.py | 175 ++++++++++-------- .../preprocessing/tests/test_pipeline.py | 32 ++-- 3 files changed, 118 insertions(+), 93 deletions(-) diff --git a/src/spikeinterface/preprocessing/__init__.py b/src/spikeinterface/preprocessing/__init__.py index fd8d8fd787..23a5bb30b1 100644 --- a/src/spikeinterface/preprocessing/__init__.py +++ b/src/spikeinterface/preprocessing/__init__.py @@ -15,8 +15,8 @@ from .pipeline import ( apply_preprocessing_pipeline, - get_preprocessing_dict_from_analyzer, - get_preprocessing_dict_from_file, + get_preprocessing_list_from_analyzer, + get_preprocessing_list_from_file, PreprocessingPipeline, ) diff --git a/src/spikeinterface/preprocessing/pipeline.py b/src/spikeinterface/preprocessing/pipeline.py index 4d90007964..fdab3767b9 100644 --- a/src/spikeinterface/preprocessing/pipeline.py +++ b/src/spikeinterface/preprocessing/pipeline.py @@ -9,48 +9,40 @@ pp_names_to_classes = {pp_function.__name__: pp_class for pp_class, pp_function in _all_preprocesser_dict.items()} -class PreprocessingPipeline: - """ - A preprocessing pipeline, containing ordered preprocessing steps. - - Parameters - ---------- - preprocessor_dict : dict - Dictionary containing preprocessing steps and their kwargs - - Examples - -------- - Generate a `PreprocessingPipeline` containing a `bandpass_filter` then a - `common_reference` step. Then apply this to a recording - - >>> from spikeinterface.preprocessing import PreprocessingPipeline - >>> preprocessor_dict = {'bandpass_filter': {'freq_max': 3000}, 'common_reference': {}} - >>> my_pipeline = PreprocessingPipeline(preprocessor_dict) - PreprocessingPipeline: Raw Recording → bandpass_filter → common_reference → Preprocessed Recording - >>> my_pipeline._apply(recording) - - """ - - def __init__(self, preprocessor_dict): +class ABCPipeline: + function_names_to_functions = dict() + function_names_to_classes = dict() + def __init__(self, preprocessor_dict_or_list): non_supported_preprocessors = [] - for preprocessor in preprocessor_dict: - if preprocessor not in pp_names_to_functions.keys(): - non_supported_preprocessors.append(preprocessor) + # convert dicts to lists + preprocessor_list = [] + if isinstance(preprocessor_dict_or_list, dict): + for key, value in preprocessor_dict_or_list.items(): + step = dict(name=key, kwargs=value) + preprocessor_list.append(step) + elif isinstance(preprocessor_dict_or_list, list): + preprocessor_list = preprocessor_dict_or_list + assert all( + isinstance(step, dict) and "name" in step and "kwargs" in step for step in preprocessor_list + ), "Each step in the preprocessor list must be a dict with 'name' and 'kwargs' keys." + + for preprocessor in preprocessor_list: + if preprocessor["name"] not in self.function_names_to_functions.keys(): + non_supported_preprocessors.append(preprocessor["name"]) if len(non_supported_preprocessors) > 0: raise TypeError( - f"The preprocessors '{non_supported_preprocessors}' are not supported by the `PreprocessingPipeline`. \ -To see the list of supported steps, run:\n>>> from spikeinterface.preprocessing.pipeline import pp_names_to_functions \ -\n>>> print(pp_names_to_functions.keys())" + f"The preprocessors '{non_supported_preprocessors}' are not supported by the `PreprocessingPipeline`. " + f"Available preprocessors are: {list(self.function_names_to_functions.keys())}" ) - self.preprocessor_dict = preprocessor_dict + self.preprocessor_list = preprocessor_list def __repr__(self): txt = "PreprocessingPipeline: \tRaw Recording \u2192 " - for preprocessor in self.preprocessor_dict: - txt += str(preprocessor) + " \u2192 " + for preprocessor in self.preprocessor_list: + txt += str(preprocessor["name"]) + " \u2192 " txt += "Preprocessed Recording" return txt @@ -100,8 +92,9 @@ def _apply(self, recording, apply_precomputed_kwargs=False): """ instantiated_recordings = {"raw": recording} - for preprocessor_name, kwargs_ in self.preprocessor_dict.items(): - kwargs = kwargs_.copy() + for step in self.preprocessor_list: + preprocessor_name = step["name"] + kwargs = step["kwargs"].copy() dont_apply_kwargs = ["recording", "parent_recording"] for k, v in kwargs.items(): @@ -123,33 +116,61 @@ def _apply(self, recording, apply_precomputed_kwargs=False): kwargs[k] = substituted_recording if not apply_precomputed_kwargs: - preprocessor_class = pp_names_to_classes[preprocessor_name] + preprocessor_class = self.function_names_to_classes[preprocessor_name] precomputable_kwarg_names = preprocessor_class._precomputable_kwarg_names dont_apply_kwargs += precomputable_kwarg_names non_rec_kwargs = {key: value for key, value in kwargs.items() if key not in dont_apply_kwargs} - pp_output = pp_names_to_functions[preprocessor_name](recording, **non_rec_kwargs) + pp_output = self.function_names_to_functions[preprocessor_name](recording, **non_rec_kwargs) recording = pp_output instantiated_recordings[preprocessor_name] = recording return recording +class PreprocessingPipeline(ABCPipeline): + """ + A preprocessing pipeline, containing ordered preprocessing steps. + + Parameters + ---------- + preprocessor_list_or_dict : dict or list + Dictionary or list containing preprocessing steps and their kwargs + + Examples + -------- + Generate a `PreprocessingPipeline` containing a `bandpass_filter` then a + `common_reference` step. Then apply this to a recording + + >>> from spikeinterface.preprocessing import PreprocessingPipeline + >>> preprocessor_dict = {'bandpass_filter': {'freq_max': 3000}, 'common_reference': {}} + >>> my_pipeline = PreprocessingPipeline(preprocessor_dict) + PreprocessingPipeline: Raw Recording → bandpass_filter → common_reference → Preprocessed Recording + >>> my_pipeline._apply(recording) + + """ + + function_names_to_functions = pp_names_to_functions + function_names_to_classes = pp_names_to_classes + + def apply_preprocessing_pipeline( - recording: BaseRecording, pipeline_or_dict: PreprocessingPipeline | dict, apply_precomputed_kwargs=True + recording_or_dict: BaseRecording | dict, + pipeline: PreprocessingPipeline | list | dict, + apply_precomputed_kwargs=True, ): """ Creates a preprocessed recording by applying the preprocessing steps in - `preprocessor_dict` to `recording`. + `pipeline` to `recording`. Parameters ---------- - recording : BaseRecording - The initial recording - pipeline_or_dict : PreprocessingPipeline | dict - Dictionary containing preprocessing steps and their kwargs, or a pipeline object. + recording_or_dict : BaseRecording | dict + The initial recording or a dictionary of recordings + pipeline : PreprocessingPipeline | list | dict + Dictionary containing preprocessing steps and their kwargs, a list of preprocessing steps, or a pipeline object. If None, the original recording is returned. - apply_precomputed_kwargs : Bool, default: False + apply_precomputed_kwargs : Bool, default: True Some preprocessing steps (e.g. Whitening) contain arguments which are computed during preprocessing. If True, we use the arguments which have already been computed. If False, we recompute them on application of the pipeline. @@ -161,30 +182,32 @@ def apply_preprocessing_pipeline( Examples -------- - Create a preprocessed recording from a generated recording and a preprocessor_dict + Create a preprocessed recording from a generated recording and a preprocessing pipeline >>> from spikeinterface.preprocessing import create_preprocessed >>> from spikeinterface.generation import generate_recording >>> recording = generate_recording() - >>> preprocessor_dict = {'bandpass_filter': {'freq_max': 3000}, 'common_reference': {}} - >>> preprocessed_recording = apply_preprocessing_pipeline(recording, preprocessor_dict) + >>> pipeline = [{'name': 'bandpass_filter', 'kwargs': {'freq_max': 3000}}, {'name': 'common_reference', 'kwargs': {}}] + >>> preprocessed_recording = apply_preprocessing_pipeline(recording, pipeline) """ - if isinstance(pipeline_or_dict, PreprocessingPipeline): - pipeline = pipeline_or_dict - elif isinstance(pipeline_or_dict, dict): - pipeline = PreprocessingPipeline(pipeline_or_dict) + if isinstance(pipeline, PreprocessingPipeline): + pipeline = pipeline + elif isinstance(pipeline, dict): + pipeline = PreprocessingPipeline(pipeline) + elif isinstance(pipeline, list): + pipeline = PreprocessingPipeline(pipeline) else: - raise TypeError("`pipeline_or_dict` must be a `PreprocessingPipeline` or a dict") + raise TypeError("`pipeline` must be a `PreprocessingPipeline`, a list, or a dict") - preprocessed_recording = pipeline._apply(recording, apply_precomputed_kwargs) + preprocessed_recording = pipeline._apply(recording_or_dict, apply_precomputed_kwargs) return preprocessed_recording -def get_preprocessing_dict_from_analyzer(analyzer_folder, format="auto", backend_options=None): +def get_preprocessing_list_from_analyzer(analyzer_folder, format="auto", backend_options=None): """ - Generates a dictionary from a saved analyzer. The dictionary can be passed to the - `PreprocessingPipeline` class to create a preprocessing pipeline. + Generates a preprocessing list from a saved analyzer. The list can be passed to the + `PreprocessingPipeline` class to create a preprocessing pipeline from the list. Parameters ---------- @@ -197,8 +220,8 @@ def get_preprocessing_dict_from_analyzer(analyzer_folder, format="auto", backend Returns ------- - preprocessing_dict : dict - The preprocessing dict extracted from the analyzer's recording. + preprocessing_list : list + The preprocessing list extracted from the analyzer's recording. """ if not is_path_remote(analyzer_folder): analyzer_folder = Path(analyzer_folder) @@ -215,7 +238,7 @@ def get_preprocessing_dict_from_analyzer(analyzer_folder, format="auto", backend raise FileNotFoundError(f"Cannot find `recording.*` file in {analyzer_folder}.") else: recording_file = recording_files[0] - preprocessing_dict = get_preprocessing_dict_from_file(recording_file) + preprocessing_list = get_preprocessing_list_from_file(recording_file) elif format == "zarr": backend_options = {} if backend_options is None else backend_options @@ -228,14 +251,14 @@ def get_preprocessing_dict_from_analyzer(analyzer_folder, format="auto", backend else: recording_dict = {} - preprocessing_dict = _make_pipeline_dict_from_recording_dict(recording_dict) + preprocessing_list = _make_pipeline_list_from_recording_dict(recording_dict) - return preprocessing_dict + return preprocessing_list -def get_preprocessing_dict_from_file(recording_dictionary_path): +def get_preprocessing_list_from_file(recording_dictionary_path): """ - Generates a preprocessing dict, passable to `apply_preprocessing_pipeline` function and + Generates a preprocessing list, passable to `apply_preprocessing_pipeline` function and `PreprocessPipeline` class, from a recording dictionary. Only extracts preprocessing steps which can be applied "globally" to any recording. @@ -248,8 +271,8 @@ def get_preprocessing_dict_from_file(recording_dictionary_path): Returns ------- - preprocessor_dict : dict - Dictionary containing preprocessing steps and their kwargs + preprocessing_list : list + List containing preprocessing steps and their kwargs, each element is a dict with keys "name" and "kwargs". """ @@ -264,20 +287,20 @@ def get_preprocessing_dict_from_file(recording_dictionary_path): with open(recording_dictionary_path, "rb") as f: recording_dict = pickle.load(f) - pipeline_dict = _make_pipeline_dict_from_recording_dict(recording_dict) - return pipeline_dict + preprocessing_list = _make_pipeline_list_from_recording_dict(recording_dict) + return preprocessing_list -def _make_pipeline_dict_from_recording_dict(recording_dict): +def _make_pipeline_list_from_recording_dict(recording_dict): """ Transforms a recording dict (created by the `dump` method of `BaseRecording`) - into a preprocessing pipeline dict. + into a preprocessing pipeline list. """ pipeline_dict_from_file = {} _ = _load_pp_from_dict(recording_dict, pipeline_dict_from_file) - pipeline_dict = {} + preprocessing_list = [] for preprocessor in pipeline_dict_from_file: preprocessor_class_name = preprocessor.split(".")[-1] @@ -292,9 +315,9 @@ def _make_pipeline_dict_from_recording_dict(recording_dict): if key not in ["recording", "parent_recording"] } - pipeline_dict[preprocessor_function.__name__] = pp_kwargs + preprocessing_list.append({"name": preprocessor_function.__name__, "kwargs": pp_kwargs}) - return pipeline_dict + return preprocessing_list def _load_pp_from_dict(prov_dict, kwargs_dict): @@ -348,10 +371,10 @@ def _get_all_kwargs_and_values(my_pipeline): """ all_kwargs = {} - for preprocessor in my_pipeline.preprocessor_dict: + for preprocessor in my_pipeline.preprocessor_list: - preprocessor_name = preprocessor.split(".")[-1] - pp_function = pp_names_to_functions[preprocessor.split(".")[-1]] + preprocessor_name = preprocessor["name"].split(".")[-1] + pp_function = my_pipeline.function_names_to_functions[preprocessor["name"].split(".")[-1]] signature = inspect.signature(pp_function) all_kwargs[preprocessor_name] = {} @@ -368,7 +391,9 @@ def _get_all_kwargs_and_values(my_pipeline): except: default_value = None - pipeline_value = my_pipeline.preprocessor_dict[preprocessor].get(par_name) + pipeline_value = my_pipeline.preprocessor_list[my_pipeline.preprocessor_list.index(preprocessor)][ + "kwargs" + ].get(par_name) if pipeline_value is None: if default_value != pipeline_value: diff --git a/src/spikeinterface/preprocessing/tests/test_pipeline.py b/src/spikeinterface/preprocessing/tests/test_pipeline.py index 37376781b7..944b5e6106 100644 --- a/src/spikeinterface/preprocessing/tests/test_pipeline.py +++ b/src/spikeinterface/preprocessing/tests/test_pipeline.py @@ -15,8 +15,8 @@ ) from spikeinterface.preprocessing.pipeline import ( pp_names_to_functions, - get_preprocessing_dict_from_file, - get_preprocessing_dict_from_analyzer, + get_preprocessing_list_from_file, + get_preprocessing_list_from_analyzer, ) @@ -129,11 +129,11 @@ def test_three_preprocessing_steps(): rec_groups.set_property(key="group", values=[0, 1]) dict_of_recs = rec_groups.split_by("group") - pp_dict_of_recs_from_pipeline = apply_preprocessing_pipeline(dict_of_recs, pipeline_dict) - pp_dict_of_recs_from_functions = whiten(bandpass_filter(common_reference(dict_of_recs)), seed=1205) + pp_list_of_recs_from_pipeline = apply_preprocessing_pipeline(dict_of_recs, pipeline_dict) + pp_list_of_recs_from_functions = whiten(bandpass_filter(common_reference(dict_of_recs)), seed=1205) - check_recordings_equal(pp_dict_of_recs_from_pipeline[0], pp_dict_of_recs_from_functions[0]) - check_recordings_equal(pp_dict_of_recs_from_pipeline[1], pp_dict_of_recs_from_functions[1]) + check_recordings_equal(pp_list_of_recs_from_pipeline[0], pp_list_of_recs_from_functions[0]) + check_recordings_equal(pp_list_of_recs_from_pipeline[1], pp_list_of_recs_from_functions[1]) def test_kwargs_are_propagated(): @@ -161,7 +161,7 @@ def test_kwargs_are_propagated(): def test_loading_provenance(create_cache_folder): """ Makes a preprocessed recording using a Pipeline and saves it. Then reloads the preprocessed - recording using `get_preprocessing_dict_from_file`, either ignoring or applying the + recording using `get_preprocessing_list_from_file`, either ignoring or applying the precomputed kwargs. These reloaded recordings should be the same as the original preprocessed recording. """ @@ -178,15 +178,15 @@ def test_loading_provenance(create_cache_folder): ) pp_rec.save_to_folder(folder=cache_folder) - loaded_pp_dict = get_preprocessing_dict_from_file(cache_folder / "provenance.pkl") + loaded_pp_list = get_preprocessing_list_from_file(cache_folder / "provenance.pkl") pipeline_rec_applying_precomputed_kwargs = apply_preprocessing_pipeline( rec, - loaded_pp_dict, + loaded_pp_list, apply_precomputed_kwargs=True, ) pipeline_rec_ignoring_precomputed_kwargs = apply_preprocessing_pipeline( - rec, loaded_pp_dict, apply_precomputed_kwargs=False + rec, loaded_pp_list, apply_precomputed_kwargs=False ) check_recordings_equal(pipeline_rec_applying_precomputed_kwargs, pp_rec) @@ -195,7 +195,7 @@ def test_loading_provenance(create_cache_folder): def test_loading_from_analyzer(create_cache_folder): """ - Tests the `get_preprocessing_dict_from_analyzer` function, which constructs a preprocessing pipeline + Tests the `get_preprocessing_list_from_analyzer` function, which constructs a preprocessing pipeline dict from a saved sorting analyzer (either binary folder or zarr). This test creates a preprocessed recording, uses this to create a sorting analyzer and saves binary and zarr versions of the analyzer. Then we generate the preprocessing dict from the analyzer, and apply it to the original recording to check that it's the same @@ -212,14 +212,14 @@ def test_loading_from_analyzer(create_cache_folder): _ = create_sorting_analyzer( sorting=sorting, recording=pp_recording, format="binary_folder", folder=analyzer_binary_folder ) - pp_dict_from_binary = get_preprocessing_dict_from_analyzer(analyzer_binary_folder) - pp_recording_from_binary = apply_preprocessing_pipeline(recording, pp_dict_from_binary) + pp_list_from_binary = get_preprocessing_list_from_analyzer(analyzer_binary_folder) + pp_recording_from_binary = apply_preprocessing_pipeline(recording, pp_list_from_binary) check_recordings_equal(pp_recording, pp_recording_from_binary) analyzer_zarr_folder = cache_folder / "zarr_format.zarr" _ = create_sorting_analyzer(sorting=sorting, recording=pp_recording, format="zarr", folder=analyzer_zarr_folder) - pp_dict_from_zarr = get_preprocessing_dict_from_analyzer(analyzer_zarr_folder) - pp_recording_from_zarr = apply_preprocessing_pipeline(recording, pp_dict_from_zarr) + pp_list_from_zarr = get_preprocessing_list_from_analyzer(analyzer_zarr_folder) + pp_recording_from_zarr = apply_preprocessing_pipeline(recording, pp_list_from_zarr) check_recordings_equal(pp_recording, pp_recording_from_zarr) @@ -269,7 +269,7 @@ def test_pipeline_recording_arg_substitution(create_cache_folder): # Test dumping the pipeline to pickle and loading it back with the correct substitution still works pp_rec_from_pipeline.dump_to_pickle(create_cache_folder / "pipeline_substitution_test.pkl") - pp_rec_from_pkl = get_preprocessing_dict_from_file(create_cache_folder / "pipeline_substitution_test.pkl") + pp_rec_from_pkl = get_preprocessing_list_from_file(create_cache_folder / "pipeline_substitution_test.pkl") pp_rec_from_pipeline_substitution = apply_preprocessing_pipeline( rec, pp_rec_from_pkl, apply_precomputed_kwargs=True ) From 5c131f675730c8355c9d903328be461cc0cf1fd1 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 3 Sep 2026 13:13:34 +0200 Subject: [PATCH 2/6] fix: params instead of kwargs and rename ABCPipeline to BasePipeline --- src/spikeinterface/preprocessing/pipeline.py | 75 +++++++++++++------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/src/spikeinterface/preprocessing/pipeline.py b/src/spikeinterface/preprocessing/pipeline.py index fdab3767b9..bda5d5bee8 100644 --- a/src/spikeinterface/preprocessing/pipeline.py +++ b/src/spikeinterface/preprocessing/pipeline.py @@ -1,5 +1,6 @@ from pathlib import Path import inspect +import warnings from spikeinterface.core import BaseRecording from spikeinterface.core.core_tools import is_dict_extractor, is_path_remote from spikeinterface.core.zarrextractors import super_zarr_open @@ -9,23 +10,34 @@ pp_names_to_classes = {pp_function.__name__: pp_class for pp_class, pp_function in _all_preprocesser_dict.items()} -class ABCPipeline: +class BasePipeline: + """ + Base processing pipeline to construct a processing pipeline from a list of processing steps and their params. + + Inherited classes should define the `function_names_to_functions` and `function_names_to_classes` attributes, + which map the names of processing steps to their corresponding functions and classes, respectively. + """ + function_names_to_functions = dict() function_names_to_classes = dict() - def __init__(self, preprocessor_dict_or_list): + def __init__(self, preprocessor_list_or_dict): non_supported_preprocessors = [] # convert dicts to lists preprocessor_list = [] - if isinstance(preprocessor_dict_or_list, dict): - for key, value in preprocessor_dict_or_list.items(): - step = dict(name=key, kwargs=value) + if isinstance(preprocessor_list_or_dict, dict): + for key, value in preprocessor_list_or_dict.items(): + step = dict(name=key, params=value) preprocessor_list.append(step) - elif isinstance(preprocessor_dict_or_list, list): - preprocessor_list = preprocessor_dict_or_list + elif isinstance(preprocessor_list_or_dict, list): + preprocessor_list = preprocessor_list_or_dict assert all( - isinstance(step, dict) and "name" in step and "kwargs" in step for step in preprocessor_list - ), "Each step in the preprocessor list must be a dict with 'name' and 'kwargs' keys." + isinstance(step, dict) and "name" in step for step in preprocessor_list + ), "Each step in the preprocessor list must be a dict with 'name' key." + + for step in preprocessor_list: + if "params" not in step: + step["params"] = {} for preprocessor in preprocessor_list: if preprocessor["name"] not in self.function_names_to_functions.keys(): @@ -33,29 +45,31 @@ def __init__(self, preprocessor_dict_or_list): if len(non_supported_preprocessors) > 0: raise TypeError( - f"The preprocessors '{non_supported_preprocessors}' are not supported by the `PreprocessingPipeline`. " + f"The preprocessors '{non_supported_preprocessors}' are not supported by the pipeline. " f"Available preprocessors are: {list(self.function_names_to_functions.keys())}" ) self.preprocessor_list = preprocessor_list def __repr__(self): - txt = "PreprocessingPipeline: \tRaw Recording \u2192 " + txt = "Pipeline: \tRaw \u2192 " for preprocessor in self.preprocessor_list: txt += str(preprocessor["name"]) + " \u2192 " - txt += "Preprocessed Recording" + txt += "Preprocessed" return txt def _repr_html_(self): - all_kwargs = _get_all_kwargs_and_values(self) + all_kwargs_list = _get_all_kwargs_and_values(self) html_text = "" html_text += "PreprocessingPipeline" html_text += "
Initial Recording
" html_text += "
" - for a, (preprocessor, kwargs) in enumerate(all_kwargs.items()): + for all_kwargs in all_kwargs_list: + preprocessor = all_kwargs["name"] + kwargs = all_kwargs["kwargs"] html_text += "
" html_text += f"{preprocessor}" @@ -94,10 +108,10 @@ def _apply(self, recording, apply_precomputed_kwargs=False): instantiated_recordings = {"raw": recording} for step in self.preprocessor_list: preprocessor_name = step["name"] - kwargs = step["kwargs"].copy() + params = step["params"].copy() dont_apply_kwargs = ["recording", "parent_recording"] - for k, v in kwargs.items(): + for k, v in params.items(): if isinstance(v, str) and "pipeline[" in v: if "recording" not in k: raise ValueError( @@ -113,22 +127,22 @@ def _apply(self, recording, apply_precomputed_kwargs=False): substituted_recording = instantiated_recordings.get(rec_name) if substituted_recording is None: raise ValueError(f"Cannot find recording '{rec_name}' from previous steps in the pipeline.") - kwargs[k] = substituted_recording + params[k] = substituted_recording if not apply_precomputed_kwargs: preprocessor_class = self.function_names_to_classes[preprocessor_name] precomputable_kwarg_names = preprocessor_class._precomputable_kwarg_names dont_apply_kwargs += precomputable_kwarg_names - non_rec_kwargs = {key: value for key, value in kwargs.items() if key not in dont_apply_kwargs} - pp_output = self.function_names_to_functions[preprocessor_name](recording, **non_rec_kwargs) + non_rec_params = {key: value for key, value in params.items() if key not in dont_apply_kwargs} + pp_output = self.function_names_to_functions[preprocessor_name](recording, **non_rec_params) recording = pp_output instantiated_recordings[preprocessor_name] = recording return recording -class PreprocessingPipeline(ABCPipeline): +class PreprocessingPipeline(BasePipeline): """ A preprocessing pipeline, containing ordered preprocessing steps. @@ -168,7 +182,7 @@ def apply_preprocessing_pipeline( recording_or_dict : BaseRecording | dict The initial recording or a dictionary of recordings pipeline : PreprocessingPipeline | list | dict - Dictionary containing preprocessing steps and their kwargs, a list of preprocessing steps, or a pipeline object. + A list of preprocessing steps, or a pipeline object. If None, the original recording is returned. apply_precomputed_kwargs : Bool, default: True Some preprocessing steps (e.g. Whitening) contain arguments which are computed @@ -193,10 +207,16 @@ def apply_preprocessing_pipeline( if isinstance(pipeline, PreprocessingPipeline): pipeline = pipeline - elif isinstance(pipeline, dict): - pipeline = PreprocessingPipeline(pipeline) elif isinstance(pipeline, list): pipeline = PreprocessingPipeline(pipeline) + elif isinstance(pipeline, dict): + warnings.warn( + "Passing a dict to `apply_preprocessing_pipeline` is deprecated and will be removed in 0.106.0. " + "Please pass a list of preprocessing steps instead.", + DeprecationWarning, + stacklevel=2, + ) + pipeline = PreprocessingPipeline(pipeline) else: raise TypeError("`pipeline` must be a `PreprocessingPipeline`, a list, or a dict") @@ -370,14 +390,14 @@ def _get_all_kwargs_and_values(my_pipeline): including the default values. """ - all_kwargs = {} + all_kwargs_list = [] for preprocessor in my_pipeline.preprocessor_list: preprocessor_name = preprocessor["name"].split(".")[-1] pp_function = my_pipeline.function_names_to_functions[preprocessor["name"].split(".")[-1]] signature = inspect.signature(pp_function) - all_kwargs[preprocessor_name] = {} + all_kwargs = {"name": preprocessor_name, "kwargs": {}} for _, value in signature.parameters.items(): par_name = str(value).split("=")[0].split(":")[0] @@ -399,6 +419,7 @@ def _get_all_kwargs_and_values(my_pipeline): if default_value != pipeline_value: pipeline_value = default_value - all_kwargs[preprocessor_name][par_name] = pipeline_value + all_kwargs["kwargs"][par_name] = pipeline_value - return all_kwargs + all_kwargs_list.append(all_kwargs) + return all_kwargs_list From edf31189d3965adf78eaf5b8ab55072cb495c2dc Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 3 Sep 2026 13:21:10 +0200 Subject: [PATCH 3/6] fix: simplify BasePipeline to only use function dict --- src/spikeinterface/core/core_tools.py | 2 ++ src/spikeinterface/preprocessing/pipeline.py | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py index ddc52e283f..9aa12036c2 100644 --- a/src/spikeinterface/core/core_tools.py +++ b/src/spikeinterface/core/core_tools.py @@ -55,6 +55,8 @@ def source_class_or_dict_of_sources_classes(*args, **kwargs): source_class_or_dict_of_sources_classes.__signature__ = inspect.signature(source_class) source_class_or_dict_of_sources_classes.__doc__ = source_class.__doc__ source_class_or_dict_of_sources_classes.__name__ = name + # propagate the _precomputable_kwarg_names attribute from the source class to the wrapper function + source_class_or_dict_of_sources_classes._precomputable_kwarg_names = source_class._precomputable_kwarg_names return source_class_or_dict_of_sources_classes diff --git a/src/spikeinterface/preprocessing/pipeline.py b/src/spikeinterface/preprocessing/pipeline.py index bda5d5bee8..35f9d732af 100644 --- a/src/spikeinterface/preprocessing/pipeline.py +++ b/src/spikeinterface/preprocessing/pipeline.py @@ -7,19 +7,17 @@ from spikeinterface.preprocessing.preprocessing_classes import preprocessor_dict, _all_preprocesser_dict pp_names_to_functions = {preprocessor.__name__: preprocessor for preprocessor in preprocessor_dict.values()} -pp_names_to_classes = {pp_function.__name__: pp_class for pp_class, pp_function in _all_preprocesser_dict.items()} class BasePipeline: """ Base processing pipeline to construct a processing pipeline from a list of processing steps and their params. - Inherited classes should define the `function_names_to_functions` and `function_names_to_classes` attributes, - which map the names of processing steps to their corresponding functions and classes, respectively. + Inherited classes should define the `function_names_to_functions` attributes, + which map the names of processing steps to their corresponding functions, respectively. """ function_names_to_functions = dict() - function_names_to_classes = dict() def __init__(self, preprocessor_list_or_dict): non_supported_preprocessors = [] @@ -130,9 +128,10 @@ def _apply(self, recording, apply_precomputed_kwargs=False): params[k] = substituted_recording if not apply_precomputed_kwargs: - preprocessor_class = self.function_names_to_classes[preprocessor_name] - precomputable_kwarg_names = preprocessor_class._precomputable_kwarg_names - dont_apply_kwargs += precomputable_kwarg_names + preprocessor_function = self.function_names_to_functions[preprocessor_name] + if hasattr(preprocessor_function, "_precomputable_kwarg_names"): + precomputable_kwarg_names = preprocessor_function._precomputable_kwarg_names + dont_apply_kwargs += precomputable_kwarg_names non_rec_params = {key: value for key, value in params.items() if key not in dont_apply_kwargs} pp_output = self.function_names_to_functions[preprocessor_name](recording, **non_rec_params) From 4d5f6f60beb86da4158896261ab6da7d734691db Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 3 Sep 2026 16:52:55 +0200 Subject: [PATCH 4/6] fix: properly remove pp_names_to_classes --- src/spikeinterface/preprocessing/pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/spikeinterface/preprocessing/pipeline.py b/src/spikeinterface/preprocessing/pipeline.py index 35f9d732af..69e550ffb6 100644 --- a/src/spikeinterface/preprocessing/pipeline.py +++ b/src/spikeinterface/preprocessing/pipeline.py @@ -164,7 +164,6 @@ class PreprocessingPipeline(BasePipeline): """ function_names_to_functions = pp_names_to_functions - function_names_to_classes = pp_names_to_classes def apply_preprocessing_pipeline( From efd31aad23112cff3e5b1d3c7e39e1d4865ee286 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 3 Sep 2026 18:07:42 +0200 Subject: [PATCH 5/6] fix: tests --- src/spikeinterface/preprocessing/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spikeinterface/preprocessing/pipeline.py b/src/spikeinterface/preprocessing/pipeline.py index 69e550ffb6..b5407c1cc1 100644 --- a/src/spikeinterface/preprocessing/pipeline.py +++ b/src/spikeinterface/preprocessing/pipeline.py @@ -333,7 +333,7 @@ def _make_pipeline_list_from_recording_dict(recording_dict): if key not in ["recording", "parent_recording"] } - preprocessing_list.append({"name": preprocessor_function.__name__, "kwargs": pp_kwargs}) + preprocessing_list.append({"name": preprocessor_function.__name__, "params": pp_kwargs}) return preprocessing_list From a07169f588415977f8c9357d24228acc91cfbe04 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Fri, 4 Sep 2026 10:03:07 +0200 Subject: [PATCH 6/6] final touches ro repr --- src/spikeinterface/preprocessing/pipeline.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/spikeinterface/preprocessing/pipeline.py b/src/spikeinterface/preprocessing/pipeline.py index b5407c1cc1..512155f7e7 100644 --- a/src/spikeinterface/preprocessing/pipeline.py +++ b/src/spikeinterface/preprocessing/pipeline.py @@ -50,10 +50,10 @@ def __init__(self, preprocessor_list_or_dict): self.preprocessor_list = preprocessor_list def __repr__(self): - txt = "Pipeline: \tRaw \u2192 " + txt = "Pipeline: \tinput \u2192 " for preprocessor in self.preprocessor_list: txt += str(preprocessor["name"]) + " \u2192 " - txt += "Preprocessed" + txt += "preprocessed" return txt def _repr_html_(self): @@ -62,7 +62,7 @@ def _repr_html_(self): html_text = "" html_text += "PreprocessingPipeline" - html_text += "
Initial Recording
" + html_text += "
input
" html_text += "
" for all_kwargs in all_kwargs_list: @@ -78,7 +78,7 @@ def _repr_html_(self): html_text += "
" html_text += """
""" - html_text += "
Preprocessed Recording
" + html_text += "
preprocessed
" html_text += "" return html_text @@ -409,9 +409,8 @@ def _get_all_kwargs_and_values(my_pipeline): except: default_value = None - pipeline_value = my_pipeline.preprocessor_list[my_pipeline.preprocessor_list.index(preprocessor)][ - "kwargs" - ].get(par_name) + preprocessor_index = my_pipeline.preprocessor_list.index(preprocessor) + pipeline_value = my_pipeline.preprocessor_list[preprocessor_index]["params"].get(par_name) if pipeline_value is None: if default_value != pipeline_value: