Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions neo/core/baseneo.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from numbers import Number

import numpy as np
import quantities as pq

ALLOWED_ANNOTATION_TYPES = (
int,
Expand All @@ -35,6 +36,25 @@ class MergeError(Exception):
pass


def _merge_quantity_arrays(quantities, axis=0):
"""Merge compatible Quantity arrays while preserving uncertainty metadata."""
reference_units = quantities[0].units
normalized = [quantity.rescale(reference_units) for quantity in quantities]
magnitudes = np.concatenate([quantity.magnitude for quantity in normalized], axis=axis)

uncertain_quantities = [isinstance(quantity, pq.UncertainQuantity) for quantity in normalized]
if any(uncertain_quantities):
if not all(uncertain_quantities):
raise ValueError(
"Cannot merge Quantity and UncertainQuantity arrays without losing uncertainty information"
)
uncertainties = [quantity.uncertainty.rescale(reference_units).magnitude for quantity in normalized]
uncertainty = np.concatenate(uncertainties, axis=axis)
return pq.UncertainQuantity(magnitudes, units=reference_units, uncertainty=uncertainty)

return magnitudes * reference_units


class NeoReadWriteError(IOError):
"""
This is the main neo-specific error that has to deal with
Expand Down Expand Up @@ -95,6 +115,8 @@ def merge_annotation(a, b):
raise TypeError(f"type({a}) {type(a)} != type({b}) {type(b)}")
if isinstance(a, dict):
return merge_annotations(a, b)
elif isinstance(a, pq.Quantity):
return _merge_quantity_arrays([a, b], axis=None)
elif isinstance(a, np.ndarray): # concatenate b to a
return np.append(a, b)
elif isinstance(a, list): # concatenate b to a
Expand Down
5 changes: 2 additions & 3 deletions neo/core/dataobject.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import quantities as pq
import numpy as np

from neo.core.baseneo import BaseNeo, _check_annotations
from neo.core.baseneo import BaseNeo, _check_annotations, _merge_quantity_arrays


def _normalize_array_annotations(value, length):
Expand Down Expand Up @@ -276,10 +276,9 @@ def _merge_array_annotations(self, other):
# Quantities need to be rescaled to common unit
if isinstance(value, pq.Quantity):
try:
other_value = other_value.rescale(value.units)
merged_array_annotations[key] = _merge_quantity_arrays([value, other_value], axis=None)
except ValueError:
raise ValueError("Could not merge array annotations " "due to different units")
merged_array_annotations[key] = np.append(value, other_value) * value.units
else:
merged_array_annotations[key] = np.append(value, other_value)

Expand Down
4 changes: 2 additions & 2 deletions neo/core/epoch.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,8 +321,8 @@ def merge(self, other):
"""
othertimes = other.times.rescale(self.times.units)
otherdurations = other.durations.rescale(self.durations.units)
times = np.hstack([self.times, othertimes]) * self.times.units
durations = np.hstack([self.durations, otherdurations]) * self.durations.units
times = np.hstack([self.times.magnitude, othertimes.magnitude]) * self.times.units
durations = np.hstack([self.durations.magnitude, otherdurations.magnitude]) * self.durations.units
labels = np.hstack([self.labels, other.labels])
kwargs = {}
for name in ("name", "description", "file_origin"):
Expand Down
2 changes: 1 addition & 1 deletion neo/core/event.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ def merge(self, other):
compatible, and Exception is raised.
"""
othertimes = other.times.rescale(self.times.units)
times = np.hstack([self.times, othertimes]) * self.times.units
times = np.hstack([self.times.magnitude, othertimes.magnitude]) * self.times.units
labels = np.hstack([self.labels, other.labels])
kwargs = {}
for name in ("name", "description", "file_origin"):
Expand Down
4 changes: 3 additions & 1 deletion neo/core/irregularlysampledsignal.py
Original file line number Diff line number Diff line change
Expand Up @@ -661,7 +661,9 @@ def concatenate(self, other, allow_overlap=False):
if other.units != self.units:
other = other.rescale(self.units)

new_times = np.hstack((self.times, other.times))
time_units = self.times.units
other_times = other.times.rescale(time_units)
new_times = np.hstack((self.times.magnitude, other_times.magnitude)) * time_units
sorting = np.argsort(new_times)
new_samples = np.vstack((self.magnitude, other.magnitude))

Expand Down
10 changes: 5 additions & 5 deletions neo/core/spiketrain.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import numpy as np
import quantities as pq
from neo.core.baseneo import BaseNeo, MergeError, merge_annotations
from neo.core.baseneo import BaseNeo, MergeError, _merge_quantity_arrays, merge_annotations
from neo.core.dataobject import DataObject, ArrayDict

# need this to avoid circular import issue
Expand Down Expand Up @@ -942,7 +942,7 @@ def merge(self, *others):
**kwargs,
)
if all(wfs):
wfs_stack = np.vstack([st.waveforms.rescale(self.waveforms.units) for st in all_spiketrains])
wfs_stack = np.vstack([st.waveforms.rescale(self.waveforms.units).magnitude for st in all_spiketrains])
wfs_stack = wfs_stack[sorting] * self.waveforms.units
train.waveforms = wfs_stack
train.segment = self.segment
Expand Down Expand Up @@ -972,11 +972,11 @@ def _merge_array_annotations(self, others, sorting=None):
for key in keys:
try:
self_ann = deepcopy(self.array_annotations[key])
other_ann = np.concatenate([deepcopy(other.array_annotations[key]) for other in others])
if isinstance(self_ann, pq.Quantity):
other_ann.rescale(self_ann.units)
arr_ann = np.concatenate([self_ann, other_ann]) * self_ann.units
quantities = [self_ann] + [deepcopy(other.array_annotations[key]) for other in others]
arr_ann = _merge_quantity_arrays(quantities, axis=0)
else:
other_ann = np.concatenate([deepcopy(other.array_annotations[key]) for other in others])
arr_ann = np.concatenate([self_ann, other_ann])
merged_array_annotations[key] = arr_ann[sorting]
# Annotation only available in 'self', must be skipped
Expand Down
18 changes: 11 additions & 7 deletions neo/core/spiketrainlist.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,16 @@ def _add_spiketrainlists(self, other, in_place=False):
if self._spiketrain_metadata["t_stop"] != other._spiketrain_metadata["t_stop"]:
raise ValueError("Incompatible t_stop")
# todo: adjust t_stop of self and other as necessary
combined_spike_time_array = np.hstack((self._spike_time_array, other._spike_time_array))
reference_units = self._spike_time_array.units
combined_spike_time_array = (
np.hstack(
(
self._spike_time_array.rescale(reference_units).magnitude,
other._spike_time_array.rescale(reference_units).magnitude,
)
)
* reference_units
)
combined_channel_id_array = np.hstack((self._channel_id_array, other._channel_id_array))
combined_channel_ids = set(list(self._all_channel_ids) + other._all_channel_ids)
if len(combined_channel_ids) != (len(self._all_channel_ids) + len(other._all_channel_ids)):
Expand Down Expand Up @@ -386,15 +395,10 @@ def multiplexed(self):
else:
channel_ids = []
spike_times = []
dim = self._items[0].units.dimensionality
for i, spiketrain in enumerate(self._items):
if hasattr(spiketrain, "load"): # proxy object
spiketrain = spiketrain.load()
if spiketrain.times.dimensionality.items() == dim.items():
# no need to rescale
spike_times.append(spiketrain.times)
else:
spike_times.append(spiketrain.times.rescale(dim))
spike_times.append(spiketrain.times.rescale(self._items[0].units).magnitude)
if "channel_id" in spiketrain.annotations and isinstance(spiketrain.annotations["channel_id"], int):
ch_id = spiketrain.annotations["channel_id"]
else:
Expand Down
3 changes: 3 additions & 0 deletions neo/io/brainwaresrcio.py
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,9 @@ def _combine_spiketrains(self, spiketrains):
if not spiketrains:
return self._default_spiketrain.copy()

# Reader-created SpikeTrains are normalized to milliseconds by this method below.
# The source decoder returns raw millisecond values, so dropping Quantity metadata
# here and rebuilding the result in milliseconds preserves the file-format values.
# get the times of the spiketrains and combine them
waveforms = [itrain.waveforms for itrain in spiketrains]
rawtrains = np.array(np.concatenate(spiketrains, axis=1))
Expand Down
51 changes: 23 additions & 28 deletions neo/io/elphyio.py
Original file line number Diff line number Diff line change
Expand Up @@ -3695,6 +3695,25 @@ def get_names(self):
# --------------------------------------------------------


def _stack_analog_signal_magnitudes(analogsignalarrays):
"""Stack analog signal channels without converting their per-channel units."""
return np.hstack([np.asarray(signal.magnitude) for signal in analogsignalarrays])


def _serialize_analog_signal_channels(analogsignalarrays):
"""Return Elphy channel metadata and RDATA bytes in each channel's native units."""
if not analogsignalarrays:
return 0, 0, b"", []

analogsignals = _stack_analog_signal_magnitudes(analogsignalarrays).T
nbchan, nbpt = analogsignals.shape
analog_data = np.array(analogsignals.flatten("F"), dtype=np.float32)
data_format = "<" + str(analogsignals.size) + "f"
serialized_analog_data = struct.pack(data_format, *analog_data)
aa_units = [signal.units for signal in analogsignalarrays for _ in range(signal.shape[1])]
return nbchan, nbpt, serialized_analog_data, aa_units


class ElphyIO(BaseIO):
"""
Class for reading from and writing to an Elphy file.
Expand Down Expand Up @@ -3831,17 +3850,13 @@ def write_block(self, block):
# print "\n\n--------------------------------------------\n"
# print "write_block() - n_segments:",len(block.segments)
for seg in block.segments:
analogsignals = 0 # init
nbchan = 0
nbpt = 0
chls = 0
analogsignal_arrays = []
Dxu = 1e-8 # 0.0000001
Rxu = 1e8 # 10000000.0
X0uSpk = 0.0
CyberTime = 0.0
aa_units = []
NbEv = []
serialized_analog_data = ""
serialized_analog_data = b""
serialized_spike_data = ""
# AnalogSignals
# Neo signalarrays are 2D numpy array where each row is an array of samples for a
Expand Down Expand Up @@ -3871,30 +3886,10 @@ def write_block(self, block):
annotations, "analogsignal", asigar.annotations.items(), asigar.name, idx
)
# array structure
_, chls = asigar.shape
# units
for _ in range(chls):
aa_units.append(asigar.units)
Dxu = asigar.sampling_period
Rxu = asigar.sampling_rate
if isinstance(analogsignals, np.ndarray):
analogsignals = np.hstack((analogsignals, asigar))
else:
analogsignals = asigar # first time
# collect and reshape all analogsignals
if isinstance(analogsignals, np.ndarray):
# transpose matrix since in Neo channels are column-wise while in Elphy are
# row-wise
analogsignals = analogsignals.T
# get dimensions
nbchan, nbpt = analogsignals.shape
# serialize AnalogSignal
analog_data_fmt = "<" + str(analogsignals.size) + "f"
# serialized flattened numpy channels in 'F'ortran style
analog_data_64 = analogsignals.flatten("F")
# elphy normally uses float32 values (for performance reasons)
analog_data = np.array(analog_data_64, dtype=np.float32)
serialized_analog_data += struct.pack(analog_data_fmt, *analog_data)
analogsignal_arrays.append(asigar)
nbchan, nbpt, serialized_analog_data, aa_units = _serialize_analog_signal_channels(analogsignal_arrays)
# SpikeTrains
# Neo spiketrains are stored as a one-dimensional array of times
# [ 0.11, 1.23, 2.34, 3.45, 4.56, 5.67, 6.78, 7.89 ... ]
Expand Down
27 changes: 27 additions & 0 deletions neo/test/coretest/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,33 @@ def test_merge_annotation__func__ndarray(self):
assert_arrays_equal(ann1, ann1c)
assert_arrays_equal(ann2, ann2c)

def test_merge_annotation__func__quantity(self):
ann1 = np.array([1, 2]) * pq.s
ann2 = np.array([500, 250]) * pq.ms

result = merge_annotation(ann1, ann2)

self.assertEqual(result.units, pq.s)
np.testing.assert_array_equal(result.magnitude, [1, 2, 0.5, 0.25])

with self.assertRaises(ValueError):
merge_annotation(ann1, np.array([1, 2]) * pq.m)

def test_merge_annotation__func__uncertain_quantity(self):
ann1 = pq.UncertainQuantity([1, 2], units=pq.s, uncertainty=[0.1, 0.2])
ann2 = pq.UncertainQuantity([500], units=pq.ms, uncertainty=[25])

result = merge_annotation(ann1, ann2)

self.assertIsInstance(result, pq.UncertainQuantity)
self.assertEqual(result.units, pq.s)
np.testing.assert_array_equal(result.magnitude, [1, 2, 0.5])
self.assertEqual(result.uncertainty.units, pq.s)
np.testing.assert_array_equal(result.uncertainty.magnitude, [0.1, 0.2, 0.025])

with self.assertRaises(ValueError):
merge_annotation(ann1, pq.UncertainQuantity([1], units=pq.m, uncertainty=[0.1]))

def test_merge_annotation__func__list(self):
ann1 = [0, 1, 2]
ann2 = [4, 5, 6]
Expand Down
44 changes: 42 additions & 2 deletions neo/test/coretest/test_dataobject.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import copy
import unittest

import numpy as np
import unittest
import quantities as pq

from neo.core.dataobject import DataObject, _normalize_array_annotations, ArrayDict
from neo.core.dataobject import ArrayDict, DataObject, _normalize_array_annotations


class Test_DataObject(unittest.TestCase):
Expand All @@ -12,6 +13,45 @@ def test(self):


class Test_array_annotations(unittest.TestCase):
def test_merge_quantity_array_annotations(self):
first = DataObject([1, 2])
first.array_annotate(quantity=[1, 2] * pq.s, plain=np.array([10, 20]))
second = DataObject([3, 4])
second.array_annotate(quantity=[500, 250] * pq.ms, plain=np.array([30, 40]))

merged = first._merge_array_annotations(second)

self.assertEqual(merged["quantity"].units, pq.s)
np.testing.assert_array_equal(merged["quantity"].magnitude, [1, 2, 0.5, 0.25])
self.assertEqual(merged["quantity"].shape, (4,))
np.testing.assert_array_equal(merged["plain"], [10, 20, 30, 40])

second.array_annotations["quantity"] = [1, 2] * pq.m
with self.assertRaisesRegex(ValueError, "Could not merge array annotations"):
first._merge_array_annotations(second)

def test_merge_uncertain_quantity_array_annotations(self):
first = DataObject([1, 2])
first.array_annotate(quantity=pq.UncertainQuantity([1, 2], units=pq.s, uncertainty=[0.1, 0.2]))
second = DataObject([3])
second.array_annotate(quantity=pq.UncertainQuantity([500], units=pq.ms, uncertainty=[25]))

merged = first._merge_array_annotations(second)["quantity"]

self.assertIsInstance(merged, pq.UncertainQuantity)
self.assertEqual(merged.units, pq.s)
np.testing.assert_array_equal(merged.magnitude, [1, 2, 0.5])
self.assertEqual(merged.uncertainty.units, pq.s)
np.testing.assert_array_equal(merged.uncertainty.magnitude, [0.1, 0.2, 0.025])

second.array_annotations["quantity"] = [500] * pq.ms
with self.assertRaisesRegex(ValueError, "Could not merge array annotations"):
first._merge_array_annotations(second)

second.array_annotations["quantity"] = pq.UncertainQuantity([1], units=pq.m, uncertainty=[0.1])
with self.assertRaisesRegex(ValueError, "Could not merge array annotations"):
first._merge_array_annotations(second)

def test_check_arr_ann(self):
# DataObject instance that handles checking
datobj = DataObject([1, 2]) # Inherits from Quantity, so some data is required
Expand Down
5 changes: 5 additions & 0 deletions neo/test/coretest/test_epoch.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,11 @@ def test_Epoch_merge(self):
)

assert_neo_object_is_compliant(epcres)
self.assertEqual(epcres.times.units, pq.ms)
self.assertEqual(epcres.durations.units, pq.us)
np.testing.assert_allclose(epcres.times.magnitude, [1.1, 1.5, 1.7, 0.0021, 0.0025, 0.0027])
np.testing.assert_allclose(epcres.durations.magnitude, [20, 40, 60, 3000, 5000, 7000])
assert_array_equal(epcres.labels, epctarg.labels)
assert_same_sub_schema(epctarg, epcres)
# Remove this, when array_annotations are added to assert_same_sub_schema
assert_arrays_equal(epcres.array_annotations["index"], np.array([10, 11, 12, 0, 1, 2]))
Expand Down
3 changes: 3 additions & 0 deletions neo/test/coretest/test_event.py
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,9 @@ def test_Event_merge(self):
)

assert_neo_object_is_compliant(evtres)
self.assertEqual(evtres.times.units, pq.ms)
np.testing.assert_allclose(evtres.times.magnitude, [1.1, 1.5, 1.7, 0.0021, 0.0025, 0.0027])
assert_array_equal(evtres.labels, evttarg.labels)
assert_same_sub_schema(evttarg, evtres)
# Remove this, when array_annotations are added to assert_same_sub_schema
assert_arrays_equal(evtres.array_annotations["index"], np.array([10, 11, 12, 0, 1, 2]))
Expand Down
10 changes: 10 additions & 0 deletions neo/test/coretest/test_irregularysampledsignal.py
Original file line number Diff line number Diff line change
Expand Up @@ -978,6 +978,16 @@ def test_concatenate_simple(self):
continue
self.assertEqual(getattr(signal1, attr[0], None), getattr(result, attr[0], None))

def test_concatenate_rescales_time_units(self):
signal1 = IrregularlySampledSignal(signal=[1, 2] * pq.mV, times=[1, 2] * pq.s)
signal2 = IrregularlySampledSignal(signal=[3, 4] * pq.mV, times=[3000, 4000] * pq.ms)

result = signal1.concatenate(signal2)

self.assertEqual(result.times.units, pq.s)
assert_array_equal(result.times.magnitude, [1, 2, 3, 4])
assert_array_equal(result.magnitude, [[1], [2], [3], [4]])

def test_concatenate_no_overlap(self):
signal1 = IrregularlySampledSignal(signal=[0, 1, 2, 3] * pq.s, times=range(4) * pq.s)
signal2 = IrregularlySampledSignal(signal=[4, 5, 6] * pq.s, times=range(4, 7) * pq.s)
Expand Down
Loading