diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md new file mode 100644 index 0000000000..7375d9a392 --- /dev/null +++ b/python/changelog.d/861.feat.md @@ -0,0 +1,17 @@ +# Bit field columns in import configs + +Bit field columns are supported in CSV, Parquet, TDMS and HDF5 imports through `bit_field_elements`. ULog and MCAP imports don't support bit fields and raise `ValueError`. + +Set `data_type` to `BIT_FIELD` on a detected column and list its elements: + +```python +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType + +config = client.data_import.detect_config("data.parquet", data_type=DataTypeKey.PARQUET_FLATDATASET) +status = config["status"] +status.data_type = ChannelDataType.BIT_FIELD +status.bit_field_elements = [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), +] +``` diff --git a/python/changelog.d/861.fix.md b/python/changelog.d/861.fix.md new file mode 100644 index 0000000000..4a3ad71877 --- /dev/null +++ b/python/changelog.d/861.fix.md @@ -0,0 +1 @@ +MCAP import configs keep `enum_types` when sent to Sift and when read back from a data import. Before, enum definitions on MCAP data columns were dropped. diff --git a/python/examples/data_import/parquet/flat_dataset/main.py b/python/examples/data_import/parquet/flat_dataset/main.py index 0020dd785e..6f13d49da0 100644 --- a/python/examples/data_import/parquet/flat_dataset/main.py +++ b/python/examples/data_import/parquet/flat_dataset/main.py @@ -46,6 +46,22 @@ # # Example: drop a column from the import # config.data_columns = [dc for dc in config.data_columns if dc.path != "channel_0"] # + # # Example: import a binary column as a bit field. detect_config can't + # # infer the bit layout, so supply it here. Each element's index is its + # # starting bit and bit_count is its width. Values must be bytes (or a + # # string of 0s and 1s, or base64), not integers; convert an integer + # # column first, e.g. pa.array([int(v).to_bytes(1, "little") for v in col], + # # pa.binary()). + # from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType + # + # status = config["status"] + # status.data_type = ChannelDataType.BIT_FIELD + # status.bit_field_elements = [ + # ChannelBitFieldElement(name="armed", index=0, bit_count=1), + # ChannelBitFieldElement(name="error_code", index=1, bit_count=3), + # ChannelBitFieldElement(name="mode", index=4, bit_count=4), + # ] + # # import_job = client.data_import.import_from_path( # "sample_data.parquet", # asset=asset_name, diff --git a/python/lib/sift_client/_internal/util/bit_field.py b/python/lib/sift_client/_internal/util/bit_field.py new file mode 100644 index 0000000000..527a501b67 --- /dev/null +++ b/python/lib/sift_client/_internal/util/bit_field.py @@ -0,0 +1,112 @@ +"""Check bit field values before upload, so a wrong encoding fails early. + +The import service reads a bit field value as raw bytes, a string of 0s and 1s, +or URL-safe base64, and fails the import on the first value it can't parse. It +only reports that after the whole file uploads. These checks read the +Parquet schema and a sample of values, and raise before the upload starts. +""" + +from __future__ import annotations + +import base64 +import binascii +import csv +import re +from typing import TYPE_CHECKING, Iterable, Mapping + +from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.data_import import ( + CsvImportConfig, + DataColumnBase, + ParquetFlatDatasetImportConfig, + ParquetSingleChannelPerRowImportConfig, +) + +if TYPE_CHECKING: + from pathlib import Path + + from sift_client.sift_types.data_import import ImportConfig + +SAMPLE_ROWS = 1000 + +_BITS = re.compile(r"[01]+\Z") +_BASE64_URL = re.compile(r"[A-Za-z0-9_-]*={0,2}\Z") + +ACCEPTED_FORMATS = "raw bytes, a string of 0s and 1s like '00110101', or URL-safe base64" + + +def check_bit_field_values(config: ImportConfig, path: Path) -> None: + """Raise if a ``BIT_FIELD`` column's values aren't in a format the import service reads.""" + if isinstance(config, CsvImportConfig): + csv_columns = { + dc.column - 1: dc + for dc in config.data_columns + if dc.data_type == ChannelDataType.BIT_FIELD + } + if csv_columns: + _check_csv(path, config.first_data_row, csv_columns) + return + + parquet_columns: dict[str, DataColumnBase] + if isinstance(config, ParquetFlatDatasetImportConfig): + parquet_columns = { + dc.path: dc for dc in config.data_columns if dc.data_type == ChannelDataType.BIT_FIELD + } + elif isinstance(config, ParquetSingleChannelPerRowImportConfig): + sc = config.single_channel + parquet_columns = ( + {sc.data_path: sc} + if sc is not None and sc.data_type == ChannelDataType.BIT_FIELD + else {} + ) + else: + return + if parquet_columns: + try: + from sift_client._internal.util.parquet import check_bit_field_columns + except ImportError as e: + raise RuntimeError( + "pyarrow is required for Parquet import. " + "Install it via `pip install sift-stack-py[parquet]`." + ) from e + check_bit_field_columns(path, parquet_columns) + + +def _is_bit_value(value: str) -> bool: + """Whether the import service can parse ``value`` as a bit field.""" + if _BITS.match(value): + return True + if len(value) % 4 or not _BASE64_URL.match(value): + return False + try: + base64.urlsafe_b64decode(value) + except (binascii.Error, ValueError): + return False + return True + + +def _check_values(dc: DataColumnBase, values: Iterable[tuple[int, str]]) -> None: + """Check sampled ``(row, value)`` pairs from one column. Empty values are skipped.""" + for row, value in values: + if value == "": + continue + if not _is_bit_value(value): + raise ValueError( + f"bit field column '{dc.name}' has value {value!r} in row {row}, which isn't " + f"{ACCEPTED_FORMATS}. To convert integers, use format(value, '08b') for an " + "8-bit status word." + ) + + +def _check_csv(path: Path, first_data_row: int, columns: Mapping[int, DataColumnBase]) -> None: + samples: dict[int, list[tuple[int, str]]] = {i: [] for i in columns} + with open(path, newline="", encoding="utf-8") as f: + for row_number, row in enumerate(csv.reader(f), start=1): + if row_number < first_data_row: + continue + if row_number >= first_data_row + SAMPLE_ROWS: + break + for i in columns: + samples[i].append((row_number, row[i] if i < len(row) else "")) + for i, dc in columns.items(): + _check_values(dc, samples[i]) diff --git a/python/lib/sift_client/_internal/util/parquet.py b/python/lib/sift_client/_internal/util/parquet.py index b69dacac48..66c021ba69 100644 --- a/python/lib/sift_client/_internal/util/parquet.py +++ b/python/lib/sift_client/_internal/util/parquet.py @@ -18,6 +18,7 @@ from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto from sift_client._internal.time import to_timestamp_nanos +from sift_client._internal.util.bit_field import ACCEPTED_FORMATS, SAMPLE_ROWS, _check_values from sift_client._internal.util.file import extract_parquet_footer from sift_client._internal.util.time_column import infer_time_column from sift_client.sift_types.channel import ChannelDataType @@ -27,11 +28,15 @@ ParquetTimeColumn, TimeFormat, TimeFormatProto, + _bit_field_elements_from_proto, _enum_types_from_proto, ) if TYPE_CHECKING: from datetime import datetime + from typing import Mapping + + from sift_client.sift_types.data_import import DataColumnBase # Separator used in column paths to address nested struct fields. Channel # names replace it with "." since Sift forbids "|" in channel names. @@ -222,6 +227,7 @@ def _parse_embedded_data_column(path: str, raw: bytes) -> ParquetDataColumn: units=config.units, description=config.description, enum_types=_enum_types_from_proto(config), + bit_field_elements=_bit_field_elements_from_proto(config), ) @@ -318,3 +324,69 @@ def detect_flat_parquet_config( footer_offset=footer_offset, footer_length=len(footer_bytes), ) + + +def check_bit_field_columns(path: Path, columns: Mapping[str, DataColumnBase]) -> None: + """Raise if a ``BIT_FIELD`` column isn't binary, or its sampled string values don't parse.""" + schema = pq.read_schema(path) + string_columns: dict[str, DataColumnBase] = {} + for column_path, dc in columns.items(): + field_type = _field_type(schema, column_path.split(PATH_SEPARATOR)) + if field_type is None: + # The import service reports columns missing from the file. + continue + if ( + pa.types.is_binary(field_type) + or pa.types.is_large_binary(field_type) + or pa.types.is_binary_view(field_type) + or pa.types.is_fixed_size_binary(field_type) + ): + continue + if ( + pa.types.is_string(field_type) + or pa.types.is_large_string(field_type) + or pa.types.is_string_view(field_type) + ): + string_columns[column_path] = dc + continue + raise ValueError( + f"bit field column '{dc.name}' is {field_type}, but bit field values must be " + f"{ACCEPTED_FORMATS}. Store the column as binary, for example " + "pa.array([int(v).to_bytes(2, 'little') for v in values], pa.binary()) for a " + "16-bit status word." + ) + + if not string_columns: + return + # Read only the sampled rows of the string columns. + pf = pq.ParquetFile(path) + top_level = sorted({p.split(PATH_SEPARATOR)[0] for p in string_columns}) + batch = next(pf.iter_batches(batch_size=SAMPLE_ROWS, columns=top_level), None) + if batch is None: + return + table = pa.Table.from_batches([batch]) + for column_path, dc in string_columns.items(): + values = _column_values(table, column_path.split(PATH_SEPARATOR)) + _check_values(dc, ((row, "" if v is None else v) for row, v in enumerate(values, start=1))) + + +def _field_type(schema: pa.Schema, parts: list[str]) -> pa.DataType | None: + index = schema.get_field_index(parts[0]) + if index == -1: + return None + field_type = schema.field(index).type + for part in parts[1:]: + if not pa.types.is_struct(field_type): + return None + child = field_type.get_field_index(part) + if child == -1: + return None + field_type = field_type.field(child).type + return field_type + + +def _column_values(table: pa.Table, parts: list[str]) -> list[str | None]: + column = table.column(parts[0]).combine_chunks() + for part in parts[1:]: + column = column.field(part) + return column.to_pylist() diff --git a/python/lib/sift_client/_tests/_internal/test_bit_field.py b/python/lib/sift_client/_tests/_internal/test_bit_field.py new file mode 100644 index 0000000000..3782e6dd76 --- /dev/null +++ b/python/lib/sift_client/_tests/_internal/test_bit_field.py @@ -0,0 +1,222 @@ +"""Tests for checking bit field values before upload.""" + +from __future__ import annotations + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from sift_client._internal.util.bit_field import SAMPLE_ROWS, check_bit_field_values +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType +from sift_client.sift_types.data_import import ( + CsvDataColumn, + CsvImportConfig, + CsvTimeColumn, + ParquetDataColumn, + ParquetFlatDatasetImportConfig, + ParquetSingleChannelConfig, + ParquetSingleChannelPerRowImportConfig, + ParquetTimeColumn, + TimeFormat, +) + +ELEMENTS = [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), +] + + +def _csv_config(first_data_row=2): + return CsvImportConfig( + asset_name="a", + first_data_row=first_data_row, + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ELEMENTS, + ), + CsvDataColumn(column=3, name="x", data_type=ChannelDataType.DOUBLE), + ], + ) + + +def _write_csv(path, values): + path.write_text("time,status,x\n" + "".join(f"t,{v},1.5\n" for v in values)) + + +class TestCsv: + @pytest.mark.parametrize( + "values", + [ + pytest.param(["00110101", "00000000", "11111111"], id="bit-strings"), + pytest.param(["0", "1", "1"], id="single-bits"), + pytest.param(["NQ==", "AA==", "_w=="], id="base64"), + pytest.param(["00110101", "", "00000001"], id="empty-cells"), + pytest.param(["0", "1", "10", "11"], id="unpadded-bit-strings"), + pytest.param(["1234", "2048"], id="digit-only-base64"), + ], + ) + def test_accepted_values_pass(self, tmp_path, values): + path = tmp_path / "data.csv" + _write_csv(path, values) + + check_bit_field_values(_csv_config(), path) + + def test_unparseable_value_raises_with_row(self, tmp_path): + path = tmp_path / "data.csv" + _write_csv(path, ["00110101", "53"]) + + with pytest.raises(ValueError, match=r"value '53' in row 3"): + check_bit_field_values(_csv_config(), path) + + def test_rows_before_first_data_row_are_skipped(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("units,raw,m\ntime,status,x\nt,00000101,1\n") + + check_bit_field_values(_csv_config(first_data_row=3), path) + + def test_only_the_sample_is_read(self, tmp_path): + path = tmp_path / "data.csv" + _write_csv(path, ["00000001"] * SAMPLE_ROWS + ["53"]) + + check_bit_field_values(_csv_config(), path) + + def test_no_bit_field_columns_skips_reading_the_file(self, tmp_path): + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[CsvDataColumn(column=2, name="x", data_type=ChannelDataType.DOUBLE)], + ) + + check_bit_field_values(config, tmp_path / "missing.csv") + + +def _parquet_config(column="status"): + return ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ + ParquetDataColumn( + path=column, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ELEMENTS, + ), + ParquetDataColumn(path="x", name="x", data_type=ChannelDataType.DOUBLE), + ], + ) + + +def _write_parquet(path, status: pa.Array): + rows = len(status) + pq.write_table( + pa.table( + { + "t": pa.array(range(rows), pa.int64()), + "status": status, + "x": pa.array([0.0] * rows), + } + ), + path, + ) + + +class TestParquet: + @pytest.mark.parametrize( + "status", + [ + pytest.param(pa.array([b"\x35", None], pa.binary()), id="binary"), + pytest.param(pa.array([b"\x35"], pa.large_binary()), id="large-binary"), + pytest.param(pa.array([b"\x35"], pa.binary(1)), id="fixed-size-binary"), + pytest.param(pa.array(["00110101", None], pa.string()), id="bit-string"), + pytest.param(pa.array(["NQ=="], pa.large_string()), id="base64"), + ], + ) + def test_accepted_types_pass(self, tmp_path, status): + path = tmp_path / "data.parquet" + _write_parquet(path, status) + + check_bit_field_values(_parquet_config(), path) + + @pytest.mark.parametrize( + ("status", "type_name"), + [ + pytest.param(pa.array([53], pa.uint8()), "uint8", id="uint8"), + pytest.param(pa.array([53], pa.int64()), "int64", id="int64"), + pytest.param(pa.array([[53]], pa.list_(pa.uint8())), "list<", id="list"), + pytest.param(pa.array([True], pa.bool_()), "bool", id="bool"), + ], + ) + def test_other_types_raise(self, tmp_path, status, type_name): + path = tmp_path / "data.parquet" + _write_parquet(path, status) + + with pytest.raises(ValueError, match=rf"'status' is {type_name}"): + check_bit_field_values(_parquet_config(), path) + + def test_unparseable_string_raises(self, tmp_path): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array(["00110101", "not bits"], pa.string())) + + with pytest.raises(ValueError, match=r"value 'not bits' in row 2"): + check_bit_field_values(_parquet_config(), path) + + def test_nested_field_type_is_checked(self, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table( + { + "t": pa.array([0], pa.int64()), + "s": pa.array([{"status": 53}], pa.struct([("status", pa.uint8())])), + "x": pa.array([0.0]), + } + ), + path, + ) + + with pytest.raises(ValueError, match="'status' is uint8"): + check_bit_field_values(_parquet_config(column="s|status"), path) + + def test_nested_string_values_are_sampled(self, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table( + { + "t": pa.array([0], pa.int64()), + "s": pa.array([{"status": "53"}], pa.struct([("status", pa.string())])), + "x": pa.array([0.0]), + } + ), + path, + ) + + with pytest.raises(ValueError, match="value '53' in row 1"): + check_bit_field_values(_parquet_config(column="s|status"), path) + + def test_missing_column_is_left_to_the_server(self, tmp_path): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array([b"\x35"], pa.binary())) + + check_bit_field_values(_parquet_config(column="absent"), path) + + def test_single_channel_per_row(self, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table({"t": pa.array([0], pa.int64()), "value": pa.array([2], pa.uint8())}), path + ) + config = ParquetSingleChannelPerRowImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + single_channel=ParquetSingleChannelConfig( + data_path="value", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ELEMENTS, + ), + ) + + with pytest.raises(ValueError, match="'status' is uint8"): + check_bit_field_values(config, path) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py index 16bc587245..c766f320b0 100644 --- a/python/lib/sift_client/_tests/_internal/test_parquet.py +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -15,7 +15,7 @@ from sift_client._internal.util.parquet import detect_flat_parquet_config from sift_client.resources import DataImportAPIAsync -from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType from sift_client.sift_types.data_import import ( DataTypeKey, TimeFormat, @@ -432,6 +432,38 @@ def test_camel_case_keys_and_numeric_enums(self, create_parquet_file): assert config["x"].data_type == ChannelDataType.ENUM assert config["x"].enum_types == {"on": 1} + def test_embedded_bit_field_elements(self, create_parquet_file): + table = _embedded_table( + [ + _embedded_field( + "t", + pa.int64(), + _TIME_CONFIG, + {"time_format": "TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS"}, + ), + _embedded_field( + "status", + pa.uint8(), + _CHANNEL_CONFIG, + { + "name": "status", + "data_type": "CHANNEL_DATA_TYPE_BIT_FIELD", + "bit_field_elements": [ + {"name": "armed", "index": 0, "bit_count": 1}, + {"name": "mode", "index": 4, "bit_count": 4}, + ], + }, + ), + ] + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert config["status"].data_type == ChannelDataType.BIT_FIELD + assert config["status"].bit_field_elements == [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), + ] + @pytest.mark.parametrize( ("fields", "match"), [ diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index a14f712c14..7fcc973e05 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2,6 +2,7 @@ from __future__ import annotations +import uuid import warnings from datetime import datetime, timezone from typing import TYPE_CHECKING, cast @@ -41,7 +42,7 @@ from sift_client._internal.util.time_column import infer_time_column from sift_client._tests.sift_types.channel_time import assert_axis_matches_channel from sift_client.resources import DataImportAPI, DataImportAPIAsync -from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType from sift_client.sift_types.data_import import ( CsvDataColumn, CsvImportConfig, @@ -1610,3 +1611,437 @@ def test_enum_types_reject_duplicate_keys(self): def test_no_enum_types_from_proto_is_none(self, csv_config): round_tripped = CsvImportConfig._from_proto(csv_config._to_proto()) assert round_tripped["cpu_util"].enum_types is None + + def test_mcap_round_trip(self): + config = McapImportConfig( + asset_name="a", + data=[ + McapDataColumn( + topic="/state", + field_path="mode", + data_type=ChannelDataType.ENUM, + enum_types=ENUMS, + ) + ], + ) + proto = config._to_proto() + assert _enum_dict(proto.data[0].channel_config) == ENUMS + assert McapImportConfig._from_proto(proto).data[0].enum_types == ENUMS + + def test_in_place_edit_is_validated_on_import(self, csv_config): + csv_config["cpu_util"].data_type = ChannelDataType.ENUM + with pytest.raises(ValueError, match="requires 'enum_types'"): + csv_config._to_proto() + + +BIT_FIELDS = [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="error_code", index=1, bit_count=3), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), +] + + +def _bit_field_list(channel_config: ChannelConfigProto) -> list[ChannelBitFieldElement]: + return [ChannelBitFieldElement._from_proto(el) for el in channel_config.bit_field_elements] + + +class TestBitFieldElements: + def test_csv_round_trip(self): + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.data_columns[2]) == BIT_FIELDS + assert CsvImportConfig._from_proto(proto)["status"].bit_field_elements == BIT_FIELDS + + def test_parquet_flat_dataset_round_trip(self): + config = ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="ts", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ + ParquetDataColumn( + path="status", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.flat_dataset.data_columns[0].channel_config) == BIT_FIELDS + round_tripped = ParquetFlatDatasetImportConfig._from_proto(proto) + assert round_tripped["status"].bit_field_elements == BIT_FIELDS + + def test_parquet_single_channel_per_row_round_trip(self): + config = ParquetSingleChannelPerRowImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="ts", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + single_channel=ParquetSingleChannelConfig( + data_path="value", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ), + ) + proto = config._to_proto() + assert _bit_field_list(proto.single_channel_per_row.single_channel.channel) == BIT_FIELDS + round_tripped = ParquetSingleChannelPerRowImportConfig._from_proto(proto) + assert round_tripped.single_channel is not None + assert round_tripped.single_channel.bit_field_elements == BIT_FIELDS + + def test_tdms_round_trip(self): + config = TdmsImportConfig( + asset_name="a", + data=[ + TdmsDataColumn( + group_name="g", + channel_name="status", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS + round_tripped = TdmsImportConfig._from_proto(proto) + assert round_tripped.data[0].bit_field_elements == BIT_FIELDS + + @pytest.mark.parametrize( + "make_column", + [ + pytest.param( + lambda **kw: UlogDataColumn( + message_name="vehicle_status", field_name="flags", **kw + ), + id="ulog", + ), + pytest.param( + lambda **kw: McapDataColumn(topic="/state", field_path="flags", **kw), + id="mcap", + ), + ], + ) + def test_unsupported_formats_reject_bit_fields(self, make_column): + with pytest.raises(ValueError, match="doesn't support ChannelDataType.BIT_FIELD"): + make_column(data_type=ChannelDataType.BIT_FIELD, bit_field_elements=BIT_FIELDS) + + column = make_column(data_type=ChannelDataType.UINT_32) + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = BIT_FIELDS + with pytest.raises(ValueError, match="doesn't support ChannelDataType.BIT_FIELD"): + column._channel_config_proto() + + def test_hdf5_to_proto(self): + config = Hdf5ImportConfig( + asset_name="a", + data=[ + Hdf5DataColumn( + name="status", + data_type=ChannelDataType.BIT_FIELD, + time_dataset="/t", + value_dataset="/status", + bit_field_elements=BIT_FIELDS, + ) + ], + time_format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS, + ) + proto = config._to_proto() + assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS + + def test_elements_require_bit_field_data_type(self): + with pytest.raises(ValueError, match="BIT_FIELD"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.UINT_32, + bit_field_elements=BIT_FIELDS, + ) + + def test_bit_field_data_type_requires_elements(self): + with pytest.raises(ValueError, match="requires 'bit_field_elements'"): + CsvDataColumn(column=2, name="status", data_type=ChannelDataType.BIT_FIELD) + + def test_rejects_duplicate_names(self): + with pytest.raises(ValueError, match="[Dd]uplicate"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=[ + ChannelBitFieldElement(name="a", index=0, bit_count=1), + ChannelBitFieldElement(name="a", index=1, bit_count=1), + ], + ) + + @pytest.mark.parametrize( + ("index", "bit_count"), + [pytest.param(-1, 1, id="negative-index"), pytest.param(0, 0, id="zero-width")], + ) + def test_rejects_invalid_range(self, index, bit_count): + with pytest.raises(ValueError, match="index >= 0 and bit_count >= 1"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=[ + ChannelBitFieldElement(name="a", index=index, bit_count=bit_count) + ], + ) + + def test_rejects_overlapping_elements(self): + with pytest.raises(ValueError, match="overlaps"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=[ + ChannelBitFieldElement(name="a", index=0, bit_count=4), + ChannelBitFieldElement(name="b", index=3, bit_count=2), + ], + ) + + def test_no_elements_from_proto_is_none(self, csv_config): + round_tripped = CsvImportConfig._from_proto(csv_config._to_proto()) + assert round_tripped["cpu_util"].bit_field_elements is None + + def test_in_place_edit_is_validated_on_import(self, parquet_config): + column = parquet_config.data_columns[0] + column.data_type = ChannelDataType.BIT_FIELD + with pytest.raises(ValueError, match="requires 'bit_field_elements'"): + parquet_config._to_proto() + column.bit_field_elements = BIT_FIELDS + proto = parquet_config._to_proto() + assert _bit_field_list(proto.flat_dataset.data_columns[0].channel_config) == BIT_FIELDS + + +class TestImportFromPathBitFieldCheck: + @pytest.mark.asyncio + async def test_unreadable_values_fail_before_upload(self, tmp_path, monkeypatch): + upload = MagicMock() + monkeypatch.setattr("sift_client.resources.data_imports.upload_file", upload) + api = DataImportAPIAsync(MagicMock()) + low_level = MagicMock() + low_level.create_from_upload = AsyncMock(return_value=("import_1", "url")) + api._low_level_client = low_level + path = tmp_path / "status.csv" + path.write_text("time,status\nt0,53\n") + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + + with pytest.raises(ValueError, match="value '53' in row 2"): + await api.import_from_path(path, config=config, show_progress=False) + + low_level.create_from_upload.assert_not_called() + upload.assert_not_called() + + +# Raw status values and their decoded elements under BIT_FIELDS: +# bit 0 is armed, bits 1-3 are error_code, bits 4-7 are mode. +STATUS_VALUES = [53, 0, 255, 1] +DECODED_STATUS = { + "armed": [1, 0, 1, 1], + "error_code": [2, 0, 7, 0], + "mode": [3, 0, 15, 0], +} + +# A 16-bit status word, to check that multi-byte values decode little-endian. +WIDE_BIT_FIELDS = [ + ChannelBitFieldElement(name="low", index=0, bit_count=8), + ChannelBitFieldElement(name="high", index=8, bit_count=8), +] +WIDE_VALUES = [0x1234, 0x00FF, 0xFF00, 1] +DECODED_WIDE = {"low": [0x34, 0xFF, 0x00, 1], "high": [0x12, 0x00, 0xFF, 0]} + +# TDMS and HDF5 store typed integers, which import as their two's complement bytes. +SIGNED_VALUES = [0x1234, -1, -256, 1] +DECODED_SIGNED = {"low": [0x34, 0xFF, 0x00, 1], "high": [0x12, 0xFF, 0xFF, 0]} +INTEGER_CASES = [ + pytest.param("uint8", STATUS_VALUES, BIT_FIELDS, DECODED_STATUS, id="uint8"), + pytest.param("uint16", WIDE_VALUES, WIDE_BIT_FIELDS, DECODED_WIDE, id="uint16"), + pytest.param("int16", SIGNED_VALUES, WIDE_BIT_FIELDS, DECODED_SIGNED, id="int16"), +] + + +def _import_and_wait(sift_client, path, config) -> DataImport: + job = sift_client.data_import.import_from_path(path, config=config, show_progress=False) + job.wait_until_complete(polling_interval_secs=1, timeout_secs=300) + data_import = job.get_data_import() + assert data_import.status == DataImportStatus.SUCCEEDED, data_import.error_message + assert job.job_status == JobStatus.FINISHED + return data_import + + +def _assert_decoded(sift_client, data_import: DataImport, elements, decoded, name="status"): + run = sift_client.data_import.get_run(data_import.id_) + channels = sift_client.channels.list_(run=run, name=name) + assert len(channels) == 1 + status = channels[0] + assert status.data_type == ChannelDataType.BIT_FIELD + assert status.bit_field_elements == elements + + data = sift_client.channels.get_data(channels=[status], run=run, show_progress=False) + for element, expected in decoded.items(): + frame = data[f"{name}.{element}"] + assert frame.iloc[:, 0].astype(int).tolist() == expected, (element, frame) + + +def _import_parquet(sift_client, tmp_path, status, elements) -> DataImport: + import pyarrow as pa + import pyarrow.parquet as pq + + path = tmp_path / "status.parquet" + start = 1_700_000_000_000_000_000 + pq.write_table( + pa.table( + { + "timestamp": pa.array( + [start + i * 1_000_000_000 for i in range(len(status))], pa.int64() + ), + "status": status, + } + ), + path, + ) + + config = sift_client.data_import.detect_config(path, data_type=DataTypeKey.PARQUET_FLATDATASET) + assert isinstance(config, ParquetFlatDatasetImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + config.time_column = ParquetTimeColumn( + path="timestamp", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS + ) + config.data_columns = [dc for dc in config.data_columns if dc.path != "timestamp"] + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = elements + return _import_and_wait(sift_client, path, config) + + +@pytest.mark.integration +class TestBitFieldImportIntegration: + @pytest.mark.parametrize( + "encoding", ["binary", "fixed-size-binary", "bit-string", "base64-string"] + ) + def test_parquet_flat_dataset(self, sift_client, tmp_path, encoding): + import base64 + + import pyarrow as pa + + raw = [bytes([v]) for v in STATUS_VALUES] + status = { + "binary": pa.array(raw, pa.binary()), + "fixed-size-binary": pa.array(raw, pa.binary(1)), + "bit-string": pa.array([format(v, "08b") for v in STATUS_VALUES], pa.string()), + "base64-string": pa.array( + [base64.urlsafe_b64encode(b).decode() for b in raw], pa.string() + ), + }[encoding] + + data_import = _import_parquet(sift_client, tmp_path, status, BIT_FIELDS) + _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) + + def test_parquet_multi_byte_status(self, sift_client, tmp_path): + import pyarrow as pa + + status = pa.array([v.to_bytes(2, "little") for v in WIDE_VALUES], pa.binary()) + + data_import = _import_parquet(sift_client, tmp_path, status, WIDE_BIT_FIELDS) + _assert_decoded(sift_client, data_import, WIDE_BIT_FIELDS, DECODED_WIDE) + + @pytest.mark.parametrize("encoding", ["bit-strings", "base64"]) + def test_csv(self, sift_client, tmp_path, encoding): + import base64 + + path = tmp_path / "status.csv" + times = pd.date_range("2024-01-01", periods=len(STATUS_VALUES), freq="s", tz="UTC") + values = ( + [format(v, "08b") for v in STATUS_VALUES] + if encoding == "bit-strings" + else [base64.urlsafe_b64encode(bytes([v])).decode() for v in STATUS_VALUES] + ) + pd.DataFrame( + {"time": [t.strftime("%Y-%m-%dT%H:%M:%S.%fZ") for t in times], "status": values} + ).to_csv(path, index=False) + + config = sift_client.data_import.detect_config( + path, time_format=TimeFormat.ABSOLUTE_RFC3339 + ) + assert isinstance(config, CsvImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = BIT_FIELDS + + data_import = _import_and_wait(sift_client, path, config) + _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) + + @pytest.mark.parametrize(("dtype", "values", "elements", "decoded"), INTEGER_CASES) + def test_tdms(self, sift_client, tmp_path, dtype, values, elements, decoded): + import numpy as np + from nptdms import ChannelObject, GroupObject, RootObject, TdmsWriter + + path = tmp_path / "status.tdms" + properties = { + "wf_start_time": np.datetime64(1_700_000_000_000_000_000, "ns"), + "wf_increment": 1.0, + "wf_start_offset": 0.0, + } + with TdmsWriter(path) as writer: + writer.write_segment( + [ + RootObject(), + GroupObject("g"), + ChannelObject("g", "status", np.array(values, dtype), properties=properties), + ] + ) + + config = sift_client.data_import.detect_config(path) + assert isinstance(config, TdmsImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + column = config["g.status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = elements + + data_import = _import_and_wait(sift_client, path, config) + _assert_decoded(sift_client, data_import, elements, decoded, name="g.status") + + @pytest.mark.parametrize(("dtype", "values", "elements", "decoded"), INTEGER_CASES) + def test_hdf5(self, sift_client, tmp_path, dtype, values, elements, decoded): + import h5py + import numpy as np + + path = tmp_path / "status.h5" + with h5py.File(path, "w") as f: + start = 1_700_000_000_000_000_000 + f["time"] = np.array([start + i * 1_000_000_000 for i in range(len(values))]) + f["status"] = np.array(values, dtype) + + config = sift_client.data_import.detect_config(path, data_type=DataTypeKey.HDF5_ONE_D) + assert isinstance(config, Hdf5ImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + config.time_format = TimeFormat.ABSOLUTE_UNIX_NANOSECONDS + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = elements + + data_import = _import_and_wait(sift_client, path, config) + _assert_decoded(sift_client, data_import, elements, decoded) diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 52e90464e9..7dab0beaac 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -5,6 +5,7 @@ from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.time import to_timestamp_nanos +from sift_client._internal.util.bit_field import check_bit_field_values from sift_client._internal.util.data_imports import ( parse_csv_detect_response, parse_parquet_detect_response, @@ -82,6 +83,12 @@ async def import_from_path( If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. + For files with ``ChannelDataType.BIT_FIELD`` columns, each value + must be raw bytes, a string of 0s and 1s, or URL-safe base64. + Integers aren't supported. Before uploading, this method checks CSV + and Parquet bit field columns. A Parquet column must be a binary or + string type, and a sample of string values must parse. + Examples: Import a CSV file with auto-detected config: @@ -153,6 +160,8 @@ async def import_from_path( Raises: FileNotFoundError: If the file does not exist. + ValueError: If a ``BIT_FIELD`` column's values aren't in a format + the import service reads, such as integers. """ path = Path(file_path) if not path.is_file(): @@ -183,6 +192,10 @@ async def import_from_path( elif not config.run_name and not config.run_id: config.run_name = path.name + # The import service only reports unreadable bit field values after the + # upload, so check them first. + await run_sync_function(check_bit_field_values, config, path) + if isinstance( config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) ): diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 80094023a7..1574056c35 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1770,6 +1770,12 @@ class DataImportAPI: If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. + For files with ``ChannelDataType.BIT_FIELD`` columns, each value + must be raw bytes, a string of 0s and 1s, or URL-safe base64. + Integers aren't supported. Before uploading, this method checks CSV + and Parquet bit field columns. A Parquet column must be a binary or + string type, and a sample of string values must parse. + Examples: Import a CSV file with auto-detected config: @@ -1841,6 +1847,8 @@ class DataImportAPI: Raises: FileNotFoundError: If the file does not exist. + ValueError: If a ``BIT_FIELD`` column's values aren't in a format + the import service reads, such as integers. """ ... diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index 0cb4c83220..b75ba51343 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from datetime import datetime, timezone from enum import Enum -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, ClassVar, Union import pandas as pd # noqa: TC002 from pydantic import BaseModel, ConfigDict, model_validator @@ -71,7 +71,7 @@ from sift_client._internal.util.util import caller_stacklevel from sift_client.errors import SiftIgnoredInputWarning from sift_client.sift_types._base import BaseType -from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType if TYPE_CHECKING: from sift_client.client import SiftClient @@ -168,20 +168,42 @@ class DataColumnBase(BaseModel, ABC): Attributes: name: Channel name. data_type: The data type of the channel values. - units: Optional units string. - description: Optional channel description. + units: Optional units string. On a bit field channel, every element + shows these units. + description: Optional channel description. On a bit field channel, + every element shows this description. enum_types: Mapping of enum name to key. Only valid when ``data_type`` is ``ChannelDataType.ENUM``. + bit_field_elements: Named bit ranges within each value. Only valid when + ``data_type`` is ``ChannelDataType.BIT_FIELD``, and only on formats + that support bit fields. Each element reads back as an unsigned + integer. Elements can't have their own units or description. """ + # The ULog and MCAP import jobs only accept the file's native scalar types + # as overrides, so they reject BIT_FIELD after the upload. + _supports_bit_fields: ClassVar[bool] = True + name: str data_type: ChannelDataType units: str = "" description: str = "" enum_types: dict[str, int] | None = None + bit_field_elements: list[ChannelBitFieldElement] | None = None @model_validator(mode="after") - def _check_enum_types(self) -> DataColumnBase: + def _check_channel_type_fields(self) -> DataColumnBase: + # mypy can't call a decorated validator directly, so the checks live in + # a plain method that _channel_config_proto can also call. + self._validate_channel_type_fields() + return self + + def _validate_channel_type_fields(self) -> None: + """Raise if ``enum_types`` or ``bit_field_elements`` don't match ``data_type``. + + Assigning fields bypasses the model validators, so this also runs when + the column is converted to proto before an import. + """ if self.data_type == ChannelDataType.ENUM or self.enum_types: if self.data_type != ChannelDataType.ENUM: raise ValueError( @@ -194,9 +216,40 @@ def _check_enum_types(self) -> DataColumnBase: ) if len(set(self.enum_types.values())) != len(self.enum_types): raise ValueError(f"'enum_types' contains duplicate keys ({self.name}).") - return self + if self.data_type == ChannelDataType.BIT_FIELD or self.bit_field_elements: + if not self._supports_bit_fields: + raise ValueError( + f"{type(self).__name__} doesn't support ChannelDataType.BIT_FIELD " + f"({self.name})." + ) + if self.data_type != ChannelDataType.BIT_FIELD: + raise ValueError( + f"'bit_field_elements' requires data_type ChannelDataType.BIT_FIELD " + f"({self.name} is {self.data_type.name})." + ) + if not self.bit_field_elements: + raise ValueError( + f"data_type ChannelDataType.BIT_FIELD requires 'bit_field_elements' " + f"({self.name})." + ) + names = [el.name for el in self.bit_field_elements] + if len(set(names)) != len(names): + raise ValueError(f"'bit_field_elements' contains duplicate names ({self.name}).") + next_free_bit = 0 + for el in sorted(self.bit_field_elements, key=lambda el: el.index): + if el.index < 0 or el.bit_count < 1: + raise ValueError( + f"bit field element '{el.name}' needs index >= 0 and bit_count >= 1 " + f"({self.name})." + ) + if el.index < next_free_bit: + raise ValueError( + f"bit field element '{el.name}' overlaps another element ({self.name})." + ) + next_free_bit = el.index + el.bit_count def _channel_config_proto(self) -> ChannelConfigProto: + self._validate_channel_type_fields() proto = ChannelConfigProto( name=self.name, data_type=self.data_type.value, @@ -207,6 +260,8 @@ def _channel_config_proto(self) -> ChannelConfigProto: proto.enum_types.extend( ChannelEnumTypeProto(name=name, key=key) for name, key in self.enum_types.items() ) + if self.bit_field_elements: + proto.bit_field_elements.extend(el._to_proto() for el in self.bit_field_elements) return proto @@ -214,6 +269,14 @@ def _enum_types_from_proto(channel_config: ChannelConfigProto) -> dict[str, int] return {e.name: e.key for e in channel_config.enum_types} or None +def _bit_field_elements_from_proto( + channel_config: ChannelConfigProto, +) -> list[ChannelBitFieldElement] | None: + return [ + ChannelBitFieldElement._from_proto(el) for el in channel_config.bit_field_elements + ] or None + + class ImportConfigBase(BaseModel, ABC): """Base class for all import configurations. @@ -389,6 +452,7 @@ def _from_proto(cls, proto: CsvConfigProto) -> CsvImportConfig: units=ch_cfg.units, description=ch_cfg.description, enum_types=_enum_types_from_proto(ch_cfg), + bit_field_elements=_bit_field_elements_from_proto(ch_cfg), ) for col_num, ch_cfg in proto.data_columns.items() ] @@ -547,6 +611,7 @@ def _from_proto( units=dc.channel_config.units, description=dc.channel_config.description, enum_types=_enum_types_from_proto(dc.channel_config), + bit_field_elements=_bit_field_elements_from_proto(dc.channel_config), ) for dc in fd.data_columns ] @@ -691,6 +756,7 @@ def _from_proto( units=sc.channel.units, description=sc.channel.description, enum_types=_enum_types_from_proto(sc.channel), + bit_field_elements=_bit_field_elements_from_proto(sc.channel), ) elif scpr.HasField("multi_channel"): mc = scpr.multi_channel @@ -872,6 +938,7 @@ def _from_proto(cls, proto: TDMSConfigProto) -> TdmsImportConfig: scaled=d.scaled if d.HasField("scaled") else None, complex_component=complex_component, enum_types=_enum_types_from_proto(ch), + bit_field_elements=_bit_field_elements_from_proto(ch), ) ) @@ -1033,6 +1100,8 @@ class UlogDataColumn(DataColumnBase): e.g. ``"sensor_accel_0.x"``. """ + _supports_bit_fields: ClassVar[bool] = False + message_name: str instance: int = 0 field_name: str = "" @@ -1220,6 +1289,8 @@ class McapDataColumn(DataColumnBase): e.g. ``"/imu/data.orientation.x"``. """ + _supports_bit_fields: ClassVar[bool] = False + topic: str field_path: str name: str = "" @@ -1322,16 +1393,14 @@ def add(dc: McapDataColumn, name: str, data_type: ChannelDataType) -> None: f"'{name}'. Rename or remove one before importing." ) taken_names[name.lower()] = dc.name + channel_config = dc._channel_config_proto() + channel_config.name = name + channel_config.data_type = data_type.value proto.data.append( McapDataConfigProto( topic=dc.topic, ros2=McapRos2SelectorProto(field_path=dc.field_path), - channel_config=ChannelConfigProto( - name=name, - data_type=data_type.value, - units=dc.units, - description=dc.description, - ), + channel_config=channel_config, ) ) @@ -1378,6 +1447,7 @@ def _from_proto(cls, proto: McapConfigProto) -> McapImportConfig: data_type=ChannelDataType(d.channel_config.data_type), units=d.channel_config.units, description=d.channel_config.description, + enum_types=_enum_types_from_proto(d.channel_config), ) for d in proto.data ]