From 7689d6702b98c3847b58b1a1748516b9495e0dea Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 2 Nov 2026 13:49:50 -0800 Subject: [PATCH 01/15] python(feat): support bit field definitions on import configs --- python/changelog.d/+bit-field-imports.feat.md | 15 ++ python/changelog.d/+mcap-enum-types.fix.md | 1 + .../data_import/parquet/flat_dataset/main.py | 13 ++ .../lib/sift_client/_internal/util/parquet.py | 2 + .../_tests/_internal/test_parquet.py | 34 ++- .../_tests/resources/test_data_imports.py | 221 +++++++++++++++++- .../lib/sift_client/sift_types/data_import.py | 73 +++++- 7 files changed, 348 insertions(+), 11 deletions(-) create mode 100644 python/changelog.d/+bit-field-imports.feat.md create mode 100644 python/changelog.d/+mcap-enum-types.fix.md diff --git a/python/changelog.d/+bit-field-imports.feat.md b/python/changelog.d/+bit-field-imports.feat.md new file mode 100644 index 0000000000..38d0db0193 --- /dev/null +++ b/python/changelog.d/+bit-field-imports.feat.md @@ -0,0 +1,15 @@ +Data columns on all import configs accept `bit_field_elements`, so you can import an integer column as a `BIT_FIELD` channel. + +```python +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType + +config = client.data_import.detect_config("data.parquet", data_type=DataTypeKey.PARQUET_FLATDATASET) +status = config["status"] +status.data_type = ChannelDataType.BIT_FIELD +status.bit_field_elements = [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), +] +``` + +A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, so a column edited in place fails before the request is sent. Multi-channel Parquet single-channel-per-row imports don't support bit fields, since channel configs there are derived per row. diff --git a/python/changelog.d/+mcap-enum-types.fix.md b/python/changelog.d/+mcap-enum-types.fix.md new file mode 100644 index 0000000000..4a3ad71877 --- /dev/null +++ b/python/changelog.d/+mcap-enum-types.fix.md @@ -0,0 +1 @@ +MCAP import configs keep `enum_types` when sent to Sift and when read back from a data import. Before, enum definitions on MCAP data columns were dropped. diff --git a/python/examples/data_import/parquet/flat_dataset/main.py b/python/examples/data_import/parquet/flat_dataset/main.py index 0020dd785e..c1e11ee330 100644 --- a/python/examples/data_import/parquet/flat_dataset/main.py +++ b/python/examples/data_import/parquet/flat_dataset/main.py @@ -46,6 +46,19 @@ # # Example: drop a column from the import # config.data_columns = [dc for dc in config.data_columns if dc.path != "channel_0"] # + # # Example: import an integer column as a bit field. Parquet stores only the + # # raw integer, so detect_config can't infer the bit layout; supply it here. + # # Each element's index is its starting bit and bit_count is its width. + # from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType + # + # status = config["status"] + # status.data_type = ChannelDataType.BIT_FIELD + # status.bit_field_elements = [ + # ChannelBitFieldElement(name="armed", index=0, bit_count=1), + # ChannelBitFieldElement(name="error_code", index=1, bit_count=3), + # ChannelBitFieldElement(name="mode", index=4, bit_count=4), + # ] + # # import_job = client.data_import.import_from_path( # "sample_data.parquet", # asset=asset_name, diff --git a/python/lib/sift_client/_internal/util/parquet.py b/python/lib/sift_client/_internal/util/parquet.py index b69dacac48..dc71da88c3 100644 --- a/python/lib/sift_client/_internal/util/parquet.py +++ b/python/lib/sift_client/_internal/util/parquet.py @@ -27,6 +27,7 @@ ParquetTimeColumn, TimeFormat, TimeFormatProto, + _bit_field_elements_from_proto, _enum_types_from_proto, ) @@ -222,6 +223,7 @@ def _parse_embedded_data_column(path: str, raw: bytes) -> ParquetDataColumn: units=config.units, description=config.description, enum_types=_enum_types_from_proto(config), + bit_field_elements=_bit_field_elements_from_proto(config), ) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py index 16bc587245..c766f320b0 100644 --- a/python/lib/sift_client/_tests/_internal/test_parquet.py +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -15,7 +15,7 @@ from sift_client._internal.util.parquet import detect_flat_parquet_config from sift_client.resources import DataImportAPIAsync -from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType from sift_client.sift_types.data_import import ( DataTypeKey, TimeFormat, @@ -432,6 +432,38 @@ def test_camel_case_keys_and_numeric_enums(self, create_parquet_file): assert config["x"].data_type == ChannelDataType.ENUM assert config["x"].enum_types == {"on": 1} + def test_embedded_bit_field_elements(self, create_parquet_file): + table = _embedded_table( + [ + _embedded_field( + "t", + pa.int64(), + _TIME_CONFIG, + {"time_format": "TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS"}, + ), + _embedded_field( + "status", + pa.uint8(), + _CHANNEL_CONFIG, + { + "name": "status", + "data_type": "CHANNEL_DATA_TYPE_BIT_FIELD", + "bit_field_elements": [ + {"name": "armed", "index": 0, "bit_count": 1}, + {"name": "mode", "index": 4, "bit_count": 4}, + ], + }, + ), + ] + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert config["status"].data_type == ChannelDataType.BIT_FIELD + assert config["status"].bit_field_elements == [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), + ] + @pytest.mark.parametrize( ("fields", "match"), [ diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index a14f712c14..01b84c26d0 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -41,7 +41,7 @@ from sift_client._internal.util.time_column import infer_time_column from sift_client._tests.sift_types.channel_time import assert_axis_matches_channel from sift_client.resources import DataImportAPI, DataImportAPIAsync -from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType from sift_client.sift_types.data_import import ( CsvDataColumn, CsvImportConfig, @@ -1610,3 +1610,222 @@ def test_enum_types_reject_duplicate_keys(self): def test_no_enum_types_from_proto_is_none(self, csv_config): round_tripped = CsvImportConfig._from_proto(csv_config._to_proto()) assert round_tripped["cpu_util"].enum_types is None + + def test_mcap_round_trip(self): + config = McapImportConfig( + asset_name="a", + data=[ + McapDataColumn( + topic="/state", + field_path="mode", + data_type=ChannelDataType.ENUM, + enum_types=ENUMS, + ) + ], + ) + proto = config._to_proto() + assert _enum_dict(proto.data[0].channel_config) == ENUMS + assert McapImportConfig._from_proto(proto).data[0].enum_types == ENUMS + + def test_in_place_edit_is_validated_on_import(self, csv_config): + csv_config["cpu_util"].data_type = ChannelDataType.ENUM + with pytest.raises(ValueError, match="requires 'enum_types'"): + csv_config._to_proto() + + +BIT_FIELDS = [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="error_code", index=1, bit_count=3), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), +] + + +def _bit_field_list(channel_config: ChannelConfigProto) -> list[ChannelBitFieldElement]: + return [ChannelBitFieldElement._from_proto(el) for el in channel_config.bit_field_elements] + + +class TestBitFieldElements: + def test_csv_round_trip(self): + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.data_columns[2]) == BIT_FIELDS + assert CsvImportConfig._from_proto(proto)["status"].bit_field_elements == BIT_FIELDS + + def test_parquet_flat_dataset_round_trip(self): + config = ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="ts", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ + ParquetDataColumn( + path="status", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.flat_dataset.data_columns[0].channel_config) == BIT_FIELDS + round_tripped = ParquetFlatDatasetImportConfig._from_proto(proto) + assert round_tripped["status"].bit_field_elements == BIT_FIELDS + + def test_parquet_single_channel_per_row_round_trip(self): + config = ParquetSingleChannelPerRowImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="ts", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + single_channel=ParquetSingleChannelConfig( + data_path="value", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ), + ) + proto = config._to_proto() + assert _bit_field_list(proto.single_channel_per_row.single_channel.channel) == BIT_FIELDS + round_tripped = ParquetSingleChannelPerRowImportConfig._from_proto(proto) + assert round_tripped.single_channel is not None + assert round_tripped.single_channel.bit_field_elements == BIT_FIELDS + + def test_tdms_round_trip(self): + config = TdmsImportConfig( + asset_name="a", + data=[ + TdmsDataColumn( + group_name="g", + channel_name="status", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS + round_tripped = TdmsImportConfig._from_proto(proto) + assert round_tripped.data[0].bit_field_elements == BIT_FIELDS + + def test_ulog_round_trip(self): + config = UlogImportConfig( + asset_name="a", + data=[ + UlogDataColumn( + message_name="vehicle_status", + field_name="flags", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS + round_tripped = UlogImportConfig._from_proto(proto) + assert round_tripped.data[0].bit_field_elements == BIT_FIELDS + + def test_mcap_round_trip(self): + config = McapImportConfig( + asset_name="a", + data=[ + McapDataColumn( + topic="/state", + field_path="flags", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + proto = config._to_proto() + assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS + assert McapImportConfig._from_proto(proto).data[0].bit_field_elements == BIT_FIELDS + + def test_hdf5_to_proto(self): + config = Hdf5ImportConfig( + asset_name="a", + data=[ + Hdf5DataColumn( + name="status", + data_type=ChannelDataType.BIT_FIELD, + time_dataset="/t", + value_dataset="/status", + bit_field_elements=BIT_FIELDS, + ) + ], + time_format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS, + ) + proto = config._to_proto() + assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS + + def test_elements_require_bit_field_data_type(self): + with pytest.raises(ValueError, match="BIT_FIELD"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.UINT_32, + bit_field_elements=BIT_FIELDS, + ) + + def test_bit_field_data_type_requires_elements(self): + with pytest.raises(ValueError, match="requires 'bit_field_elements'"): + CsvDataColumn(column=2, name="status", data_type=ChannelDataType.BIT_FIELD) + + def test_rejects_duplicate_names(self): + with pytest.raises(ValueError, match="[Dd]uplicate"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=[ + ChannelBitFieldElement(name="a", index=0, bit_count=1), + ChannelBitFieldElement(name="a", index=1, bit_count=1), + ], + ) + + @pytest.mark.parametrize( + ("index", "bit_count"), + [pytest.param(-1, 1, id="negative-index"), pytest.param(0, 0, id="zero-width")], + ) + def test_rejects_invalid_range(self, index, bit_count): + with pytest.raises(ValueError, match="index >= 0 and bit_count >= 1"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=[ + ChannelBitFieldElement(name="a", index=index, bit_count=bit_count) + ], + ) + + def test_rejects_overlapping_elements(self): + with pytest.raises(ValueError, match="overlaps"): + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=[ + ChannelBitFieldElement(name="a", index=0, bit_count=4), + ChannelBitFieldElement(name="b", index=3, bit_count=2), + ], + ) + + def test_no_elements_from_proto_is_none(self, csv_config): + round_tripped = CsvImportConfig._from_proto(csv_config._to_proto()) + assert round_tripped["cpu_util"].bit_field_elements is None + + def test_in_place_edit_is_validated_on_import(self, parquet_config): + column = parquet_config.data_columns[0] + column.data_type = ChannelDataType.BIT_FIELD + with pytest.raises(ValueError, match="requires 'bit_field_elements'"): + parquet_config._to_proto() + column.bit_field_elements = BIT_FIELDS + proto = parquet_config._to_proto() + assert _bit_field_list(proto.flat_dataset.data_columns[0].channel_config) == BIT_FIELDS diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index 0cb4c83220..4ff8f327db 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -71,7 +71,7 @@ from sift_client._internal.util.util import caller_stacklevel from sift_client.errors import SiftIgnoredInputWarning from sift_client.sift_types._base import BaseType -from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType if TYPE_CHECKING: from sift_client.client import SiftClient @@ -172,6 +172,8 @@ class DataColumnBase(BaseModel, ABC): description: Optional channel description. enum_types: Mapping of enum name to key. Only valid when ``data_type`` is ``ChannelDataType.ENUM``. + bit_field_elements: Named bit ranges within each value. Only valid when + ``data_type`` is ``ChannelDataType.BIT_FIELD``. """ name: str @@ -179,9 +181,21 @@ class DataColumnBase(BaseModel, ABC): units: str = "" description: str = "" enum_types: dict[str, int] | None = None + bit_field_elements: list[ChannelBitFieldElement] | None = None @model_validator(mode="after") - def _check_enum_types(self) -> DataColumnBase: + def _check_channel_type_fields(self) -> DataColumnBase: + # mypy can't call a decorated validator directly, so the checks live in + # a plain method that _channel_config_proto can also call. + self._validate_channel_type_fields() + return self + + def _validate_channel_type_fields(self) -> None: + """Raise if ``enum_types`` or ``bit_field_elements`` don't match ``data_type``. + + Assigning fields bypasses the model validators, so this also runs when + the column is converted to proto before an import. + """ if self.data_type == ChannelDataType.ENUM or self.enum_types: if self.data_type != ChannelDataType.ENUM: raise ValueError( @@ -194,9 +208,35 @@ def _check_enum_types(self) -> DataColumnBase: ) if len(set(self.enum_types.values())) != len(self.enum_types): raise ValueError(f"'enum_types' contains duplicate keys ({self.name}).") - return self + if self.data_type == ChannelDataType.BIT_FIELD or self.bit_field_elements: + if self.data_type != ChannelDataType.BIT_FIELD: + raise ValueError( + f"'bit_field_elements' requires data_type ChannelDataType.BIT_FIELD " + f"({self.name} is {self.data_type.name})." + ) + if not self.bit_field_elements: + raise ValueError( + f"data_type ChannelDataType.BIT_FIELD requires 'bit_field_elements' " + f"({self.name})." + ) + names = [el.name for el in self.bit_field_elements] + if len(set(names)) != len(names): + raise ValueError(f"'bit_field_elements' contains duplicate names ({self.name}).") + next_free_bit = 0 + for el in sorted(self.bit_field_elements, key=lambda el: el.index): + if el.index < 0 or el.bit_count < 1: + raise ValueError( + f"bit field element '{el.name}' needs index >= 0 and bit_count >= 1 " + f"({self.name})." + ) + if el.index < next_free_bit: + raise ValueError( + f"bit field element '{el.name}' overlaps another element ({self.name})." + ) + next_free_bit = el.index + el.bit_count def _channel_config_proto(self) -> ChannelConfigProto: + self._validate_channel_type_fields() proto = ChannelConfigProto( name=self.name, data_type=self.data_type.value, @@ -207,6 +247,8 @@ def _channel_config_proto(self) -> ChannelConfigProto: proto.enum_types.extend( ChannelEnumTypeProto(name=name, key=key) for name, key in self.enum_types.items() ) + if self.bit_field_elements: + proto.bit_field_elements.extend(el._to_proto() for el in self.bit_field_elements) return proto @@ -214,6 +256,14 @@ def _enum_types_from_proto(channel_config: ChannelConfigProto) -> dict[str, int] return {e.name: e.key for e in channel_config.enum_types} or None +def _bit_field_elements_from_proto( + channel_config: ChannelConfigProto, +) -> list[ChannelBitFieldElement] | None: + return [ + ChannelBitFieldElement._from_proto(el) for el in channel_config.bit_field_elements + ] or None + + class ImportConfigBase(BaseModel, ABC): """Base class for all import configurations. @@ -389,6 +439,7 @@ def _from_proto(cls, proto: CsvConfigProto) -> CsvImportConfig: units=ch_cfg.units, description=ch_cfg.description, enum_types=_enum_types_from_proto(ch_cfg), + bit_field_elements=_bit_field_elements_from_proto(ch_cfg), ) for col_num, ch_cfg in proto.data_columns.items() ] @@ -547,6 +598,7 @@ def _from_proto( units=dc.channel_config.units, description=dc.channel_config.description, enum_types=_enum_types_from_proto(dc.channel_config), + bit_field_elements=_bit_field_elements_from_proto(dc.channel_config), ) for dc in fd.data_columns ] @@ -691,6 +743,7 @@ def _from_proto( units=sc.channel.units, description=sc.channel.description, enum_types=_enum_types_from_proto(sc.channel), + bit_field_elements=_bit_field_elements_from_proto(sc.channel), ) elif scpr.HasField("multi_channel"): mc = scpr.multi_channel @@ -872,6 +925,7 @@ def _from_proto(cls, proto: TDMSConfigProto) -> TdmsImportConfig: scaled=d.scaled if d.HasField("scaled") else None, complex_component=complex_component, enum_types=_enum_types_from_proto(ch), + bit_field_elements=_bit_field_elements_from_proto(ch), ) ) @@ -1156,6 +1210,7 @@ def _from_proto(cls, proto: UlogConfigProto) -> UlogImportConfig: units=d.channel_config.units, description=d.channel_config.description, enum_types=_enum_types_from_proto(d.channel_config), + bit_field_elements=_bit_field_elements_from_proto(d.channel_config), ) for d in proto.data ] @@ -1322,16 +1377,14 @@ def add(dc: McapDataColumn, name: str, data_type: ChannelDataType) -> None: f"'{name}'. Rename or remove one before importing." ) taken_names[name.lower()] = dc.name + channel_config = dc._channel_config_proto() + channel_config.name = name + channel_config.data_type = data_type.value proto.data.append( McapDataConfigProto( topic=dc.topic, ros2=McapRos2SelectorProto(field_path=dc.field_path), - channel_config=ChannelConfigProto( - name=name, - data_type=data_type.value, - units=dc.units, - description=dc.description, - ), + channel_config=channel_config, ) ) @@ -1378,6 +1431,8 @@ def _from_proto(cls, proto: McapConfigProto) -> McapImportConfig: data_type=ChannelDataType(d.channel_config.data_type), units=d.channel_config.units, description=d.channel_config.description, + enum_types=_enum_types_from_proto(d.channel_config), + bit_field_elements=_bit_field_elements_from_proto(d.channel_config), ) for d in proto.data ] From 8c254dd7c64a8418e92c54459d49c8e59addb475 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 2 Nov 2026 13:55:08 -0800 Subject: [PATCH 02/15] python(chore): name changelog fragments after the PR --- python/changelog.d/{+bit-field-imports.feat.md => 861.feat.md} | 0 python/changelog.d/{+mcap-enum-types.fix.md => 861.fix.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/changelog.d/{+bit-field-imports.feat.md => 861.feat.md} (100%) rename python/changelog.d/{+mcap-enum-types.fix.md => 861.fix.md} (100%) diff --git a/python/changelog.d/+bit-field-imports.feat.md b/python/changelog.d/861.feat.md similarity index 100% rename from python/changelog.d/+bit-field-imports.feat.md rename to python/changelog.d/861.feat.md diff --git a/python/changelog.d/+mcap-enum-types.fix.md b/python/changelog.d/861.fix.md similarity index 100% rename from python/changelog.d/+mcap-enum-types.fix.md rename to python/changelog.d/861.fix.md From bc54be6d9a172873570edfa834d2209a10685123 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 2 Nov 2026 18:30:00 -0800 Subject: [PATCH 03/15] python(feat): convert integer bit field columns before upload --- python/changelog.d/861.feat.md | 2 + .../data_import/parquet/flat_dataset/main.py | 2 + .../sift_client/_internal/util/bit_field.py | 305 +++++++++++++++ .../_tests/_internal/test_bit_field.py | 348 ++++++++++++++++++ .../_tests/resources/test_data_imports.py | 223 +++++++++++ .../lib/sift_client/resources/data_imports.py | 56 ++- .../resources/sync_stubs/__init__.pyi | 7 + 7 files changed, 927 insertions(+), 16 deletions(-) create mode 100644 python/lib/sift_client/_internal/util/bit_field.py create mode 100644 python/lib/sift_client/_tests/_internal/test_bit_field.py diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index 38d0db0193..1fec63dc39 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -13,3 +13,5 @@ status.bit_field_elements = [ ``` A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, so a column edited in place fails before the request is sent. Multi-channel Parquet single-channel-per-row imports don't support bit fields, since channel configs there are derived per row. + +The import service reads bit field values as bytes, a string of 0s and 1s, or base64. When a CSV or Parquet `BIT_FIELD` column holds integers, `import_from_path` converts the values and uploads a converted copy from a temporary directory. Your file isn't changed. Parquet integers become little-endian bytes at the column's width, and CSV integers become fixed-width bit strings at the width the elements span. Nested Parquet integer columns aren't converted and raise an error, as does a CSV column of only 0s and 1s in values of different lengths, since `10` could mean ten or two. diff --git a/python/examples/data_import/parquet/flat_dataset/main.py b/python/examples/data_import/parquet/flat_dataset/main.py index c1e11ee330..f3b16555c8 100644 --- a/python/examples/data_import/parquet/flat_dataset/main.py +++ b/python/examples/data_import/parquet/flat_dataset/main.py @@ -49,6 +49,8 @@ # # Example: import an integer column as a bit field. Parquet stores only the # # raw integer, so detect_config can't infer the bit layout; supply it here. # # Each element's index is its starting bit and bit_count is its width. + # # import_from_path converts the integers to the bytes the import service + # # reads and uploads a converted copy; the file itself isn't changed. # from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType # # status = config["status"] diff --git a/python/lib/sift_client/_internal/util/bit_field.py b/python/lib/sift_client/_internal/util/bit_field.py new file mode 100644 index 0000000000..de93f41c23 --- /dev/null +++ b/python/lib/sift_client/_internal/util/bit_field.py @@ -0,0 +1,305 @@ +"""Convert integer bit field columns to an encoding the import service accepts. + +The import service reads bit field values as raw bytes, a string of 0s and 1s, or +base64, and rejects integers. Status words are usually stored as integers, so when +a ``BIT_FIELD`` column holds integers the client uploads a converted copy of the +file instead. The original file and config are never changed. +""" + +from __future__ import annotations + +import csv +import re +import shutil +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Iterable, Mapping, Union + +from sift_client._internal.util.file import extract_parquet_footer +from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.data_import import ( + CsvImportConfig, + DataColumnBase, + ParquetFlatDatasetImportConfig, + ParquetSingleChannelPerRowImportConfig, +) + +if TYPE_CHECKING: + import pyarrow as pa + + from sift_client.sift_types.data_import import ImportConfig + +ParquetImportConfig = Union[ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig] + +# Matches the separator and metadata key in ``_internal/util/parquet.py``, which +# imports pyarrow at module load and so can't be imported here unconditionally. +_PATH_SEPARATOR = "|" +_EMBEDDED_CHANNEL_CONFIG_KEY = b"sift_channel_config" + +_DECIMAL = re.compile(r"-?[0-9]+\Z") +_BITS = re.compile(r"[01]+\Z") + + +def convert_integer_bit_fields( + config: ImportConfig, path: Path +) -> tuple[ImportConfig, Path] | None: + """Return a config and a converted copy of ``path`` to upload instead of the originals. + + Returns ``None`` when no ``BIT_FIELD`` column holds integers, so the file is + uploaded as is. The converted copy is written to a new temporary directory, + which the caller removes after the upload. + """ + if isinstance(config, CsvImportConfig): + columns = { + dc.column - 1: dc + for dc in config.data_columns + if dc.data_type == ChannelDataType.BIT_FIELD + } + if not columns: + return None + _validate(columns.values()) + to_convert = _csv_columns_to_convert(path, config.first_data_row, columns) + if not to_convert: + return None + return config, _write_converted(path, lambda out: _write_csv(path, out, config, to_convert)) + + parquet_columns: dict[str, DataColumnBase] + if isinstance(config, ParquetFlatDatasetImportConfig): + parquet_columns = { + dc.path: dc for dc in config.data_columns if dc.data_type == ChannelDataType.BIT_FIELD + } + elif isinstance(config, ParquetSingleChannelPerRowImportConfig): + sc = config.single_channel + parquet_columns = ( + {sc.data_path: sc} + if sc is not None and sc.data_type == ChannelDataType.BIT_FIELD + else {} + ) + else: + return None + if not parquet_columns: + return None + _validate(parquet_columns.values()) + return _convert_parquet(config, path, parquet_columns) + + +def _validate(columns: Iterable[DataColumnBase]) -> None: + # Catch a bad layout here instead of failing on an empty or invalid element + # list partway through a conversion. + for dc in columns: + dc._validate_channel_type_fields() + + +def _bits_used(dc: DataColumnBase) -> int: + assert dc.bit_field_elements + return max(el.index + el.bit_count for el in dc.bit_field_elements) + + +def _write_converted(path: Path, write) -> Path: + out = Path(tempfile.mkdtemp(prefix="sift-import-")) / path.name + try: + write(out) + except BaseException: + shutil.rmtree(out.parent, ignore_errors=True) + raise + return out + + +def _csv_columns_to_convert( + path: Path, first_data_row: int, columns: Mapping[int, DataColumnBase] +) -> dict[int, int]: + """Return the byte width to convert at for each CSV column that holds integers. + + A column is converted when every value is a decimal integer and at least one + isn't also a valid bit string. Bit strings and base64 are sent unchanged. + """ + all_decimal = dict.fromkeys(columns, True) + has_integer = dict.fromkeys(columns, False) + bit_widths: dict[int, set[int]] = {i: set() for i in columns} + with open(path, newline="", encoding="utf-8") as f: + for row_number, row in enumerate(csv.reader(f), start=1): + if row_number < first_data_row: + continue + for i in columns: + cell = row[i].strip() if i < len(row) else "" + if not cell: + continue + if not _DECIMAL.match(cell): + all_decimal[i] = False + elif _BITS.match(cell): + bit_widths[i].add(len(cell)) + else: + has_integer[i] = True + + to_convert = {} + for i, dc in columns.items(): + if not all_decimal[i]: + continue + if has_integer[i]: + to_convert[i] = (_bits_used(dc) + 7) // 8 + elif len(bit_widths[i]) > 1: + raise ValueError( + f"can't tell whether bit field column '{dc.name}' holds integers or bit " + "strings: it only has 0s and 1s, in values of different lengths, so a value " + "like '10' reads as either 10 or 2. Write the column as fixed-width bit " + "strings, like '00001010', before importing." + ) + return to_convert + + +def _csv_cell(cell: str, width: int, dc: DataColumnBase, row_number: int) -> str: + if not cell.strip(): + return cell + value = int(cell) + bits = width * 8 + if value < 0: + value += 1 << bits + if not 0 <= value < 1 << bits: + raise ValueError( + f"value {cell.strip()} in row {row_number} of bit field column '{dc.name}' " + f"doesn't fit in the {bits} bits its elements span." + ) + return format(value, f"0{bits}b") + + +def _write_csv(src: Path, out: Path, config: CsvImportConfig, to_convert: dict[int, int]) -> None: + columns = {dc.column - 1: dc for dc in config.data_columns} + with open(src, newline="", encoding="utf-8") as fin, open( + out, "w", newline="", encoding="utf-8" + ) as fout: + writer = csv.writer(fout, lineterminator="\n") + for row_number, row in enumerate(csv.reader(fin), start=1): + if row_number >= config.first_data_row: + for i, width in to_convert.items(): + if i < len(row): + row[i] = _csv_cell(row[i], width, columns[i], row_number) + writer.writerow(row) + + +def _convert_parquet( + config: ParquetImportConfig, path: Path, columns: Mapping[str, DataColumnBase] +) -> tuple[ImportConfig, Path] | None: + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as e: + raise RuntimeError( + "pyarrow is required for Parquet import. " + "Install it via `pip install sift-stack-py[parquet]`." + ) from e + + try: + pf = pq.ParquetFile(path) + except Exception as e: + raise ValueError( + f"failed to read {path.name} to check its bit field columns. Encrypted " + "Parquet files aren't supported; store bit field columns as binary instead." + ) from e + schema = pf.schema_arrow + + to_convert: dict[int, int] = {} + for column_path, dc in columns.items(): + index = schema.get_field_index(column_path) + if index == -1: + leaf = _nested_leaf_type(schema, column_path) + if leaf is not None and pa.types.is_integer(leaf): + raise ValueError( + f"bit field column '{dc.name}' is a nested integer field, which isn't " + "converted automatically. Store it as binary before importing." + ) + continue + field_type = schema.field(index).type + if not pa.types.is_integer(field_type): + continue + if _bits_used(dc) > field_type.bit_width: + raise ValueError( + f"bit field elements of '{dc.name}' span {_bits_used(dc)} bits, but the " + f"column is {field_type} ({field_type.bit_width} bits)." + ) + to_convert[index] = field_type.bit_width // 8 + if not to_convert: + return None + + new_schema = schema + for index in to_convert: + old = schema.field(index) + # An embedded config still describes the old integer type. The config sent + # with the import is authoritative, so drop it from the converted column. + metadata = { + k: v for k, v in (old.metadata or {}).items() if k != _EMBEDDED_CHANNEL_CONFIG_KEY + } + new_schema = new_schema.set( + index, pa.field(old.name, pa.binary(), nullable=old.nullable, metadata=metadata or None) + ) + + codecs = ( + { + pf.metadata.row_group(0).column(c).compression.lower() + for c in range(pf.metadata.num_columns) + } + if pf.num_row_groups + else set() + ) + compression = codecs.pop() if len(codecs) == 1 else "snappy" + + def write(out: Path) -> None: + # Write each source row group as its own row group so the layout, and the + # footer that describes it, stays the same apart from the converted columns. + with pq.ParquetWriter(out, new_schema, compression=compression) as writer: + for rg in range(pf.num_row_groups): + table = pf.read_row_group(rg) + for index, width in to_convert.items(): + table = table.set_column( + index, + new_schema.field(index), + _integers_to_bytes(table.column(index), width), + ) + writer.write_table( + table.replace_schema_metadata(new_schema.metadata), + row_group_size=max(table.num_rows, 1), + ) + + out = _write_converted(path, write) + upload_config = config.model_copy(deep=True) + # Footer values from detect_config point into the original file. + footer_bytes, footer_offset = extract_parquet_footer(out) + upload_config.footer_offset = footer_offset + upload_config.footer_length = len(footer_bytes) + return upload_config, out + + +def _nested_leaf_type(schema: pa.Schema, column_path: str) -> pa.DataType | None: + import pyarrow as pa + + parts = column_path.split(_PATH_SEPARATOR) + index = schema.get_field_index(parts[0]) + if index == -1: + return None + field_type = schema.field(index).type + for part in parts[1:]: + if not pa.types.is_struct(field_type): + return None + child = field_type.get_field_index(part) + if child == -1: + return None + field_type = field_type.field(child).type + return field_type + + +def _integers_to_bytes(column: pa.ChunkedArray, width: int) -> pa.Array: + """Encode integers as little-endian bytes, ``width`` bytes each, keeping nulls.""" + import numpy as np + import pyarrow as pa + import pyarrow.compute as pc + + values = column.combine_chunks() + # Signed values keep their two's complement bit pattern. + raw = pc.fill_null(values, 0).to_numpy(zero_copy_only=False) + little_endian = np.ascontiguousarray(raw, dtype=raw.dtype.newbyteorder("<")) + encoded = pa.FixedSizeBinaryArray.from_buffers( + pa.binary(width), len(little_endian), [None, pa.py_buffer(little_endian.tobytes())] + ).cast(pa.binary()) + if values.null_count: + # pyarrow's stubs don't declare its generated compute functions. + encoded = pc.if_else(values.is_valid(), encoded, pa.scalar(None, pa.binary())) # type: ignore[attr-defined] + return encoded diff --git a/python/lib/sift_client/_tests/_internal/test_bit_field.py b/python/lib/sift_client/_tests/_internal/test_bit_field.py new file mode 100644 index 0000000000..4d4637d59e --- /dev/null +++ b/python/lib/sift_client/_tests/_internal/test_bit_field.py @@ -0,0 +1,348 @@ +"""Tests for converting integer bit field columns before upload.""" + +from __future__ import annotations + +import os +import shutil +import tempfile + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from sift_client._internal.util.bit_field import convert_integer_bit_fields +from sift_client._internal.util.file import extract_parquet_footer +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType +from sift_client.sift_types.data_import import ( + CsvDataColumn, + CsvImportConfig, + CsvTimeColumn, + ParquetDataColumn, + ParquetFlatDatasetImportConfig, + ParquetSingleChannelConfig, + ParquetSingleChannelPerRowImportConfig, + ParquetTimeColumn, + TimeFormat, +) + +ONE_BYTE = [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), +] +TWO_BYTES = [ + ChannelBitFieldElement(name="low", index=0, bit_count=8), + ChannelBitFieldElement(name="high", index=8, bit_count=8), +] +CHANNEL_CONFIG_KEY = b"sift_channel_config" + + +@pytest.fixture +def converted_dirs(): + """Remove the temporary directories conversions create.""" + dirs = [] + yield dirs + for d in dirs: + shutil.rmtree(d) + + +def _convert(config, path, converted_dirs): + result = convert_integer_bit_fields(config, path) + assert result is not None + converted_dirs.append(result[1].parent) + return result + + +def _parquet_config(path, column="status", elements=ONE_BYTE): + footer, offset = extract_parquet_footer(path) + return ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ + ParquetDataColumn( + path=column, + name=column, + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=elements, + ), + ParquetDataColumn(path="x", name="x", data_type=ChannelDataType.DOUBLE), + ], + footer_offset=offset, + footer_length=len(footer), + ) + + +def _write_parquet(path, status: pa.Array, **kwargs): + rows = len(status) + pq.write_table( + pa.table( + { + "t": pa.array(range(rows), pa.int64()), + "status": status, + "x": pa.array([float(i) for i in range(rows)]), + } + ), + path, + **kwargs, + ) + + +class TestParquet: + @pytest.mark.parametrize( + ("arrow_type", "values", "expected"), + [ + pytest.param(pa.uint8(), [53, 0, 255], [b"\x35", b"\x00", b"\xff"], id="uint8"), + pytest.param(pa.int8(), [-1, 5], [b"\xff", b"\x05"], id="int8-twos-complement"), + pytest.param( + pa.uint16(), [0x1234, 1], [b"\x34\x12", b"\x01\x00"], id="uint16-little-endian" + ), + pytest.param(pa.int64(), [-2], [b"\xfe" + b"\xff" * 7], id="int64"), + ], + ) + def test_integers_become_little_endian_bytes( + self, tmp_path, converted_dirs, arrow_type, values, expected + ): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array(values, arrow_type)) + + _, out = _convert(_parquet_config(path), path, converted_dirs) + table = pq.read_table(out) + assert table.schema.field("status").type == pa.binary() + assert table.column("status").to_pylist() == expected + assert table.column("x").to_pylist() == pq.read_table(path).column("x").to_pylist() + + def test_nulls_are_kept(self, tmp_path, converted_dirs): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array([1, None, 3], pa.uint8())) + + _, out = _convert(_parquet_config(path), path, converted_dirs) + + assert pq.read_table(out).column("status").to_pylist() == [b"\x01", None, b"\x03"] + + def test_footer_is_recomputed_on_a_copy_of_the_config(self, tmp_path, converted_dirs): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array([1, 2, 3], pa.uint8())) + config = _parquet_config(path) + original = (config.footer_offset, config.footer_length) + + upload_config, out = _convert(config, path, converted_dirs) + + footer, offset = extract_parquet_footer(out) + assert (upload_config.footer_offset, upload_config.footer_length) == (offset, len(footer)) + assert (config.footer_offset, config.footer_length) == original + assert out.name == path.name + assert out.parent != path.parent + + def test_layout_and_metadata_are_kept(self, tmp_path, converted_dirs): + path = tmp_path / "data.parquet" + schema = pa.schema( + [ + pa.field("t", pa.int64()), + pa.field( + "status", + pa.uint8(), + metadata={CHANNEL_CONFIG_KEY: b"{}", b"keep": b"yes"}, + ), + pa.field("x", pa.float64()), + ], + metadata={b"origin": b"test"}, + ) + table = pa.table( + {"t": list(range(30)), "status": [i % 256 for i in range(30)], "x": [0.0] * 30}, + schema=schema, + ) + pq.write_table(table, path, row_group_size=10, compression="zstd") + + _, out = _convert(_parquet_config(path), path, converted_dirs) + + converted = pq.ParquetFile(out) + assert converted.num_row_groups == 3 + assert { + converted.metadata.row_group(0).column(c).compression + for c in range(converted.metadata.num_columns) + } == {"ZSTD"} + assert converted.schema_arrow.metadata == {b"origin": b"test"} + # The stale embedded channel config is dropped; other field metadata stays. + assert converted.schema_arrow.field("status").metadata == {b"keep": b"yes"} + + @pytest.mark.parametrize( + "status", + [ + pytest.param(pa.array([b"\x01"], pa.binary()), id="binary"), + pytest.param(pa.array(["AQ=="], pa.string()), id="base64-string"), + ], + ) + def test_non_integer_columns_are_uploaded_as_is(self, tmp_path, status): + path = tmp_path / "data.parquet" + _write_parquet(path, status) + + assert convert_integer_bit_fields(_parquet_config(path), path) is None + + def test_no_bit_field_columns_skips_reading_the_file(self, tmp_path): + path = tmp_path / "not-parquet.parquet" + path.write_bytes(b"not a parquet file") + config = ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ParquetDataColumn(path="x", name="x", data_type=ChannelDataType.DOUBLE)], + ) + + assert convert_integer_bit_fields(config, path) is None + + def test_elements_wider_than_the_integer_raise(self, tmp_path): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array([1], pa.uint8())) + + with pytest.raises(ValueError, match="span 16 bits"): + convert_integer_bit_fields(_parquet_config(path, elements=TWO_BYTES), path) + + def test_nested_integer_column_raises(self, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table( + { + "t": pa.array([0], pa.int64()), + "s": pa.array([{"status": 1}], pa.struct([("status", pa.uint8())])), + "x": pa.array([0.0]), + } + ), + path, + ) + + with pytest.raises(ValueError, match="nested integer"): + convert_integer_bit_fields(_parquet_config(path, column="s|status"), path) + + def test_unreadable_file_raises(self, tmp_path): + path = tmp_path / "data.parquet" + path.write_bytes(b"not a parquet file") + + with pytest.raises(ValueError, match="failed to read data.parquet"): + convert_integer_bit_fields(_parquet_config_without_footer(), path) + + def test_single_channel_per_row(self, tmp_path, converted_dirs): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table({"t": pa.array([0, 1], pa.int64()), "value": pa.array([2, 3], pa.uint8())}), + path, + ) + config = ParquetSingleChannelPerRowImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + single_channel=ParquetSingleChannelConfig( + data_path="value", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ONE_BYTE, + ), + ) + + _, out = _convert(config, path, converted_dirs) + + assert pq.read_table(out).column("value").to_pylist() == [b"\x02", b"\x03"] + + +def _parquet_config_without_footer(): + return ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ + ParquetDataColumn( + path="status", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ONE_BYTE, + ) + ], + ) + + +def _csv_config(elements=ONE_BYTE, first_data_row=2): + return CsvImportConfig( + asset_name="a", + first_data_row=first_data_row, + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=elements, + ), + CsvDataColumn(column=3, name="x", data_type=ChannelDataType.DOUBLE), + ], + ) + + +class TestCsv: + def test_integers_become_fixed_width_bit_strings(self, tmp_path, converted_dirs): + path = tmp_path / "data.csv" + path.write_text("time,status,x\nt0,53,1.5\nt1,0,2\nt2,,3\nt3,-1,4\n") + + config, out = _convert(_csv_config(), path, converted_dirs) + + assert out.read_text() == ( + "time,status,x\nt0,00110101,1.5\nt1,00000000,2\nt2,,3\nt3,11111111,4\n" + ) + assert path.read_text().startswith("time,status,x\nt0,53,") + + def test_width_follows_the_elements(self, tmp_path, converted_dirs): + path = tmp_path / "data.csv" + path.write_text("time,status,x\nt0,258,1\n") + + _, out = _convert(_csv_config(elements=TWO_BYTES), path, converted_dirs) + + assert out.read_text() == "time,status,x\nt0,0000000100000010,1\n" + + def test_rows_before_the_first_data_row_are_kept(self, tmp_path, converted_dirs): + path = tmp_path / "data.csv" + path.write_text("units,raw,m\ntime,status,x\nt0,5,1\n") + + _, out = _convert(_csv_config(first_data_row=3), path, converted_dirs) + + assert out.read_text() == "units,raw,m\ntime,status,x\nt0,00000101,1\n" + + @pytest.mark.parametrize( + "values", + [ + pytest.param(["00110101", "00000000"], id="bit-strings"), + pytest.param(["0", "1", "1"], id="single-bits"), + pytest.param(["NQ==", "AA=="], id="base64"), + ], + ) + def test_encoded_values_are_uploaded_as_is(self, tmp_path, values): + path = tmp_path / "data.csv" + path.write_text("time,status,x\n" + "".join(f"t,{v},1\n" for v in values)) + + assert convert_integer_bit_fields(_csv_config(), path) is None + + def test_ambiguous_bit_strings_raise(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("time,status,x\nt0,0,1\nt1,10,1\nt2,11,1\n") + + with pytest.raises(ValueError, match="can't tell whether"): + convert_integer_bit_fields(_csv_config(), path) + + def test_value_too_large_raises(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("time,status,x\nt0,256,1\n") + + with pytest.raises(ValueError, match="value 256 in row 2"): + convert_integer_bit_fields(_csv_config(), path) + + def test_failed_conversion_leaves_no_temp_dir(self, tmp_path, monkeypatch): + created = [] + real_mkdtemp = tempfile.mkdtemp + + def tracking_mkdtemp(*args, **kwargs): + d = real_mkdtemp(*args, **kwargs) + created.append(d) + return d + + monkeypatch.setattr(tempfile, "mkdtemp", tracking_mkdtemp) + path = tmp_path / "data.csv" + path.write_text("time,status,x\nt0,5,1\nt1,300,1\n") + + with pytest.raises(ValueError, match="value 300 in row 3"): + convert_integer_bit_fields(_csv_config(), path) + + assert created + assert not any(os.path.exists(d) for d in created) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 01b84c26d0..8c97eb28f1 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2,6 +2,7 @@ from __future__ import annotations +import uuid import warnings from datetime import datetime, timezone from typing import TYPE_CHECKING, cast @@ -1829,3 +1830,225 @@ def test_in_place_edit_is_validated_on_import(self, parquet_config): column.bit_field_elements = BIT_FIELDS proto = parquet_config._to_proto() assert _bit_field_list(proto.flat_dataset.data_columns[0].channel_config) == BIT_FIELDS + + +class TestImportFromPathBitFieldConversion: + def _api(self, monkeypatch, uploads, fail=False): + def fake_upload(url, path, **kwargs): + uploads.append((path, path.read_bytes())) + if fail: + raise RuntimeError("upload failed") + return {"jobId": "j1"} + + monkeypatch.setattr("sift_client.resources.data_imports.upload_file", fake_upload) + api = DataImportAPIAsync(MagicMock()) + api._low_level_client = MagicMock() + api._low_level_client.create_from_upload = AsyncMock(return_value=("import_1", "url")) + api.client.async_.jobs.get = AsyncMock(return_value=MagicMock()) + return api + + def _parquet(self, tmp_path): + import pyarrow as pa + import pyarrow.parquet as pq + + path = tmp_path / "status.parquet" + pq.write_table( + pa.table({"t": pa.array([0, 1], pa.int64()), "status": pa.array([53, 1], pa.uint8())}), + path, + ) + config = ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ + ParquetDataColumn( + path="status", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + return path, config + + @pytest.mark.asyncio + async def test_uploads_converted_copy_and_cleans_up(self, tmp_path, monkeypatch): + import pyarrow as pa + import pyarrow.parquet as pq + + from sift_client._internal.util.file import extract_parquet_footer + + uploads: list = [] + api = self._api(monkeypatch, uploads) + path, config = self._parquet(tmp_path) + + await api.import_from_path(path, config=config, show_progress=False) + + ((uploaded_path, uploaded_bytes),) = uploads + assert uploaded_path != path + assert not uploaded_path.parent.exists() + converted = tmp_path / "uploaded.parquet" + converted.write_bytes(uploaded_bytes) + assert pq.read_table(converted).column("status").to_pylist() == [b"\x35", b"\x01"] + assert pq.read_schema(path).field("status").type == pa.uint8() + + low_level = cast("MagicMock", api._low_level_client) + (sent_config,) = low_level.create_from_upload.call_args.args + footer, offset = extract_parquet_footer(converted) + assert (sent_config.footer_offset, sent_config.footer_length) == (offset, len(footer)) + assert sent_config is not config + assert config.run_name == "status.parquet" + + @pytest.mark.asyncio + async def test_failed_upload_still_cleans_up(self, tmp_path, monkeypatch): + uploads: list = [] + api = self._api(monkeypatch, uploads, fail=True) + path, config = self._parquet(tmp_path) + + with pytest.raises(RuntimeError, match="upload failed"): + await api.import_from_path(path, config=config, show_progress=False) + + ((uploaded_path, _),) = uploads + assert not uploaded_path.parent.exists() + + @pytest.mark.asyncio + async def test_columns_without_integers_upload_the_original(self, tmp_path, monkeypatch): + uploads: list = [] + api = self._api(monkeypatch, uploads) + path = tmp_path / "status.csv" + path.write_text("time,status\nt0,00110101\n") + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + + await api.import_from_path(path, config=config, show_progress=False) + + ((uploaded_path, _),) = uploads + assert uploaded_path == path + + +# Raw status values and their decoded elements under BIT_FIELDS: +# bit 0 is armed, bits 1-3 are error_code, bits 4-7 are mode. +STATUS_VALUES = [53, 0, 255, 1] +DECODED_STATUS = { + "armed": [1, 0, 1, 1], + "error_code": [2, 0, 7, 0], + "mode": [3, 0, 15, 0], +} + +# A 16-bit status word, to check that multi-byte values decode little-endian. +WIDE_BIT_FIELDS = [ + ChannelBitFieldElement(name="low", index=0, bit_count=8), + ChannelBitFieldElement(name="high", index=8, bit_count=8), +] +WIDE_VALUES = [0x1234, 0x00FF, 0xFF00, 1] +DECODED_WIDE = {"low": [0x34, 0xFF, 0x00, 1], "high": [0x12, 0x00, 0xFF, 0]} + + +def _import_and_wait(sift_client, path, config) -> DataImport: + job = sift_client.data_import.import_from_path(path, config=config, show_progress=False) + job.wait_until_complete(polling_interval_secs=1, timeout_secs=300) + data_import = job.get_data_import() + assert data_import.status == DataImportStatus.SUCCEEDED, data_import.error_message + assert job.job_status == JobStatus.FINISHED + return data_import + + +def _assert_decoded(sift_client, data_import: DataImport, elements, decoded): + run = sift_client.data_import.get_run(data_import.id_) + channels = sift_client.channels.list_(run=run, name="status") + assert len(channels) == 1 + status = channels[0] + assert status.data_type == ChannelDataType.BIT_FIELD + assert status.bit_field_elements == elements + + data = sift_client.channels.get_data(channels=[status], run=run, show_progress=False) + for element, expected in decoded.items(): + frame = data[f"status.{element}"] + assert frame.iloc[:, 0].astype(int).tolist() == expected, (element, frame) + + +def _import_parquet(sift_client, tmp_path, status, elements) -> DataImport: + import pyarrow as pa + import pyarrow.parquet as pq + + path = tmp_path / "status.parquet" + start = 1_700_000_000_000_000_000 + pq.write_table( + pa.table( + { + "timestamp": pa.array( + [start + i * 1_000_000_000 for i in range(len(status))], pa.int64() + ), + "status": status, + } + ), + path, + ) + + config = sift_client.data_import.detect_config(path, data_type=DataTypeKey.PARQUET_FLATDATASET) + assert isinstance(config, ParquetFlatDatasetImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + config.time_column = ParquetTimeColumn( + path="timestamp", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS + ) + config.data_columns = [dc for dc in config.data_columns if dc.path != "timestamp"] + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = elements + return _import_and_wait(sift_client, path, config) + + +@pytest.mark.integration +class TestBitFieldImportIntegration: + @pytest.mark.parametrize("encoding", ["uint8", "int64", "binary"]) + def test_parquet_flat_dataset(self, sift_client, tmp_path, encoding): + import pyarrow as pa + + status = { + "uint8": pa.array(STATUS_VALUES, pa.uint8()), + "int64": pa.array(STATUS_VALUES, pa.int64()), + "binary": pa.array([bytes([v]) for v in STATUS_VALUES], pa.binary()), + }[encoding] + + data_import = _import_parquet(sift_client, tmp_path, status, BIT_FIELDS) + _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) + + def test_parquet_multi_byte_status(self, sift_client, tmp_path): + import pyarrow as pa + + status = pa.array(WIDE_VALUES, pa.uint16()) + + data_import = _import_parquet(sift_client, tmp_path, status, WIDE_BIT_FIELDS) + _assert_decoded(sift_client, data_import, WIDE_BIT_FIELDS, DECODED_WIDE) + + @pytest.mark.parametrize("encoding", ["integers", "bit-strings"]) + def test_csv(self, sift_client, tmp_path, encoding): + path = tmp_path / "status.csv" + times = pd.date_range("2024-01-01", periods=len(STATUS_VALUES), freq="s", tz="UTC") + values = ( + STATUS_VALUES if encoding == "integers" else [format(v, "08b") for v in STATUS_VALUES] + ) + pd.DataFrame( + {"time": [t.strftime("%Y-%m-%dT%H:%M:%S.%fZ") for t in times], "status": values} + ).to_csv(path, index=False) + + config = sift_client.data_import.detect_config( + path, time_format=TimeFormat.ABSOLUTE_RFC3339 + ) + assert isinstance(config, CsvImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = BIT_FIELDS + + data_import = _import_and_wait(sift_client, path, config) + _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 52e90464e9..5f34c07b3b 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,10 +1,12 @@ from __future__ import annotations +import shutil from pathlib import Path from typing import TYPE_CHECKING from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.time import to_timestamp_nanos +from sift_client._internal.util.bit_field import convert_integer_bit_fields from sift_client._internal.util.data_imports import ( parse_csv_detect_response, parse_parquet_detect_response, @@ -82,6 +84,11 @@ async def import_from_path( If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. + The import service reads ``BIT_FIELD`` values as bytes, a string of + 0s and 1s, or base64. When a CSV or Parquet ``BIT_FIELD`` column holds + integers, the client converts them and uploads the converted copy + from a temporary directory. The original file isn't changed. + Examples: Import a CSV file with auto-detected config: @@ -153,6 +160,8 @@ async def import_from_path( Raises: FileNotFoundError: If the file does not exist. + ValueError: If a ``BIT_FIELD`` column can't be converted, for example + when its values don't fit the bits its elements span. """ path = Path(file_path) if not path.is_file(): @@ -183,24 +192,39 @@ async def import_from_path( elif not config.run_name and not config.run_id: config.run_name = path.name - if isinstance( - config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) - ): - await prepare_parquet_config(config, path) - - if show_progress is None: - show_progress = self._show_progress() - - data_import_id, upload_url = await self._low_level_client.create_from_upload(config) + # The import service rejects integer bit field values, so BIT_FIELD columns + # stored as integers are uploaded from a converted copy of the file. + upload_config: ImportConfig = config + upload_path = path + converted = await run_sync_function(lambda: convert_integer_bit_fields(upload_config, path)) + if converted is not None: + upload_config, upload_path = converted + + try: + if isinstance( + upload_config, + (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig), + ): + await prepare_parquet_config(upload_config, upload_path) + + if show_progress is None: + show_progress = self._show_progress() + + data_import_id, upload_url = await self._low_level_client.create_from_upload( + upload_config + ) - response = await run_sync_function( - lambda: upload_file( - upload_url, - path, - rest_client=self.client.rest_client, - show_progress=show_progress, + response = await run_sync_function( + lambda: upload_file( + upload_url, + upload_path, + rest_client=self.client.rest_client, + show_progress=show_progress, + ) ) - ) + finally: + if upload_path != path: + shutil.rmtree(upload_path.parent, ignore_errors=True) job_id = response.get("jobId") if not job_id: raise RuntimeError("Upload succeeded but server response did not include a job ID.") diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 80094023a7..d63fa8f766 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1770,6 +1770,11 @@ class DataImportAPI: If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. + The import service reads ``BIT_FIELD`` values as bytes, a string of + 0s and 1s, or base64. When a CSV or Parquet ``BIT_FIELD`` column holds + integers, the client converts them and uploads the converted copy + from a temporary directory. The original file isn't changed. + Examples: Import a CSV file with auto-detected config: @@ -1841,6 +1846,8 @@ class DataImportAPI: Raises: FileNotFoundError: If the file does not exist. + ValueError: If a ``BIT_FIELD`` column can't be converted, for example + when its values don't fit the bits its elements span. """ ... From cf84e63bab21f0e2e24bb06dfc74bb9223151c3b Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 2 Nov 2026 18:52:22 -0800 Subject: [PATCH 04/15] Revert "python(feat): convert integer bit field columns before upload" This reverts commit bc54be6d9a172873570edfa834d2209a10685123. --- python/changelog.d/861.feat.md | 2 - .../data_import/parquet/flat_dataset/main.py | 2 - .../sift_client/_internal/util/bit_field.py | 305 --------------- .../_tests/_internal/test_bit_field.py | 348 ------------------ .../_tests/resources/test_data_imports.py | 223 ----------- .../lib/sift_client/resources/data_imports.py | 56 +-- .../resources/sync_stubs/__init__.pyi | 7 - 7 files changed, 16 insertions(+), 927 deletions(-) delete mode 100644 python/lib/sift_client/_internal/util/bit_field.py delete mode 100644 python/lib/sift_client/_tests/_internal/test_bit_field.py diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index 1fec63dc39..38d0db0193 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -13,5 +13,3 @@ status.bit_field_elements = [ ``` A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, so a column edited in place fails before the request is sent. Multi-channel Parquet single-channel-per-row imports don't support bit fields, since channel configs there are derived per row. - -The import service reads bit field values as bytes, a string of 0s and 1s, or base64. When a CSV or Parquet `BIT_FIELD` column holds integers, `import_from_path` converts the values and uploads a converted copy from a temporary directory. Your file isn't changed. Parquet integers become little-endian bytes at the column's width, and CSV integers become fixed-width bit strings at the width the elements span. Nested Parquet integer columns aren't converted and raise an error, as does a CSV column of only 0s and 1s in values of different lengths, since `10` could mean ten or two. diff --git a/python/examples/data_import/parquet/flat_dataset/main.py b/python/examples/data_import/parquet/flat_dataset/main.py index f3b16555c8..c1e11ee330 100644 --- a/python/examples/data_import/parquet/flat_dataset/main.py +++ b/python/examples/data_import/parquet/flat_dataset/main.py @@ -49,8 +49,6 @@ # # Example: import an integer column as a bit field. Parquet stores only the # # raw integer, so detect_config can't infer the bit layout; supply it here. # # Each element's index is its starting bit and bit_count is its width. - # # import_from_path converts the integers to the bytes the import service - # # reads and uploads a converted copy; the file itself isn't changed. # from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType # # status = config["status"] diff --git a/python/lib/sift_client/_internal/util/bit_field.py b/python/lib/sift_client/_internal/util/bit_field.py deleted file mode 100644 index de93f41c23..0000000000 --- a/python/lib/sift_client/_internal/util/bit_field.py +++ /dev/null @@ -1,305 +0,0 @@ -"""Convert integer bit field columns to an encoding the import service accepts. - -The import service reads bit field values as raw bytes, a string of 0s and 1s, or -base64, and rejects integers. Status words are usually stored as integers, so when -a ``BIT_FIELD`` column holds integers the client uploads a converted copy of the -file instead. The original file and config are never changed. -""" - -from __future__ import annotations - -import csv -import re -import shutil -import tempfile -from pathlib import Path -from typing import TYPE_CHECKING, Iterable, Mapping, Union - -from sift_client._internal.util.file import extract_parquet_footer -from sift_client.sift_types.channel import ChannelDataType -from sift_client.sift_types.data_import import ( - CsvImportConfig, - DataColumnBase, - ParquetFlatDatasetImportConfig, - ParquetSingleChannelPerRowImportConfig, -) - -if TYPE_CHECKING: - import pyarrow as pa - - from sift_client.sift_types.data_import import ImportConfig - -ParquetImportConfig = Union[ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig] - -# Matches the separator and metadata key in ``_internal/util/parquet.py``, which -# imports pyarrow at module load and so can't be imported here unconditionally. -_PATH_SEPARATOR = "|" -_EMBEDDED_CHANNEL_CONFIG_KEY = b"sift_channel_config" - -_DECIMAL = re.compile(r"-?[0-9]+\Z") -_BITS = re.compile(r"[01]+\Z") - - -def convert_integer_bit_fields( - config: ImportConfig, path: Path -) -> tuple[ImportConfig, Path] | None: - """Return a config and a converted copy of ``path`` to upload instead of the originals. - - Returns ``None`` when no ``BIT_FIELD`` column holds integers, so the file is - uploaded as is. The converted copy is written to a new temporary directory, - which the caller removes after the upload. - """ - if isinstance(config, CsvImportConfig): - columns = { - dc.column - 1: dc - for dc in config.data_columns - if dc.data_type == ChannelDataType.BIT_FIELD - } - if not columns: - return None - _validate(columns.values()) - to_convert = _csv_columns_to_convert(path, config.first_data_row, columns) - if not to_convert: - return None - return config, _write_converted(path, lambda out: _write_csv(path, out, config, to_convert)) - - parquet_columns: dict[str, DataColumnBase] - if isinstance(config, ParquetFlatDatasetImportConfig): - parquet_columns = { - dc.path: dc for dc in config.data_columns if dc.data_type == ChannelDataType.BIT_FIELD - } - elif isinstance(config, ParquetSingleChannelPerRowImportConfig): - sc = config.single_channel - parquet_columns = ( - {sc.data_path: sc} - if sc is not None and sc.data_type == ChannelDataType.BIT_FIELD - else {} - ) - else: - return None - if not parquet_columns: - return None - _validate(parquet_columns.values()) - return _convert_parquet(config, path, parquet_columns) - - -def _validate(columns: Iterable[DataColumnBase]) -> None: - # Catch a bad layout here instead of failing on an empty or invalid element - # list partway through a conversion. - for dc in columns: - dc._validate_channel_type_fields() - - -def _bits_used(dc: DataColumnBase) -> int: - assert dc.bit_field_elements - return max(el.index + el.bit_count for el in dc.bit_field_elements) - - -def _write_converted(path: Path, write) -> Path: - out = Path(tempfile.mkdtemp(prefix="sift-import-")) / path.name - try: - write(out) - except BaseException: - shutil.rmtree(out.parent, ignore_errors=True) - raise - return out - - -def _csv_columns_to_convert( - path: Path, first_data_row: int, columns: Mapping[int, DataColumnBase] -) -> dict[int, int]: - """Return the byte width to convert at for each CSV column that holds integers. - - A column is converted when every value is a decimal integer and at least one - isn't also a valid bit string. Bit strings and base64 are sent unchanged. - """ - all_decimal = dict.fromkeys(columns, True) - has_integer = dict.fromkeys(columns, False) - bit_widths: dict[int, set[int]] = {i: set() for i in columns} - with open(path, newline="", encoding="utf-8") as f: - for row_number, row in enumerate(csv.reader(f), start=1): - if row_number < first_data_row: - continue - for i in columns: - cell = row[i].strip() if i < len(row) else "" - if not cell: - continue - if not _DECIMAL.match(cell): - all_decimal[i] = False - elif _BITS.match(cell): - bit_widths[i].add(len(cell)) - else: - has_integer[i] = True - - to_convert = {} - for i, dc in columns.items(): - if not all_decimal[i]: - continue - if has_integer[i]: - to_convert[i] = (_bits_used(dc) + 7) // 8 - elif len(bit_widths[i]) > 1: - raise ValueError( - f"can't tell whether bit field column '{dc.name}' holds integers or bit " - "strings: it only has 0s and 1s, in values of different lengths, so a value " - "like '10' reads as either 10 or 2. Write the column as fixed-width bit " - "strings, like '00001010', before importing." - ) - return to_convert - - -def _csv_cell(cell: str, width: int, dc: DataColumnBase, row_number: int) -> str: - if not cell.strip(): - return cell - value = int(cell) - bits = width * 8 - if value < 0: - value += 1 << bits - if not 0 <= value < 1 << bits: - raise ValueError( - f"value {cell.strip()} in row {row_number} of bit field column '{dc.name}' " - f"doesn't fit in the {bits} bits its elements span." - ) - return format(value, f"0{bits}b") - - -def _write_csv(src: Path, out: Path, config: CsvImportConfig, to_convert: dict[int, int]) -> None: - columns = {dc.column - 1: dc for dc in config.data_columns} - with open(src, newline="", encoding="utf-8") as fin, open( - out, "w", newline="", encoding="utf-8" - ) as fout: - writer = csv.writer(fout, lineterminator="\n") - for row_number, row in enumerate(csv.reader(fin), start=1): - if row_number >= config.first_data_row: - for i, width in to_convert.items(): - if i < len(row): - row[i] = _csv_cell(row[i], width, columns[i], row_number) - writer.writerow(row) - - -def _convert_parquet( - config: ParquetImportConfig, path: Path, columns: Mapping[str, DataColumnBase] -) -> tuple[ImportConfig, Path] | None: - try: - import pyarrow as pa - import pyarrow.parquet as pq - except ImportError as e: - raise RuntimeError( - "pyarrow is required for Parquet import. " - "Install it via `pip install sift-stack-py[parquet]`." - ) from e - - try: - pf = pq.ParquetFile(path) - except Exception as e: - raise ValueError( - f"failed to read {path.name} to check its bit field columns. Encrypted " - "Parquet files aren't supported; store bit field columns as binary instead." - ) from e - schema = pf.schema_arrow - - to_convert: dict[int, int] = {} - for column_path, dc in columns.items(): - index = schema.get_field_index(column_path) - if index == -1: - leaf = _nested_leaf_type(schema, column_path) - if leaf is not None and pa.types.is_integer(leaf): - raise ValueError( - f"bit field column '{dc.name}' is a nested integer field, which isn't " - "converted automatically. Store it as binary before importing." - ) - continue - field_type = schema.field(index).type - if not pa.types.is_integer(field_type): - continue - if _bits_used(dc) > field_type.bit_width: - raise ValueError( - f"bit field elements of '{dc.name}' span {_bits_used(dc)} bits, but the " - f"column is {field_type} ({field_type.bit_width} bits)." - ) - to_convert[index] = field_type.bit_width // 8 - if not to_convert: - return None - - new_schema = schema - for index in to_convert: - old = schema.field(index) - # An embedded config still describes the old integer type. The config sent - # with the import is authoritative, so drop it from the converted column. - metadata = { - k: v for k, v in (old.metadata or {}).items() if k != _EMBEDDED_CHANNEL_CONFIG_KEY - } - new_schema = new_schema.set( - index, pa.field(old.name, pa.binary(), nullable=old.nullable, metadata=metadata or None) - ) - - codecs = ( - { - pf.metadata.row_group(0).column(c).compression.lower() - for c in range(pf.metadata.num_columns) - } - if pf.num_row_groups - else set() - ) - compression = codecs.pop() if len(codecs) == 1 else "snappy" - - def write(out: Path) -> None: - # Write each source row group as its own row group so the layout, and the - # footer that describes it, stays the same apart from the converted columns. - with pq.ParquetWriter(out, new_schema, compression=compression) as writer: - for rg in range(pf.num_row_groups): - table = pf.read_row_group(rg) - for index, width in to_convert.items(): - table = table.set_column( - index, - new_schema.field(index), - _integers_to_bytes(table.column(index), width), - ) - writer.write_table( - table.replace_schema_metadata(new_schema.metadata), - row_group_size=max(table.num_rows, 1), - ) - - out = _write_converted(path, write) - upload_config = config.model_copy(deep=True) - # Footer values from detect_config point into the original file. - footer_bytes, footer_offset = extract_parquet_footer(out) - upload_config.footer_offset = footer_offset - upload_config.footer_length = len(footer_bytes) - return upload_config, out - - -def _nested_leaf_type(schema: pa.Schema, column_path: str) -> pa.DataType | None: - import pyarrow as pa - - parts = column_path.split(_PATH_SEPARATOR) - index = schema.get_field_index(parts[0]) - if index == -1: - return None - field_type = schema.field(index).type - for part in parts[1:]: - if not pa.types.is_struct(field_type): - return None - child = field_type.get_field_index(part) - if child == -1: - return None - field_type = field_type.field(child).type - return field_type - - -def _integers_to_bytes(column: pa.ChunkedArray, width: int) -> pa.Array: - """Encode integers as little-endian bytes, ``width`` bytes each, keeping nulls.""" - import numpy as np - import pyarrow as pa - import pyarrow.compute as pc - - values = column.combine_chunks() - # Signed values keep their two's complement bit pattern. - raw = pc.fill_null(values, 0).to_numpy(zero_copy_only=False) - little_endian = np.ascontiguousarray(raw, dtype=raw.dtype.newbyteorder("<")) - encoded = pa.FixedSizeBinaryArray.from_buffers( - pa.binary(width), len(little_endian), [None, pa.py_buffer(little_endian.tobytes())] - ).cast(pa.binary()) - if values.null_count: - # pyarrow's stubs don't declare its generated compute functions. - encoded = pc.if_else(values.is_valid(), encoded, pa.scalar(None, pa.binary())) # type: ignore[attr-defined] - return encoded diff --git a/python/lib/sift_client/_tests/_internal/test_bit_field.py b/python/lib/sift_client/_tests/_internal/test_bit_field.py deleted file mode 100644 index 4d4637d59e..0000000000 --- a/python/lib/sift_client/_tests/_internal/test_bit_field.py +++ /dev/null @@ -1,348 +0,0 @@ -"""Tests for converting integer bit field columns before upload.""" - -from __future__ import annotations - -import os -import shutil -import tempfile - -import pyarrow as pa -import pyarrow.parquet as pq -import pytest - -from sift_client._internal.util.bit_field import convert_integer_bit_fields -from sift_client._internal.util.file import extract_parquet_footer -from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType -from sift_client.sift_types.data_import import ( - CsvDataColumn, - CsvImportConfig, - CsvTimeColumn, - ParquetDataColumn, - ParquetFlatDatasetImportConfig, - ParquetSingleChannelConfig, - ParquetSingleChannelPerRowImportConfig, - ParquetTimeColumn, - TimeFormat, -) - -ONE_BYTE = [ - ChannelBitFieldElement(name="armed", index=0, bit_count=1), - ChannelBitFieldElement(name="mode", index=4, bit_count=4), -] -TWO_BYTES = [ - ChannelBitFieldElement(name="low", index=0, bit_count=8), - ChannelBitFieldElement(name="high", index=8, bit_count=8), -] -CHANNEL_CONFIG_KEY = b"sift_channel_config" - - -@pytest.fixture -def converted_dirs(): - """Remove the temporary directories conversions create.""" - dirs = [] - yield dirs - for d in dirs: - shutil.rmtree(d) - - -def _convert(config, path, converted_dirs): - result = convert_integer_bit_fields(config, path) - assert result is not None - converted_dirs.append(result[1].parent) - return result - - -def _parquet_config(path, column="status", elements=ONE_BYTE): - footer, offset = extract_parquet_footer(path) - return ParquetFlatDatasetImportConfig( - asset_name="a", - time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), - data_columns=[ - ParquetDataColumn( - path=column, - name=column, - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=elements, - ), - ParquetDataColumn(path="x", name="x", data_type=ChannelDataType.DOUBLE), - ], - footer_offset=offset, - footer_length=len(footer), - ) - - -def _write_parquet(path, status: pa.Array, **kwargs): - rows = len(status) - pq.write_table( - pa.table( - { - "t": pa.array(range(rows), pa.int64()), - "status": status, - "x": pa.array([float(i) for i in range(rows)]), - } - ), - path, - **kwargs, - ) - - -class TestParquet: - @pytest.mark.parametrize( - ("arrow_type", "values", "expected"), - [ - pytest.param(pa.uint8(), [53, 0, 255], [b"\x35", b"\x00", b"\xff"], id="uint8"), - pytest.param(pa.int8(), [-1, 5], [b"\xff", b"\x05"], id="int8-twos-complement"), - pytest.param( - pa.uint16(), [0x1234, 1], [b"\x34\x12", b"\x01\x00"], id="uint16-little-endian" - ), - pytest.param(pa.int64(), [-2], [b"\xfe" + b"\xff" * 7], id="int64"), - ], - ) - def test_integers_become_little_endian_bytes( - self, tmp_path, converted_dirs, arrow_type, values, expected - ): - path = tmp_path / "data.parquet" - _write_parquet(path, pa.array(values, arrow_type)) - - _, out = _convert(_parquet_config(path), path, converted_dirs) - table = pq.read_table(out) - assert table.schema.field("status").type == pa.binary() - assert table.column("status").to_pylist() == expected - assert table.column("x").to_pylist() == pq.read_table(path).column("x").to_pylist() - - def test_nulls_are_kept(self, tmp_path, converted_dirs): - path = tmp_path / "data.parquet" - _write_parquet(path, pa.array([1, None, 3], pa.uint8())) - - _, out = _convert(_parquet_config(path), path, converted_dirs) - - assert pq.read_table(out).column("status").to_pylist() == [b"\x01", None, b"\x03"] - - def test_footer_is_recomputed_on_a_copy_of_the_config(self, tmp_path, converted_dirs): - path = tmp_path / "data.parquet" - _write_parquet(path, pa.array([1, 2, 3], pa.uint8())) - config = _parquet_config(path) - original = (config.footer_offset, config.footer_length) - - upload_config, out = _convert(config, path, converted_dirs) - - footer, offset = extract_parquet_footer(out) - assert (upload_config.footer_offset, upload_config.footer_length) == (offset, len(footer)) - assert (config.footer_offset, config.footer_length) == original - assert out.name == path.name - assert out.parent != path.parent - - def test_layout_and_metadata_are_kept(self, tmp_path, converted_dirs): - path = tmp_path / "data.parquet" - schema = pa.schema( - [ - pa.field("t", pa.int64()), - pa.field( - "status", - pa.uint8(), - metadata={CHANNEL_CONFIG_KEY: b"{}", b"keep": b"yes"}, - ), - pa.field("x", pa.float64()), - ], - metadata={b"origin": b"test"}, - ) - table = pa.table( - {"t": list(range(30)), "status": [i % 256 for i in range(30)], "x": [0.0] * 30}, - schema=schema, - ) - pq.write_table(table, path, row_group_size=10, compression="zstd") - - _, out = _convert(_parquet_config(path), path, converted_dirs) - - converted = pq.ParquetFile(out) - assert converted.num_row_groups == 3 - assert { - converted.metadata.row_group(0).column(c).compression - for c in range(converted.metadata.num_columns) - } == {"ZSTD"} - assert converted.schema_arrow.metadata == {b"origin": b"test"} - # The stale embedded channel config is dropped; other field metadata stays. - assert converted.schema_arrow.field("status").metadata == {b"keep": b"yes"} - - @pytest.mark.parametrize( - "status", - [ - pytest.param(pa.array([b"\x01"], pa.binary()), id="binary"), - pytest.param(pa.array(["AQ=="], pa.string()), id="base64-string"), - ], - ) - def test_non_integer_columns_are_uploaded_as_is(self, tmp_path, status): - path = tmp_path / "data.parquet" - _write_parquet(path, status) - - assert convert_integer_bit_fields(_parquet_config(path), path) is None - - def test_no_bit_field_columns_skips_reading_the_file(self, tmp_path): - path = tmp_path / "not-parquet.parquet" - path.write_bytes(b"not a parquet file") - config = ParquetFlatDatasetImportConfig( - asset_name="a", - time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), - data_columns=[ParquetDataColumn(path="x", name="x", data_type=ChannelDataType.DOUBLE)], - ) - - assert convert_integer_bit_fields(config, path) is None - - def test_elements_wider_than_the_integer_raise(self, tmp_path): - path = tmp_path / "data.parquet" - _write_parquet(path, pa.array([1], pa.uint8())) - - with pytest.raises(ValueError, match="span 16 bits"): - convert_integer_bit_fields(_parquet_config(path, elements=TWO_BYTES), path) - - def test_nested_integer_column_raises(self, tmp_path): - path = tmp_path / "data.parquet" - pq.write_table( - pa.table( - { - "t": pa.array([0], pa.int64()), - "s": pa.array([{"status": 1}], pa.struct([("status", pa.uint8())])), - "x": pa.array([0.0]), - } - ), - path, - ) - - with pytest.raises(ValueError, match="nested integer"): - convert_integer_bit_fields(_parquet_config(path, column="s|status"), path) - - def test_unreadable_file_raises(self, tmp_path): - path = tmp_path / "data.parquet" - path.write_bytes(b"not a parquet file") - - with pytest.raises(ValueError, match="failed to read data.parquet"): - convert_integer_bit_fields(_parquet_config_without_footer(), path) - - def test_single_channel_per_row(self, tmp_path, converted_dirs): - path = tmp_path / "data.parquet" - pq.write_table( - pa.table({"t": pa.array([0, 1], pa.int64()), "value": pa.array([2, 3], pa.uint8())}), - path, - ) - config = ParquetSingleChannelPerRowImportConfig( - asset_name="a", - time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), - single_channel=ParquetSingleChannelConfig( - data_path="value", - name="status", - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=ONE_BYTE, - ), - ) - - _, out = _convert(config, path, converted_dirs) - - assert pq.read_table(out).column("value").to_pylist() == [b"\x02", b"\x03"] - - -def _parquet_config_without_footer(): - return ParquetFlatDatasetImportConfig( - asset_name="a", - time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), - data_columns=[ - ParquetDataColumn( - path="status", - name="status", - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=ONE_BYTE, - ) - ], - ) - - -def _csv_config(elements=ONE_BYTE, first_data_row=2): - return CsvImportConfig( - asset_name="a", - first_data_row=first_data_row, - time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), - data_columns=[ - CsvDataColumn( - column=2, - name="status", - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=elements, - ), - CsvDataColumn(column=3, name="x", data_type=ChannelDataType.DOUBLE), - ], - ) - - -class TestCsv: - def test_integers_become_fixed_width_bit_strings(self, tmp_path, converted_dirs): - path = tmp_path / "data.csv" - path.write_text("time,status,x\nt0,53,1.5\nt1,0,2\nt2,,3\nt3,-1,4\n") - - config, out = _convert(_csv_config(), path, converted_dirs) - - assert out.read_text() == ( - "time,status,x\nt0,00110101,1.5\nt1,00000000,2\nt2,,3\nt3,11111111,4\n" - ) - assert path.read_text().startswith("time,status,x\nt0,53,") - - def test_width_follows_the_elements(self, tmp_path, converted_dirs): - path = tmp_path / "data.csv" - path.write_text("time,status,x\nt0,258,1\n") - - _, out = _convert(_csv_config(elements=TWO_BYTES), path, converted_dirs) - - assert out.read_text() == "time,status,x\nt0,0000000100000010,1\n" - - def test_rows_before_the_first_data_row_are_kept(self, tmp_path, converted_dirs): - path = tmp_path / "data.csv" - path.write_text("units,raw,m\ntime,status,x\nt0,5,1\n") - - _, out = _convert(_csv_config(first_data_row=3), path, converted_dirs) - - assert out.read_text() == "units,raw,m\ntime,status,x\nt0,00000101,1\n" - - @pytest.mark.parametrize( - "values", - [ - pytest.param(["00110101", "00000000"], id="bit-strings"), - pytest.param(["0", "1", "1"], id="single-bits"), - pytest.param(["NQ==", "AA=="], id="base64"), - ], - ) - def test_encoded_values_are_uploaded_as_is(self, tmp_path, values): - path = tmp_path / "data.csv" - path.write_text("time,status,x\n" + "".join(f"t,{v},1\n" for v in values)) - - assert convert_integer_bit_fields(_csv_config(), path) is None - - def test_ambiguous_bit_strings_raise(self, tmp_path): - path = tmp_path / "data.csv" - path.write_text("time,status,x\nt0,0,1\nt1,10,1\nt2,11,1\n") - - with pytest.raises(ValueError, match="can't tell whether"): - convert_integer_bit_fields(_csv_config(), path) - - def test_value_too_large_raises(self, tmp_path): - path = tmp_path / "data.csv" - path.write_text("time,status,x\nt0,256,1\n") - - with pytest.raises(ValueError, match="value 256 in row 2"): - convert_integer_bit_fields(_csv_config(), path) - - def test_failed_conversion_leaves_no_temp_dir(self, tmp_path, monkeypatch): - created = [] - real_mkdtemp = tempfile.mkdtemp - - def tracking_mkdtemp(*args, **kwargs): - d = real_mkdtemp(*args, **kwargs) - created.append(d) - return d - - monkeypatch.setattr(tempfile, "mkdtemp", tracking_mkdtemp) - path = tmp_path / "data.csv" - path.write_text("time,status,x\nt0,5,1\nt1,300,1\n") - - with pytest.raises(ValueError, match="value 300 in row 3"): - convert_integer_bit_fields(_csv_config(), path) - - assert created - assert not any(os.path.exists(d) for d in created) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 8c97eb28f1..01b84c26d0 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2,7 +2,6 @@ from __future__ import annotations -import uuid import warnings from datetime import datetime, timezone from typing import TYPE_CHECKING, cast @@ -1830,225 +1829,3 @@ def test_in_place_edit_is_validated_on_import(self, parquet_config): column.bit_field_elements = BIT_FIELDS proto = parquet_config._to_proto() assert _bit_field_list(proto.flat_dataset.data_columns[0].channel_config) == BIT_FIELDS - - -class TestImportFromPathBitFieldConversion: - def _api(self, monkeypatch, uploads, fail=False): - def fake_upload(url, path, **kwargs): - uploads.append((path, path.read_bytes())) - if fail: - raise RuntimeError("upload failed") - return {"jobId": "j1"} - - monkeypatch.setattr("sift_client.resources.data_imports.upload_file", fake_upload) - api = DataImportAPIAsync(MagicMock()) - api._low_level_client = MagicMock() - api._low_level_client.create_from_upload = AsyncMock(return_value=("import_1", "url")) - api.client.async_.jobs.get = AsyncMock(return_value=MagicMock()) - return api - - def _parquet(self, tmp_path): - import pyarrow as pa - import pyarrow.parquet as pq - - path = tmp_path / "status.parquet" - pq.write_table( - pa.table({"t": pa.array([0, 1], pa.int64()), "status": pa.array([53, 1], pa.uint8())}), - path, - ) - config = ParquetFlatDatasetImportConfig( - asset_name="a", - time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), - data_columns=[ - ParquetDataColumn( - path="status", - name="status", - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=BIT_FIELDS, - ) - ], - ) - return path, config - - @pytest.mark.asyncio - async def test_uploads_converted_copy_and_cleans_up(self, tmp_path, monkeypatch): - import pyarrow as pa - import pyarrow.parquet as pq - - from sift_client._internal.util.file import extract_parquet_footer - - uploads: list = [] - api = self._api(monkeypatch, uploads) - path, config = self._parquet(tmp_path) - - await api.import_from_path(path, config=config, show_progress=False) - - ((uploaded_path, uploaded_bytes),) = uploads - assert uploaded_path != path - assert not uploaded_path.parent.exists() - converted = tmp_path / "uploaded.parquet" - converted.write_bytes(uploaded_bytes) - assert pq.read_table(converted).column("status").to_pylist() == [b"\x35", b"\x01"] - assert pq.read_schema(path).field("status").type == pa.uint8() - - low_level = cast("MagicMock", api._low_level_client) - (sent_config,) = low_level.create_from_upload.call_args.args - footer, offset = extract_parquet_footer(converted) - assert (sent_config.footer_offset, sent_config.footer_length) == (offset, len(footer)) - assert sent_config is not config - assert config.run_name == "status.parquet" - - @pytest.mark.asyncio - async def test_failed_upload_still_cleans_up(self, tmp_path, monkeypatch): - uploads: list = [] - api = self._api(monkeypatch, uploads, fail=True) - path, config = self._parquet(tmp_path) - - with pytest.raises(RuntimeError, match="upload failed"): - await api.import_from_path(path, config=config, show_progress=False) - - ((uploaded_path, _),) = uploads - assert not uploaded_path.parent.exists() - - @pytest.mark.asyncio - async def test_columns_without_integers_upload_the_original(self, tmp_path, monkeypatch): - uploads: list = [] - api = self._api(monkeypatch, uploads) - path = tmp_path / "status.csv" - path.write_text("time,status\nt0,00110101\n") - config = CsvImportConfig( - asset_name="a", - time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), - data_columns=[ - CsvDataColumn( - column=2, - name="status", - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=BIT_FIELDS, - ) - ], - ) - - await api.import_from_path(path, config=config, show_progress=False) - - ((uploaded_path, _),) = uploads - assert uploaded_path == path - - -# Raw status values and their decoded elements under BIT_FIELDS: -# bit 0 is armed, bits 1-3 are error_code, bits 4-7 are mode. -STATUS_VALUES = [53, 0, 255, 1] -DECODED_STATUS = { - "armed": [1, 0, 1, 1], - "error_code": [2, 0, 7, 0], - "mode": [3, 0, 15, 0], -} - -# A 16-bit status word, to check that multi-byte values decode little-endian. -WIDE_BIT_FIELDS = [ - ChannelBitFieldElement(name="low", index=0, bit_count=8), - ChannelBitFieldElement(name="high", index=8, bit_count=8), -] -WIDE_VALUES = [0x1234, 0x00FF, 0xFF00, 1] -DECODED_WIDE = {"low": [0x34, 0xFF, 0x00, 1], "high": [0x12, 0x00, 0xFF, 0]} - - -def _import_and_wait(sift_client, path, config) -> DataImport: - job = sift_client.data_import.import_from_path(path, config=config, show_progress=False) - job.wait_until_complete(polling_interval_secs=1, timeout_secs=300) - data_import = job.get_data_import() - assert data_import.status == DataImportStatus.SUCCEEDED, data_import.error_message - assert job.job_status == JobStatus.FINISHED - return data_import - - -def _assert_decoded(sift_client, data_import: DataImport, elements, decoded): - run = sift_client.data_import.get_run(data_import.id_) - channels = sift_client.channels.list_(run=run, name="status") - assert len(channels) == 1 - status = channels[0] - assert status.data_type == ChannelDataType.BIT_FIELD - assert status.bit_field_elements == elements - - data = sift_client.channels.get_data(channels=[status], run=run, show_progress=False) - for element, expected in decoded.items(): - frame = data[f"status.{element}"] - assert frame.iloc[:, 0].astype(int).tolist() == expected, (element, frame) - - -def _import_parquet(sift_client, tmp_path, status, elements) -> DataImport: - import pyarrow as pa - import pyarrow.parquet as pq - - path = tmp_path / "status.parquet" - start = 1_700_000_000_000_000_000 - pq.write_table( - pa.table( - { - "timestamp": pa.array( - [start + i * 1_000_000_000 for i in range(len(status))], pa.int64() - ), - "status": status, - } - ), - path, - ) - - config = sift_client.data_import.detect_config(path, data_type=DataTypeKey.PARQUET_FLATDATASET) - assert isinstance(config, ParquetFlatDatasetImportConfig) - config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" - config.time_column = ParquetTimeColumn( - path="timestamp", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS - ) - config.data_columns = [dc for dc in config.data_columns if dc.path != "timestamp"] - column = config["status"] - column.data_type = ChannelDataType.BIT_FIELD - column.bit_field_elements = elements - return _import_and_wait(sift_client, path, config) - - -@pytest.mark.integration -class TestBitFieldImportIntegration: - @pytest.mark.parametrize("encoding", ["uint8", "int64", "binary"]) - def test_parquet_flat_dataset(self, sift_client, tmp_path, encoding): - import pyarrow as pa - - status = { - "uint8": pa.array(STATUS_VALUES, pa.uint8()), - "int64": pa.array(STATUS_VALUES, pa.int64()), - "binary": pa.array([bytes([v]) for v in STATUS_VALUES], pa.binary()), - }[encoding] - - data_import = _import_parquet(sift_client, tmp_path, status, BIT_FIELDS) - _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) - - def test_parquet_multi_byte_status(self, sift_client, tmp_path): - import pyarrow as pa - - status = pa.array(WIDE_VALUES, pa.uint16()) - - data_import = _import_parquet(sift_client, tmp_path, status, WIDE_BIT_FIELDS) - _assert_decoded(sift_client, data_import, WIDE_BIT_FIELDS, DECODED_WIDE) - - @pytest.mark.parametrize("encoding", ["integers", "bit-strings"]) - def test_csv(self, sift_client, tmp_path, encoding): - path = tmp_path / "status.csv" - times = pd.date_range("2024-01-01", periods=len(STATUS_VALUES), freq="s", tz="UTC") - values = ( - STATUS_VALUES if encoding == "integers" else [format(v, "08b") for v in STATUS_VALUES] - ) - pd.DataFrame( - {"time": [t.strftime("%Y-%m-%dT%H:%M:%S.%fZ") for t in times], "status": values} - ).to_csv(path, index=False) - - config = sift_client.data_import.detect_config( - path, time_format=TimeFormat.ABSOLUTE_RFC3339 - ) - assert isinstance(config, CsvImportConfig) - config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" - column = config["status"] - column.data_type = ChannelDataType.BIT_FIELD - column.bit_field_elements = BIT_FIELDS - - data_import = _import_and_wait(sift_client, path, config) - _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 5f34c07b3b..52e90464e9 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,12 +1,10 @@ from __future__ import annotations -import shutil from pathlib import Path from typing import TYPE_CHECKING from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.time import to_timestamp_nanos -from sift_client._internal.util.bit_field import convert_integer_bit_fields from sift_client._internal.util.data_imports import ( parse_csv_detect_response, parse_parquet_detect_response, @@ -84,11 +82,6 @@ async def import_from_path( If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. - The import service reads ``BIT_FIELD`` values as bytes, a string of - 0s and 1s, or base64. When a CSV or Parquet ``BIT_FIELD`` column holds - integers, the client converts them and uploads the converted copy - from a temporary directory. The original file isn't changed. - Examples: Import a CSV file with auto-detected config: @@ -160,8 +153,6 @@ async def import_from_path( Raises: FileNotFoundError: If the file does not exist. - ValueError: If a ``BIT_FIELD`` column can't be converted, for example - when its values don't fit the bits its elements span. """ path = Path(file_path) if not path.is_file(): @@ -192,39 +183,24 @@ async def import_from_path( elif not config.run_name and not config.run_id: config.run_name = path.name - # The import service rejects integer bit field values, so BIT_FIELD columns - # stored as integers are uploaded from a converted copy of the file. - upload_config: ImportConfig = config - upload_path = path - converted = await run_sync_function(lambda: convert_integer_bit_fields(upload_config, path)) - if converted is not None: - upload_config, upload_path = converted - - try: - if isinstance( - upload_config, - (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig), - ): - await prepare_parquet_config(upload_config, upload_path) - - if show_progress is None: - show_progress = self._show_progress() - - data_import_id, upload_url = await self._low_level_client.create_from_upload( - upload_config - ) + if isinstance( + config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) + ): + await prepare_parquet_config(config, path) - response = await run_sync_function( - lambda: upload_file( - upload_url, - upload_path, - rest_client=self.client.rest_client, - show_progress=show_progress, - ) + if show_progress is None: + show_progress = self._show_progress() + + data_import_id, upload_url = await self._low_level_client.create_from_upload(config) + + response = await run_sync_function( + lambda: upload_file( + upload_url, + path, + rest_client=self.client.rest_client, + show_progress=show_progress, ) - finally: - if upload_path != path: - shutil.rmtree(upload_path.parent, ignore_errors=True) + ) job_id = response.get("jobId") if not job_id: raise RuntimeError("Upload succeeded but server response did not include a job ID.") diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index d63fa8f766..80094023a7 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1770,11 +1770,6 @@ class DataImportAPI: If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. - The import service reads ``BIT_FIELD`` values as bytes, a string of - 0s and 1s, or base64. When a CSV or Parquet ``BIT_FIELD`` column holds - integers, the client converts them and uploads the converted copy - from a temporary directory. The original file isn't changed. - Examples: Import a CSV file with auto-detected config: @@ -1846,8 +1841,6 @@ class DataImportAPI: Raises: FileNotFoundError: If the file does not exist. - ValueError: If a ``BIT_FIELD`` column can't be converted, for example - when its values don't fit the bits its elements span. """ ... From f65a5a99ec5bfa164d45ed3e3e1433dac7b1c0d7 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 2 Nov 2026 18:58:26 -0800 Subject: [PATCH 05/15] python(feat): check bit field values before upload --- python/changelog.d/861.feat.md | 6 +- .../data_import/parquet/flat_dataset/main.py | 9 +- .../sift_client/_internal/util/bit_field.py | 210 +++++++++++++++ .../_tests/_internal/test_bit_field.py | 242 ++++++++++++++++++ .../_tests/resources/test_data_imports.py | 163 ++++++++++++ .../lib/sift_client/resources/data_imports.py | 13 + .../resources/sync_stubs/__init__.pyi | 7 + 7 files changed, 646 insertions(+), 4 deletions(-) create mode 100644 python/lib/sift_client/_internal/util/bit_field.py create mode 100644 python/lib/sift_client/_tests/_internal/test_bit_field.py diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index 38d0db0193..e5059b2da7 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -1,4 +1,4 @@ -Data columns on all import configs accept `bit_field_elements`, so you can import an integer column as a `BIT_FIELD` channel. +Data columns on all import configs accept `bit_field_elements`, so a column can be imported as a `BIT_FIELD` channel. ```python from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType @@ -12,4 +12,8 @@ status.bit_field_elements = [ ] ``` +Bit field values must be raw bytes, a string of 0s and 1s like `00110101`, or URL-safe base64. Bytes are little-endian, so bit 8 is the lowest bit of the second byte. Integers aren't accepted. Convert an integer column before importing, for example with `int(v).to_bytes(2, "little")` for a 16-bit status word in Parquet, or `format(v, "08b")` for an 8-bit value in CSV. + +`import_from_path` checks `BIT_FIELD` columns before uploading. A Parquet column must be a binary or string type, and a sample of CSV or Parquet string values must parse. A column of digits that only parse by accident, like `10` (read as 2) or `1234` (read as base64), raises too, since those would import as the wrong values. + A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, so a column edited in place fails before the request is sent. Multi-channel Parquet single-channel-per-row imports don't support bit fields, since channel configs there are derived per row. diff --git a/python/examples/data_import/parquet/flat_dataset/main.py b/python/examples/data_import/parquet/flat_dataset/main.py index c1e11ee330..6f13d49da0 100644 --- a/python/examples/data_import/parquet/flat_dataset/main.py +++ b/python/examples/data_import/parquet/flat_dataset/main.py @@ -46,9 +46,12 @@ # # Example: drop a column from the import # config.data_columns = [dc for dc in config.data_columns if dc.path != "channel_0"] # - # # Example: import an integer column as a bit field. Parquet stores only the - # # raw integer, so detect_config can't infer the bit layout; supply it here. - # # Each element's index is its starting bit and bit_count is its width. + # # Example: import a binary column as a bit field. detect_config can't + # # infer the bit layout, so supply it here. Each element's index is its + # # starting bit and bit_count is its width. Values must be bytes (or a + # # string of 0s and 1s, or base64), not integers; convert an integer + # # column first, e.g. pa.array([int(v).to_bytes(1, "little") for v in col], + # # pa.binary()). # from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType # # status = config["status"] diff --git a/python/lib/sift_client/_internal/util/bit_field.py b/python/lib/sift_client/_internal/util/bit_field.py new file mode 100644 index 0000000000..d5fb89ef6f --- /dev/null +++ b/python/lib/sift_client/_internal/util/bit_field.py @@ -0,0 +1,210 @@ +"""Check bit field values before upload, so a wrong encoding fails early. + +The import service reads a bit field value as raw bytes, a string of 0s and 1s, +or URL-safe base64, and fails the import on the first value it can't parse. It +only reports that after the whole file is uploaded, and integers that happen to +parse, like ``10`` or ``1234``, import as the wrong value with no error. These +checks read the Parquet schema and a sample of values, and raise before the +upload starts. +""" + +from __future__ import annotations + +import base64 +import binascii +import csv +import re +from typing import TYPE_CHECKING, Iterable, Mapping + +from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.data_import import ( + CsvImportConfig, + DataColumnBase, + ParquetFlatDatasetImportConfig, + ParquetSingleChannelPerRowImportConfig, +) + +if TYPE_CHECKING: + from pathlib import Path + + import pyarrow as pa + + from sift_client.sift_types.data_import import ImportConfig + +# Matches ``PATH_SEPARATOR`` in ``_internal/util/parquet.py``, which imports +# pyarrow at module load and so can't be imported here unconditionally. +_PATH_SEPARATOR = "|" + +SAMPLE_ROWS = 1000 + +_BITS = re.compile(r"[01]+\Z") +_DECIMAL = re.compile(r"-?[0-9]+\Z") +_BASE64_URL = re.compile(r"[A-Za-z0-9_-]*={0,2}\Z") + +ACCEPTED_FORMATS = "raw bytes, a string of 0s and 1s like '00110101', or URL-safe base64" + + +def check_bit_field_values(config: ImportConfig, path: Path) -> None: + """Raise if a ``BIT_FIELD`` column's values aren't in a format the import service reads.""" + if isinstance(config, CsvImportConfig): + csv_columns = { + dc.column - 1: dc + for dc in config.data_columns + if dc.data_type == ChannelDataType.BIT_FIELD + } + if csv_columns: + _check_csv(path, config.first_data_row, csv_columns) + return + + parquet_columns: dict[str, DataColumnBase] + if isinstance(config, ParquetFlatDatasetImportConfig): + parquet_columns = { + dc.path: dc for dc in config.data_columns if dc.data_type == ChannelDataType.BIT_FIELD + } + elif isinstance(config, ParquetSingleChannelPerRowImportConfig): + sc = config.single_channel + parquet_columns = ( + {sc.data_path: sc} + if sc is not None and sc.data_type == ChannelDataType.BIT_FIELD + else {} + ) + else: + return + if parquet_columns: + _check_parquet(path, parquet_columns) + + +def _is_bit_value(value: str) -> bool: + """Whether the import service can parse ``value`` as a bit field.""" + if _BITS.match(value): + return True + if len(value) % 4 or not _BASE64_URL.match(value): + return False + try: + base64.urlsafe_b64decode(value) + except (binascii.Error, ValueError): + return False + return True + + +def _check_values(dc: DataColumnBase, values: Iterable[tuple[int, str]]) -> None: + """Check sampled ``(row, value)`` pairs from one column. Empty values are skipped.""" + all_decimal = True + bit_widths: set[int] = set() + seen = False + for row, value in values: + if value == "": + continue + seen = True + if not _is_bit_value(value): + raise ValueError( + f"bit field column '{dc.name}' has value {value!r} in row {row}, which isn't " + f"{ACCEPTED_FORMATS}." + ) + if not _DECIMAL.match(value): + all_decimal = False + elif _BITS.match(value): + bit_widths.add(len(value)) + else: + # All digits, and not 0s and 1s, so it only parsed as base64. + bit_widths.add(-1) + # A column of only digits is either fixed-width bit strings or integers that + # happen to parse, like 10 (read as 2) or 1234 (read as base64). + if seen and all_decimal and (-1 in bit_widths or len(bit_widths) > 1): + raise ValueError( + f"bit field column '{dc.name}' looks like it holds integers, which import as the " + f"wrong values. Write each value as {ACCEPTED_FORMATS}, for example " + "format(value, '08b') for an 8-bit status word." + ) + + +def _check_csv(path: Path, first_data_row: int, columns: Mapping[int, DataColumnBase]) -> None: + samples: dict[int, list[tuple[int, str]]] = {i: [] for i in columns} + with open(path, newline="", encoding="utf-8") as f: + for row_number, row in enumerate(csv.reader(f), start=1): + if row_number < first_data_row: + continue + if row_number >= first_data_row + SAMPLE_ROWS: + break + for i in columns: + samples[i].append((row_number, row[i] if i < len(row) else "")) + for i, dc in columns.items(): + _check_values(dc, samples[i]) + + +def _check_parquet(path: Path, columns: Mapping[str, DataColumnBase]) -> None: + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as e: + raise RuntimeError( + "pyarrow is required for Parquet import. " + "Install it via `pip install sift-stack-py[parquet]`." + ) from e + + schema = pq.read_schema(path) + string_columns: dict[str, DataColumnBase] = {} + for column_path, dc in columns.items(): + field_type = _field_type(schema, column_path) + if field_type is None: + # The import service reports columns missing from the file. + continue + if ( + pa.types.is_binary(field_type) + or pa.types.is_large_binary(field_type) + or pa.types.is_binary_view(field_type) + or pa.types.is_fixed_size_binary(field_type) + ): + continue + if ( + pa.types.is_string(field_type) + or pa.types.is_large_string(field_type) + or pa.types.is_string_view(field_type) + ): + string_columns[column_path] = dc + continue + raise ValueError( + f"bit field column '{dc.name}' is {field_type}, but bit field values must be " + f"{ACCEPTED_FORMATS}. Store the column as binary, for example " + "pa.array([int(v).to_bytes(2, 'little') for v in values], pa.binary()) for a " + "16-bit status word." + ) + + if not string_columns: + return + # Read only the sampled rows of the string columns. + pf = pq.ParquetFile(path) + top_level = sorted({p.split(_PATH_SEPARATOR)[0] for p in string_columns}) + batch = next(pf.iter_batches(batch_size=SAMPLE_ROWS, columns=top_level), None) + if batch is None: + return + table = pa.Table.from_batches([batch]) + for column_path, dc in string_columns.items(): + values = _column_values(table, column_path) + _check_values(dc, ((row, "" if v is None else v) for row, v in enumerate(values, start=1))) + + +def _field_type(schema: pa.Schema, column_path: str) -> pa.DataType | None: + import pyarrow as pa + + parts = column_path.split(_PATH_SEPARATOR) + index = schema.get_field_index(parts[0]) + if index == -1: + return None + field_type = schema.field(index).type + for part in parts[1:]: + if not pa.types.is_struct(field_type): + return None + child = field_type.get_field_index(part) + if child == -1: + return None + field_type = field_type.field(child).type + return field_type + + +def _column_values(table: pa.Table, column_path: str) -> list[str | None]: + parts = column_path.split(_PATH_SEPARATOR) + column = table.column(parts[0]).combine_chunks() + for part in parts[1:]: + column = column.field(part) + return column.to_pylist() diff --git a/python/lib/sift_client/_tests/_internal/test_bit_field.py b/python/lib/sift_client/_tests/_internal/test_bit_field.py new file mode 100644 index 0000000000..cab8d9e783 --- /dev/null +++ b/python/lib/sift_client/_tests/_internal/test_bit_field.py @@ -0,0 +1,242 @@ +"""Tests for checking bit field values before upload.""" + +from __future__ import annotations + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from sift_client._internal.util.bit_field import SAMPLE_ROWS, check_bit_field_values +from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType +from sift_client.sift_types.data_import import ( + CsvDataColumn, + CsvImportConfig, + CsvTimeColumn, + ParquetDataColumn, + ParquetFlatDatasetImportConfig, + ParquetSingleChannelConfig, + ParquetSingleChannelPerRowImportConfig, + ParquetTimeColumn, + TimeFormat, +) + +ELEMENTS = [ + ChannelBitFieldElement(name="armed", index=0, bit_count=1), + ChannelBitFieldElement(name="mode", index=4, bit_count=4), +] + + +def _csv_config(first_data_row=2): + return CsvImportConfig( + asset_name="a", + first_data_row=first_data_row, + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ELEMENTS, + ), + CsvDataColumn(column=3, name="x", data_type=ChannelDataType.DOUBLE), + ], + ) + + +def _write_csv(path, values): + path.write_text("time,status,x\n" + "".join(f"t,{v},1.5\n" for v in values)) + + +class TestCsv: + @pytest.mark.parametrize( + "values", + [ + pytest.param(["00110101", "00000000", "11111111"], id="bit-strings"), + pytest.param(["0", "1", "1"], id="single-bits"), + pytest.param(["NQ==", "AA==", "_w=="], id="base64"), + pytest.param(["00110101", "", "00000001"], id="empty-cells"), + ], + ) + def test_accepted_values_pass(self, tmp_path, values): + path = tmp_path / "data.csv" + _write_csv(path, values) + + check_bit_field_values(_csv_config(), path) + + def test_unparseable_value_raises_with_row(self, tmp_path): + path = tmp_path / "data.csv" + _write_csv(path, ["00110101", "53"]) + + with pytest.raises(ValueError, match=r"value '53' in row 3"): + check_bit_field_values(_csv_config(), path) + + @pytest.mark.parametrize( + "values", + [ + pytest.param(["0", "1", "10", "11"], id="reads-as-bits"), + pytest.param(["1234", "2048"], id="reads-as-base64"), + pytest.param(["-123", "1000"], id="negative-reads-as-base64"), + ], + ) + def test_integers_that_parse_raise(self, tmp_path, values): + path = tmp_path / "data.csv" + _write_csv(path, values) + + with pytest.raises(ValueError, match="looks like it holds integers"): + check_bit_field_values(_csv_config(), path) + + def test_rows_before_first_data_row_are_skipped(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("units,raw,m\ntime,status,x\nt,00000101,1\n") + + check_bit_field_values(_csv_config(first_data_row=3), path) + + def test_only_the_sample_is_read(self, tmp_path): + path = tmp_path / "data.csv" + _write_csv(path, ["00000001"] * SAMPLE_ROWS + ["53"]) + + check_bit_field_values(_csv_config(), path) + + def test_no_bit_field_columns_skips_reading_the_file(self, tmp_path): + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[CsvDataColumn(column=2, name="x", data_type=ChannelDataType.DOUBLE)], + ) + + check_bit_field_values(config, tmp_path / "missing.csv") + + +def _parquet_config(column="status"): + return ParquetFlatDatasetImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[ + ParquetDataColumn( + path=column, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ELEMENTS, + ), + ParquetDataColumn(path="x", name="x", data_type=ChannelDataType.DOUBLE), + ], + ) + + +def _write_parquet(path, status: pa.Array): + rows = len(status) + pq.write_table( + pa.table( + { + "t": pa.array(range(rows), pa.int64()), + "status": status, + "x": pa.array([0.0] * rows), + } + ), + path, + ) + + +class TestParquet: + @pytest.mark.parametrize( + "status", + [ + pytest.param(pa.array([b"\x35", None], pa.binary()), id="binary"), + pytest.param(pa.array([b"\x35"], pa.large_binary()), id="large-binary"), + pytest.param(pa.array([b"\x35"], pa.binary(1)), id="fixed-size-binary"), + pytest.param(pa.array(["00110101", None], pa.string()), id="bit-string"), + pytest.param(pa.array(["NQ=="], pa.large_string()), id="base64"), + ], + ) + def test_accepted_types_pass(self, tmp_path, status): + path = tmp_path / "data.parquet" + _write_parquet(path, status) + + check_bit_field_values(_parquet_config(), path) + + @pytest.mark.parametrize( + ("status", "type_name"), + [ + pytest.param(pa.array([53], pa.uint8()), "uint8", id="uint8"), + pytest.param(pa.array([53], pa.int64()), "int64", id="int64"), + pytest.param(pa.array([[53]], pa.list_(pa.uint8())), "list<", id="list"), + pytest.param(pa.array([True], pa.bool_()), "bool", id="bool"), + ], + ) + def test_other_types_raise(self, tmp_path, status, type_name): + path = tmp_path / "data.parquet" + _write_parquet(path, status) + + with pytest.raises(ValueError, match=rf"'status' is {type_name}"): + check_bit_field_values(_parquet_config(), path) + + def test_unparseable_string_raises(self, tmp_path): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array(["00110101", "not bits"], pa.string())) + + with pytest.raises(ValueError, match=r"value 'not bits' in row 2"): + check_bit_field_values(_parquet_config(), path) + + def test_integer_strings_raise(self, tmp_path): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array(["0", "1", "10"], pa.string())) + + with pytest.raises(ValueError, match="looks like it holds integers"): + check_bit_field_values(_parquet_config(), path) + + def test_nested_field_type_is_checked(self, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table( + { + "t": pa.array([0], pa.int64()), + "s": pa.array([{"status": 53}], pa.struct([("status", pa.uint8())])), + "x": pa.array([0.0]), + } + ), + path, + ) + + with pytest.raises(ValueError, match="'status' is uint8"): + check_bit_field_values(_parquet_config(column="s|status"), path) + + def test_nested_string_values_are_sampled(self, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table( + { + "t": pa.array([0], pa.int64()), + "s": pa.array([{"status": "53"}], pa.struct([("status", pa.string())])), + "x": pa.array([0.0]), + } + ), + path, + ) + + with pytest.raises(ValueError, match="value '53' in row 1"): + check_bit_field_values(_parquet_config(column="s|status"), path) + + def test_missing_column_is_left_to_the_server(self, tmp_path): + path = tmp_path / "data.parquet" + _write_parquet(path, pa.array([b"\x35"], pa.binary())) + + check_bit_field_values(_parquet_config(column="absent"), path) + + def test_single_channel_per_row(self, tmp_path): + path = tmp_path / "data.parquet" + pq.write_table( + pa.table({"t": pa.array([0], pa.int64()), "value": pa.array([2], pa.uint8())}), path + ) + config = ParquetSingleChannelPerRowImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + single_channel=ParquetSingleChannelConfig( + data_path="value", + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=ELEMENTS, + ), + ) + + with pytest.raises(ValueError, match="'status' is uint8"): + check_bit_field_values(config, path) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 01b84c26d0..6bcbd37f63 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2,6 +2,7 @@ from __future__ import annotations +import uuid import warnings from datetime import datetime, timezone from typing import TYPE_CHECKING, cast @@ -1829,3 +1830,165 @@ def test_in_place_edit_is_validated_on_import(self, parquet_config): column.bit_field_elements = BIT_FIELDS proto = parquet_config._to_proto() assert _bit_field_list(proto.flat_dataset.data_columns[0].channel_config) == BIT_FIELDS + + +class TestImportFromPathBitFieldCheck: + @pytest.mark.asyncio + async def test_unreadable_values_fail_before_upload(self, tmp_path, monkeypatch): + upload = MagicMock() + monkeypatch.setattr("sift_client.resources.data_imports.upload_file", upload) + api = DataImportAPIAsync(MagicMock()) + low_level = MagicMock() + low_level.create_from_upload = AsyncMock(return_value=("import_1", "url")) + api._low_level_client = low_level + path = tmp_path / "status.csv" + path.write_text("time,status\nt0,53\n") + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[ + CsvDataColumn( + column=2, + name="status", + data_type=ChannelDataType.BIT_FIELD, + bit_field_elements=BIT_FIELDS, + ) + ], + ) + + with pytest.raises(ValueError, match="value '53' in row 2"): + await api.import_from_path(path, config=config, show_progress=False) + + low_level.create_from_upload.assert_not_called() + upload.assert_not_called() + + +# Raw status values and their decoded elements under BIT_FIELDS: +# bit 0 is armed, bits 1-3 are error_code, bits 4-7 are mode. +STATUS_VALUES = [53, 0, 255, 1] +DECODED_STATUS = { + "armed": [1, 0, 1, 1], + "error_code": [2, 0, 7, 0], + "mode": [3, 0, 15, 0], +} + +# A 16-bit status word, to check that multi-byte values decode little-endian. +WIDE_BIT_FIELDS = [ + ChannelBitFieldElement(name="low", index=0, bit_count=8), + ChannelBitFieldElement(name="high", index=8, bit_count=8), +] +WIDE_VALUES = [0x1234, 0x00FF, 0xFF00, 1] +DECODED_WIDE = {"low": [0x34, 0xFF, 0x00, 1], "high": [0x12, 0x00, 0xFF, 0]} + + +def _import_and_wait(sift_client, path, config) -> DataImport: + job = sift_client.data_import.import_from_path(path, config=config, show_progress=False) + job.wait_until_complete(polling_interval_secs=1, timeout_secs=300) + data_import = job.get_data_import() + assert data_import.status == DataImportStatus.SUCCEEDED, data_import.error_message + assert job.job_status == JobStatus.FINISHED + return data_import + + +def _assert_decoded(sift_client, data_import: DataImport, elements, decoded): + run = sift_client.data_import.get_run(data_import.id_) + channels = sift_client.channels.list_(run=run, name="status") + assert len(channels) == 1 + status = channels[0] + assert status.data_type == ChannelDataType.BIT_FIELD + assert status.bit_field_elements == elements + + data = sift_client.channels.get_data(channels=[status], run=run, show_progress=False) + for element, expected in decoded.items(): + frame = data[f"status.{element}"] + assert frame.iloc[:, 0].astype(int).tolist() == expected, (element, frame) + + +def _import_parquet(sift_client, tmp_path, status, elements) -> DataImport: + import pyarrow as pa + import pyarrow.parquet as pq + + path = tmp_path / "status.parquet" + start = 1_700_000_000_000_000_000 + pq.write_table( + pa.table( + { + "timestamp": pa.array( + [start + i * 1_000_000_000 for i in range(len(status))], pa.int64() + ), + "status": status, + } + ), + path, + ) + + config = sift_client.data_import.detect_config(path, data_type=DataTypeKey.PARQUET_FLATDATASET) + assert isinstance(config, ParquetFlatDatasetImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + config.time_column = ParquetTimeColumn( + path="timestamp", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS + ) + config.data_columns = [dc for dc in config.data_columns if dc.path != "timestamp"] + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = elements + return _import_and_wait(sift_client, path, config) + + +@pytest.mark.integration +class TestBitFieldImportIntegration: + @pytest.mark.parametrize( + "encoding", ["binary", "fixed-size-binary", "bit-string", "base64-string"] + ) + def test_parquet_flat_dataset(self, sift_client, tmp_path, encoding): + import base64 + + import pyarrow as pa + + raw = [bytes([v]) for v in STATUS_VALUES] + status = { + "binary": pa.array(raw, pa.binary()), + "fixed-size-binary": pa.array(raw, pa.binary(1)), + "bit-string": pa.array([format(v, "08b") for v in STATUS_VALUES], pa.string()), + "base64-string": pa.array( + [base64.urlsafe_b64encode(b).decode() for b in raw], pa.string() + ), + }[encoding] + + data_import = _import_parquet(sift_client, tmp_path, status, BIT_FIELDS) + _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) + + def test_parquet_multi_byte_status(self, sift_client, tmp_path): + import pyarrow as pa + + status = pa.array([v.to_bytes(2, "little") for v in WIDE_VALUES], pa.binary()) + + data_import = _import_parquet(sift_client, tmp_path, status, WIDE_BIT_FIELDS) + _assert_decoded(sift_client, data_import, WIDE_BIT_FIELDS, DECODED_WIDE) + + @pytest.mark.parametrize("encoding", ["bit-strings", "base64"]) + def test_csv(self, sift_client, tmp_path, encoding): + import base64 + + path = tmp_path / "status.csv" + times = pd.date_range("2024-01-01", periods=len(STATUS_VALUES), freq="s", tz="UTC") + values = ( + [format(v, "08b") for v in STATUS_VALUES] + if encoding == "bit-strings" + else [base64.urlsafe_b64encode(bytes([v])).decode() for v in STATUS_VALUES] + ) + pd.DataFrame( + {"time": [t.strftime("%Y-%m-%dT%H:%M:%S.%fZ") for t in times], "status": values} + ).to_csv(path, index=False) + + config = sift_client.data_import.detect_config( + path, time_format=TimeFormat.ABSOLUTE_RFC3339 + ) + assert isinstance(config, CsvImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = BIT_FIELDS + + data_import = _import_and_wait(sift_client, path, config) + _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 52e90464e9..e0e5851c5f 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -5,6 +5,7 @@ from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.time import to_timestamp_nanos +from sift_client._internal.util.bit_field import check_bit_field_values from sift_client._internal.util.data_imports import ( parse_csv_detect_response, parse_parquet_detect_response, @@ -82,6 +83,11 @@ async def import_from_path( If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. + ``BIT_FIELD`` values must be raw bytes, a string of 0s and 1s, or + URL-safe base64. Before uploading, CSV and Parquet ``BIT_FIELD`` + columns are checked: Parquet columns must be a binary or string + type, and a sample of string values must parse. + Examples: Import a CSV file with auto-detected config: @@ -153,6 +159,8 @@ async def import_from_path( Raises: FileNotFoundError: If the file does not exist. + ValueError: If a ``BIT_FIELD`` column's values aren't in a format + the import service reads, such as integers. """ path = Path(file_path) if not path.is_file(): @@ -183,6 +191,11 @@ async def import_from_path( elif not config.run_name and not config.run_id: config.run_name = path.name + # The import service only reports unreadable bit field values after the + # upload, so check them first. + checked_config: ImportConfig = config + await run_sync_function(lambda: check_bit_field_values(checked_config, path)) + if isinstance( config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) ): diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 80094023a7..3f6c12ec1f 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1770,6 +1770,11 @@ class DataImportAPI: If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. + ``BIT_FIELD`` values must be raw bytes, a string of 0s and 1s, or + URL-safe base64. Before uploading, CSV and Parquet ``BIT_FIELD`` + columns are checked: Parquet columns must be a binary or string + type, and a sample of string values must parse. + Examples: Import a CSV file with auto-detected config: @@ -1841,6 +1846,8 @@ class DataImportAPI: Raises: FileNotFoundError: If the file does not exist. + ValueError: If a ``BIT_FIELD`` column's values aren't in a format + the import service reads, such as integers. """ ... From dc92e06291af84ee3d0a23b15e4b9535ff966fa0 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Tue, 3 Nov 2026 13:04:08 -0800 Subject: [PATCH 06/15] python(refactor): accept bit field values that parse --- python/changelog.d/861.feat.md | 2 +- .../sift_client/_internal/util/bit_field.py | 29 +++---------------- .../_tests/_internal/test_bit_field.py | 24 ++------------- 3 files changed, 7 insertions(+), 48 deletions(-) diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index e5059b2da7..b4f95f6db8 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -14,6 +14,6 @@ status.bit_field_elements = [ Bit field values must be raw bytes, a string of 0s and 1s like `00110101`, or URL-safe base64. Bytes are little-endian, so bit 8 is the lowest bit of the second byte. Integers aren't accepted. Convert an integer column before importing, for example with `int(v).to_bytes(2, "little")` for a 16-bit status word in Parquet, or `format(v, "08b")` for an 8-bit value in CSV. -`import_from_path` checks `BIT_FIELD` columns before uploading. A Parquet column must be a binary or string type, and a sample of CSV or Parquet string values must parse. A column of digits that only parse by accident, like `10` (read as 2) or `1234` (read as base64), raises too, since those would import as the wrong values. +`import_from_path` checks `BIT_FIELD` columns before uploading. A Parquet column must be a binary or string type, and a sample of CSV or Parquet string values must parse. A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, so a column edited in place fails before the request is sent. Multi-channel Parquet single-channel-per-row imports don't support bit fields, since channel configs there are derived per row. diff --git a/python/lib/sift_client/_internal/util/bit_field.py b/python/lib/sift_client/_internal/util/bit_field.py index d5fb89ef6f..2927a9be3c 100644 --- a/python/lib/sift_client/_internal/util/bit_field.py +++ b/python/lib/sift_client/_internal/util/bit_field.py @@ -2,10 +2,8 @@ The import service reads a bit field value as raw bytes, a string of 0s and 1s, or URL-safe base64, and fails the import on the first value it can't parse. It -only reports that after the whole file is uploaded, and integers that happen to -parse, like ``10`` or ``1234``, import as the wrong value with no error. These -checks read the Parquet schema and a sample of values, and raise before the -upload starts. +only reports that after the whole file is uploaded. These checks read the +Parquet schema and a sample of values, and raise before the upload starts. """ from __future__ import annotations @@ -38,7 +36,6 @@ SAMPLE_ROWS = 1000 _BITS = re.compile(r"[01]+\Z") -_DECIMAL = re.compile(r"-?[0-9]+\Z") _BASE64_URL = re.compile(r"[A-Za-z0-9_-]*={0,2}\Z") ACCEPTED_FORMATS = "raw bytes, a string of 0s and 1s like '00110101', or URL-safe base64" @@ -89,33 +86,15 @@ def _is_bit_value(value: str) -> bool: def _check_values(dc: DataColumnBase, values: Iterable[tuple[int, str]]) -> None: """Check sampled ``(row, value)`` pairs from one column. Empty values are skipped.""" - all_decimal = True - bit_widths: set[int] = set() - seen = False for row, value in values: if value == "": continue - seen = True if not _is_bit_value(value): raise ValueError( f"bit field column '{dc.name}' has value {value!r} in row {row}, which isn't " - f"{ACCEPTED_FORMATS}." + f"{ACCEPTED_FORMATS}. To convert integers, use format(value, '08b') for an " + "8-bit status word." ) - if not _DECIMAL.match(value): - all_decimal = False - elif _BITS.match(value): - bit_widths.add(len(value)) - else: - # All digits, and not 0s and 1s, so it only parsed as base64. - bit_widths.add(-1) - # A column of only digits is either fixed-width bit strings or integers that - # happen to parse, like 10 (read as 2) or 1234 (read as base64). - if seen and all_decimal and (-1 in bit_widths or len(bit_widths) > 1): - raise ValueError( - f"bit field column '{dc.name}' looks like it holds integers, which import as the " - f"wrong values. Write each value as {ACCEPTED_FORMATS}, for example " - "format(value, '08b') for an 8-bit status word." - ) def _check_csv(path: Path, first_data_row: int, columns: Mapping[int, DataColumnBase]) -> None: diff --git a/python/lib/sift_client/_tests/_internal/test_bit_field.py b/python/lib/sift_client/_tests/_internal/test_bit_field.py index cab8d9e783..3782e6dd76 100644 --- a/python/lib/sift_client/_tests/_internal/test_bit_field.py +++ b/python/lib/sift_client/_tests/_internal/test_bit_field.py @@ -55,6 +55,8 @@ class TestCsv: pytest.param(["0", "1", "1"], id="single-bits"), pytest.param(["NQ==", "AA==", "_w=="], id="base64"), pytest.param(["00110101", "", "00000001"], id="empty-cells"), + pytest.param(["0", "1", "10", "11"], id="unpadded-bit-strings"), + pytest.param(["1234", "2048"], id="digit-only-base64"), ], ) def test_accepted_values_pass(self, tmp_path, values): @@ -70,21 +72,6 @@ def test_unparseable_value_raises_with_row(self, tmp_path): with pytest.raises(ValueError, match=r"value '53' in row 3"): check_bit_field_values(_csv_config(), path) - @pytest.mark.parametrize( - "values", - [ - pytest.param(["0", "1", "10", "11"], id="reads-as-bits"), - pytest.param(["1234", "2048"], id="reads-as-base64"), - pytest.param(["-123", "1000"], id="negative-reads-as-base64"), - ], - ) - def test_integers_that_parse_raise(self, tmp_path, values): - path = tmp_path / "data.csv" - _write_csv(path, values) - - with pytest.raises(ValueError, match="looks like it holds integers"): - check_bit_field_values(_csv_config(), path) - def test_rows_before_first_data_row_are_skipped(self, tmp_path): path = tmp_path / "data.csv" path.write_text("units,raw,m\ntime,status,x\nt,00000101,1\n") @@ -177,13 +164,6 @@ def test_unparseable_string_raises(self, tmp_path): with pytest.raises(ValueError, match=r"value 'not bits' in row 2"): check_bit_field_values(_parquet_config(), path) - def test_integer_strings_raise(self, tmp_path): - path = tmp_path / "data.parquet" - _write_parquet(path, pa.array(["0", "1", "10"], pa.string())) - - with pytest.raises(ValueError, match="looks like it holds integers"): - check_bit_field_values(_parquet_config(), path) - def test_nested_field_type_is_checked(self, tmp_path): path = tmp_path / "data.parquet" pq.write_table( From 5ca2ed6e3bbb5bf561aa7075fc1c60490c5dc7f7 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Tue, 3 Nov 2026 13:29:16 -0800 Subject: [PATCH 07/15] python(refactor): reuse the Parquet path separator in bit field checks --- .../sift_client/_internal/util/bit_field.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/python/lib/sift_client/_internal/util/bit_field.py b/python/lib/sift_client/_internal/util/bit_field.py index 2927a9be3c..57d2cfa05f 100644 --- a/python/lib/sift_client/_internal/util/bit_field.py +++ b/python/lib/sift_client/_internal/util/bit_field.py @@ -29,10 +29,6 @@ from sift_client.sift_types.data_import import ImportConfig -# Matches ``PATH_SEPARATOR`` in ``_internal/util/parquet.py``, which imports -# pyarrow at module load and so can't be imported here unconditionally. -_PATH_SEPARATOR = "|" - SAMPLE_ROWS = 1000 _BITS = re.compile(r"[01]+\Z") @@ -115,6 +111,8 @@ def _check_parquet(path: Path, columns: Mapping[str, DataColumnBase]) -> None: try: import pyarrow as pa import pyarrow.parquet as pq + + from sift_client._internal.util.parquet import PATH_SEPARATOR except ImportError as e: raise RuntimeError( "pyarrow is required for Parquet import. " @@ -124,7 +122,7 @@ def _check_parquet(path: Path, columns: Mapping[str, DataColumnBase]) -> None: schema = pq.read_schema(path) string_columns: dict[str, DataColumnBase] = {} for column_path, dc in columns.items(): - field_type = _field_type(schema, column_path) + field_type = _field_type(schema, column_path.split(PATH_SEPARATOR)) if field_type is None: # The import service reports columns missing from the file. continue @@ -153,20 +151,19 @@ def _check_parquet(path: Path, columns: Mapping[str, DataColumnBase]) -> None: return # Read only the sampled rows of the string columns. pf = pq.ParquetFile(path) - top_level = sorted({p.split(_PATH_SEPARATOR)[0] for p in string_columns}) + top_level = sorted({p.split(PATH_SEPARATOR)[0] for p in string_columns}) batch = next(pf.iter_batches(batch_size=SAMPLE_ROWS, columns=top_level), None) if batch is None: return table = pa.Table.from_batches([batch]) for column_path, dc in string_columns.items(): - values = _column_values(table, column_path) + values = _column_values(table, column_path.split(PATH_SEPARATOR)) _check_values(dc, ((row, "" if v is None else v) for row, v in enumerate(values, start=1))) -def _field_type(schema: pa.Schema, column_path: str) -> pa.DataType | None: +def _field_type(schema: pa.Schema, parts: list[str]) -> pa.DataType | None: import pyarrow as pa - parts = column_path.split(_PATH_SEPARATOR) index = schema.get_field_index(parts[0]) if index == -1: return None @@ -181,8 +178,7 @@ def _field_type(schema: pa.Schema, column_path: str) -> pa.DataType | None: return field_type -def _column_values(table: pa.Table, column_path: str) -> list[str | None]: - parts = column_path.split(_PATH_SEPARATOR) +def _column_values(table: pa.Table, parts: list[str]) -> list[str | None]: column = table.column(parts[0]).combine_chunks() for part in parts[1:]: column = column.field(part) From 7d9fab327328392814fd486425a91d7632cb00f8 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Tue, 3 Nov 2026 13:35:50 -0800 Subject: [PATCH 08/15] python(feat): reject bit fields on ULog and MCAP columns --- python/changelog.d/861.feat.md | 4 +- .../_tests/resources/test_data_imports.py | 54 ++++++++----------- .../lib/sift_client/sift_types/data_import.py | 17 +++++- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index b4f95f6db8..e510fc0523 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -1,4 +1,4 @@ -Data columns on all import configs accept `bit_field_elements`, so a column can be imported as a `BIT_FIELD` channel. +CSV, Parquet, TDMS and HDF5 data columns accept `bit_field_elements`, so a column can be imported as a `BIT_FIELD` channel. ULog and MCAP columns raise `ValueError` if set to `BIT_FIELD`, since those imports can't override a channel to a bit field. ```python from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType @@ -12,7 +12,7 @@ status.bit_field_elements = [ ] ``` -Bit field values must be raw bytes, a string of 0s and 1s like `00110101`, or URL-safe base64. Bytes are little-endian, so bit 8 is the lowest bit of the second byte. Integers aren't accepted. Convert an integer column before importing, for example with `int(v).to_bytes(2, "little")` for a 16-bit status word in Parquet, or `format(v, "08b")` for an 8-bit value in CSV. +In CSV and Parquet files, bit field values must be raw bytes, a string of 0s and 1s like `00110101`, or URL-safe base64. Bytes are little-endian, so bit 8 is the lowest bit of the second byte. Integers aren't accepted. Convert an integer column before importing, for example with `int(v).to_bytes(2, "little")` for a 16-bit status word in Parquet, or `format(v, "08b")` for an 8-bit value in CSV. `import_from_path` checks `BIT_FIELD` columns before uploading. A Parquet column must be a binary or string type, and a sample of CSV or Parquet string values must parse. diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 6bcbd37f63..7e5896379c 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -1716,38 +1716,30 @@ def test_tdms_round_trip(self): round_tripped = TdmsImportConfig._from_proto(proto) assert round_tripped.data[0].bit_field_elements == BIT_FIELDS - def test_ulog_round_trip(self): - config = UlogImportConfig( - asset_name="a", - data=[ - UlogDataColumn( - message_name="vehicle_status", - field_name="flags", - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=BIT_FIELDS, - ) - ], - ) - proto = config._to_proto() - assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS - round_tripped = UlogImportConfig._from_proto(proto) - assert round_tripped.data[0].bit_field_elements == BIT_FIELDS + @pytest.mark.parametrize( + "make_column", + [ + pytest.param( + lambda **kw: UlogDataColumn( + message_name="vehicle_status", field_name="flags", **kw + ), + id="ulog", + ), + pytest.param( + lambda **kw: McapDataColumn(topic="/state", field_path="flags", **kw), + id="mcap", + ), + ], + ) + def test_unsupported_formats_reject_bit_fields(self, make_column): + with pytest.raises(ValueError, match="doesn't support ChannelDataType.BIT_FIELD"): + make_column(data_type=ChannelDataType.BIT_FIELD, bit_field_elements=BIT_FIELDS) - def test_mcap_round_trip(self): - config = McapImportConfig( - asset_name="a", - data=[ - McapDataColumn( - topic="/state", - field_path="flags", - data_type=ChannelDataType.BIT_FIELD, - bit_field_elements=BIT_FIELDS, - ) - ], - ) - proto = config._to_proto() - assert _bit_field_list(proto.data[0].channel_config) == BIT_FIELDS - assert McapImportConfig._from_proto(proto).data[0].bit_field_elements == BIT_FIELDS + column = make_column(data_type=ChannelDataType.UINT_32) + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = BIT_FIELDS + with pytest.raises(ValueError, match="doesn't support ChannelDataType.BIT_FIELD"): + column._channel_config_proto() def test_hdf5_to_proto(self): config = Hdf5ImportConfig( diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index 4ff8f327db..ba5288a4ad 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from datetime import datetime, timezone from enum import Enum -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, ClassVar, Union import pandas as pd # noqa: TC002 from pydantic import BaseModel, ConfigDict, model_validator @@ -173,9 +173,13 @@ class DataColumnBase(BaseModel, ABC): enum_types: Mapping of enum name to key. Only valid when ``data_type`` is ``ChannelDataType.ENUM``. bit_field_elements: Named bit ranges within each value. Only valid when - ``data_type`` is ``ChannelDataType.BIT_FIELD``. + ``data_type`` is ``ChannelDataType.BIT_FIELD``, and only on formats + that support bit fields. """ + # ULog and MCAP imports can't override a channel to BIT_FIELD. + _supports_bit_fields: ClassVar[bool] = True + name: str data_type: ChannelDataType units: str = "" @@ -209,6 +213,11 @@ def _validate_channel_type_fields(self) -> None: if len(set(self.enum_types.values())) != len(self.enum_types): raise ValueError(f"'enum_types' contains duplicate keys ({self.name}).") if self.data_type == ChannelDataType.BIT_FIELD or self.bit_field_elements: + if not self._supports_bit_fields: + raise ValueError( + f"{type(self).__name__} doesn't support ChannelDataType.BIT_FIELD " + f"({self.name})." + ) if self.data_type != ChannelDataType.BIT_FIELD: raise ValueError( f"'bit_field_elements' requires data_type ChannelDataType.BIT_FIELD " @@ -1087,6 +1096,8 @@ class UlogDataColumn(DataColumnBase): e.g. ``"sensor_accel_0.x"``. """ + _supports_bit_fields: ClassVar[bool] = False + message_name: str instance: int = 0 field_name: str = "" @@ -1275,6 +1286,8 @@ class McapDataColumn(DataColumnBase): e.g. ``"/imu/data.orientation.x"``. """ + _supports_bit_fields: ClassVar[bool] = False + topic: str field_path: str name: str = "" From 42c7d4dc541e9cc6b496c01a87238b9c59f9434c Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Tue, 3 Nov 2026 14:05:46 -0800 Subject: [PATCH 09/15] python(test): cover TDMS and HDF5 bit field imports --- .../_tests/resources/test_data_imports.py | 67 ++++++++++++++++++- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 7e5896379c..7fcc973e05 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -1872,6 +1872,15 @@ async def test_unreadable_values_fail_before_upload(self, tmp_path, monkeypatch) WIDE_VALUES = [0x1234, 0x00FF, 0xFF00, 1] DECODED_WIDE = {"low": [0x34, 0xFF, 0x00, 1], "high": [0x12, 0x00, 0xFF, 0]} +# TDMS and HDF5 store typed integers, which import as their two's complement bytes. +SIGNED_VALUES = [0x1234, -1, -256, 1] +DECODED_SIGNED = {"low": [0x34, 0xFF, 0x00, 1], "high": [0x12, 0xFF, 0xFF, 0]} +INTEGER_CASES = [ + pytest.param("uint8", STATUS_VALUES, BIT_FIELDS, DECODED_STATUS, id="uint8"), + pytest.param("uint16", WIDE_VALUES, WIDE_BIT_FIELDS, DECODED_WIDE, id="uint16"), + pytest.param("int16", SIGNED_VALUES, WIDE_BIT_FIELDS, DECODED_SIGNED, id="int16"), +] + def _import_and_wait(sift_client, path, config) -> DataImport: job = sift_client.data_import.import_from_path(path, config=config, show_progress=False) @@ -1882,9 +1891,9 @@ def _import_and_wait(sift_client, path, config) -> DataImport: return data_import -def _assert_decoded(sift_client, data_import: DataImport, elements, decoded): +def _assert_decoded(sift_client, data_import: DataImport, elements, decoded, name="status"): run = sift_client.data_import.get_run(data_import.id_) - channels = sift_client.channels.list_(run=run, name="status") + channels = sift_client.channels.list_(run=run, name=name) assert len(channels) == 1 status = channels[0] assert status.data_type == ChannelDataType.BIT_FIELD @@ -1892,7 +1901,7 @@ def _assert_decoded(sift_client, data_import: DataImport, elements, decoded): data = sift_client.channels.get_data(channels=[status], run=run, show_progress=False) for element, expected in decoded.items(): - frame = data[f"status.{element}"] + frame = data[f"{name}.{element}"] assert frame.iloc[:, 0].astype(int).tolist() == expected, (element, frame) @@ -1984,3 +1993,55 @@ def test_csv(self, sift_client, tmp_path, encoding): data_import = _import_and_wait(sift_client, path, config) _assert_decoded(sift_client, data_import, BIT_FIELDS, DECODED_STATUS) + + @pytest.mark.parametrize(("dtype", "values", "elements", "decoded"), INTEGER_CASES) + def test_tdms(self, sift_client, tmp_path, dtype, values, elements, decoded): + import numpy as np + from nptdms import ChannelObject, GroupObject, RootObject, TdmsWriter + + path = tmp_path / "status.tdms" + properties = { + "wf_start_time": np.datetime64(1_700_000_000_000_000_000, "ns"), + "wf_increment": 1.0, + "wf_start_offset": 0.0, + } + with TdmsWriter(path) as writer: + writer.write_segment( + [ + RootObject(), + GroupObject("g"), + ChannelObject("g", "status", np.array(values, dtype), properties=properties), + ] + ) + + config = sift_client.data_import.detect_config(path) + assert isinstance(config, TdmsImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + column = config["g.status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = elements + + data_import = _import_and_wait(sift_client, path, config) + _assert_decoded(sift_client, data_import, elements, decoded, name="g.status") + + @pytest.mark.parametrize(("dtype", "values", "elements", "decoded"), INTEGER_CASES) + def test_hdf5(self, sift_client, tmp_path, dtype, values, elements, decoded): + import h5py + import numpy as np + + path = tmp_path / "status.h5" + with h5py.File(path, "w") as f: + start = 1_700_000_000_000_000_000 + f["time"] = np.array([start + i * 1_000_000_000 for i in range(len(values))]) + f["status"] = np.array(values, dtype) + + config = sift_client.data_import.detect_config(path, data_type=DataTypeKey.HDF5_ONE_D) + assert isinstance(config, Hdf5ImportConfig) + config.asset_name = f"bit-field-import-{uuid.uuid4().hex[:8]}" + config.time_format = TimeFormat.ABSOLUTE_UNIX_NANOSECONDS + column = config["status"] + column.data_type = ChannelDataType.BIT_FIELD + column.bit_field_elements = elements + + data_import = _import_and_wait(sift_client, path, config) + _assert_decoded(sift_client, data_import, elements, decoded) From fceddf88b2648e599df3a1df3174162ae4164055 Mon Sep 17 00:00:00 2001 From: alden-sift Date: Thu, 5 Nov 2026 10:27:55 -0800 Subject: [PATCH 10/15] Update python/lib/sift_client/resources/data_imports.py Co-authored-by: Alex Luck --- python/lib/sift_client/resources/data_imports.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index e0e5851c5f..fdfcba4b6c 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -193,8 +193,7 @@ async def import_from_path( # The import service only reports unreadable bit field values after the # upload, so check them first. - checked_config: ImportConfig = config - await run_sync_function(lambda: check_bit_field_values(checked_config, path)) + await run_sync_function(check_bit_field_values, config, path) if isinstance( config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) From dbc5c419b3b9aa42eb6e69f5c9279f8328378526 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Thu, 5 Nov 2026 13:34:09 -0800 Subject: [PATCH 11/15] python(refactor): drop bit field reads from ULog and MCAP configs --- python/lib/sift_client/sift_types/data_import.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index ba5288a4ad..bcf0a78cfb 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -1221,7 +1221,6 @@ def _from_proto(cls, proto: UlogConfigProto) -> UlogImportConfig: units=d.channel_config.units, description=d.channel_config.description, enum_types=_enum_types_from_proto(d.channel_config), - bit_field_elements=_bit_field_elements_from_proto(d.channel_config), ) for d in proto.data ] @@ -1445,7 +1444,6 @@ def _from_proto(cls, proto: McapConfigProto) -> McapImportConfig: units=d.channel_config.units, description=d.channel_config.description, enum_types=_enum_types_from_proto(d.channel_config), - bit_field_elements=_bit_field_elements_from_proto(d.channel_config), ) for d in proto.data ] From a865d0ce2b02394e2e135e237fe029e115c39dc9 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Thu, 5 Nov 2026 13:34:12 -0800 Subject: [PATCH 12/15] python(refactor): move Parquet bit field checks into the parquet util --- .../sift_client/_internal/util/bit_field.py | 91 ++----------------- .../lib/sift_client/_internal/util/parquet.py | 70 ++++++++++++++ 2 files changed, 79 insertions(+), 82 deletions(-) diff --git a/python/lib/sift_client/_internal/util/bit_field.py b/python/lib/sift_client/_internal/util/bit_field.py index 57d2cfa05f..527a501b67 100644 --- a/python/lib/sift_client/_internal/util/bit_field.py +++ b/python/lib/sift_client/_internal/util/bit_field.py @@ -2,7 +2,7 @@ The import service reads a bit field value as raw bytes, a string of 0s and 1s, or URL-safe base64, and fails the import on the first value it can't parse. It -only reports that after the whole file is uploaded. These checks read the +only reports that after the whole file uploads. These checks read the Parquet schema and a sample of values, and raise before the upload starts. """ @@ -25,8 +25,6 @@ if TYPE_CHECKING: from pathlib import Path - import pyarrow as pa - from sift_client.sift_types.data_import import ImportConfig SAMPLE_ROWS = 1000 @@ -64,7 +62,14 @@ def check_bit_field_values(config: ImportConfig, path: Path) -> None: else: return if parquet_columns: - _check_parquet(path, parquet_columns) + try: + from sift_client._internal.util.parquet import check_bit_field_columns + except ImportError as e: + raise RuntimeError( + "pyarrow is required for Parquet import. " + "Install it via `pip install sift-stack-py[parquet]`." + ) from e + check_bit_field_columns(path, parquet_columns) def _is_bit_value(value: str) -> bool: @@ -105,81 +110,3 @@ def _check_csv(path: Path, first_data_row: int, columns: Mapping[int, DataColumn samples[i].append((row_number, row[i] if i < len(row) else "")) for i, dc in columns.items(): _check_values(dc, samples[i]) - - -def _check_parquet(path: Path, columns: Mapping[str, DataColumnBase]) -> None: - try: - import pyarrow as pa - import pyarrow.parquet as pq - - from sift_client._internal.util.parquet import PATH_SEPARATOR - except ImportError as e: - raise RuntimeError( - "pyarrow is required for Parquet import. " - "Install it via `pip install sift-stack-py[parquet]`." - ) from e - - schema = pq.read_schema(path) - string_columns: dict[str, DataColumnBase] = {} - for column_path, dc in columns.items(): - field_type = _field_type(schema, column_path.split(PATH_SEPARATOR)) - if field_type is None: - # The import service reports columns missing from the file. - continue - if ( - pa.types.is_binary(field_type) - or pa.types.is_large_binary(field_type) - or pa.types.is_binary_view(field_type) - or pa.types.is_fixed_size_binary(field_type) - ): - continue - if ( - pa.types.is_string(field_type) - or pa.types.is_large_string(field_type) - or pa.types.is_string_view(field_type) - ): - string_columns[column_path] = dc - continue - raise ValueError( - f"bit field column '{dc.name}' is {field_type}, but bit field values must be " - f"{ACCEPTED_FORMATS}. Store the column as binary, for example " - "pa.array([int(v).to_bytes(2, 'little') for v in values], pa.binary()) for a " - "16-bit status word." - ) - - if not string_columns: - return - # Read only the sampled rows of the string columns. - pf = pq.ParquetFile(path) - top_level = sorted({p.split(PATH_SEPARATOR)[0] for p in string_columns}) - batch = next(pf.iter_batches(batch_size=SAMPLE_ROWS, columns=top_level), None) - if batch is None: - return - table = pa.Table.from_batches([batch]) - for column_path, dc in string_columns.items(): - values = _column_values(table, column_path.split(PATH_SEPARATOR)) - _check_values(dc, ((row, "" if v is None else v) for row, v in enumerate(values, start=1))) - - -def _field_type(schema: pa.Schema, parts: list[str]) -> pa.DataType | None: - import pyarrow as pa - - index = schema.get_field_index(parts[0]) - if index == -1: - return None - field_type = schema.field(index).type - for part in parts[1:]: - if not pa.types.is_struct(field_type): - return None - child = field_type.get_field_index(part) - if child == -1: - return None - field_type = field_type.field(child).type - return field_type - - -def _column_values(table: pa.Table, parts: list[str]) -> list[str | None]: - column = table.column(parts[0]).combine_chunks() - for part in parts[1:]: - column = column.field(part) - return column.to_pylist() diff --git a/python/lib/sift_client/_internal/util/parquet.py b/python/lib/sift_client/_internal/util/parquet.py index dc71da88c3..66c021ba69 100644 --- a/python/lib/sift_client/_internal/util/parquet.py +++ b/python/lib/sift_client/_internal/util/parquet.py @@ -18,6 +18,7 @@ from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto from sift_client._internal.time import to_timestamp_nanos +from sift_client._internal.util.bit_field import ACCEPTED_FORMATS, SAMPLE_ROWS, _check_values from sift_client._internal.util.file import extract_parquet_footer from sift_client._internal.util.time_column import infer_time_column from sift_client.sift_types.channel import ChannelDataType @@ -33,6 +34,9 @@ if TYPE_CHECKING: from datetime import datetime + from typing import Mapping + + from sift_client.sift_types.data_import import DataColumnBase # Separator used in column paths to address nested struct fields. Channel # names replace it with "." since Sift forbids "|" in channel names. @@ -320,3 +324,69 @@ def detect_flat_parquet_config( footer_offset=footer_offset, footer_length=len(footer_bytes), ) + + +def check_bit_field_columns(path: Path, columns: Mapping[str, DataColumnBase]) -> None: + """Raise if a ``BIT_FIELD`` column isn't binary, or its sampled string values don't parse.""" + schema = pq.read_schema(path) + string_columns: dict[str, DataColumnBase] = {} + for column_path, dc in columns.items(): + field_type = _field_type(schema, column_path.split(PATH_SEPARATOR)) + if field_type is None: + # The import service reports columns missing from the file. + continue + if ( + pa.types.is_binary(field_type) + or pa.types.is_large_binary(field_type) + or pa.types.is_binary_view(field_type) + or pa.types.is_fixed_size_binary(field_type) + ): + continue + if ( + pa.types.is_string(field_type) + or pa.types.is_large_string(field_type) + or pa.types.is_string_view(field_type) + ): + string_columns[column_path] = dc + continue + raise ValueError( + f"bit field column '{dc.name}' is {field_type}, but bit field values must be " + f"{ACCEPTED_FORMATS}. Store the column as binary, for example " + "pa.array([int(v).to_bytes(2, 'little') for v in values], pa.binary()) for a " + "16-bit status word." + ) + + if not string_columns: + return + # Read only the sampled rows of the string columns. + pf = pq.ParquetFile(path) + top_level = sorted({p.split(PATH_SEPARATOR)[0] for p in string_columns}) + batch = next(pf.iter_batches(batch_size=SAMPLE_ROWS, columns=top_level), None) + if batch is None: + return + table = pa.Table.from_batches([batch]) + for column_path, dc in string_columns.items(): + values = _column_values(table, column_path.split(PATH_SEPARATOR)) + _check_values(dc, ((row, "" if v is None else v) for row, v in enumerate(values, start=1))) + + +def _field_type(schema: pa.Schema, parts: list[str]) -> pa.DataType | None: + index = schema.get_field_index(parts[0]) + if index == -1: + return None + field_type = schema.field(index).type + for part in parts[1:]: + if not pa.types.is_struct(field_type): + return None + child = field_type.get_field_index(part) + if child == -1: + return None + field_type = field_type.field(child).type + return field_type + + +def _column_values(table: pa.Table, parts: list[str]) -> list[str | None]: + column = table.column(parts[0]).combine_chunks() + for part in parts[1:]: + column = column.field(part) + return column.to_pylist() From cb935df180a04b1827ab651bd6015892827e8e03 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Thu, 5 Nov 2026 13:34:12 -0800 Subject: [PATCH 13/15] python(docs): document bit field units, descriptions and import checks --- python/changelog.d/861.feat.md | 8 +++++--- python/lib/sift_client/resources/data_imports.py | 9 +++++---- .../sift_client/resources/sync_stubs/__init__.pyi | 9 +++++---- python/lib/sift_client/sift_types/data_import.py | 12 ++++++++---- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index e510fc0523..952f700e99 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -1,4 +1,4 @@ -CSV, Parquet, TDMS and HDF5 data columns accept `bit_field_elements`, so a column can be imported as a `BIT_FIELD` channel. ULog and MCAP columns raise `ValueError` if set to `BIT_FIELD`, since those imports can't override a channel to a bit field. +Bit field columns are supported in CSV, Parquet, TDMS and HDF5 imports through `bit_field_elements`. ULog and MCAP imports don't support bit fields and raise `ValueError`. ```python from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType @@ -12,8 +12,10 @@ status.bit_field_elements = [ ] ``` -In CSV and Parquet files, bit field values must be raw bytes, a string of 0s and 1s like `00110101`, or URL-safe base64. Bytes are little-endian, so bit 8 is the lowest bit of the second byte. Integers aren't accepted. Convert an integer column before importing, for example with `int(v).to_bytes(2, "little")` for a 16-bit status word in Parquet, or `format(v, "08b")` for an 8-bit value in CSV. +In CSV and Parquet files, bit field values must be raw bytes, a string of 0s and 1s like `00110101`, or URL-safe base64. Bytes are little-endian, so bit 8 is the lowest bit of the second byte. Integers aren't supported. Convert an integer column before importing, for example with `int(v).to_bytes(2, "little")` for a 16-bit status word in Parquet, or `format(v, "08b")` for an 8-bit value in CSV. + +Each element reads back as an unsigned integer and shows the column's `units` and `description`. Elements can't have their own units or description. `import_from_path` checks `BIT_FIELD` columns before uploading. A Parquet column must be a binary or string type, and a sample of CSV or Parquet string values must parse. -A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, so a column edited in place fails before the request is sent. Multi-channel Parquet single-channel-per-row imports don't support bit fields, since channel configs there are derived per row. +A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, including on columns edited in place. Multi-channel Parquet single-channel-per-row imports don't support bit fields. diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index fdfcba4b6c..7dab0beaac 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -83,10 +83,11 @@ async def import_from_path( If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. - ``BIT_FIELD`` values must be raw bytes, a string of 0s and 1s, or - URL-safe base64. Before uploading, CSV and Parquet ``BIT_FIELD`` - columns are checked: Parquet columns must be a binary or string - type, and a sample of string values must parse. + For files with ``ChannelDataType.BIT_FIELD`` columns, each value + must be raw bytes, a string of 0s and 1s, or URL-safe base64. + Integers aren't supported. Before uploading, this method checks CSV + and Parquet bit field columns. A Parquet column must be a binary or + string type, and a sample of string values must parse. Examples: Import a CSV file with auto-detected config: diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 3f6c12ec1f..1574056c35 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1770,10 +1770,11 @@ class DataImportAPI: If neither ``run`` nor ``run_name`` is provided (and none is set on the config), ``run_name`` defaults to the filename. - ``BIT_FIELD`` values must be raw bytes, a string of 0s and 1s, or - URL-safe base64. Before uploading, CSV and Parquet ``BIT_FIELD`` - columns are checked: Parquet columns must be a binary or string - type, and a sample of string values must parse. + For files with ``ChannelDataType.BIT_FIELD`` columns, each value + must be raw bytes, a string of 0s and 1s, or URL-safe base64. + Integers aren't supported. Before uploading, this method checks CSV + and Parquet bit field columns. A Parquet column must be a binary or + string type, and a sample of string values must parse. Examples: Import a CSV file with auto-detected config: diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index bcf0a78cfb..b75ba51343 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -168,16 +168,20 @@ class DataColumnBase(BaseModel, ABC): Attributes: name: Channel name. data_type: The data type of the channel values. - units: Optional units string. - description: Optional channel description. + units: Optional units string. On a bit field channel, every element + shows these units. + description: Optional channel description. On a bit field channel, + every element shows this description. enum_types: Mapping of enum name to key. Only valid when ``data_type`` is ``ChannelDataType.ENUM``. bit_field_elements: Named bit ranges within each value. Only valid when ``data_type`` is ``ChannelDataType.BIT_FIELD``, and only on formats - that support bit fields. + that support bit fields. Each element reads back as an unsigned + integer. Elements can't have their own units or description. """ - # ULog and MCAP imports can't override a channel to BIT_FIELD. + # The ULog and MCAP import jobs only accept the file's native scalar types + # as overrides, so they reject BIT_FIELD after the upload. _supports_bit_fields: ClassVar[bool] = True name: str From 44676c677e9ace070207f2154a4cbea90ba27135 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Tue, 10 Nov 2026 07:19:39 -0800 Subject: [PATCH 14/15] python(docs): make the bit field changelog fragment a highlight --- python/changelog.d/861.feat.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index 952f700e99..011c8581e8 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -1,5 +1,9 @@ +# Bit field columns in import configs + Bit field columns are supported in CSV, Parquet, TDMS and HDF5 imports through `bit_field_elements`. ULog and MCAP imports don't support bit fields and raise `ValueError`. +Set `data_type` to `BIT_FIELD` on a detected column and list its elements: + ```python from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType From e53d6332f7ababdaef8f9fb2c7c9b3282f82ac00 Mon Sep 17 00:00:00 2001 From: alden-sift Date: Tue, 10 Nov 2026 10:21:54 -0800 Subject: [PATCH 15/15] Simplify changelog Removed detailed explanation about bit field values and import requirements from the changelog. See relevant docstrings instead --- python/changelog.d/861.feat.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/changelog.d/861.feat.md b/python/changelog.d/861.feat.md index 011c8581e8..7375d9a392 100644 --- a/python/changelog.d/861.feat.md +++ b/python/changelog.d/861.feat.md @@ -15,11 +15,3 @@ status.bit_field_elements = [ ChannelBitFieldElement(name="mode", index=4, bit_count=4), ] ``` - -In CSV and Parquet files, bit field values must be raw bytes, a string of 0s and 1s like `00110101`, or URL-safe base64. Bytes are little-endian, so bit 8 is the lowest bit of the second byte. Integers aren't supported. Convert an integer column before importing, for example with `int(v).to_bytes(2, "little")` for a 16-bit status word in Parquet, or `format(v, "08b")` for an 8-bit value in CSV. - -Each element reads back as an unsigned integer and shows the column's `units` and `description`. Elements can't have their own units or description. - -`import_from_path` checks `BIT_FIELD` columns before uploading. A Parquet column must be a binary or string type, and a sample of CSV or Parquet string values must parse. - -A column must set `bit_field_elements` exactly when its data type is `BIT_FIELD`. Element names must be unique, each element needs `index >= 0` and `bit_count >= 1`, and elements can't overlap. Import configs now re-check `enum_types` and `bit_field_elements` before an import, including on columns edited in place. Multi-channel Parquet single-channel-per-row imports don't support bit fields.