Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions python/changelog.d/861.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Bit field columns in import configs

Bit field columns are supported in CSV, Parquet, TDMS and HDF5 imports through `bit_field_elements`. ULog and MCAP imports don't support bit fields and raise `ValueError`.

Set `data_type` to `BIT_FIELD` on a detected column and list its elements:

```python
from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType

config = client.data_import.detect_config("data.parquet", data_type=DataTypeKey.PARQUET_FLATDATASET)
status = config["status"]
status.data_type = ChannelDataType.BIT_FIELD
status.bit_field_elements = [
ChannelBitFieldElement(name="armed", index=0, bit_count=1),
ChannelBitFieldElement(name="mode", index=4, bit_count=4),
]
```
1 change: 1 addition & 0 deletions python/changelog.d/861.fix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
MCAP import configs keep `enum_types` when sent to Sift and when read back from a data import. Before, enum definitions on MCAP data columns were dropped.
16 changes: 16 additions & 0 deletions python/examples/data_import/parquet/flat_dataset/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,22 @@
# # Example: drop a column from the import
# config.data_columns = [dc for dc in config.data_columns if dc.path != "channel_0"]
#
# # Example: import a binary column as a bit field. detect_config can't
# # infer the bit layout, so supply it here. Each element's index is its
# # starting bit and bit_count is its width. Values must be bytes (or a
# # string of 0s and 1s, or base64), not integers; convert an integer
# # column first, e.g. pa.array([int(v).to_bytes(1, "little") for v in col],
# # pa.binary()).
# from sift_client.sift_types.channel import ChannelBitFieldElement, ChannelDataType
#
# status = config["status"]
# status.data_type = ChannelDataType.BIT_FIELD
# status.bit_field_elements = [
# ChannelBitFieldElement(name="armed", index=0, bit_count=1),
# ChannelBitFieldElement(name="error_code", index=1, bit_count=3),
# ChannelBitFieldElement(name="mode", index=4, bit_count=4),
# ]
#
# import_job = client.data_import.import_from_path(
# "sample_data.parquet",
# asset=asset_name,
Expand Down
112 changes: 112 additions & 0 deletions python/lib/sift_client/_internal/util/bit_field.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
"""Check bit field values before upload, so a wrong encoding fails early.

The import service reads a bit field value as raw bytes, a string of 0s and 1s,
or URL-safe base64, and fails the import on the first value it can't parse. It
only reports that after the whole file uploads. These checks read the
Parquet schema and a sample of values, and raise before the upload starts.
"""

from __future__ import annotations

import base64
import binascii
import csv
import re
from typing import TYPE_CHECKING, Iterable, Mapping

from sift_client.sift_types.channel import ChannelDataType
from sift_client.sift_types.data_import import (
CsvImportConfig,
DataColumnBase,
ParquetFlatDatasetImportConfig,
ParquetSingleChannelPerRowImportConfig,
)

if TYPE_CHECKING:
from pathlib import Path

from sift_client.sift_types.data_import import ImportConfig

SAMPLE_ROWS = 1000

_BITS = re.compile(r"[01]+\Z")
_BASE64_URL = re.compile(r"[A-Za-z0-9_-]*={0,2}\Z")

ACCEPTED_FORMATS = "raw bytes, a string of 0s and 1s like '00110101', or URL-safe base64"


def check_bit_field_values(config: ImportConfig, path: Path) -> None:
"""Raise if a ``BIT_FIELD`` column's values aren't in a format the import service reads."""
if isinstance(config, CsvImportConfig):
csv_columns = {
dc.column - 1: dc
for dc in config.data_columns
if dc.data_type == ChannelDataType.BIT_FIELD
}
if csv_columns:
_check_csv(path, config.first_data_row, csv_columns)
return

parquet_columns: dict[str, DataColumnBase]
if isinstance(config, ParquetFlatDatasetImportConfig):
parquet_columns = {
dc.path: dc for dc in config.data_columns if dc.data_type == ChannelDataType.BIT_FIELD
}
elif isinstance(config, ParquetSingleChannelPerRowImportConfig):
sc = config.single_channel
parquet_columns = (
{sc.data_path: sc}
if sc is not None and sc.data_type == ChannelDataType.BIT_FIELD
else {}
)
else:
return
if parquet_columns:
try:
from sift_client._internal.util.parquet import check_bit_field_columns
except ImportError as e:
raise RuntimeError(
"pyarrow is required for Parquet import. "
"Install it via `pip install sift-stack-py[parquet]`."
) from e
check_bit_field_columns(path, parquet_columns)


def _is_bit_value(value: str) -> bool:
"""Whether the import service can parse ``value`` as a bit field."""
if _BITS.match(value):
return True
if len(value) % 4 or not _BASE64_URL.match(value):
return False
try:
base64.urlsafe_b64decode(value)
except (binascii.Error, ValueError):
return False
return True


def _check_values(dc: DataColumnBase, values: Iterable[tuple[int, str]]) -> None:
"""Check sampled ``(row, value)`` pairs from one column. Empty values are skipped."""
for row, value in values:
if value == "":
continue
if not _is_bit_value(value):
raise ValueError(
f"bit field column '{dc.name}' has value {value!r} in row {row}, which isn't "
f"{ACCEPTED_FORMATS}. To convert integers, use format(value, '08b') for an "
"8-bit status word."
)


def _check_csv(path: Path, first_data_row: int, columns: Mapping[int, DataColumnBase]) -> None:
samples: dict[int, list[tuple[int, str]]] = {i: [] for i in columns}
with open(path, newline="", encoding="utf-8") as f:
for row_number, row in enumerate(csv.reader(f), start=1):
if row_number < first_data_row:
continue
if row_number >= first_data_row + SAMPLE_ROWS:
break
for i in columns:
samples[i].append((row_number, row[i] if i < len(row) else ""))
for i, dc in columns.items():
_check_values(dc, samples[i])
72 changes: 72 additions & 0 deletions python/lib/sift_client/_internal/util/parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto

from sift_client._internal.time import to_timestamp_nanos
from sift_client._internal.util.bit_field import ACCEPTED_FORMATS, SAMPLE_ROWS, _check_values
from sift_client._internal.util.file import extract_parquet_footer
from sift_client._internal.util.time_column import infer_time_column
from sift_client.sift_types.channel import ChannelDataType
Expand All @@ -27,11 +28,15 @@
ParquetTimeColumn,
TimeFormat,
TimeFormatProto,
_bit_field_elements_from_proto,
_enum_types_from_proto,
)

if TYPE_CHECKING:
from datetime import datetime
from typing import Mapping

from sift_client.sift_types.data_import import DataColumnBase

# Separator used in column paths to address nested struct fields. Channel
# names replace it with "." since Sift forbids "|" in channel names.
Expand Down Expand Up @@ -222,6 +227,7 @@ def _parse_embedded_data_column(path: str, raw: bytes) -> ParquetDataColumn:
units=config.units,
description=config.description,
enum_types=_enum_types_from_proto(config),
bit_field_elements=_bit_field_elements_from_proto(config),
)


Expand Down Expand Up @@ -318,3 +324,69 @@ def detect_flat_parquet_config(
footer_offset=footer_offset,
footer_length=len(footer_bytes),
)


def check_bit_field_columns(path: Path, columns: Mapping[str, DataColumnBase]) -> None:
"""Raise if a ``BIT_FIELD`` column isn't binary, or its sampled string values don't parse."""
schema = pq.read_schema(path)
string_columns: dict[str, DataColumnBase] = {}
for column_path, dc in columns.items():
field_type = _field_type(schema, column_path.split(PATH_SEPARATOR))
if field_type is None:
# The import service reports columns missing from the file.
continue
if (
pa.types.is_binary(field_type)
or pa.types.is_large_binary(field_type)
or pa.types.is_binary_view(field_type)
or pa.types.is_fixed_size_binary(field_type)
):
continue
if (
pa.types.is_string(field_type)
or pa.types.is_large_string(field_type)
or pa.types.is_string_view(field_type)
):
string_columns[column_path] = dc
continue
raise ValueError(
f"bit field column '{dc.name}' is {field_type}, but bit field values must be "
f"{ACCEPTED_FORMATS}. Store the column as binary, for example "
"pa.array([int(v).to_bytes(2, 'little') for v in values], pa.binary()) for a "
"16-bit status word."
)

if not string_columns:
return
# Read only the sampled rows of the string columns.
pf = pq.ParquetFile(path)
top_level = sorted({p.split(PATH_SEPARATOR)[0] for p in string_columns})
batch = next(pf.iter_batches(batch_size=SAMPLE_ROWS, columns=top_level), None)
if batch is None:
return
table = pa.Table.from_batches([batch])
for column_path, dc in string_columns.items():
values = _column_values(table, column_path.split(PATH_SEPARATOR))
_check_values(dc, ((row, "" if v is None else v) for row, v in enumerate(values, start=1)))


def _field_type(schema: pa.Schema, parts: list[str]) -> pa.DataType | None:
index = schema.get_field_index(parts[0])
if index == -1:
return None
field_type = schema.field(index).type
for part in parts[1:]:
if not pa.types.is_struct(field_type):
return None
child = field_type.get_field_index(part)
if child == -1:
return None
field_type = field_type.field(child).type
return field_type


def _column_values(table: pa.Table, parts: list[str]) -> list[str | None]:
column = table.column(parts[0]).combine_chunks()
for part in parts[1:]:
column = column.field(part)
return column.to_pylist()
Loading
Loading