Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions python/changelog.d/848.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false) and a list of paths. When you have more than one file for a run, pass them in one call. That upload is one import into one run. Pass a mapping of path to config to set a file's config yourself. Files you leave out of the mapping are detected. A finished import stops its run and fires run-stop automations unless you set `leave_run_open`, then call `run.stop()` after the last import.
147 changes: 128 additions & 19 deletions python/lib/sift_client/_internal/low_level_wrappers/data_imports.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
from __future__ import annotations

from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any, NamedTuple, cast

from sift.data_imports.v2.data_imports_pb2 import (
DATA_TYPE_KEY_CSV,
DATA_TYPE_KEY_HDF5,
DATA_TYPE_KEY_MCAP,
DATA_TYPE_KEY_PARQUET_FLATDATASET,
DATA_TYPE_KEY_PARQUET_SINGLE_CHANNEL_PER_ROW,
DATA_TYPE_KEY_TDMS,
DATA_TYPE_KEY_ULOG,
BatchConfig,
BatchFileConfig,
CreateDataImportFromUploadRequest,
CreateDataImportFromUploadResponse,
DetectConfigRequest,
Expand All @@ -29,32 +38,82 @@
from sift_client.transport import WithGrpcClient

if TYPE_CHECKING:
from collections.abc import Mapping

from sift.data_imports.v2.data_imports_pb2 import DataTypeKey

from sift_client.transport.grpc_transport import GrpcClient


class _ConfigUploadField(NamedTuple):
"""An import config's upload oneof and batch type key.

Parquet layouts share one oneof and use different type keys.
"""

config_type: type[ImportConfig]
field_name: str
type_key: DataTypeKey.ValueType


_CONFIG_UPLOAD_FIELDS: tuple[_ConfigUploadField, ...] = (
_ConfigUploadField(CsvImportConfig, "csv_config", DATA_TYPE_KEY_CSV),
_ConfigUploadField(
ParquetFlatDatasetImportConfig, "parquet_config", DATA_TYPE_KEY_PARQUET_FLATDATASET
),
_ConfigUploadField(
ParquetSingleChannelPerRowImportConfig,
"parquet_config",
DATA_TYPE_KEY_PARQUET_SINGLE_CHANNEL_PER_ROW,
),
_ConfigUploadField(TdmsImportConfig, "tdms_config", DATA_TYPE_KEY_TDMS),
_ConfigUploadField(Hdf5ImportConfig, "hdf5_config", DATA_TYPE_KEY_HDF5),
_ConfigUploadField(UlogImportConfig, "ulog_config", DATA_TYPE_KEY_ULOG),
_ConfigUploadField(McapImportConfig, "mcap_config", DATA_TYPE_KEY_MCAP),
)


def _config_upload_field(config: ImportConfig) -> _ConfigUploadField:
"""Return the upload field for a config."""
for field in _CONFIG_UPLOAD_FIELDS:
if isinstance(config, field.config_type):
return field
raise TypeError(f"Unsupported import config type: {type(config).__name__}")


def _set_config_on_request(
request: CreateDataImportFromUploadRequest,
config: ImportConfig,
) -> None:
"""Set the appropriate config field on a proto request based on the config type."""
if isinstance(config, CsvImportConfig):
request.csv_config.CopyFrom(config._to_proto())
elif isinstance(
config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig)
):
request.parquet_config.CopyFrom(config._to_proto())
elif isinstance(config, TdmsImportConfig):
request.tdms_config.CopyFrom(config._to_proto())
elif isinstance(config, Hdf5ImportConfig):
request.hdf5_config.CopyFrom(config._to_proto())
elif isinstance(config, UlogImportConfig):
request.ulog_config.CopyFrom(config._to_proto())
elif isinstance(config, McapImportConfig):
request.mcap_config.CopyFrom(config._to_proto())
else:
raise TypeError(f"Unsupported import config type: {type(config).__name__}")
"""Set the config field on an upload request for this config type."""
field = _config_upload_field(config)
getattr(request, field.field_name).CopyFrom(config._to_proto())


def _batch_file_config(filename: str, config: ImportConfig) -> BatchFileConfig:
"""Build one archive member's config."""
if isinstance(config, UlogImportConfig) and (config.info_keys or config.param_keys):
raise ValueError(
"info_keys and param_keys are not supported for ULog files in a "
f"batch import ({filename}). Import the file on its own to attach "
"its metadata to a run."
)
if isinstance(config, McapImportConfig) and config.metadata_records:
raise ValueError(
"metadata_records are not supported for MCAP files in a batch "
f"import ({filename}). Import the file on its own to attach its "
"metadata to a run."
)
field = _config_upload_field(config)
file_proto = config._to_proto()
# The batch message owns the asset and run. A per-file value that
# disagrees with it is rejected.
file_proto.asset_name = ""
file_proto.run_name = ""
file_proto.run_id = ""
batch_file = BatchFileConfig(type=field.type_key)
getattr(batch_file, field.field_name).CopyFrom(file_proto)
return batch_file


class DataImportsLowLevelClient(LowLevelClientBase, WithGrpcClient):
Expand All @@ -66,17 +125,67 @@ class DataImportsLowLevelClient(LowLevelClientBase, WithGrpcClient):
def __init__(self, grpc_client: GrpcClient):
WithGrpcClient.__init__(self, grpc_client=grpc_client)

async def create_from_upload(self, config: ImportConfig) -> tuple[str, str]:
async def create_from_upload(
self, config: ImportConfig, *, leave_run_open: bool = False
) -> tuple[str, str]:
"""Create a data import and get back a presigned upload URL.

Args:
config: The import configuration.
leave_run_open: If true, the import does not mark its run stopped.
Run-stop automations do not fire. Set this when more data is
coming to the run, then stop the run after the last import.
Ignored when the import has no run.

Returns:
A tuple of (data_import_id, upload_url).
"""
request = CreateDataImportFromUploadRequest()
_set_config_on_request(request, config)
request.leave_run_open = leave_run_open
return await self._create_upload(request)

async def create_batch_from_upload(
self,
file_configs: Mapping[str, ImportConfig],
*,
asset_name: str,
run_name: str | None = None,
run_id: str | None = None,
leave_run_open: bool = False,
) -> tuple[str, str]:
"""Create one import for an archive of files and return its upload URL.

``file_configs`` maps each archive member name to that file's import
config. The asset and run are set on the batch, not on the files.

Args:
file_configs: Archive member name to the config for that file.
asset_name: Asset every file in the archive imports into.
run_name: Name for a new run shared by every file. Ignored when
``run_id`` is set.
run_id: ID of an existing run every file appends to.
leave_run_open: If true, the import does not mark its run stopped.
Run-stop automations do not fire. Ignored when the import has
no run.

Returns:
A tuple of (data_import_id, upload_url).
"""
if not file_configs:
raise ValueError("A batch import needs at least one file.")
batch = BatchConfig(
asset_name=asset_name,
run_name=run_name or "",
run_id=run_id or "",
)
for filename, config in file_configs.items():
batch.file_configs[filename].CopyFrom(_batch_file_config(filename, config))
request = CreateDataImportFromUploadRequest(batch_config=batch)
request.leave_run_open = leave_run_open
return await self._create_upload(request)

async def _create_upload(self, request: CreateDataImportFromUploadRequest) -> tuple[str, str]:
response = await self._grpc_client.get_stub(
DataImportServiceStub
).CreateDataImportFromUpload(request)
Expand Down
56 changes: 54 additions & 2 deletions python/lib/sift_client/_internal/util/data_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

from __future__ import annotations

from collections.abc import Mapping
from pathlib import Path
from typing import TYPE_CHECKING

from sift_client._internal.time import to_timestamp_pb
Expand All @@ -12,6 +14,7 @@
from sift_client.sift_types.data_import import (
EXTENSION_TO_DATA_TYPE_KEY,
CsvImportConfig,
ImportConfigBase,
ParquetFlatDatasetImportConfig,
ParquetSingleChannelPerRowImportConfig,
ParquetTimeColumn,
Expand All @@ -20,9 +23,8 @@

if TYPE_CHECKING:
from datetime import datetime
from pathlib import Path

from sift_client.sift_types.data_import import DataTypeKey
from sift_client.sift_types.data_import import DataTypeKey, ImportConfig


_RELATIVE_FORMAT_VALUES = {f.value for f in TimeFormat if f.is_relative}
Expand Down Expand Up @@ -129,3 +131,53 @@ async def prepare_parquet_config(
footer_bytes, footer_offset = await run_sync_function(lambda: extract_parquet_footer(path))
config.footer_offset = footer_offset
config.footer_length = len(footer_bytes)


def _config_key_matches(path: Path, key: str | Path) -> bool:
"""Whether a mapping key names this path.

A key matches the path itself, or the file name when the key has no
directory. Names in one import are distinct, so a file name matches
one path.
"""
key_path = key if isinstance(key, Path) else Path(key)
if path == key_path:
return True
return key_path.parent == Path(".") and key_path.name == path.name


def _supplied_configs(
paths: list[Path],
config: ImportConfig | Mapping[str | Path, ImportConfig] | None,
) -> dict[Path, ImportConfig | None]:
"""Return each path's config. None means detect that file.

One import config applies to a single path. A mapping key is a path
from the list, or that file's name. A path with no entry is detected.
"""
if config is None:
return dict.fromkeys(paths)
if isinstance(config, ImportConfigBase):
if len(paths) != 1:
raise ValueError(
"'config' applies to one file. Pass a mapping of path to "
"config for a list, or omit it to detect each file."
)
return {paths[0]: config}
if not isinstance(config, Mapping):
raise ValueError("'config' must be an import config or a mapping of path to config.")

assigned: dict[Path, ImportConfig] = {}
for key, value in config.items():
if not isinstance(value, ImportConfigBase):
raise ValueError(f"Config for {key!s} must be an import config.")
matches = [path for path in paths if _config_key_matches(path, key)]
if not matches:
raise ValueError(f"Config key {key!s} does not match a file in this import.")
if len(matches) > 1:
raise ValueError(f"Config key {key!s} matches more than one file.")
path = matches[0]
if path in assigned:
raise ValueError(f"More than one config matches {path}.")
assigned[path] = value
return {path: assigned.get(path) for path in paths}
7 changes: 7 additions & 0 deletions python/lib/sift_client/_internal/util/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,3 +226,10 @@ def extract_parquet_footer(path: Path) -> tuple[bytes, int]:
f.seek(-(footer_len + 8), 2)
footer_bytes = f.read(footer_len)
return footer_bytes, os.path.getsize(path) - len(footer_bytes) - 8


def _write_import_zip(archive: Path, paths: list[Path]) -> None:
"""Pack each file into the archive under its own file name."""
with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zip_file:
for path in paths:
zip_file.write(path, arcname=path.name)
Loading
Loading