diff --git a/python/changelog.d/848.feat.md b/python/changelog.d/848.feat.md new file mode 100644 index 000000000..6977a3b7e --- /dev/null +++ b/python/changelog.d/848.feat.md @@ -0,0 +1 @@ +`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false) and a list of paths. When you have more than one file for a run, pass them in one call. That upload is one import into one run. Pass a mapping of path to config to set a file's config yourself. Files you leave out of the mapping are detected. A finished import stops its run and fires run-stop automations unless you set `leave_run_open`, then call `run.stop()` after the last import. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py index a5a7c5f63..31cffb22f 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py @@ -1,8 +1,17 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NamedTuple, cast from sift.data_imports.v2.data_imports_pb2 import ( + DATA_TYPE_KEY_CSV, + DATA_TYPE_KEY_HDF5, + DATA_TYPE_KEY_MCAP, + DATA_TYPE_KEY_PARQUET_FLATDATASET, + DATA_TYPE_KEY_PARQUET_SINGLE_CHANNEL_PER_ROW, + DATA_TYPE_KEY_TDMS, + DATA_TYPE_KEY_ULOG, + BatchConfig, + BatchFileConfig, CreateDataImportFromUploadRequest, CreateDataImportFromUploadResponse, DetectConfigRequest, @@ -29,32 +38,82 @@ from sift_client.transport import WithGrpcClient if TYPE_CHECKING: + from collections.abc import Mapping + from sift.data_imports.v2.data_imports_pb2 import DataTypeKey from sift_client.transport.grpc_transport import GrpcClient +class _ConfigUploadField(NamedTuple): + """An import config's upload oneof and batch type key. + + Parquet layouts share one oneof and use different type keys. + """ + + config_type: type[ImportConfig] + field_name: str + type_key: DataTypeKey.ValueType + + +_CONFIG_UPLOAD_FIELDS: tuple[_ConfigUploadField, ...] = ( + _ConfigUploadField(CsvImportConfig, "csv_config", DATA_TYPE_KEY_CSV), + _ConfigUploadField( + ParquetFlatDatasetImportConfig, "parquet_config", DATA_TYPE_KEY_PARQUET_FLATDATASET + ), + _ConfigUploadField( + ParquetSingleChannelPerRowImportConfig, + "parquet_config", + DATA_TYPE_KEY_PARQUET_SINGLE_CHANNEL_PER_ROW, + ), + _ConfigUploadField(TdmsImportConfig, "tdms_config", DATA_TYPE_KEY_TDMS), + _ConfigUploadField(Hdf5ImportConfig, "hdf5_config", DATA_TYPE_KEY_HDF5), + _ConfigUploadField(UlogImportConfig, "ulog_config", DATA_TYPE_KEY_ULOG), + _ConfigUploadField(McapImportConfig, "mcap_config", DATA_TYPE_KEY_MCAP), +) + + +def _config_upload_field(config: ImportConfig) -> _ConfigUploadField: + """Return the upload field for a config.""" + for field in _CONFIG_UPLOAD_FIELDS: + if isinstance(config, field.config_type): + return field + raise TypeError(f"Unsupported import config type: {type(config).__name__}") + + def _set_config_on_request( request: CreateDataImportFromUploadRequest, config: ImportConfig, ) -> None: - """Set the appropriate config field on a proto request based on the config type.""" - if isinstance(config, CsvImportConfig): - request.csv_config.CopyFrom(config._to_proto()) - elif isinstance( - config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) - ): - request.parquet_config.CopyFrom(config._to_proto()) - elif isinstance(config, TdmsImportConfig): - request.tdms_config.CopyFrom(config._to_proto()) - elif isinstance(config, Hdf5ImportConfig): - request.hdf5_config.CopyFrom(config._to_proto()) - elif isinstance(config, UlogImportConfig): - request.ulog_config.CopyFrom(config._to_proto()) - elif isinstance(config, McapImportConfig): - request.mcap_config.CopyFrom(config._to_proto()) - else: - raise TypeError(f"Unsupported import config type: {type(config).__name__}") + """Set the config field on an upload request for this config type.""" + field = _config_upload_field(config) + getattr(request, field.field_name).CopyFrom(config._to_proto()) + + +def _batch_file_config(filename: str, config: ImportConfig) -> BatchFileConfig: + """Build one archive member's config.""" + if isinstance(config, UlogImportConfig) and (config.info_keys or config.param_keys): + raise ValueError( + "info_keys and param_keys are not supported for ULog files in a " + f"batch import ({filename}). Import the file on its own to attach " + "its metadata to a run." + ) + if isinstance(config, McapImportConfig) and config.metadata_records: + raise ValueError( + "metadata_records are not supported for MCAP files in a batch " + f"import ({filename}). Import the file on its own to attach its " + "metadata to a run." + ) + field = _config_upload_field(config) + file_proto = config._to_proto() + # The batch message owns the asset and run. A per-file value that + # disagrees with it is rejected. + file_proto.asset_name = "" + file_proto.run_name = "" + file_proto.run_id = "" + batch_file = BatchFileConfig(type=field.type_key) + getattr(batch_file, field.field_name).CopyFrom(file_proto) + return batch_file class DataImportsLowLevelClient(LowLevelClientBase, WithGrpcClient): @@ -66,17 +125,67 @@ class DataImportsLowLevelClient(LowLevelClientBase, WithGrpcClient): def __init__(self, grpc_client: GrpcClient): WithGrpcClient.__init__(self, grpc_client=grpc_client) - async def create_from_upload(self, config: ImportConfig) -> tuple[str, str]: + async def create_from_upload( + self, config: ImportConfig, *, leave_run_open: bool = False + ) -> tuple[str, str]: """Create a data import and get back a presigned upload URL. Args: config: The import configuration. + leave_run_open: If true, the import does not mark its run stopped. + Run-stop automations do not fire. Set this when more data is + coming to the run, then stop the run after the last import. + Ignored when the import has no run. Returns: A tuple of (data_import_id, upload_url). """ request = CreateDataImportFromUploadRequest() _set_config_on_request(request, config) + request.leave_run_open = leave_run_open + return await self._create_upload(request) + + async def create_batch_from_upload( + self, + file_configs: Mapping[str, ImportConfig], + *, + asset_name: str, + run_name: str | None = None, + run_id: str | None = None, + leave_run_open: bool = False, + ) -> tuple[str, str]: + """Create one import for an archive of files and return its upload URL. + + ``file_configs`` maps each archive member name to that file's import + config. The asset and run are set on the batch, not on the files. + + Args: + file_configs: Archive member name to the config for that file. + asset_name: Asset every file in the archive imports into. + run_name: Name for a new run shared by every file. Ignored when + ``run_id`` is set. + run_id: ID of an existing run every file appends to. + leave_run_open: If true, the import does not mark its run stopped. + Run-stop automations do not fire. Ignored when the import has + no run. + + Returns: + A tuple of (data_import_id, upload_url). + """ + if not file_configs: + raise ValueError("A batch import needs at least one file.") + batch = BatchConfig( + asset_name=asset_name, + run_name=run_name or "", + run_id=run_id or "", + ) + for filename, config in file_configs.items(): + batch.file_configs[filename].CopyFrom(_batch_file_config(filename, config)) + request = CreateDataImportFromUploadRequest(batch_config=batch) + request.leave_run_open = leave_run_open + return await self._create_upload(request) + + async def _create_upload(self, request: CreateDataImportFromUploadRequest) -> tuple[str, str]: response = await self._grpc_client.get_stub( DataImportServiceStub ).CreateDataImportFromUpload(request) diff --git a/python/lib/sift_client/_internal/util/data_imports.py b/python/lib/sift_client/_internal/util/data_imports.py index e5a5af447..8e89cb60e 100644 --- a/python/lib/sift_client/_internal/util/data_imports.py +++ b/python/lib/sift_client/_internal/util/data_imports.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Mapping +from pathlib import Path from typing import TYPE_CHECKING from sift_client._internal.time import to_timestamp_pb @@ -12,6 +14,7 @@ from sift_client.sift_types.data_import import ( EXTENSION_TO_DATA_TYPE_KEY, CsvImportConfig, + ImportConfigBase, ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig, ParquetTimeColumn, @@ -20,9 +23,8 @@ if TYPE_CHECKING: from datetime import datetime - from pathlib import Path - from sift_client.sift_types.data_import import DataTypeKey + from sift_client.sift_types.data_import import DataTypeKey, ImportConfig _RELATIVE_FORMAT_VALUES = {f.value for f in TimeFormat if f.is_relative} @@ -129,3 +131,53 @@ async def prepare_parquet_config( footer_bytes, footer_offset = await run_sync_function(lambda: extract_parquet_footer(path)) config.footer_offset = footer_offset config.footer_length = len(footer_bytes) + + +def _config_key_matches(path: Path, key: str | Path) -> bool: + """Whether a mapping key names this path. + + A key matches the path itself, or the file name when the key has no + directory. Names in one import are distinct, so a file name matches + one path. + """ + key_path = key if isinstance(key, Path) else Path(key) + if path == key_path: + return True + return key_path.parent == Path(".") and key_path.name == path.name + + +def _supplied_configs( + paths: list[Path], + config: ImportConfig | Mapping[str | Path, ImportConfig] | None, +) -> dict[Path, ImportConfig | None]: + """Return each path's config. None means detect that file. + + One import config applies to a single path. A mapping key is a path + from the list, or that file's name. A path with no entry is detected. + """ + if config is None: + return dict.fromkeys(paths) + if isinstance(config, ImportConfigBase): + if len(paths) != 1: + raise ValueError( + "'config' applies to one file. Pass a mapping of path to " + "config for a list, or omit it to detect each file." + ) + return {paths[0]: config} + if not isinstance(config, Mapping): + raise ValueError("'config' must be an import config or a mapping of path to config.") + + assigned: dict[Path, ImportConfig] = {} + for key, value in config.items(): + if not isinstance(value, ImportConfigBase): + raise ValueError(f"Config for {key!s} must be an import config.") + matches = [path for path in paths if _config_key_matches(path, key)] + if not matches: + raise ValueError(f"Config key {key!s} does not match a file in this import.") + if len(matches) > 1: + raise ValueError(f"Config key {key!s} matches more than one file.") + path = matches[0] + if path in assigned: + raise ValueError(f"More than one config matches {path}.") + assigned[path] = value + return {path: assigned.get(path) for path in paths} diff --git a/python/lib/sift_client/_internal/util/file.py b/python/lib/sift_client/_internal/util/file.py index b1606d1b5..eb8a0e298 100644 --- a/python/lib/sift_client/_internal/util/file.py +++ b/python/lib/sift_client/_internal/util/file.py @@ -226,3 +226,10 @@ def extract_parquet_footer(path: Path) -> tuple[bytes, int]: f.seek(-(footer_len + 8), 2) footer_bytes = f.read(footer_len) return footer_bytes, os.path.getsize(path) - len(footer_bytes) - 8 + + +def _write_import_zip(archive: Path, paths: list[Path]) -> None: + """Pack each file into the archive under its own file name.""" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zip_file: + for path in paths: + zip_file.write(path, arcname=path.name) diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py index ba2f0939c..bfb80d3ee 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py @@ -2,12 +2,30 @@ from __future__ import annotations +from unittest.mock import MagicMock + import pytest -from sift.data_imports.v2.data_imports_pb2 import CreateDataImportFromUploadRequest +from sift.data_imports.v2.data_imports_pb2 import ( + DATA_TYPE_KEY_CSV, + DATA_TYPE_KEY_MCAP, + CreateDataImportFromUploadRequest, + CreateDataImportFromUploadResponse, +) -from sift_client._internal.low_level_wrappers.data_imports import _set_config_on_request +from sift_client._internal.low_level_wrappers.data_imports import ( + DataImportsLowLevelClient, + _set_config_on_request, +) from sift_client.sift_types.channel import ChannelDataType -from sift_client.sift_types.data_import import McapDataColumn, McapImportConfig +from sift_client.sift_types.data_import import ( + CsvDataColumn, + CsvImportConfig, + CsvTimeColumn, + McapDataColumn, + McapImportConfig, + TimeFormat, + UlogImportConfig, +) class TestSetConfigOnRequest: @@ -32,3 +50,83 @@ def test_unknown_config_type_raises(self): request = CreateDataImportFromUploadRequest() with pytest.raises(TypeError, match="Unsupported import config type"): _set_config_on_request(request, object()) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_create_from_upload_sets_leave_run_open(): + """leave_run_open is copied onto the upload request.""" + captured: dict[str, CreateDataImportFromUploadRequest] = {} + + class _Stub: + async def CreateDataImportFromUpload(self, request): # noqa: N802 + captured["request"] = request + return CreateDataImportFromUploadResponse(data_import_id="id", upload_url="url") + + client = DataImportsLowLevelClient(grpc_client=MagicMock()) + client._grpc_client.get_stub = MagicMock(return_value=_Stub()) + config = McapImportConfig(asset_name="a", data=[]) + + await client.create_from_upload(config, leave_run_open=True) + + assert captured["request"].leave_run_open is True + + +def _csv_config() -> CsvImportConfig: + return CsvImportConfig( + asset_name="from-file", + run_name="from-file", + run_id="from-file", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[CsvDataColumn(column=2, name="v", data_type=ChannelDataType.DOUBLE)], + ) + + +@pytest.mark.asyncio +async def test_create_batch_from_upload_sets_batch_config(): + """A batch upload carries one config per archive member and the shared run.""" + captured: dict[str, CreateDataImportFromUploadRequest] = {} + + class _Stub: + async def CreateDataImportFromUpload(self, request): # noqa: N802 + captured["request"] = request + return CreateDataImportFromUploadResponse(data_import_id="id", upload_url="url") + + client = DataImportsLowLevelClient(grpc_client=MagicMock()) + client._grpc_client.get_stub = MagicMock(return_value=_Stub()) + + await client.create_batch_from_upload( + { + "a.csv": _csv_config(), + "log.mcap": McapImportConfig(asset_name="from-file", data=[]), + }, + asset_name="shared-asset", + run_id="run-1", + leave_run_open=True, + ) + + request = captured["request"] + assert request.leave_run_open is True + assert not request.HasField("csv_config") + batch = request.batch_config + assert batch.asset_name == "shared-asset" + assert batch.run_id == "run-1" + assert batch.file_configs["a.csv"].type == DATA_TYPE_KEY_CSV + assert batch.file_configs["a.csv"].csv_config.asset_name == "" + assert batch.file_configs["a.csv"].csv_config.run_id == "" + assert batch.file_configs["a.csv"].csv_config.data_columns[2].name == "v" + assert batch.file_configs["log.mcap"].type == DATA_TYPE_KEY_MCAP + assert batch.file_configs["log.mcap"].mcap_config.asset_name == "" + + +@pytest.mark.asyncio +async def test_create_batch_from_upload_rejects_empty_and_run_metadata(): + client = DataImportsLowLevelClient(grpc_client=MagicMock()) + + with pytest.raises(ValueError, match="at least one file"): + await client.create_batch_from_upload({}, asset_name="a") + + with pytest.raises(ValueError, match="info_keys"): + await client.create_batch_from_upload( + {"log.ulg": UlogImportConfig(asset_name="a", info_keys=["git_hash"])}, + asset_name="a", + ) diff --git a/python/lib/sift_client/_tests/_internal/util/test_data_imports.py b/python/lib/sift_client/_tests/_internal/util/test_data_imports.py new file mode 100644 index 000000000..e19d65f04 --- /dev/null +++ b/python/lib/sift_client/_tests/_internal/util/test_data_imports.py @@ -0,0 +1,68 @@ +"""Tests for data import helpers that do not need a client.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sift_client._internal.util.data_imports import _config_key_matches, _supplied_configs +from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.data_import import ( + CsvDataColumn, + CsvImportConfig, + CsvTimeColumn, + TimeFormat, +) + + +def _csv(name: str = "v") -> CsvImportConfig: + return CsvImportConfig( + asset_name="asset", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[CsvDataColumn(column=2, name=name, data_type=ChannelDataType.DOUBLE)], + ) + + +class TestConfigKeyMatches: + def test_matches_the_path_or_a_bare_file_name(self): + path = Path("dir/a.csv") + + assert _config_key_matches(path, path) + assert _config_key_matches(path, "a.csv") + assert not _config_key_matches(path, "other/a.csv") + assert not _config_key_matches(path, "b.csv") + + +class TestSuppliedConfigs: + def test_none_detects_every_path(self): + paths = [Path("a.csv"), Path("b.csv")] + + assert _supplied_configs(paths, None) == {paths[0]: None, paths[1]: None} + + def test_one_config_applies_to_one_path(self): + path = Path("a.csv") + config = _csv() + + assert _supplied_configs([path], config) == {path: config} + + def test_one_config_rejects_several_paths(self): + with pytest.raises(ValueError, match="one file"): + _supplied_configs([Path("a.csv"), Path("b.csv")], _csv()) + + def test_mapping_assigns_by_name_and_leaves_the_rest_to_detect(self): + first = Path("dir/a.csv") + second = Path("dir/b.csv") + config = _csv("supplied") + + assigned = _supplied_configs([first, second], {"a.csv": config}) + + assert assigned[first] is config + assert assigned[second] is None + + def test_unknown_key_and_two_keys_for_one_path_raise(self): + path = Path("a.csv") + with pytest.raises(ValueError, match="does not match"): + _supplied_configs([path], {"missing.csv": _csv()}) + with pytest.raises(ValueError, match="More than one config"): + _supplied_configs([path], {path: _csv(), "a.csv": _csv()}) diff --git a/python/lib/sift_client/_tests/_internal/util/test_file.py b/python/lib/sift_client/_tests/_internal/util/test_file.py index 5c4f1401f..139cd8a09 100644 --- a/python/lib/sift_client/_tests/_internal/util/test_file.py +++ b/python/lib/sift_client/_tests/_internal/util/test_file.py @@ -1,11 +1,17 @@ from __future__ import annotations +import zipfile from unittest.mock import MagicMock import pytest from requests import HTTPError, Request, Response -from sift_client._internal.util.file import download_file, read_url_bytes, upload_file +from sift_client._internal.util.file import ( + _write_import_zip, + download_file, + read_url_bytes, + upload_file, +) @pytest.fixture @@ -146,3 +152,19 @@ def test_returns_body_without_session_auth(self): assert data == b"hello world" assert captured["url"] == "https://example.test/signed" assert captured["headers"] == {"Authorization": None} + + +class TestWriteImportZip: + def test_packs_each_file_under_its_name(self, tmp_path): + first = tmp_path / "a.csv" + second = tmp_path / "nested" / "b.csv" + second.parent.mkdir() + first.write_text("a") + second.write_text("b") + archive = tmp_path / "batch.zip" + + _write_import_zip(archive, [first, second]) + + with zipfile.ZipFile(archive) as zip_file: + assert zip_file.read("a.csv") == b"a" + assert zip_file.read("b.csv") == b"b" diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 7fcc973e0..75aa9e465 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -647,7 +647,14 @@ async def _import(self, tmp_path, filename, detected): job = MagicMock() api.client.async_.jobs.get = AsyncMock(return_value=job) - assert await api.import_from_path(path, asset="my_asset", show_progress=False) is job + assert ( + await api.import_from_path( + path, asset="my_asset", show_progress=False, leave_run_open=True + ) + is job + ) + api._low_level_client.create_from_upload.assert_awaited() + assert api._low_level_client.create_from_upload.await_args.kwargs["leave_run_open"] is True @pytest.mark.asyncio async def test_mcap_data_cleared(self, tmp_path, monkeypatch): @@ -2045,3 +2052,209 @@ def test_hdf5(self, sift_client, tmp_path, dtype, values, elements, decoded): data_import = _import_and_wait(sift_client, path, config) _assert_decoded(sift_client, data_import, elements, decoded) + + +class TestImportFromPathBatch: + """A list of paths is one zip upload. A single path stays a normal import.""" + + def _csv(self) -> CsvImportConfig: + return CsvImportConfig( + asset_name="", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[CsvDataColumn(column=2, name="v", data_type=ChannelDataType.DOUBLE)], + ) + + def _api(self): + detect = AsyncMock(return_value=self._csv()) + create_from_upload = AsyncMock(return_value=("import_1", "url")) + create_batch_from_upload = AsyncMock(return_value=("import_1", "url")) + low_level = MagicMock() + low_level.create_from_upload = create_from_upload + low_level.create_batch_from_upload = create_batch_from_upload + api = DataImportAPIAsync(MagicMock()) + api.detect_config = detect + api._low_level_client = low_level + api.client.async_.jobs.get = AsyncMock(return_value=MagicMock()) + return api, detect, create_from_upload, create_batch_from_upload + + @pytest.mark.asyncio + async def test_list_uploads_one_zip(self, tmp_path, monkeypatch): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + captured: dict[str, list[str]] = {} + + def upload(_url, file_path, **_kwargs): + import zipfile + + with zipfile.ZipFile(file_path) as archive: + captured["names"] = archive.namelist() + return {"jobId": "j1"} + + monkeypatch.setattr("sift_client.resources.data_imports.upload_file", upload) + api, _, create_from_upload, create_batch = self._api() + + await api.import_from_path( + [first, second], asset="my_asset", show_progress=False, leave_run_open=True + ) + + assert captured["names"] == ["a.csv", "b.csv"] + create_from_upload.assert_not_awaited() + call = create_batch.await_args + assert call is not None + args, kwargs = call + assert set(args[0]) == {"a.csv", "b.csv"} + assert kwargs["asset_name"] == "my_asset" + assert kwargs["run_name"] == "a.csv" + assert kwargs["run_id"] is None + assert kwargs["leave_run_open"] is True + + @pytest.mark.asyncio + async def test_one_path_stays_a_single_import(self, tmp_path, monkeypatch): + path = tmp_path / "a.csv" + path.write_text("t,v\n") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api, _, create_from_upload, create_batch = self._api() + + await api.import_from_path([path], asset="my_asset", show_progress=False) + + create_from_upload.assert_awaited() + create_batch.assert_not_awaited() + + @pytest.mark.asyncio + async def test_duplicate_names_and_config_are_rejected(self, tmp_path): + first = tmp_path / "nested" / "a.csv" + first.parent.mkdir() + first.write_text("t,v\n") + second = tmp_path / "a.csv" + second.write_text("t,v\n") + api, _, _, _ = self._api() + + with pytest.raises(ValueError, match="distinct names"): + await api.import_from_path([first, second], asset="my_asset", show_progress=False) + with pytest.raises(ValueError, match="config"): + await api.import_from_path( + [first, tmp_path / "b.csv"], + asset="my_asset", + config=self._csv(), + show_progress=False, + ) + with pytest.raises(ValueError, match="At least one"): + await api.import_from_path([], asset="my_asset", show_progress=False) + + @pytest.mark.asyncio + async def test_data_type_skips_a_known_extension(self, tmp_path, monkeypatch): + csv_path = tmp_path / "a.csv" + parquet_path = tmp_path / "b.parquet" + csv_path.write_text("t,v\n") + parquet_path.write_bytes(b"") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api, detect, _, _ = self._api() + + await api.import_from_path( + [csv_path, parquet_path], + asset="my_asset", + data_type=DataTypeKey.PARQUET_FLATDATASET, + show_progress=False, + ) + + calls = {call.args[0]: call.kwargs["data_type"] for call in detect.await_args_list} + assert calls[csv_path] is None + assert calls[parquet_path] is DataTypeKey.PARQUET_FLATDATASET + + @pytest.mark.asyncio + async def test_missing_asset_raises_before_detect(self, tmp_path): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + api, detect, _, _ = self._api() + + with pytest.raises(ValueError, match="asset"): + await api.import_from_path([first, second], show_progress=False) + + detect.assert_not_awaited() + + @pytest.mark.asyncio + async def test_mapping_sets_one_file_and_detects_the_other(self, tmp_path, monkeypatch): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api, detect, create_from_upload, create_batch = self._api() + supplied = self._csv() + supplied.data_columns[0].name = "supplied" + detected = self._csv() + detected.data_columns[0].name = "detected" + detect.return_value = detected + + await api.import_from_path( + [first, second], + asset="my_asset", + config={first.name: supplied}, + show_progress=False, + ) + + detect.assert_awaited_once() + assert detect.await_args is not None + assert detect.await_args.args[0] == second + create_from_upload.assert_not_awaited() + call = create_batch.await_args + assert call is not None + configs = call.args[0] + assert configs["a.csv"] is supplied + assert configs["b.csv"] is detected + assert call.kwargs["asset_name"] == "my_asset" + + @pytest.mark.asyncio + async def test_mapping_rejects_an_unknown_key_and_a_bare_config(self, tmp_path): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + api, detect, _, _ = self._api() + + with pytest.raises(ValueError, match="does not match"): + await api.import_from_path( + [first, second], + asset="my_asset", + config={"missing.csv": self._csv()}, + show_progress=False, + ) + with pytest.raises(ValueError, match="one file"): + await api.import_from_path( + [first, second], + asset="my_asset", + config=self._csv(), + show_progress=False, + ) + detect.assert_not_awaited() + + @pytest.mark.asyncio + async def test_one_path_accepts_a_config_or_a_mapping(self, tmp_path, monkeypatch): + path = tmp_path / "a.csv" + path.write_text("t,v\n") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api, detect, create_from_upload, create_batch = self._api() + supplied = self._csv() + + await api.import_from_path([path], asset="my_asset", config=supplied, show_progress=False) + await api.import_from_path( + path, asset="my_asset", config={path.name: supplied}, show_progress=False + ) + + detect.assert_not_awaited() + create_batch.assert_not_awaited() + assert create_from_upload.await_count == 2 + assert create_from_upload.await_args_list[0].args[0] is supplied + assert create_from_upload.await_args_list[1].args[0] is supplied diff --git a/python/lib/sift_client/_tests/sift_types/test_run.py b/python/lib/sift_client/_tests/sift_types/test_run.py index b94b59f56..55e0a8ff3 100644 --- a/python/lib/sift_client/_tests/sift_types/test_run.py +++ b/python/lib/sift_client/_tests/sift_types/test_run.py @@ -176,6 +176,33 @@ def test_bounds_match_channel_timestamp_resolution(self): assert unbounded.start_time is None assert unbounded.stop_time is None + def test_import_data_passes_a_list_of_paths(self, mock_run): + paths = ["a.csv", "b.csv"] + mock_run.import_data(paths, leave_run_open=True) + mock_run.client.data_import.import_from_path.assert_called_once_with( + paths, + asset=None, + config=None, + data_type=None, + run=mock_run, + show_progress=None, + leave_run_open=True, + ) + + def test_import_data_passes_a_config_mapping(self, mock_run): + paths = ["a.csv", "b.csv"] + configs = {"a.csv": object()} + mock_run.import_data(paths, config=configs) + mock_run.client.data_import.import_from_path.assert_called_once_with( + paths, + asset=None, + config=configs, + data_type=None, + run=mock_run, + show_progress=None, + leave_run_open=False, + ) + def test_assets_property_calls_client(self, mock_run, mock_client): """Test that assets property calls client.assets.list_ with correct parameters.""" mock_client.assets.list_.return_value = [] diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 7dab0beaa..2c900c25a 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,5 +1,6 @@ from __future__ import annotations +import tempfile from pathlib import Path from typing import TYPE_CHECKING @@ -7,17 +8,19 @@ from sift_client._internal.time import to_timestamp_nanos from sift_client._internal.util.bit_field import check_bit_field_values from sift_client._internal.util.data_imports import ( + _supplied_configs, parse_csv_detect_response, parse_parquet_detect_response, prepare_parquet_config, resolve_data_type_key, ) from sift_client._internal.util.executor import run_sync_function -from sift_client._internal.util.file import extract_parquet_footer, upload_file +from sift_client._internal.util.file import _write_import_zip, extract_parquet_footer, upload_file from sift_client.resources._base import ResourceBase from sift_client.sift_types.asset import Asset from sift_client.sift_types.data_import import ( DATA_TYPE_KEY_TO_PROTO, + EXTENSION_TO_DATA_TYPE_KEY, DataImport, DataTypeKey, ImportConfig, @@ -32,7 +35,9 @@ from sift_client.util import cel_utils as cel if TYPE_CHECKING: + from collections.abc import Mapping from datetime import datetime + from typing import Sequence import pandas as pd @@ -41,6 +46,71 @@ from sift_client.sift_types.job import Job +def _run_id_and_name( + run: Run | str | None, + run_name: str | None, + *, + default_name: str | None, +) -> tuple[str | None, str | None]: + """Return the run id and run name to import into. + + ``run`` and ``run_name`` are mutually exclusive. When neither is set, + the run name is ``default_name``. + """ + if run is not None and run_name is not None: + raise ValueError("'run' and 'run_name' are mutually exclusive.") + if run is not None: + return (run._id_or_error if isinstance(run, Run) else run), None + if run_name is not None: + return None, run_name + return None, default_name + + +def _import_target( + paths: list[Path], + configs: list[ImportConfig], + asset: Asset | str | None, + run: Run | str | None, + run_name: str | None, +) -> tuple[str, str | None, str | None]: + """Return the asset name, run id, and run name for this import. + + One file uses the asset on its config when ``asset`` is omitted. Several + files require ``asset``. The run comes from ``run`` or ``run_name``. When + neither is set, one file keeps the run on its config, and several files + use the first file's name. + """ + if asset is None: + asset_name = None + elif isinstance(asset, Asset): + asset_name = asset.name + else: + asset_name = asset + if len(paths) == 1: + config = configs[0] + if asset_name is None: + if not config.asset_name: + raise ValueError("'asset' is required when not set on the config.") + asset_name = config.asset_name + default_name = None if config.run_name or config.run_id else paths[0].name + else: + if asset_name is None: + raise ValueError("'asset' is required to import more than one file.") + default_name = paths[0].name + resolved_run_id, resolved_run_name = _run_id_and_name(run, run_name, default_name=default_name) + return asset_name, resolved_run_id, resolved_run_name + + +def _paths_from_argument(file_path: str | Path | Sequence[str | Path]) -> list[Path]: + """Return the paths to import. + + A string is one path. A list or other sequence is one path per entry. + """ + if isinstance(file_path, (str, Path)): + return [Path(file_path)] + return [Path(path) for path in file_path] + + class DataImportAPIAsync(ResourceBase): """High-level API for importing data into Sift.""" @@ -57,18 +127,19 @@ def __init__(self, sift_client: SiftClient): async def import_from_path( self, - file_path: str | Path, + file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, - config: ImportConfig | None = None, + config: ImportConfig | Mapping[str | Path, ImportConfig] | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, relative_start_time: pd.Timestamp | datetime | str | None = None, run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, + leave_run_open: bool = False, ) -> Job: - """Import data from a local file. + """Import data from a local file, or from a list of files. Creates a data import on the server, uploads the file, and returns a ``Job`` handle after uploading the file. The import processes @@ -76,6 +147,15 @@ async def import_from_path( ``job.wait_until_complete()`` only if you need to confirm completion before proceeding. + If you have more than one file for the same run, pass the paths in + one call. The client packs them into one zip and imports them + together into one asset and one run. Each file is detected on its + own, unless you pass a config for that path. ``data_type`` applies + only to detected files whose extension does not name a format, such + as Parquet and HDF5. Use ``leave_run_open`` when more files are + still coming, then call ``Run.stop`` after the last import. A + finished import stops its run unless that flag is set. + When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). When ``asset`` is provided it overrides the config value; @@ -115,17 +195,37 @@ async def import_from_path( config=config, ) + Set a config for one file in a list, and detect the other: + + paths = ["part-1.csv", "part-2.csv"] + configs = {path: client.data_import.detect_config(path) for path in paths} + configs["part-1.csv"]["temperature"].data_type = ChannelDataType.FLOAT + + job = client.data_import.import_from_path( + paths, + asset=my_asset, + config=configs, + ) + Args: - file_path: Path to the local file to import. + file_path: Path to the local file to import, or a list of paths + for one run. Pass every file in one call when they belong + together. Two or more paths are one zip. Each archive member + is the file's name, so the names must be distinct. A list of + one path imports that file directly. asset: Asset object or asset name to import data into. Optional - when ``config`` already has ``asset_name`` set. - config: Import configuration describing the file format and column - mapping. When provided, ``data_type`` is ignored. If omitted, - the config is auto-detected via ``detect_config`` (for ULog - and MCAP the detected channel list is dropped so every channel - in the file is imported). You can call ``detect_config`` - yourself to inspect and modify the config before passing it - here. + when ``config`` already has ``asset_name`` set. Required for + a list of paths. + config: Import configuration for one file, or a mapping of path + to configuration. A mapping key is a path from ``file_path`` + or that file's name. A file with no entry is detected. A key + that matches no file raises ``ValueError``. One config applies + to a single path. When a file's config is provided, + ``data_type`` is ignored for that file. If omitted, the config + is auto-detected via ``detect_config`` (for ULog and MCAP the + detected channel list is dropped so every channel in the file + is imported). You can call ``detect_config`` yourself to + inspect and modify the config before passing it here. data_type: Explicit data type key. Required for formats with multiple supported layouts (Parquet, HDF5) where the file extension alone is ambiguous. Only used when ``config`` is @@ -149,9 +249,15 @@ async def import_from_path( run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if - neither ``run`` nor ``run_name`` is set. + neither ``run`` nor ``run_name`` is set. For a list of paths, + the default is the first file's name. show_progress: If True, display a progress spinner during upload. Defaults to True for sync, False for async. + leave_run_open: If true, the import does not mark its run stopped, + so run-stop automations do not fire. Set this when more data + is coming to the run, then call ``Run.stop`` after the last + import. Ignored when the import has no run. Defaults to false, + which stops the run when the import finishes. Returns: A ``Job`` handle for the pending import. Call @@ -161,51 +267,86 @@ async def import_from_path( Raises: FileNotFoundError: If the file does not exist. ValueError: If a ``BIT_FIELD`` column's values aren't in a format - the import service reads, such as integers. + the import service reads, such as integers. Also if a list of + paths is empty, contains duplicate file names, or omits + ``asset``. Also if one config is passed with two or more + paths, or a config mapping names a path that is not in the + list. """ - path = Path(file_path) - if not path.is_file(): - raise FileNotFoundError(f"File not found: {file_path}") - - if config is None: - config = await self.detect_config( - file_path, - data_type=data_type, - time_format=time_format, - relative_start_time=relative_start_time, - ) - config._validate_relative_start_time() - if isinstance(config, (UlogImportConfig, McapImportConfig)): - # An empty channel list imports every channel - config.data = [] - - if asset is not None: - config.asset_name = asset.name if isinstance(asset, Asset) else asset - elif not config.asset_name: - raise ValueError("'asset' is required when not set on the config.") - if run is not None and run_name is not None: - raise ValueError("'run' and 'run_name' are mutually exclusive.") - if run is not None: - config.run_id = run._id_or_error if isinstance(run, Run) else run - elif run_name is not None: - config.run_name = run_name - elif not config.run_name and not config.run_id: - config.run_name = path.name - - # The import service only reports unreadable bit field values after the - # upload, so check them first. - await run_sync_function(check_bit_field_values, config, path) - - if isinstance( - config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) - ): - await prepare_parquet_config(config, path) - + paths = _paths_from_argument(file_path) + if not paths: + raise ValueError("At least one file path is required.") if show_progress is None: show_progress = self._show_progress() + supplied = _supplied_configs(paths, config) + names = [path.name for path in paths] + if len(set(names)) != len(names): + raise ValueError( + f"Files in one import must have distinct names. Got {', '.join(names)}." + ) + for path in paths: + if not path.is_file(): + raise FileNotFoundError(f"File not found: {path}") + if len(paths) > 1 and asset is None: + raise ValueError("'asset' is required to import more than one file.") + prepared: list[ImportConfig] = [] + for path in paths: + # A known extension selects its format, so data_type can describe + # the Parquet or HDF5 files in a mixed list. One file keeps the + # override the caller passed. + file_data_type = data_type + if len(paths) > 1 and path.suffix.lower() in EXTENSION_TO_DATA_TYPE_KEY: + file_data_type = None + prepared.append( + await self._config_for_upload( + path, + supplied[path], + data_type=file_data_type, + time_format=time_format, + relative_start_time=relative_start_time, + ) + ) + + asset_name, run_id, resolved_run_name = _import_target( + paths, prepared, asset, run, run_name + ) + if len(paths) == 1: + file_config = prepared[0] + file_config.asset_name = asset_name + if run_id is not None: + file_config.run_id = run_id + elif resolved_run_name is not None: + file_config.run_name = resolved_run_name + data_import_id, upload_url = await self._low_level_client.create_from_upload( + file_config, leave_run_open=leave_run_open + ) + return await self._job_for_upload( + paths[0], data_import_id, upload_url, show_progress=show_progress + ) - data_import_id, upload_url = await self._low_level_client.create_from_upload(config) + named = {path.name: file_config for path, file_config in zip(paths, prepared)} + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "batch.zip" + await run_sync_function(_write_import_zip, archive, paths) + data_import_id, upload_url = await self._low_level_client.create_batch_from_upload( + named, + asset_name=asset_name, + run_name=resolved_run_name, + run_id=run_id, + leave_run_open=leave_run_open, + ) + return await self._job_for_upload( + archive, data_import_id, upload_url, show_progress=show_progress + ) + async def _job_for_upload( + self, + path: Path, + data_import_id: str, + upload_url: str, + *, + show_progress: bool, + ) -> Job: response = await run_sync_function( lambda: upload_file( upload_url, @@ -223,6 +364,63 @@ async def import_from_path( job._set_data_import_id(data_import_id) return job + async def _config_for_upload( + self, + path: Path, + config: ImportConfig | None, + *, + data_type: DataTypeKey | None, + time_format: TimeFormat | None, + relative_start_time: pd.Timestamp | datetime | str | None, + ) -> ImportConfig: + """Use the supplied config, or detect one, then prepare the file. + + A detected ULog or MCAP config drops its channel list so the server + imports every channel. A supplied config keeps the channels you set. + """ + if config is None: + config = await self._detect_import_config( + path, + data_type=data_type, + time_format=time_format, + relative_start_time=relative_start_time, + ) + await self._prepare_import_file(config, path) + return config + + async def _detect_import_config( + self, + path: Path, + *, + data_type: DataTypeKey | None, + time_format: TimeFormat | None, + relative_start_time: pd.Timestamp | datetime | str | None, + ) -> ImportConfig: + """Detect a file's import config.""" + config = await self.detect_config( + path, + data_type=data_type, + time_format=time_format, + relative_start_time=relative_start_time, + ) + config._validate_relative_start_time() + if isinstance(config, (UlogImportConfig, McapImportConfig)): + # An empty channel list imports every channel. + config.data = [] + return config + + async def _prepare_import_file(self, config: ImportConfig, path: Path) -> None: + """Check bit field values and fill a Parquet footer before upload. + + The import service reports unreadable bit field values only after + the upload. + """ + await run_sync_function(check_bit_field_values, config, path) + if isinstance( + config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) + ): + await prepare_parquet_config(config, path) + async def get(self, data_import_id: str) -> DataImport: """Get a data import by ID. diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 3f368f2af..67e4d11a8 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: import re - from collections.abc import Iterable + from collections.abc import Iterable, Mapping from datetime import datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO, Sequence @@ -1745,18 +1745,19 @@ class DataImportAPI: def import_from_path( self, - file_path: str | Path, + file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, - config: ImportConfig | None = None, + config: ImportConfig | Mapping[str | Path, ImportConfig] | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, relative_start_time: pd.Timestamp | datetime | str | None = None, run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, + leave_run_open: bool = False, ) -> Job: - """Import data from a local file. + """Import data from a local file, or from a list of files. Creates a data import on the server, uploads the file, and returns a ``Job`` handle after uploading the file. The import processes @@ -1764,6 +1765,15 @@ class DataImportAPI: ``job.wait_until_complete()`` only if you need to confirm completion before proceeding. + If you have more than one file for the same run, pass the paths in + one call. The client packs them into one zip and imports them + together into one asset and one run. Each file is detected on its + own, unless you pass a config for that path. ``data_type`` applies + only to detected files whose extension does not name a format, such + as Parquet and HDF5. Use ``leave_run_open`` when more files are + still coming, then call ``Run.stop`` after the last import. A + finished import stops its run unless that flag is set. + When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). When ``asset`` is provided it overrides the config value; @@ -1803,17 +1813,37 @@ class DataImportAPI: config=config, ) + Set a config for one file in a list, and detect the other: + + paths = ["part-1.csv", "part-2.csv"] + configs = {path: client.data_import.detect_config(path) for path in paths} + configs["part-1.csv"]["temperature"].data_type = ChannelDataType.FLOAT + + job = client.data_import.import_from_path( + paths, + asset=my_asset, + config=configs, + ) + Args: - file_path: Path to the local file to import. + file_path: Path to the local file to import, or a list of paths + for one run. Pass every file in one call when they belong + together. Two or more paths are one zip. Each archive member + is the file's name, so the names must be distinct. A list of + one path imports that file directly. asset: Asset object or asset name to import data into. Optional - when ``config`` already has ``asset_name`` set. - config: Import configuration describing the file format and column - mapping. When provided, ``data_type`` is ignored. If omitted, - the config is auto-detected via ``detect_config`` (for ULog - and MCAP the detected channel list is dropped so every channel - in the file is imported). You can call ``detect_config`` - yourself to inspect and modify the config before passing it - here. + when ``config`` already has ``asset_name`` set. Required for + a list of paths. + config: Import configuration for one file, or a mapping of path + to configuration. A mapping key is a path from ``file_path`` + or that file's name. A file with no entry is detected. A key + that matches no file raises ``ValueError``. One config applies + to a single path. When a file's config is provided, + ``data_type`` is ignored for that file. If omitted, the config + is auto-detected via ``detect_config`` (for ULog and MCAP the + detected channel list is dropped so every channel in the file + is imported). You can call ``detect_config`` yourself to + inspect and modify the config before passing it here. data_type: Explicit data type key. Required for formats with multiple supported layouts (Parquet, HDF5) where the file extension alone is ambiguous. Only used when ``config`` is @@ -1837,9 +1867,15 @@ class DataImportAPI: run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if - neither ``run`` nor ``run_name`` is set. + neither ``run`` nor ``run_name`` is set. For a list of paths, + the default is the first file's name. show_progress: If True, display a progress spinner during upload. Defaults to True for sync, False for async. + leave_run_open: If true, the import does not mark its run stopped, + so run-stop automations do not fire. Set this when more data + is coming to the run, then call ``Run.stop`` after the last + import. Ignored when the import has no run. Defaults to false, + which stops the run when the import finishes. Returns: A ``Job`` handle for the pending import. Call @@ -1849,7 +1885,11 @@ class DataImportAPI: Raises: FileNotFoundError: If the file does not exist. ValueError: If a ``BIT_FIELD`` column's values aren't in a format - the import service reads, such as integers. + the import service reads, such as integers. Also if a list of + paths is empty, contains duplicate file names, or omits + ``asset``. Also if one config is passed with two or more + paths, or a config mapping names a path that is not in the + list. """ ... diff --git a/python/lib/sift_client/sift_types/run.py b/python/lib/sift_client/sift_types/run.py index 796db2ec4..f2df1028e 100644 --- a/python/lib/sift_client/sift_types/run.py +++ b/python/lib/sift_client/sift_types/run.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: from pathlib import Path + from typing import Mapping, Sequence from sift_stream_bindings import RunFormPy @@ -153,25 +154,38 @@ def stop(self) -> Run: def import_data( self, - file_path: str | Path, + file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, - config: ImportConfig | None = None, + config: ImportConfig | Mapping[str | Path, ImportConfig] | None = None, data_type: DataTypeKey | None = None, show_progress: bool | None = None, + leave_run_open: bool = False, ) -> Job: - """Import data from a file into this run. + """Import data from a file, or a list of files, into this run. Convenience method that calls ``client.data_import.import_from_path`` with this run pre-filled. If the run has exactly one asset, ``asset`` is inferred automatically. + If you have more than one file for this run, pass the paths in one + call. That is one import. A finished import stops this run unless + ``leave_run_open`` is true. Set that when more files are still + coming, then call ``stop`` after the last import. + Args: - file_path: Path to the local file to import. + file_path: Path to the local file to import, or a list of paths. + Pass every file in one call when they belong together. Two + or more paths are uploaded as one zip. asset: Asset object or asset name to import data into. - config: Import configuration. Auto-detected if omitted. + config: Import configuration for one file, or a mapping of path + to configuration. A file with no entry is detected. + Auto-detected when omitted. data_type: Explicit data type key for ambiguous formats. show_progress: Display a progress spinner during upload. + leave_run_open: If true, the import does not mark this run + stopped, so run-stop automations do not fire. Defaults to + false. Returns: A ``Job`` handle for the pending import. @@ -186,6 +200,7 @@ def import_data( data_type=data_type, run=self, show_progress=show_progress, + leave_run_open=leave_run_open, )