From 996acf3132ecded653c50bf11542ecf188a34011 Mon Sep 17 00:00:00 2001 From: Ian Later Date: Tue, 20 Oct 2026 15:50:20 -0700 Subject: [PATCH 1/9] python(feat): keep a run open across file imports Several files can land in one run before run-stop automations fire. The Python upload helpers take leave_run_open, and sift-cli import commands take --leave-run-open. --- python/changelog.d/+leave-run-open.feat.md | 1 + .../low_level_wrappers/data_imports.py | 9 ++++- .../low_level_wrappers/test_data_imports.py | 33 +++++++++++++++++-- .../_tests/resources/test_data_imports.py | 9 ++++- .../lib/sift_client/resources/data_imports.py | 10 +++++- python/lib/sift_client/sift_types/run.py | 9 +++++ rust/crates/sift_cli/CHANGELOG.md | 4 +++ .../assets/docs/src/data/importing.md | 11 +++++++ rust/crates/sift_cli/src/cli/mod.rs | 18 ++++++++++ rust/crates/sift_cli/src/cmd/import/csv.rs | 6 ++++ .../sift_cli/src/cmd/import/hdf5/import.rs | 9 +++-- .../sift_cli/src/cmd/import/hdf5/tests.rs | 1 + .../sift_cli/src/cmd/import/mcap/import.rs | 9 +++-- .../sift_cli/src/cmd/import/mcap/tests.rs | 1 + .../import/parquet/channel_per_row_dataset.rs | 1 + .../src/cmd/import/parquet/flat_dataset.rs | 1 + .../sift_cli/src/cmd/import/parquet/tests.rs | 2 ++ .../src/cmd/import/tdms/detect_tdms_config.rs | 9 +++-- .../sift_cli/src/cmd/import/tdms/tests.rs | 1 + .../sift_cli/src/cmd/import/ulog/import.rs | 9 +++-- .../sift_cli/src/cmd/import/ulog/tests.rs | 1 + 21 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 python/changelog.d/+leave-run-open.feat.md diff --git a/python/changelog.d/+leave-run-open.feat.md b/python/changelog.d/+leave-run-open.feat.md new file mode 100644 index 0000000000..d6152ed602 --- /dev/null +++ b/python/changelog.d/+leave-run-open.feat.md @@ -0,0 +1 @@ +`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false). A finished import stops its run and fires run-stop automations unless you set it, then call `run.stop()` after the last import. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py index a5a7c5f63d..d87fae5fe0 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py @@ -66,17 +66,24 @@ class DataImportsLowLevelClient(LowLevelClientBase, WithGrpcClient): def __init__(self, grpc_client: GrpcClient): WithGrpcClient.__init__(self, grpc_client=grpc_client) - async def create_from_upload(self, config: ImportConfig) -> tuple[str, str]: + async def create_from_upload( + self, config: ImportConfig, *, leave_run_open: bool = False + ) -> tuple[str, str]: """Create a data import and get back a presigned upload URL. Args: config: The import configuration. + leave_run_open: If true, the import does not mark its run stopped. + Run-stop automations do not fire. Set this when more data is + coming to the run, then stop the run after the last import. + Ignored when the import has no run. Returns: A tuple of (data_import_id, upload_url). """ request = CreateDataImportFromUploadRequest() _set_config_on_request(request, config) + request.leave_run_open = leave_run_open response = await self._grpc_client.get_stub( DataImportServiceStub ).CreateDataImportFromUpload(request) diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py index ba2f0939c5..0e3cc635f4 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py @@ -2,10 +2,18 @@ from __future__ import annotations -import pytest -from sift.data_imports.v2.data_imports_pb2 import CreateDataImportFromUploadRequest +from unittest.mock import MagicMock -from sift_client._internal.low_level_wrappers.data_imports import _set_config_on_request +import pytest +from sift.data_imports.v2.data_imports_pb2 import ( + CreateDataImportFromUploadRequest, + CreateDataImportFromUploadResponse, +) + +from sift_client._internal.low_level_wrappers.data_imports import ( + DataImportsLowLevelClient, + _set_config_on_request, +) from sift_client.sift_types.channel import ChannelDataType from sift_client.sift_types.data_import import McapDataColumn, McapImportConfig @@ -32,3 +40,22 @@ def test_unknown_config_type_raises(self): request = CreateDataImportFromUploadRequest() with pytest.raises(TypeError, match="Unsupported import config type"): _set_config_on_request(request, object()) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_create_from_upload_sets_leave_run_open(): + """leave_run_open is copied onto the upload request.""" + captured: dict[str, CreateDataImportFromUploadRequest] = {} + + class _Stub: + async def CreateDataImportFromUpload(self, request): # noqa: N802 + captured["request"] = request + return CreateDataImportFromUploadResponse(data_import_id="id", upload_url="url") + + client = DataImportsLowLevelClient(grpc_client=MagicMock()) + client._grpc_client.get_stub = MagicMock(return_value=_Stub()) + config = McapImportConfig(asset_name="a", data=[]) + + await client.create_from_upload(config, leave_run_open=True) + + assert captured["request"].leave_run_open is True diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 7fcc973e05..583af7e2ff 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -647,7 +647,14 @@ async def _import(self, tmp_path, filename, detected): job = MagicMock() api.client.async_.jobs.get = AsyncMock(return_value=job) - assert await api.import_from_path(path, asset="my_asset", show_progress=False) is job + assert ( + await api.import_from_path( + path, asset="my_asset", show_progress=False, leave_run_open=True + ) + is job + ) + api._low_level_client.create_from_upload.assert_awaited() + assert api._low_level_client.create_from_upload.await_args.kwargs["leave_run_open"] is True @pytest.mark.asyncio async def test_mcap_data_cleared(self, tmp_path, monkeypatch): diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 7dab0beaac..f17162186a 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -67,6 +67,7 @@ async def import_from_path( run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, + leave_run_open: bool = False, ) -> Job: """Import data from a local file. @@ -152,6 +153,11 @@ async def import_from_path( neither ``run`` nor ``run_name`` is set. show_progress: If True, display a progress spinner during upload. Defaults to True for sync, False for async. + leave_run_open: If true, the import does not mark its run stopped, + so run-stop automations do not fire. Set this when more data + is coming to the run, then call ``Run.stop`` after the last + import. Ignored when the import has no run. Defaults to false, + which stops the run when the import finishes. Returns: A ``Job`` handle for the pending import. Call @@ -204,7 +210,9 @@ async def import_from_path( if show_progress is None: show_progress = self._show_progress() - data_import_id, upload_url = await self._low_level_client.create_from_upload(config) + data_import_id, upload_url = await self._low_level_client.create_from_upload( + config, leave_run_open=leave_run_open + ) response = await run_sync_function( lambda: upload_file( diff --git a/python/lib/sift_client/sift_types/run.py b/python/lib/sift_client/sift_types/run.py index 796db2ec42..536e570be5 100644 --- a/python/lib/sift_client/sift_types/run.py +++ b/python/lib/sift_client/sift_types/run.py @@ -159,6 +159,7 @@ def import_data( config: ImportConfig | None = None, data_type: DataTypeKey | None = None, show_progress: bool | None = None, + leave_run_open: bool = False, ) -> Job: """Import data from a file into this run. @@ -166,12 +167,19 @@ def import_data( with this run pre-filled. If the run has exactly one asset, ``asset`` is inferred automatically. + A finished import stops this run unless ``leave_run_open`` is true. + Set that when more files are coming, then call ``stop`` after the + last import. + Args: file_path: Path to the local file to import. asset: Asset object or asset name to import data into. config: Import configuration. Auto-detected if omitted. data_type: Explicit data type key for ambiguous formats. show_progress: Display a progress spinner during upload. + leave_run_open: If true, the import does not mark this run + stopped, so run-stop automations do not fire. Defaults to + false. Returns: A ``Job`` handle for the pending import. @@ -186,6 +194,7 @@ def import_data( data_type=data_type, run=self, show_progress=show_progress, + leave_run_open=leave_run_open, ) diff --git a/rust/crates/sift_cli/CHANGELOG.md b/rust/crates/sift_cli/CHANGELOG.md index 5504985f9e..405e693027 100644 --- a/rust/crates/sift_cli/CHANGELOG.md +++ b/rust/crates/sift_cli/CHANGELOG.md @@ -7,6 +7,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### What's New +- `import` accepts `--leave-run-open`. A finished import marks its run stopped + and fires run-stop automations unless the flag is set. Stop the run after + the last import. The flag does nothing when the import has no run. + ## [v0.9.0] - November 12, 2026 ### What's New diff --git a/rust/crates/sift_cli/assets/docs/src/data/importing.md b/rust/crates/sift_cli/assets/docs/src/data/importing.md index 9ef21c1e7b..a76ecc0cca 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/importing.md +++ b/rust/crates/sift_cli/assets/docs/src/data/importing.md @@ -53,6 +53,17 @@ queryable immediately afterward. sift-cli import csv ./telemetry.csv --asset rover-1 --wait ``` +### Leaving a run open + +A finished import marks its run stopped, which fires run-stop automations. +Pass `--leave-run-open` when more files are coming to that run. Stop the run +after the last import. The flag does nothing when the import has no run. + +```sh +sift-cli import csv ./part-1.csv --asset rover-1 --run sortie-4 --leave-run-open +sift-cli import csv ./part-2.csv --asset rover-1 --run sortie-4 +``` + ### Time formats Imports that parse a time column accept a `--time-format` describing how diff --git a/rust/crates/sift_cli/src/cli/mod.rs b/rust/crates/sift_cli/src/cli/mod.rs index 2387c9319a..e1c3ffcbca 100644 --- a/rust/crates/sift_cli/src/cli/mod.rs +++ b/rust/crates/sift_cli/src/cli/mod.rs @@ -486,6 +486,15 @@ pub struct ImportCsvArgs { /// Preview the parsed schema without uploading #[arg(short, long)] pub preview: bool, + + /// Leave the run open when the import finishes. + /// + /// By default a finished import marks its run stopped, which fires + /// run-stop automations. Set this when more files are coming to the run, + /// then stop the run after the last import. Ignored when the import has + /// no run. + #[arg(long)] + pub leave_run_open: bool, } #[derive(clap::Args)] @@ -544,6 +553,15 @@ pub struct CommonImportArgs { /// Preview the parsed schema without uploading #[arg(short, long)] pub preview: bool, + + /// Leave the run open when the import finishes. + /// + /// By default a finished import marks its run stopped, which fires + /// run-stop automations. Set this when more files are coming to the run, + /// then stop the run after the last import. Ignored when the import has + /// no run. + #[arg(long)] + pub leave_run_open: bool, } #[derive(Subcommand)] diff --git a/rust/crates/sift_cli/src/cmd/import/csv.rs b/rust/crates/sift_cli/src/cmd/import/csv.rs index a6a882bbe2..51d292b2c2 100644 --- a/rust/crates/sift_cli/src/cmd/import/csv.rs +++ b/rust/crates/sift_cli/src/cmd/import/csv.rs @@ -345,6 +345,7 @@ fn create_data_import_request( }), ..Default::default() }), + leave_run_open: args.leave_run_open, ..Default::default() }) } @@ -390,10 +391,12 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, + leave_run_open: true, }, ) .expect("expected Result::Ok"); + assert!(req.leave_run_open); let csv_config = req.csv_config.expect("expected Option::Some"); assert_eq!(String::from("test_asset"), csv_config.asset_name); assert!(csv_config.run_id.is_empty()); @@ -440,6 +443,7 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, + leave_run_open: false, }, ) .expect("expected Result::Ok"); @@ -484,6 +488,7 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, + leave_run_open: false, }, ) .expect("expected Result::Ok"); @@ -544,6 +549,7 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, + leave_run_open: false, }, ) .expect("expected Result::Ok"); diff --git a/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs b/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs index 803883eefe..d3e189431f 100644 --- a/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs +++ b/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs @@ -98,7 +98,8 @@ pub async fn run(ctx: Context, args: ImportHdf5Args) -> Result { let file = File::open(&args.common.path).context("failed to open hdf5 file")?; - let create_data_import_req = create_data_import_request(hdf5_config); + let create_data_import_req = + create_data_import_request(hdf5_config, args.common.leave_run_open); let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -154,9 +155,13 @@ pub fn build_hdf5_config(args: &ImportHdf5Args) -> Result { }) } -fn create_data_import_request(config: Hdf5Config) -> CreateDataImportFromUploadRequest { +fn create_data_import_request( + config: Hdf5Config, + leave_run_open: bool, +) -> CreateDataImportFromUploadRequest { CreateDataImportFromUploadRequest { hdf5_config: Some(config), + leave_run_open, ..Default::default() } } diff --git a/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs b/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs index 1f05fa45a4..54b5a49608 100644 --- a/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs @@ -23,6 +23,7 @@ fn make_args() -> ImportHdf5Args { run_id: None, wait: false, preview: false, + leave_run_open: false, }, schema: Hdf5Schema::OneD, time_format: Some(TimeFormat::AbsoluteRfc3339), diff --git a/rust/crates/sift_cli/src/cmd/import/mcap/import.rs b/rust/crates/sift_cli/src/cmd/import/mcap/import.rs index 99096b4588..60da12a361 100644 --- a/rust/crates/sift_cli/src/cmd/import/mcap/import.rs +++ b/rust/crates/sift_cli/src/cmd/import/mcap/import.rs @@ -72,7 +72,8 @@ pub async fn run(ctx: Context, args: ImportMcapArgs) -> Result { let file = File::open(&args.common.path).context("failed to open mcap file")?; - let create_data_import_req = create_data_import_request(mcap_config); + let create_data_import_req = + create_data_import_request(mcap_config, args.common.leave_run_open); let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -179,9 +180,13 @@ pub fn channel_configs( configs } -fn create_data_import_request(config: McapConfig) -> CreateDataImportFromUploadRequest { +fn create_data_import_request( + config: McapConfig, + leave_run_open: bool, +) -> CreateDataImportFromUploadRequest { CreateDataImportFromUploadRequest { mcap_config: Some(config), + leave_run_open, ..Default::default() } } diff --git a/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs b/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs index bee76998b3..1469b2b622 100644 --- a/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs @@ -24,6 +24,7 @@ fn make_args() -> ImportMcapArgs { run_id: None, wait: false, preview: false, + leave_run_open: false, }, relative_start_time: None, metadata_record: Vec::new(), diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs b/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs index 5d7ccc738b..a4f0f8f8e4 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs @@ -155,6 +155,7 @@ fn create_data_import_request( .into(), config: config.config, }), + leave_run_open: args.common.leave_run_open, ..Default::default() }) } diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs b/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs index 4fbba63017..8e7adc36c8 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs @@ -255,6 +255,7 @@ fn create_data_import_request( .into(), config: config.config, }), + leave_run_open: args.common.leave_run_open, ..Default::default() }; Ok(req) diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs index 53939c2234..39fc0572e4 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs @@ -28,6 +28,7 @@ fn make_test_args(time_path: Option<&str>, time_format: Option) -> F run_id: None, wait: false, preview: false, + leave_run_open: false, }, channel_path: vec![], data_type: vec![], @@ -551,6 +552,7 @@ fn make_channel_per_row_args( run_id: None, wait: false, preview: false, + leave_run_open: false, }, mode, time_path: time_path.map(|s| s.to_string()), diff --git a/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs b/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs index 62d4cf54dd..9e4ae6ce4a 100644 --- a/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs +++ b/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs @@ -59,7 +59,8 @@ pub async fn run(ctx: Context, args: ImportTdmsArgs) -> Result { let file = File::open(&args.common.path).context("failed to open tdms file")?; let create_data_import_req = - create_data_import_request(tdms_config).context("failed to create data import req")?; + create_data_import_request(tdms_config, args.common.leave_run_open) + .context("failed to create data import req")?; let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -133,9 +134,13 @@ pub fn build_tdms_config(args: &ImportTdmsArgs) -> Result { }) } -fn create_data_import_request(config: TdmsConfig) -> Result { +fn create_data_import_request( + config: TdmsConfig, + leave_run_open: bool, +) -> Result { let req = CreateDataImportFromUploadRequest { tdms_config: Some(config), + leave_run_open, ..Default::default() }; Ok(req) diff --git a/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs b/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs index 2b90cff5f6..31b5c7e6b6 100644 --- a/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs @@ -23,6 +23,7 @@ fn make_args() -> ImportTdmsArgs { run_id: None, wait: false, preview: false, + leave_run_open: false, }, start_time_override: None, fallback_method: TdmsFallbackMethod::FailOnError, diff --git a/rust/crates/sift_cli/src/cmd/import/ulog/import.rs b/rust/crates/sift_cli/src/cmd/import/ulog/import.rs index ea80b46785..fa0d37ea28 100644 --- a/rust/crates/sift_cli/src/cmd/import/ulog/import.rs +++ b/rust/crates/sift_cli/src/cmd/import/ulog/import.rs @@ -55,7 +55,8 @@ pub async fn run(ctx: Context, args: ImportUlogArgs) -> Result { let file = File::open(&args.common.path).context("failed to open ulog file")?; - let create_data_import_req = create_data_import_request(ulog_config); + let create_data_import_req = + create_data_import_request(ulog_config, args.common.leave_run_open); let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -114,9 +115,13 @@ pub fn build_ulog_config(args: &ImportUlogArgs) -> Result { }) } -fn create_data_import_request(config: UlogConfig) -> CreateDataImportFromUploadRequest { +fn create_data_import_request( + config: UlogConfig, + leave_run_open: bool, +) -> CreateDataImportFromUploadRequest { CreateDataImportFromUploadRequest { ulog_config: Some(config), + leave_run_open, ..Default::default() } } diff --git a/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs b/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs index fa66a8d781..b16b21f60d 100644 --- a/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs @@ -19,6 +19,7 @@ fn make_args() -> ImportUlogArgs { run_id: None, wait: false, preview: false, + leave_run_open: false, }, relative_start_time: None, info_key: Vec::new(), From e88624777eb56c2a5dd98630bf3e72946cbc21d7 Mon Sep 17 00:00:00 2001 From: Ian Later Date: Thu, 22 Oct 2026 15:56:42 -0700 Subject: [PATCH 2/9] python(fix): regenerate the data import sync stub The sync DataImportAPI type comes from the generated stub. Without leave_run_open there, mypy rejects the argument at the Run.import_data call site. --- python/lib/sift_client/resources/sync_stubs/__init__.pyi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 3f368f2af7..aff39992b8 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1755,6 +1755,7 @@ class DataImportAPI: run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, + leave_run_open: bool = False, ) -> Job: """Import data from a local file. @@ -1840,6 +1841,11 @@ class DataImportAPI: neither ``run`` nor ``run_name`` is set. show_progress: If True, display a progress spinner during upload. Defaults to True for sync, False for async. + leave_run_open: If true, the import does not mark its run stopped, + so run-stop automations do not fire. Set this when more data + is coming to the run, then call ``Run.stop`` after the last + import. Ignored when the import has no run. Defaults to false, + which stops the run when the import finishes. Returns: A ``Job`` handle for the pending import. Call From b94873371b6a08f6a6854dea8ac58c99e3b2ba7a Mon Sep 17 00:00:00 2001 From: Ian Later Date: Thu, 12 Nov 2026 17:40:09 -0800 Subject: [PATCH 3/9] python(chore): name the changelog fragment after the pull request --- python/changelog.d/{+leave-run-open.feat.md => 848.feat.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python/changelog.d/{+leave-run-open.feat.md => 848.feat.md} (100%) diff --git a/python/changelog.d/+leave-run-open.feat.md b/python/changelog.d/848.feat.md similarity index 100% rename from python/changelog.d/+leave-run-open.feat.md rename to python/changelog.d/848.feat.md From 45af50faad460f3217b3a89dea5aaeb1c7ee3b1b Mon Sep 17 00:00:00 2001 From: Ian Later Date: Thu, 12 Nov 2026 22:26:16 -0800 Subject: [PATCH 4/9] python(feat): import a list of files as one upload Callers can pass several paths to one import so those files share a run. --- python/changelog.d/848.feat.1.md | 1 + .../low_level_wrappers/data_imports.py | 125 +++++++++-- .../low_level_wrappers/test_data_imports.py | 73 +++++- .../_tests/resources/test_data_imports.py | 123 ++++++++++ .../sift_client/_tests/sift_types/test_run.py | 13 ++ .../lib/sift_client/resources/data_imports.py | 210 ++++++++++++++++-- .../resources/sync_stubs/__init__.pyi | 28 ++- python/lib/sift_client/sift_types/run.py | 14 +- 8 files changed, 533 insertions(+), 54 deletions(-) create mode 100644 python/changelog.d/848.feat.1.md diff --git a/python/changelog.d/848.feat.1.md b/python/changelog.d/848.feat.1.md new file mode 100644 index 0000000000..262985278d --- /dev/null +++ b/python/changelog.d/848.feat.1.md @@ -0,0 +1 @@ +`import_from_path` and `Run.import_data` accept a list of paths and upload those files as one import into one run. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py index d87fae5fe0..266b306b3f 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py @@ -3,6 +3,15 @@ from typing import TYPE_CHECKING, Any, cast from sift.data_imports.v2.data_imports_pb2 import ( + DATA_TYPE_KEY_CSV, + DATA_TYPE_KEY_HDF5, + DATA_TYPE_KEY_MCAP, + DATA_TYPE_KEY_PARQUET_FLATDATASET, + DATA_TYPE_KEY_PARQUET_SINGLE_CHANNEL_PER_ROW, + DATA_TYPE_KEY_TDMS, + DATA_TYPE_KEY_ULOG, + BatchConfig, + BatchFileConfig, CreateDataImportFromUploadRequest, CreateDataImportFromUploadResponse, DetectConfigRequest, @@ -29,32 +38,71 @@ from sift_client.transport import WithGrpcClient if TYPE_CHECKING: + from collections.abc import Mapping + from sift.data_imports.v2.data_imports_pb2 import DataTypeKey from sift_client.transport.grpc_transport import GrpcClient +# Config class, the oneof on the upload request, and the batch type key. +# Parquet layouts share one oneof and use different type keys. +_CONFIG_UPLOAD_FIELDS: tuple[tuple[type[ImportConfig], str, int], ...] = ( + (CsvImportConfig, "csv_config", DATA_TYPE_KEY_CSV), + (ParquetFlatDatasetImportConfig, "parquet_config", DATA_TYPE_KEY_PARQUET_FLATDATASET), + ( + ParquetSingleChannelPerRowImportConfig, + "parquet_config", + DATA_TYPE_KEY_PARQUET_SINGLE_CHANNEL_PER_ROW, + ), + (TdmsImportConfig, "tdms_config", DATA_TYPE_KEY_TDMS), + (Hdf5ImportConfig, "hdf5_config", DATA_TYPE_KEY_HDF5), + (UlogImportConfig, "ulog_config", DATA_TYPE_KEY_ULOG), + (McapImportConfig, "mcap_config", DATA_TYPE_KEY_MCAP), +) + + +def _config_upload_field(config: ImportConfig) -> tuple[str, int]: + """Return the upload oneof name and batch type key for a config.""" + for config_type, field_name, type_key in _CONFIG_UPLOAD_FIELDS: + if isinstance(config, config_type): + return field_name, type_key + raise TypeError(f"Unsupported import config type: {type(config).__name__}") + + def _set_config_on_request( request: CreateDataImportFromUploadRequest, config: ImportConfig, ) -> None: - """Set the appropriate config field on a proto request based on the config type.""" - if isinstance(config, CsvImportConfig): - request.csv_config.CopyFrom(config._to_proto()) - elif isinstance( - config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) - ): - request.parquet_config.CopyFrom(config._to_proto()) - elif isinstance(config, TdmsImportConfig): - request.tdms_config.CopyFrom(config._to_proto()) - elif isinstance(config, Hdf5ImportConfig): - request.hdf5_config.CopyFrom(config._to_proto()) - elif isinstance(config, UlogImportConfig): - request.ulog_config.CopyFrom(config._to_proto()) - elif isinstance(config, McapImportConfig): - request.mcap_config.CopyFrom(config._to_proto()) - else: - raise TypeError(f"Unsupported import config type: {type(config).__name__}") + """Set the config field on an upload request for this config type.""" + field_name, _type_key = _config_upload_field(config) + getattr(request, field_name).CopyFrom(config._to_proto()) + + +def _batch_file_config(filename: str, config: ImportConfig) -> BatchFileConfig: + """Build one archive member's config.""" + if isinstance(config, UlogImportConfig) and (config.info_keys or config.param_keys): + raise ValueError( + "info_keys and param_keys are not supported for ULog files in a " + f"batch import ({filename}). Import the file on its own to attach " + "its metadata to a run." + ) + if isinstance(config, McapImportConfig) and config.metadata_records: + raise ValueError( + "metadata_records are not supported for MCAP files in a batch " + f"import ({filename}). Import the file on its own to attach its " + "metadata to a run." + ) + field_name, type_key = _config_upload_field(config) + file_proto = config._to_proto() + # The batch message owns the asset and run. A per-file value that + # disagrees with it is rejected. + file_proto.asset_name = "" + file_proto.run_name = "" + file_proto.run_id = "" + batch_file = BatchFileConfig(type=cast("DataTypeKey.ValueType", type_key)) + getattr(batch_file, field_name).CopyFrom(file_proto) + return batch_file class DataImportsLowLevelClient(LowLevelClientBase, WithGrpcClient): @@ -84,6 +132,49 @@ async def create_from_upload( request = CreateDataImportFromUploadRequest() _set_config_on_request(request, config) request.leave_run_open = leave_run_open + return await self._create_upload(request) + + async def create_batch_from_upload( + self, + file_configs: Mapping[str, ImportConfig], + *, + asset_name: str, + run_name: str | None = None, + run_id: str | None = None, + leave_run_open: bool = False, + ) -> tuple[str, str]: + """Create one import for an archive of files and return its upload URL. + + ``file_configs`` maps each archive member name to that file's import + config. The asset and run are set on the batch, not on the files. + + Args: + file_configs: Archive member name to the config for that file. + asset_name: Asset every file in the archive imports into. + run_name: Name for a new run shared by every file. Ignored when + ``run_id`` is set. + run_id: ID of an existing run every file appends to. + leave_run_open: If true, the import does not mark its run stopped. + Run-stop automations do not fire. Ignored when the import has + no run. + + Returns: + A tuple of (data_import_id, upload_url). + """ + if not file_configs: + raise ValueError("A batch import needs at least one file.") + batch = BatchConfig( + asset_name=asset_name, + run_name=run_name or "", + run_id=run_id or "", + ) + for filename, config in file_configs.items(): + batch.file_configs[filename].CopyFrom(_batch_file_config(filename, config)) + request = CreateDataImportFromUploadRequest(batch_config=batch) + request.leave_run_open = leave_run_open + return await self._create_upload(request) + + async def _create_upload(self, request: CreateDataImportFromUploadRequest) -> tuple[str, str]: response = await self._grpc_client.get_stub( DataImportServiceStub ).CreateDataImportFromUpload(request) diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py index 0e3cc635f4..bfb80d3eeb 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_data_imports.py @@ -6,6 +6,8 @@ import pytest from sift.data_imports.v2.data_imports_pb2 import ( + DATA_TYPE_KEY_CSV, + DATA_TYPE_KEY_MCAP, CreateDataImportFromUploadRequest, CreateDataImportFromUploadResponse, ) @@ -15,7 +17,15 @@ _set_config_on_request, ) from sift_client.sift_types.channel import ChannelDataType -from sift_client.sift_types.data_import import McapDataColumn, McapImportConfig +from sift_client.sift_types.data_import import ( + CsvDataColumn, + CsvImportConfig, + CsvTimeColumn, + McapDataColumn, + McapImportConfig, + TimeFormat, + UlogImportConfig, +) class TestSetConfigOnRequest: @@ -59,3 +69,64 @@ async def CreateDataImportFromUpload(self, request): # noqa: N802 await client.create_from_upload(config, leave_run_open=True) assert captured["request"].leave_run_open is True + + +def _csv_config() -> CsvImportConfig: + return CsvImportConfig( + asset_name="from-file", + run_name="from-file", + run_id="from-file", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[CsvDataColumn(column=2, name="v", data_type=ChannelDataType.DOUBLE)], + ) + + +@pytest.mark.asyncio +async def test_create_batch_from_upload_sets_batch_config(): + """A batch upload carries one config per archive member and the shared run.""" + captured: dict[str, CreateDataImportFromUploadRequest] = {} + + class _Stub: + async def CreateDataImportFromUpload(self, request): # noqa: N802 + captured["request"] = request + return CreateDataImportFromUploadResponse(data_import_id="id", upload_url="url") + + client = DataImportsLowLevelClient(grpc_client=MagicMock()) + client._grpc_client.get_stub = MagicMock(return_value=_Stub()) + + await client.create_batch_from_upload( + { + "a.csv": _csv_config(), + "log.mcap": McapImportConfig(asset_name="from-file", data=[]), + }, + asset_name="shared-asset", + run_id="run-1", + leave_run_open=True, + ) + + request = captured["request"] + assert request.leave_run_open is True + assert not request.HasField("csv_config") + batch = request.batch_config + assert batch.asset_name == "shared-asset" + assert batch.run_id == "run-1" + assert batch.file_configs["a.csv"].type == DATA_TYPE_KEY_CSV + assert batch.file_configs["a.csv"].csv_config.asset_name == "" + assert batch.file_configs["a.csv"].csv_config.run_id == "" + assert batch.file_configs["a.csv"].csv_config.data_columns[2].name == "v" + assert batch.file_configs["log.mcap"].type == DATA_TYPE_KEY_MCAP + assert batch.file_configs["log.mcap"].mcap_config.asset_name == "" + + +@pytest.mark.asyncio +async def test_create_batch_from_upload_rejects_empty_and_run_metadata(): + client = DataImportsLowLevelClient(grpc_client=MagicMock()) + + with pytest.raises(ValueError, match="at least one file"): + await client.create_batch_from_upload({}, asset_name="a") + + with pytest.raises(ValueError, match="info_keys"): + await client.create_batch_from_upload( + {"log.ulg": UlogImportConfig(asset_name="a", info_keys=["git_hash"])}, + asset_name="a", + ) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 583af7e2ff..1a94475526 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2052,3 +2052,126 @@ def test_hdf5(self, sift_client, tmp_path, dtype, values, elements, decoded): data_import = _import_and_wait(sift_client, path, config) _assert_decoded(sift_client, data_import, elements, decoded) + + +class TestImportFromPathBatch: + """A list of paths is one zip upload. A single path stays a normal import.""" + + def _csv(self) -> CsvImportConfig: + return CsvImportConfig( + asset_name="", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[CsvDataColumn(column=2, name="v", data_type=ChannelDataType.DOUBLE)], + ) + + def _api(self): + api = DataImportAPIAsync(MagicMock()) + api.detect_config = AsyncMock(return_value=self._csv()) + api._low_level_client = MagicMock() + api._low_level_client.create_from_upload = AsyncMock(return_value=("import_1", "url")) + api._low_level_client.create_batch_from_upload = AsyncMock(return_value=("import_1", "url")) + api.client.async_.jobs.get = AsyncMock(return_value=MagicMock()) + return api + + @pytest.mark.asyncio + async def test_list_uploads_one_zip(self, tmp_path, monkeypatch): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + captured: dict[str, list[str]] = {} + + def upload(_url, file_path, **_kwargs): + import zipfile + + with zipfile.ZipFile(file_path) as archive: + captured["names"] = archive.namelist() + return {"jobId": "j1"} + + monkeypatch.setattr("sift_client.resources.data_imports.upload_file", upload) + api = self._api() + + await api.import_from_path( + [first, second], asset="my_asset", show_progress=False, leave_run_open=True + ) + + assert captured["names"] == ["a.csv", "b.csv"] + api._low_level_client.create_from_upload.assert_not_awaited() + args, kwargs = api._low_level_client.create_batch_from_upload.await_args + assert set(args[0]) == {"a.csv", "b.csv"} + assert kwargs["asset_name"] == "my_asset" + assert kwargs["run_name"] == "a.csv" + assert kwargs["run_id"] is None + assert kwargs["leave_run_open"] is True + + @pytest.mark.asyncio + async def test_one_path_stays_a_single_import(self, tmp_path, monkeypatch): + path = tmp_path / "a.csv" + path.write_text("t,v\n") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api = self._api() + + await api.import_from_path([path], asset="my_asset", show_progress=False) + + api._low_level_client.create_from_upload.assert_awaited() + api._low_level_client.create_batch_from_upload.assert_not_awaited() + + @pytest.mark.asyncio + async def test_duplicate_names_and_config_are_rejected(self, tmp_path): + first = tmp_path / "nested" / "a.csv" + first.parent.mkdir() + first.write_text("t,v\n") + second = tmp_path / "a.csv" + second.write_text("t,v\n") + api = self._api() + + with pytest.raises(ValueError, match="distinct names"): + await api.import_from_path([first, second], asset="my_asset", show_progress=False) + with pytest.raises(ValueError, match="config"): + await api.import_from_path( + [first, tmp_path / "b.csv"], + asset="my_asset", + config=self._csv(), + show_progress=False, + ) + with pytest.raises(ValueError, match="At least one"): + await api.import_from_path([], asset="my_asset", show_progress=False) + + @pytest.mark.asyncio + async def test_data_type_skips_a_known_extension(self, tmp_path, monkeypatch): + csv_path = tmp_path / "a.csv" + parquet_path = tmp_path / "b.parquet" + csv_path.write_text("t,v\n") + parquet_path.write_bytes(b"") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api = self._api() + + await api.import_from_path( + [csv_path, parquet_path], + asset="my_asset", + data_type=DataTypeKey.PARQUET_FLATDATASET, + show_progress=False, + ) + + calls = { + call.args[0]: call.kwargs["data_type"] for call in api.detect_config.await_args_list + } + assert calls[csv_path] is None + assert calls[parquet_path] is DataTypeKey.PARQUET_FLATDATASET + + @pytest.mark.asyncio + async def test_missing_asset_raises_before_detect(self, tmp_path): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + api = self._api() + + with pytest.raises(ValueError, match="asset"): + await api.import_from_path([first, second], show_progress=False) + + api.detect_config.assert_not_awaited() diff --git a/python/lib/sift_client/_tests/sift_types/test_run.py b/python/lib/sift_client/_tests/sift_types/test_run.py index b94b59f567..49246abadc 100644 --- a/python/lib/sift_client/_tests/sift_types/test_run.py +++ b/python/lib/sift_client/_tests/sift_types/test_run.py @@ -176,6 +176,19 @@ def test_bounds_match_channel_timestamp_resolution(self): assert unbounded.start_time is None assert unbounded.stop_time is None + def test_import_data_passes_a_list_of_paths(self, mock_run): + paths = ["a.csv", "b.csv"] + mock_run.import_data(paths, leave_run_open=True) + mock_run.client.data_import.import_from_path.assert_called_once_with( + paths, + asset=None, + config=None, + data_type=None, + run=mock_run, + show_progress=None, + leave_run_open=True, + ) + def test_assets_property_calls_client(self, mock_run, mock_client): """Test that assets property calls client.assets.list_ with correct parameters.""" mock_client.assets.list_.return_value = [] diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index f17162186a..ea720f48e7 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,5 +1,8 @@ from __future__ import annotations +import shutil +import tempfile +import zipfile from pathlib import Path from typing import TYPE_CHECKING @@ -18,6 +21,7 @@ from sift_client.sift_types.asset import Asset from sift_client.sift_types.data_import import ( DATA_TYPE_KEY_TO_PROTO, + EXTENSION_TO_DATA_TYPE_KEY, DataImport, DataTypeKey, ImportConfig, @@ -33,6 +37,7 @@ if TYPE_CHECKING: from datetime import datetime + from typing import Sequence import pandas as pd @@ -41,6 +46,23 @@ from sift_client.sift_types.job import Job +def _paths_from_argument(file_path: str | Path | Sequence[str | Path]) -> list[Path]: + """Return the paths to import. + + A string is one path. A list or other sequence is one path per entry. + """ + if isinstance(file_path, (str, Path)): + return [Path(file_path)] + return [Path(path) for path in file_path] + + +def _write_import_zip(archive: Path, paths: list[Path]) -> None: + """Pack each file into the archive under its own file name.""" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zip_file: + for path in paths: + zip_file.write(path, arcname=path.name) + + class DataImportAPIAsync(ResourceBase): """High-level API for importing data into Sift.""" @@ -57,7 +79,7 @@ def __init__(self, sift_client: SiftClient): async def import_from_path( self, - file_path: str | Path, + file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, config: ImportConfig | None = None, @@ -69,7 +91,7 @@ async def import_from_path( show_progress: bool | None = None, leave_run_open: bool = False, ) -> Job: - """Import data from a local file. + """Import data from a local file, or from a list of files. Creates a data import on the server, uploads the file, and returns a ``Job`` handle after uploading the file. The import processes @@ -77,6 +99,12 @@ async def import_from_path( ``job.wait_until_complete()`` only if you need to confirm completion before proceeding. + A list of two or more paths is one import. The files are packed into + a zip and imported together into one asset and one run. Each file is + detected on its own. ``data_type`` applies only to files whose + extension does not name a format, such as Parquet and HDF5. The run + stops when that import finishes unless ``leave_run_open`` is true. + When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). When ``asset`` is provided it overrides the config value; @@ -117,16 +145,21 @@ async def import_from_path( ) Args: - file_path: Path to the local file to import. + file_path: Path to the local file to import, or a list of paths. + Two or more paths are uploaded as one zip. Each archive member + is the file's name, so the names must be distinct. A list of + one path imports that file directly. asset: Asset object or asset name to import data into. Optional - when ``config`` already has ``asset_name`` set. + when ``config`` already has ``asset_name`` set. Required for + a list of paths. config: Import configuration describing the file format and column mapping. When provided, ``data_type`` is ignored. If omitted, the config is auto-detected via ``detect_config`` (for ULog and MCAP the detected channel list is dropped so every channel in the file is imported). You can call ``detect_config`` yourself to inspect and modify the config before passing it - here. + here. A list of paths detects each file, so ``config`` is + only used for a single path. data_type: Explicit data type key. Required for formats with multiple supported layouts (Parquet, HDF5) where the file extension alone is ambiguous. Only used when ``config`` is @@ -150,7 +183,8 @@ async def import_from_path( run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if - neither ``run`` nor ``run_name`` is set. + neither ``run`` nor ``run_name`` is set. For a list of paths, + the default is the first file's name. show_progress: If True, display a progress spinner during upload. Defaults to True for sync, False for async. leave_run_open: If true, the import does not mark its run stopped, @@ -167,23 +201,37 @@ async def import_from_path( Raises: FileNotFoundError: If the file does not exist. ValueError: If a ``BIT_FIELD`` column's values aren't in a format - the import service reads, such as integers. + the import service reads, such as integers. Also if a list of + paths is empty, contains duplicate file names, omits + ``asset``, or is passed with ``config``. """ - path = Path(file_path) + paths = _paths_from_argument(file_path) + if not paths: + raise ValueError("At least one file path is required.") + if len(paths) > 1: + return await self._import_many( + paths, + asset=asset, + config=config, + data_type=data_type, + time_format=time_format, + relative_start_time=relative_start_time, + run=run, + run_name=run_name, + show_progress=show_progress, + leave_run_open=leave_run_open, + ) + path = paths[0] if not path.is_file(): - raise FileNotFoundError(f"File not found: {file_path}") + raise FileNotFoundError(f"File not found: {path}") if config is None: - config = await self.detect_config( - file_path, + config = await self._detect_import_config( + path, data_type=data_type, time_format=time_format, relative_start_time=relative_start_time, ) - config._validate_relative_start_time() - if isinstance(config, (UlogImportConfig, McapImportConfig)): - # An empty channel list imports every channel - config.data = [] if asset is not None: config.asset_name = asset.name if isinstance(asset, Asset) else asset @@ -198,14 +246,7 @@ async def import_from_path( elif not config.run_name and not config.run_id: config.run_name = path.name - # The import service only reports unreadable bit field values after the - # upload, so check them first. - await run_sync_function(check_bit_field_values, config, path) - - if isinstance( - config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) - ): - await prepare_parquet_config(config, path) + await self._prepare_import_file(config, path) if show_progress is None: show_progress = self._show_progress() @@ -214,6 +255,96 @@ async def import_from_path( config, leave_run_open=leave_run_open ) + return await self._job_for_upload( + path, data_import_id, upload_url, show_progress=show_progress + ) + + async def _import_many( + self, + paths: list[Path], + *, + asset: Asset | str | None, + config: ImportConfig | None, + data_type: DataTypeKey | None, + time_format: TimeFormat | None, + relative_start_time: pd.Timestamp | datetime | str | None, + run: Run | str | None, + run_name: str | None, + show_progress: bool | None, + leave_run_open: bool, + ) -> Job: + """Detect each file, zip them, and upload the archive as one import.""" + if config is not None: + raise ValueError( + "'config' applies to one file. Omit it to detect each file in the list." + ) + names = [path.name for path in paths] + if len(set(names)) != len(names): + raise ValueError( + f"Files in one import must have distinct names. Got {', '.join(names)}." + ) + if asset is None: + raise ValueError("'asset' is required to import more than one file.") + if run is not None and run_name is not None: + raise ValueError("'run' and 'run_name' are mutually exclusive.") + asset_name = asset.name if isinstance(asset, Asset) else asset + if run is not None: + resolved_run_id = run._id_or_error if isinstance(run, Run) else run + resolved_run_name = None + elif run_name is not None: + resolved_run_id = None + resolved_run_name = run_name + else: + resolved_run_id = None + resolved_run_name = paths[0].name + for path in paths: + if not path.is_file(): + raise FileNotFoundError(f"File not found: {path}") + + file_configs: dict[str, ImportConfig] = {} + for path in paths: + # A known extension selects its format, so data_type can describe + # the Parquet or HDF5 files in a mixed list. + file_data_type = ( + None if path.suffix.lower() in EXTENSION_TO_DATA_TYPE_KEY else data_type + ) + detected = await self._detect_import_config( + path, + data_type=file_data_type, + time_format=time_format, + relative_start_time=relative_start_time, + ) + await self._prepare_import_file(detected, path) + file_configs[path.name] = detected + + if show_progress is None: + show_progress = self._show_progress() + + directory = Path(tempfile.mkdtemp()) + archive = directory / "batch.zip" + try: + await run_sync_function(_write_import_zip, archive, paths) + data_import_id, upload_url = await self._low_level_client.create_batch_from_upload( + file_configs, + asset_name=asset_name, + run_name=resolved_run_name, + run_id=resolved_run_id, + leave_run_open=leave_run_open, + ) + return await self._job_for_upload( + archive, data_import_id, upload_url, show_progress=show_progress + ) + finally: + shutil.rmtree(directory, ignore_errors=True) + + async def _job_for_upload( + self, + path: Path, + data_import_id: str, + upload_url: str, + *, + show_progress: bool, + ) -> Job: response = await run_sync_function( lambda: upload_file( upload_url, @@ -231,6 +362,39 @@ async def import_from_path( job._set_data_import_id(data_import_id) return job + async def _detect_import_config( + self, + path: Path, + *, + data_type: DataTypeKey | None, + time_format: TimeFormat | None, + relative_start_time: pd.Timestamp | datetime | str | None, + ) -> ImportConfig: + """Detect a file's import config.""" + config = await self.detect_config( + path, + data_type=data_type, + time_format=time_format, + relative_start_time=relative_start_time, + ) + config._validate_relative_start_time() + if isinstance(config, (UlogImportConfig, McapImportConfig)): + # An empty channel list imports every channel. + config.data = [] + return config + + async def _prepare_import_file(self, config: ImportConfig, path: Path) -> None: + """Check bit field values and fill a Parquet footer before upload. + + The import service reports unreadable bit field values only after + the upload. + """ + await run_sync_function(check_bit_field_values, config, path) + if isinstance( + config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) + ): + await prepare_parquet_config(config, path) + async def get(self, data_import_id: str) -> DataImport: """Get a data import by ID. diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index aff39992b8..dee5106c42 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1745,7 +1745,7 @@ class DataImportAPI: def import_from_path( self, - file_path: str | Path, + file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, config: ImportConfig | None = None, @@ -1757,7 +1757,7 @@ class DataImportAPI: show_progress: bool | None = None, leave_run_open: bool = False, ) -> Job: - """Import data from a local file. + """Import data from a local file, or from a list of files. Creates a data import on the server, uploads the file, and returns a ``Job`` handle after uploading the file. The import processes @@ -1765,6 +1765,12 @@ class DataImportAPI: ``job.wait_until_complete()`` only if you need to confirm completion before proceeding. + A list of two or more paths is one import. The files are packed into + a zip and imported together into one asset and one run. Each file is + detected on its own. ``data_type`` applies only to files whose + extension does not name a format, such as Parquet and HDF5. The run + stops when that import finishes unless ``leave_run_open`` is true. + When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). When ``asset`` is provided it overrides the config value; @@ -1805,16 +1811,21 @@ class DataImportAPI: ) Args: - file_path: Path to the local file to import. + file_path: Path to the local file to import, or a list of paths. + Two or more paths are uploaded as one zip. Each archive member + is the file's name, so the names must be distinct. A list of + one path imports that file directly. asset: Asset object or asset name to import data into. Optional - when ``config`` already has ``asset_name`` set. + when ``config`` already has ``asset_name`` set. Required for + a list of paths. config: Import configuration describing the file format and column mapping. When provided, ``data_type`` is ignored. If omitted, the config is auto-detected via ``detect_config`` (for ULog and MCAP the detected channel list is dropped so every channel in the file is imported). You can call ``detect_config`` yourself to inspect and modify the config before passing it - here. + here. A list of paths detects each file, so ``config`` is + only used for a single path. data_type: Explicit data type key. Required for formats with multiple supported layouts (Parquet, HDF5) where the file extension alone is ambiguous. Only used when ``config`` is @@ -1838,7 +1849,8 @@ class DataImportAPI: run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if - neither ``run`` nor ``run_name`` is set. + neither ``run`` nor ``run_name`` is set. For a list of paths, + the default is the first file's name. show_progress: If True, display a progress spinner during upload. Defaults to True for sync, False for async. leave_run_open: If true, the import does not mark its run stopped, @@ -1855,7 +1867,9 @@ class DataImportAPI: Raises: FileNotFoundError: If the file does not exist. ValueError: If a ``BIT_FIELD`` column's values aren't in a format - the import service reads, such as integers. + the import service reads, such as integers. Also if a list of + paths is empty, contains duplicate file names, omits + ``asset``, or is passed with ``config``. """ ... diff --git a/python/lib/sift_client/sift_types/run.py b/python/lib/sift_client/sift_types/run.py index 536e570be5..ec67d96452 100644 --- a/python/lib/sift_client/sift_types/run.py +++ b/python/lib/sift_client/sift_types/run.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: from pathlib import Path + from typing import Sequence from sift_stream_bindings import RunFormPy @@ -153,7 +154,7 @@ def stop(self) -> Run: def import_data( self, - file_path: str | Path, + file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, config: ImportConfig | None = None, @@ -161,18 +162,19 @@ def import_data( show_progress: bool | None = None, leave_run_open: bool = False, ) -> Job: - """Import data from a file into this run. + """Import data from a file, or a list of files, into this run. Convenience method that calls ``client.data_import.import_from_path`` with this run pre-filled. If the run has exactly one asset, ``asset`` is inferred automatically. - A finished import stops this run unless ``leave_run_open`` is true. - Set that when more files are coming, then call ``stop`` after the - last import. + A list of two or more paths is one import into this run. A finished + import stops this run unless ``leave_run_open`` is true. Set that + when more files are coming, then call ``stop`` after the last import. Args: - file_path: Path to the local file to import. + file_path: Path to the local file to import, or a list of paths. + Two or more paths are uploaded as one zip. asset: Asset object or asset name to import data into. config: Import configuration. Auto-detected if omitted. data_type: Explicit data type key for ambiguous formats. From 48f871ece9f91f951d148e6cf894e6371d2a83e4 Mon Sep 17 00:00:00 2001 From: Ian Later Date: Thu, 12 Nov 2026 23:52:20 -0800 Subject: [PATCH 5/9] python(chore): share import run resolution and one changelog entry Both import paths resolved the run the same way. The batch tests asserted on the typed client methods, which pyright rejects. --- python/changelog.d/848.feat.1.md | 1 - python/changelog.d/848.feat.md | 2 +- .../low_level_wrappers/data_imports.py | 53 ++++++++++------- .../_tests/resources/test_data_imports.py | 40 +++++++------ .../lib/sift_client/resources/data_imports.py | 58 ++++++++++--------- 5 files changed, 87 insertions(+), 67 deletions(-) delete mode 100644 python/changelog.d/848.feat.1.md diff --git a/python/changelog.d/848.feat.1.md b/python/changelog.d/848.feat.1.md deleted file mode 100644 index 262985278d..0000000000 --- a/python/changelog.d/848.feat.1.md +++ /dev/null @@ -1 +0,0 @@ -`import_from_path` and `Run.import_data` accept a list of paths and upload those files as one import into one run. diff --git a/python/changelog.d/848.feat.md b/python/changelog.d/848.feat.md index d6152ed602..11fd57f8e8 100644 --- a/python/changelog.d/848.feat.md +++ b/python/changelog.d/848.feat.md @@ -1 +1 @@ -`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false). A finished import stops its run and fires run-stop automations unless you set it, then call `run.stop()` after the last import. +`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false) and a list of paths. A finished import stops its run and fires run-stop automations unless you set `leave_run_open`, then call `run.stop()` after the last import. A list of two or more paths uploads as one import into one run. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py index 266b306b3f..2bd3599c4d 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NamedTuple, cast from sift.data_imports.v2.data_imports_pb2 import ( DATA_TYPE_KEY_CSV, @@ -45,28 +45,39 @@ from sift_client.transport.grpc_transport import GrpcClient -# Config class, the oneof on the upload request, and the batch type key. -# Parquet layouts share one oneof and use different type keys. -_CONFIG_UPLOAD_FIELDS: tuple[tuple[type[ImportConfig], str, int], ...] = ( - (CsvImportConfig, "csv_config", DATA_TYPE_KEY_CSV), - (ParquetFlatDatasetImportConfig, "parquet_config", DATA_TYPE_KEY_PARQUET_FLATDATASET), - ( +class _ConfigUploadField(NamedTuple): + """An import config's upload oneof and batch type key. + + Parquet layouts share one oneof and use different type keys. + """ + + config_type: type[ImportConfig] + field_name: str + type_key: int + + +_CONFIG_UPLOAD_FIELDS: tuple[_ConfigUploadField, ...] = ( + _ConfigUploadField(CsvImportConfig, "csv_config", DATA_TYPE_KEY_CSV), + _ConfigUploadField( + ParquetFlatDatasetImportConfig, "parquet_config", DATA_TYPE_KEY_PARQUET_FLATDATASET + ), + _ConfigUploadField( ParquetSingleChannelPerRowImportConfig, "parquet_config", DATA_TYPE_KEY_PARQUET_SINGLE_CHANNEL_PER_ROW, ), - (TdmsImportConfig, "tdms_config", DATA_TYPE_KEY_TDMS), - (Hdf5ImportConfig, "hdf5_config", DATA_TYPE_KEY_HDF5), - (UlogImportConfig, "ulog_config", DATA_TYPE_KEY_ULOG), - (McapImportConfig, "mcap_config", DATA_TYPE_KEY_MCAP), + _ConfigUploadField(TdmsImportConfig, "tdms_config", DATA_TYPE_KEY_TDMS), + _ConfigUploadField(Hdf5ImportConfig, "hdf5_config", DATA_TYPE_KEY_HDF5), + _ConfigUploadField(UlogImportConfig, "ulog_config", DATA_TYPE_KEY_ULOG), + _ConfigUploadField(McapImportConfig, "mcap_config", DATA_TYPE_KEY_MCAP), ) -def _config_upload_field(config: ImportConfig) -> tuple[str, int]: - """Return the upload oneof name and batch type key for a config.""" - for config_type, field_name, type_key in _CONFIG_UPLOAD_FIELDS: - if isinstance(config, config_type): - return field_name, type_key +def _config_upload_field(config: ImportConfig) -> _ConfigUploadField: + """Return the upload field for a config.""" + for field in _CONFIG_UPLOAD_FIELDS: + if isinstance(config, field.config_type): + return field raise TypeError(f"Unsupported import config type: {type(config).__name__}") @@ -75,8 +86,8 @@ def _set_config_on_request( config: ImportConfig, ) -> None: """Set the config field on an upload request for this config type.""" - field_name, _type_key = _config_upload_field(config) - getattr(request, field_name).CopyFrom(config._to_proto()) + field = _config_upload_field(config) + getattr(request, field.field_name).CopyFrom(config._to_proto()) def _batch_file_config(filename: str, config: ImportConfig) -> BatchFileConfig: @@ -93,15 +104,15 @@ def _batch_file_config(filename: str, config: ImportConfig) -> BatchFileConfig: f"import ({filename}). Import the file on its own to attach its " "metadata to a run." ) - field_name, type_key = _config_upload_field(config) + field = _config_upload_field(config) file_proto = config._to_proto() # The batch message owns the asset and run. A per-file value that # disagrees with it is rejected. file_proto.asset_name = "" file_proto.run_name = "" file_proto.run_id = "" - batch_file = BatchFileConfig(type=cast("DataTypeKey.ValueType", type_key)) - getattr(batch_file, field_name).CopyFrom(file_proto) + batch_file = BatchFileConfig(type=cast("DataTypeKey.ValueType", field.type_key)) + getattr(batch_file, field.field_name).CopyFrom(file_proto) return batch_file diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 1a94475526..1d04a3b30b 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2065,13 +2065,17 @@ def _csv(self) -> CsvImportConfig: ) def _api(self): + detect = AsyncMock(return_value=self._csv()) + create_from_upload = AsyncMock(return_value=("import_1", "url")) + create_batch_from_upload = AsyncMock(return_value=("import_1", "url")) + low_level = MagicMock() + low_level.create_from_upload = create_from_upload + low_level.create_batch_from_upload = create_batch_from_upload api = DataImportAPIAsync(MagicMock()) - api.detect_config = AsyncMock(return_value=self._csv()) - api._low_level_client = MagicMock() - api._low_level_client.create_from_upload = AsyncMock(return_value=("import_1", "url")) - api._low_level_client.create_batch_from_upload = AsyncMock(return_value=("import_1", "url")) + api.detect_config = detect + api._low_level_client = low_level api.client.async_.jobs.get = AsyncMock(return_value=MagicMock()) - return api + return api, detect, create_from_upload, create_batch_from_upload @pytest.mark.asyncio async def test_list_uploads_one_zip(self, tmp_path, monkeypatch): @@ -2089,15 +2093,17 @@ def upload(_url, file_path, **_kwargs): return {"jobId": "j1"} monkeypatch.setattr("sift_client.resources.data_imports.upload_file", upload) - api = self._api() + api, _, create_from_upload, create_batch = self._api() await api.import_from_path( [first, second], asset="my_asset", show_progress=False, leave_run_open=True ) assert captured["names"] == ["a.csv", "b.csv"] - api._low_level_client.create_from_upload.assert_not_awaited() - args, kwargs = api._low_level_client.create_batch_from_upload.await_args + create_from_upload.assert_not_awaited() + call = create_batch.await_args + assert call is not None + args, kwargs = call assert set(args[0]) == {"a.csv", "b.csv"} assert kwargs["asset_name"] == "my_asset" assert kwargs["run_name"] == "a.csv" @@ -2111,12 +2117,12 @@ async def test_one_path_stays_a_single_import(self, tmp_path, monkeypatch): monkeypatch.setattr( "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} ) - api = self._api() + api, _, create_from_upload, create_batch = self._api() await api.import_from_path([path], asset="my_asset", show_progress=False) - api._low_level_client.create_from_upload.assert_awaited() - api._low_level_client.create_batch_from_upload.assert_not_awaited() + create_from_upload.assert_awaited() + create_batch.assert_not_awaited() @pytest.mark.asyncio async def test_duplicate_names_and_config_are_rejected(self, tmp_path): @@ -2125,7 +2131,7 @@ async def test_duplicate_names_and_config_are_rejected(self, tmp_path): first.write_text("t,v\n") second = tmp_path / "a.csv" second.write_text("t,v\n") - api = self._api() + api, _, _, _ = self._api() with pytest.raises(ValueError, match="distinct names"): await api.import_from_path([first, second], asset="my_asset", show_progress=False) @@ -2148,7 +2154,7 @@ async def test_data_type_skips_a_known_extension(self, tmp_path, monkeypatch): monkeypatch.setattr( "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} ) - api = self._api() + api, detect, _, _ = self._api() await api.import_from_path( [csv_path, parquet_path], @@ -2157,9 +2163,7 @@ async def test_data_type_skips_a_known_extension(self, tmp_path, monkeypatch): show_progress=False, ) - calls = { - call.args[0]: call.kwargs["data_type"] for call in api.detect_config.await_args_list - } + calls = {call.args[0]: call.kwargs["data_type"] for call in detect.await_args_list} assert calls[csv_path] is None assert calls[parquet_path] is DataTypeKey.PARQUET_FLATDATASET @@ -2169,9 +2173,9 @@ async def test_missing_asset_raises_before_detect(self, tmp_path): second = tmp_path / "b.csv" first.write_text("t,v\n") second.write_text("t,v\n") - api = self._api() + api, detect, _, _ = self._api() with pytest.raises(ValueError, match="asset"): await api.import_from_path([first, second], show_progress=False) - api.detect_config.assert_not_awaited() + detect.assert_not_awaited() diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index ea720f48e7..960bdd3728 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -46,6 +46,26 @@ from sift_client.sift_types.job import Job +def _run_id_and_name( + run: Run | str | None, + run_name: str | None, + *, + default_name: str | None, +) -> tuple[str | None, str | None]: + """Return the run id and run name to import into. + + ``run`` and ``run_name`` are mutually exclusive. When neither is set, + the run name is ``default_name``. + """ + if run is not None and run_name is not None: + raise ValueError("'run' and 'run_name' are mutually exclusive.") + if run is not None: + return (run._id_or_error if isinstance(run, Run) else run), None + if run_name is not None: + return None, run_name + return None, default_name + + def _paths_from_argument(file_path: str | Path | Sequence[str | Path]) -> list[Path]: """Return the paths to import. @@ -208,6 +228,8 @@ async def import_from_path( paths = _paths_from_argument(file_path) if not paths: raise ValueError("At least one file path is required.") + if show_progress is None: + show_progress = self._show_progress() if len(paths) > 1: return await self._import_many( paths, @@ -237,20 +259,15 @@ async def import_from_path( config.asset_name = asset.name if isinstance(asset, Asset) else asset elif not config.asset_name: raise ValueError("'asset' is required when not set on the config.") - if run is not None and run_name is not None: - raise ValueError("'run' and 'run_name' are mutually exclusive.") - if run is not None: - config.run_id = run._id_or_error if isinstance(run, Run) else run - elif run_name is not None: - config.run_name = run_name - elif not config.run_name and not config.run_id: - config.run_name = path.name + default_name = None if config.run_name or config.run_id else path.name + run_id, resolved_run_name = _run_id_and_name(run, run_name, default_name=default_name) + if run_id is not None: + config.run_id = run_id + elif resolved_run_name is not None: + config.run_name = resolved_run_name await self._prepare_import_file(config, path) - if show_progress is None: - show_progress = self._show_progress() - data_import_id, upload_url = await self._low_level_client.create_from_upload( config, leave_run_open=leave_run_open ) @@ -270,7 +287,7 @@ async def _import_many( relative_start_time: pd.Timestamp | datetime | str | None, run: Run | str | None, run_name: str | None, - show_progress: bool | None, + show_progress: bool, leave_run_open: bool, ) -> Job: """Detect each file, zip them, and upload the archive as one import.""" @@ -285,18 +302,10 @@ async def _import_many( ) if asset is None: raise ValueError("'asset' is required to import more than one file.") - if run is not None and run_name is not None: - raise ValueError("'run' and 'run_name' are mutually exclusive.") asset_name = asset.name if isinstance(asset, Asset) else asset - if run is not None: - resolved_run_id = run._id_or_error if isinstance(run, Run) else run - resolved_run_name = None - elif run_name is not None: - resolved_run_id = None - resolved_run_name = run_name - else: - resolved_run_id = None - resolved_run_name = paths[0].name + resolved_run_id, resolved_run_name = _run_id_and_name( + run, run_name, default_name=paths[0].name + ) for path in paths: if not path.is_file(): raise FileNotFoundError(f"File not found: {path}") @@ -317,9 +326,6 @@ async def _import_many( await self._prepare_import_file(detected, path) file_configs[path.name] = detected - if show_progress is None: - show_progress = self._show_progress() - directory = Path(tempfile.mkdtemp()) archive = directory / "batch.zip" try: From 4f7eacc9ad431cc200e0599662b36db243145f6d Mon Sep 17 00:00:00 2001 From: Ian Later Date: Fri, 13 Nov 2026 12:05:17 -0800 Subject: [PATCH 6/9] docs: prefer one import when several files share a run Callers who already have the files were told to upload them one at a time. Pass the paths together instead. --- python/changelog.d/848.feat.md | 2 +- .../lib/sift_client/resources/data_imports.py | 18 +++++++++------- .../resources/sync_stubs/__init__.pyi | 18 +++++++++------- python/lib/sift_client/sift_types/run.py | 10 +++++---- .../assets/docs/src/data/import-csv.md | 3 +++ .../assets/docs/src/data/import-mcap.md | 3 +++ .../assets/docs/src/data/import-tdms.md | 3 +++ .../assets/docs/src/data/import-ulog.md | 3 +++ .../assets/docs/src/data/importing.md | 21 +++++++++++++++---- 9 files changed, 58 insertions(+), 23 deletions(-) diff --git a/python/changelog.d/848.feat.md b/python/changelog.d/848.feat.md index 11fd57f8e8..0ee9053c17 100644 --- a/python/changelog.d/848.feat.md +++ b/python/changelog.d/848.feat.md @@ -1 +1 @@ -`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false) and a list of paths. A finished import stops its run and fires run-stop automations unless you set `leave_run_open`, then call `run.stop()` after the last import. A list of two or more paths uploads as one import into one run. +`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false) and a list of paths. When you have more than one file for a run, pass them in one call. That upload is one import into one run. A finished import stops its run and fires run-stop automations unless you set `leave_run_open`, then call `run.stop()` after the last import. diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 960bdd3728..149245b80e 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -119,11 +119,14 @@ async def import_from_path( ``job.wait_until_complete()`` only if you need to confirm completion before proceeding. - A list of two or more paths is one import. The files are packed into - a zip and imported together into one asset and one run. Each file is - detected on its own. ``data_type`` applies only to files whose - extension does not name a format, such as Parquet and HDF5. The run - stops when that import finishes unless ``leave_run_open`` is true. + If you have more than one file for the same run, pass the paths in + one call. The client packs them into one zip and imports them + together into one asset and one run. Each file is detected on its + own. ``data_type`` applies only to files whose extension does not + name a format, such as Parquet and HDF5. Use ``leave_run_open`` + when more files are still coming, then call ``Run.stop`` after the + last import. A finished import stops its run unless that flag is + set. When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). @@ -165,8 +168,9 @@ async def import_from_path( ) Args: - file_path: Path to the local file to import, or a list of paths. - Two or more paths are uploaded as one zip. Each archive member + file_path: Path to the local file to import, or a list of paths + for one run. Pass every file in one call when they belong + together. Two or more paths are one zip. Each archive member is the file's name, so the names must be distinct. A list of one path imports that file directly. asset: Asset object or asset name to import data into. Optional diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index dee5106c42..18e4dfa4a3 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1765,11 +1765,14 @@ class DataImportAPI: ``job.wait_until_complete()`` only if you need to confirm completion before proceeding. - A list of two or more paths is one import. The files are packed into - a zip and imported together into one asset and one run. Each file is - detected on its own. ``data_type`` applies only to files whose - extension does not name a format, such as Parquet and HDF5. The run - stops when that import finishes unless ``leave_run_open`` is true. + If you have more than one file for the same run, pass the paths in + one call. The client packs them into one zip and imports them + together into one asset and one run. Each file is detected on its + own. ``data_type`` applies only to files whose extension does not + name a format, such as Parquet and HDF5. Use ``leave_run_open`` + when more files are still coming, then call ``Run.stop`` after the + last import. A finished import stops its run unless that flag is + set. When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). @@ -1811,8 +1814,9 @@ class DataImportAPI: ) Args: - file_path: Path to the local file to import, or a list of paths. - Two or more paths are uploaded as one zip. Each archive member + file_path: Path to the local file to import, or a list of paths + for one run. Pass every file in one call when they belong + together. Two or more paths are one zip. Each archive member is the file's name, so the names must be distinct. A list of one path imports that file directly. asset: Asset object or asset name to import data into. Optional diff --git a/python/lib/sift_client/sift_types/run.py b/python/lib/sift_client/sift_types/run.py index ec67d96452..e2bbfd9b47 100644 --- a/python/lib/sift_client/sift_types/run.py +++ b/python/lib/sift_client/sift_types/run.py @@ -168,13 +168,15 @@ def import_data( with this run pre-filled. If the run has exactly one asset, ``asset`` is inferred automatically. - A list of two or more paths is one import into this run. A finished - import stops this run unless ``leave_run_open`` is true. Set that - when more files are coming, then call ``stop`` after the last import. + If you have more than one file for this run, pass the paths in one + call. That is one import. A finished import stops this run unless + ``leave_run_open`` is true. Set that when more files are still + coming, then call ``stop`` after the last import. Args: file_path: Path to the local file to import, or a list of paths. - Two or more paths are uploaded as one zip. + Pass every file in one call when they belong together. Two + or more paths are uploaded as one zip. asset: Asset object or asset name to import data into. config: Import configuration. Auto-detected if omitted. data_type: Explicit data type key for ambiguous formats. diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-csv.md b/rust/crates/sift_cli/assets/docs/src/data/import-csv.md index e63171df6c..fbfe8a184f 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-csv.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-csv.md @@ -32,6 +32,9 @@ sift-cli import csv ./telemetry.csv \ --wait ``` +If you have more than one file for that run, import them together. See +[Several files in one run](./importing.md#several-files-in-one-run). + ## Pointing at the right rows and time column If your file has metadata rows above the header, or the timestamp is not the diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md b/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md index c666949ae7..6dadaef198 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md @@ -27,6 +27,9 @@ sift-cli import mcap ./recording.mcap \ --wait ``` +If you have more than one file for that run, import them together. See +[Several files in one run](./importing.md#several-files-in-one-run). + ## Previewing the detected channels `--preview` (`-p`) parses the file locally and prints the channels the import diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md b/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md index c1de6ca6a7..a6bcc3f118 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md @@ -23,6 +23,9 @@ sift-cli import tdms ./measurement.tdms \ --wait ``` +If you have more than one file for that run, import them together. See +[Several files in one run](./importing.md#several-files-in-one-run). + ## Channels with missing timing When a channel lacks timing metadata, choose how the CLI should respond with diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md b/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md index d313c8a17f..b26e592f45 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md @@ -24,6 +24,9 @@ sift-cli import ulog ./flight.ulg \ --wait ``` +If you have more than one file for that run, import them together. See +[Several files in one run](./importing.md#several-files-in-one-run). + ## Anchoring timestamps ULog records boot-relative timestamps. The import anchors them to UTC using diff --git a/rust/crates/sift_cli/assets/docs/src/data/importing.md b/rust/crates/sift_cli/assets/docs/src/data/importing.md index a76ecc0cca..3122da8b82 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/importing.md +++ b/rust/crates/sift_cli/assets/docs/src/data/importing.md @@ -53,11 +53,24 @@ queryable immediately afterward. sift-cli import csv ./telemetry.csv --asset rover-1 --wait ``` -### Leaving a run open +### Several files in one run + +If the files are available together, import them in one call from the Python +client. Pass the paths to `client.data_import.import_from_path` or +`Run.import_data`. That upload is one import into one run. + +```python +client.data_import.import_from_path( + ["part-1.csv", "part-2.csv"], + asset="rover-1", + run_name="sortie-4", +) +``` -A finished import marks its run stopped, which fires run-stop automations. -Pass `--leave-run-open` when more files are coming to that run. Stop the run -after the last import. The flag does nothing when the import has no run. +The CLI imports one file per command. When a later file is not available yet, +pass `--leave-run-open`. A finished import then does not mark the run stopped +and does not fire run-stop automations. Omit the flag on the last import. +The flag does nothing when the import has no run. ```sh sift-cli import csv ./part-1.csv --asset rover-1 --run sortie-4 --leave-run-open From 870e8ab1f8b9a8ee351f444d7ee0b344c82c5baf Mon Sep 17 00:00:00 2001 From: Ian Later Date: Fri, 13 Nov 2026 14:48:39 -0800 Subject: [PATCH 7/9] python(feat): set a config per file in one import A list of paths still uploads as one zip. A mapping of path to config overrides detection for the files it names. The batch upload also drops the manual temp-directory cleanup and the data-type cast. --- python/changelog.d/848.feat.md | 2 +- .../low_level_wrappers/data_imports.py | 4 +- .../_tests/resources/test_data_imports.py | 79 +++++++++ .../sift_client/_tests/sift_types/test_run.py | 14 ++ .../lib/sift_client/resources/data_imports.py | 156 +++++++++++++----- .../resources/sync_stubs/__init__.pyi | 50 ++++-- python/lib/sift_client/sift_types/run.py | 8 +- 7 files changed, 246 insertions(+), 67 deletions(-) diff --git a/python/changelog.d/848.feat.md b/python/changelog.d/848.feat.md index 0ee9053c17..6977a3b7e3 100644 --- a/python/changelog.d/848.feat.md +++ b/python/changelog.d/848.feat.md @@ -1 +1 @@ -`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false) and a list of paths. When you have more than one file for a run, pass them in one call. That upload is one import into one run. A finished import stops its run and fires run-stop automations unless you set `leave_run_open`, then call `run.stop()` after the last import. +`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false) and a list of paths. When you have more than one file for a run, pass them in one call. That upload is one import into one run. Pass a mapping of path to config to set a file's config yourself. Files you leave out of the mapping are detected. A finished import stops its run and fires run-stop automations unless you set `leave_run_open`, then call `run.stop()` after the last import. diff --git a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py index 2bd3599c4d..31cffb22f1 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/data_imports.py @@ -53,7 +53,7 @@ class _ConfigUploadField(NamedTuple): config_type: type[ImportConfig] field_name: str - type_key: int + type_key: DataTypeKey.ValueType _CONFIG_UPLOAD_FIELDS: tuple[_ConfigUploadField, ...] = ( @@ -111,7 +111,7 @@ def _batch_file_config(filename: str, config: ImportConfig) -> BatchFileConfig: file_proto.asset_name = "" file_proto.run_name = "" file_proto.run_id = "" - batch_file = BatchFileConfig(type=cast("DataTypeKey.ValueType", field.type_key)) + batch_file = BatchFileConfig(type=field.type_key) getattr(batch_file, field.field_name).CopyFrom(file_proto) return batch_file diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 1d04a3b30b..75aa9e4650 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2179,3 +2179,82 @@ async def test_missing_asset_raises_before_detect(self, tmp_path): await api.import_from_path([first, second], show_progress=False) detect.assert_not_awaited() + + @pytest.mark.asyncio + async def test_mapping_sets_one_file_and_detects_the_other(self, tmp_path, monkeypatch): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api, detect, create_from_upload, create_batch = self._api() + supplied = self._csv() + supplied.data_columns[0].name = "supplied" + detected = self._csv() + detected.data_columns[0].name = "detected" + detect.return_value = detected + + await api.import_from_path( + [first, second], + asset="my_asset", + config={first.name: supplied}, + show_progress=False, + ) + + detect.assert_awaited_once() + assert detect.await_args is not None + assert detect.await_args.args[0] == second + create_from_upload.assert_not_awaited() + call = create_batch.await_args + assert call is not None + configs = call.args[0] + assert configs["a.csv"] is supplied + assert configs["b.csv"] is detected + assert call.kwargs["asset_name"] == "my_asset" + + @pytest.mark.asyncio + async def test_mapping_rejects_an_unknown_key_and_a_bare_config(self, tmp_path): + first = tmp_path / "a.csv" + second = tmp_path / "b.csv" + first.write_text("t,v\n") + second.write_text("t,v\n") + api, detect, _, _ = self._api() + + with pytest.raises(ValueError, match="does not match"): + await api.import_from_path( + [first, second], + asset="my_asset", + config={"missing.csv": self._csv()}, + show_progress=False, + ) + with pytest.raises(ValueError, match="one file"): + await api.import_from_path( + [first, second], + asset="my_asset", + config=self._csv(), + show_progress=False, + ) + detect.assert_not_awaited() + + @pytest.mark.asyncio + async def test_one_path_accepts_a_config_or_a_mapping(self, tmp_path, monkeypatch): + path = tmp_path / "a.csv" + path.write_text("t,v\n") + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *_a, **_k: {"jobId": "j1"} + ) + api, detect, create_from_upload, create_batch = self._api() + supplied = self._csv() + + await api.import_from_path([path], asset="my_asset", config=supplied, show_progress=False) + await api.import_from_path( + path, asset="my_asset", config={path.name: supplied}, show_progress=False + ) + + detect.assert_not_awaited() + create_batch.assert_not_awaited() + assert create_from_upload.await_count == 2 + assert create_from_upload.await_args_list[0].args[0] is supplied + assert create_from_upload.await_args_list[1].args[0] is supplied diff --git a/python/lib/sift_client/_tests/sift_types/test_run.py b/python/lib/sift_client/_tests/sift_types/test_run.py index 49246abadc..55e0a8ff35 100644 --- a/python/lib/sift_client/_tests/sift_types/test_run.py +++ b/python/lib/sift_client/_tests/sift_types/test_run.py @@ -189,6 +189,20 @@ def test_import_data_passes_a_list_of_paths(self, mock_run): leave_run_open=True, ) + def test_import_data_passes_a_config_mapping(self, mock_run): + paths = ["a.csv", "b.csv"] + configs = {"a.csv": object()} + mock_run.import_data(paths, config=configs) + mock_run.client.data_import.import_from_path.assert_called_once_with( + paths, + asset=None, + config=configs, + data_type=None, + run=mock_run, + show_progress=None, + leave_run_open=False, + ) + def test_assets_property_calls_client(self, mock_run, mock_client): """Test that assets property calls client.assets.list_ with correct parameters.""" mock_client.assets.list_.return_value = [] diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 149245b80e..7513b1e633 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,8 +1,8 @@ from __future__ import annotations -import shutil import tempfile import zipfile +from collections.abc import Mapping from pathlib import Path from typing import TYPE_CHECKING @@ -25,6 +25,7 @@ DataImport, DataTypeKey, ImportConfig, + ImportConfigBase, McapImportConfig, ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig, @@ -66,6 +67,56 @@ def _run_id_and_name( return None, default_name +def _config_key_matches(path: Path, key: str | Path) -> bool: + """Whether a mapping key names this path. + + A key matches the path itself, or the file name when the key has no + directory. Names in one import are distinct, so a file name matches + one path. + """ + key_path = key if isinstance(key, Path) else Path(key) + if path == key_path: + return True + return key_path.parent == Path(".") and key_path.name == path.name + + +def _supplied_configs( + paths: list[Path], + config: ImportConfig | Mapping[str | Path, ImportConfig] | None, +) -> dict[Path, ImportConfig | None]: + """Return each path's config. None means detect that file. + + One import config applies to a single path. A mapping key is a path + from the list, or that file's name. A path with no entry is detected. + """ + if config is None: + return dict.fromkeys(paths) + if isinstance(config, ImportConfigBase): + if len(paths) != 1: + raise ValueError( + "'config' applies to one file. Pass a mapping of path to " + "config for a list, or omit it to detect each file." + ) + return {paths[0]: config} + if not isinstance(config, Mapping): + raise ValueError("'config' must be an import config or a mapping of path to config.") + + assigned: dict[Path, ImportConfig] = {} + for key, value in config.items(): + if not isinstance(value, ImportConfigBase): + raise ValueError(f"Config for {key!s} must be an import config.") + matches = [path for path in paths if _config_key_matches(path, key)] + if not matches: + raise ValueError(f"Config key {key!s} does not match a file in this import.") + if len(matches) > 1: + raise ValueError(f"Config key {key!s} matches more than one file.") + path = matches[0] + if path in assigned: + raise ValueError(f"More than one config matches {path}.") + assigned[path] = value + return {path: assigned.get(path) for path in paths} + + def _paths_from_argument(file_path: str | Path | Sequence[str | Path]) -> list[Path]: """Return the paths to import. @@ -102,7 +153,7 @@ async def import_from_path( file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, - config: ImportConfig | None = None, + config: ImportConfig | Mapping[str | Path, ImportConfig] | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, relative_start_time: pd.Timestamp | datetime | str | None = None, @@ -122,11 +173,11 @@ async def import_from_path( If you have more than one file for the same run, pass the paths in one call. The client packs them into one zip and imports them together into one asset and one run. Each file is detected on its - own. ``data_type`` applies only to files whose extension does not - name a format, such as Parquet and HDF5. Use ``leave_run_open`` - when more files are still coming, then call ``Run.stop`` after the - last import. A finished import stops its run unless that flag is - set. + own, unless you pass a config for that path. ``data_type`` applies + only to detected files whose extension does not name a format, such + as Parquet and HDF5. Use ``leave_run_open`` when more files are + still coming, then call ``Run.stop`` after the last import. A + finished import stops its run unless that flag is set. When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). @@ -167,6 +218,18 @@ async def import_from_path( config=config, ) + Set a config for one file in a list, and detect the other: + + paths = ["part-1.csv", "part-2.csv"] + configs = {path: client.data_import.detect_config(path) for path in paths} + configs["part-1.csv"]["temperature"].data_type = ChannelDataType.FLOAT + + job = client.data_import.import_from_path( + paths, + asset=my_asset, + config=configs, + ) + Args: file_path: Path to the local file to import, or a list of paths for one run. Pass every file in one call when they belong @@ -176,14 +239,16 @@ async def import_from_path( asset: Asset object or asset name to import data into. Optional when ``config`` already has ``asset_name`` set. Required for a list of paths. - config: Import configuration describing the file format and column - mapping. When provided, ``data_type`` is ignored. If omitted, - the config is auto-detected via ``detect_config`` (for ULog - and MCAP the detected channel list is dropped so every channel - in the file is imported). You can call ``detect_config`` - yourself to inspect and modify the config before passing it - here. A list of paths detects each file, so ``config`` is - only used for a single path. + config: Import configuration for one file, or a mapping of path + to configuration. A mapping key is a path from ``file_path`` + or that file's name. A file with no entry is detected. A key + that matches no file raises ``ValueError``. One config applies + to a single path. When a file's config is provided, + ``data_type`` is ignored for that file. If omitted, the config + is auto-detected via ``detect_config`` (for ULog and MCAP the + detected channel list is dropped so every channel in the file + is imported). You can call ``detect_config`` yourself to + inspect and modify the config before passing it here. data_type: Explicit data type key. Required for formats with multiple supported layouts (Parquet, HDF5) where the file extension alone is ambiguous. Only used when ``config`` is @@ -226,19 +291,22 @@ async def import_from_path( FileNotFoundError: If the file does not exist. ValueError: If a ``BIT_FIELD`` column's values aren't in a format the import service reads, such as integers. Also if a list of - paths is empty, contains duplicate file names, omits - ``asset``, or is passed with ``config``. + paths is empty, contains duplicate file names, or omits + ``asset``. Also if one config is passed with two or more + paths, or a config mapping names a path that is not in the + list. """ paths = _paths_from_argument(file_path) if not paths: raise ValueError("At least one file path is required.") if show_progress is None: show_progress = self._show_progress() + supplied = _supplied_configs(paths, config) if len(paths) > 1: return await self._import_many( paths, asset=asset, - config=config, + file_configs=supplied, data_type=data_type, time_format=time_format, relative_start_time=relative_start_time, @@ -248,6 +316,7 @@ async def import_from_path( leave_run_open=leave_run_open, ) path = paths[0] + config = supplied[path] if not path.is_file(): raise FileNotFoundError(f"File not found: {path}") @@ -285,7 +354,7 @@ async def _import_many( paths: list[Path], *, asset: Asset | str | None, - config: ImportConfig | None, + file_configs: dict[Path, ImportConfig | None], data_type: DataTypeKey | None, time_format: TimeFormat | None, relative_start_time: pd.Timestamp | datetime | str | None, @@ -294,11 +363,11 @@ async def _import_many( show_progress: bool, leave_run_open: bool, ) -> Job: - """Detect each file, zip them, and upload the archive as one import.""" - if config is not None: - raise ValueError( - "'config' applies to one file. Omit it to detect each file in the list." - ) + """Zip the files and upload the archive as one import. + + A path with no config is detected. ``asset``, ``run``, and + ``run_name`` apply to the whole import. + """ names = [path.name for path in paths] if len(set(names)) != len(names): raise ValueError( @@ -314,28 +383,29 @@ async def _import_many( if not path.is_file(): raise FileNotFoundError(f"File not found: {path}") - file_configs: dict[str, ImportConfig] = {} + resolved: dict[str, ImportConfig] = {} for path in paths: - # A known extension selects its format, so data_type can describe - # the Parquet or HDF5 files in a mixed list. - file_data_type = ( - None if path.suffix.lower() in EXTENSION_TO_DATA_TYPE_KEY else data_type - ) - detected = await self._detect_import_config( - path, - data_type=file_data_type, - time_format=time_format, - relative_start_time=relative_start_time, - ) - await self._prepare_import_file(detected, path) - file_configs[path.name] = detected + supplied = file_configs[path] + if supplied is None: + # A known extension selects its format, so data_type can describe + # the Parquet or HDF5 files in a mixed list. + file_data_type = ( + None if path.suffix.lower() in EXTENSION_TO_DATA_TYPE_KEY else data_type + ) + supplied = await self._detect_import_config( + path, + data_type=file_data_type, + time_format=time_format, + relative_start_time=relative_start_time, + ) + await self._prepare_import_file(supplied, path) + resolved[path.name] = supplied - directory = Path(tempfile.mkdtemp()) - archive = directory / "batch.zip" - try: + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "batch.zip" await run_sync_function(_write_import_zip, archive, paths) data_import_id, upload_url = await self._low_level_client.create_batch_from_upload( - file_configs, + resolved, asset_name=asset_name, run_name=resolved_run_name, run_id=resolved_run_id, @@ -344,8 +414,6 @@ async def _import_many( return await self._job_for_upload( archive, data_import_id, upload_url, show_progress=show_progress ) - finally: - shutil.rmtree(directory, ignore_errors=True) async def _job_for_upload( self, diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 18e4dfa4a3..67e4d11a87 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: import re - from collections.abc import Iterable + from collections.abc import Iterable, Mapping from datetime import datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO, Sequence @@ -1748,7 +1748,7 @@ class DataImportAPI: file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, - config: ImportConfig | None = None, + config: ImportConfig | Mapping[str | Path, ImportConfig] | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, relative_start_time: pd.Timestamp | datetime | str | None = None, @@ -1768,11 +1768,11 @@ class DataImportAPI: If you have more than one file for the same run, pass the paths in one call. The client packs them into one zip and imports them together into one asset and one run. Each file is detected on its - own. ``data_type`` applies only to files whose extension does not - name a format, such as Parquet and HDF5. Use ``leave_run_open`` - when more files are still coming, then call ``Run.stop`` after the - last import. A finished import stops its run unless that flag is - set. + own, unless you pass a config for that path. ``data_type`` applies + only to detected files whose extension does not name a format, such + as Parquet and HDF5. Use ``leave_run_open`` when more files are + still coming, then call ``Run.stop`` after the last import. A + finished import stops its run unless that flag is set. When ``config`` is omitted the file format is auto-detected via ``detect_config`` (CSV, Parquet, HDF5, TDMS, ULog, and MCAP). @@ -1813,6 +1813,18 @@ class DataImportAPI: config=config, ) + Set a config for one file in a list, and detect the other: + + paths = ["part-1.csv", "part-2.csv"] + configs = {path: client.data_import.detect_config(path) for path in paths} + configs["part-1.csv"]["temperature"].data_type = ChannelDataType.FLOAT + + job = client.data_import.import_from_path( + paths, + asset=my_asset, + config=configs, + ) + Args: file_path: Path to the local file to import, or a list of paths for one run. Pass every file in one call when they belong @@ -1822,14 +1834,16 @@ class DataImportAPI: asset: Asset object or asset name to import data into. Optional when ``config`` already has ``asset_name`` set. Required for a list of paths. - config: Import configuration describing the file format and column - mapping. When provided, ``data_type`` is ignored. If omitted, - the config is auto-detected via ``detect_config`` (for ULog - and MCAP the detected channel list is dropped so every channel - in the file is imported). You can call ``detect_config`` - yourself to inspect and modify the config before passing it - here. A list of paths detects each file, so ``config`` is - only used for a single path. + config: Import configuration for one file, or a mapping of path + to configuration. A mapping key is a path from ``file_path`` + or that file's name. A file with no entry is detected. A key + that matches no file raises ``ValueError``. One config applies + to a single path. When a file's config is provided, + ``data_type`` is ignored for that file. If omitted, the config + is auto-detected via ``detect_config`` (for ULog and MCAP the + detected channel list is dropped so every channel in the file + is imported). You can call ``detect_config`` yourself to + inspect and modify the config before passing it here. data_type: Explicit data type key. Required for formats with multiple supported layouts (Parquet, HDF5) where the file extension alone is ambiguous. Only used when ``config`` is @@ -1872,8 +1886,10 @@ class DataImportAPI: FileNotFoundError: If the file does not exist. ValueError: If a ``BIT_FIELD`` column's values aren't in a format the import service reads, such as integers. Also if a list of - paths is empty, contains duplicate file names, omits - ``asset``, or is passed with ``config``. + paths is empty, contains duplicate file names, or omits + ``asset``. Also if one config is passed with two or more + paths, or a config mapping names a path that is not in the + list. """ ... diff --git a/python/lib/sift_client/sift_types/run.py b/python/lib/sift_client/sift_types/run.py index e2bbfd9b47..f2df1028e3 100644 --- a/python/lib/sift_client/sift_types/run.py +++ b/python/lib/sift_client/sift_types/run.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: from pathlib import Path - from typing import Sequence + from typing import Mapping, Sequence from sift_stream_bindings import RunFormPy @@ -157,7 +157,7 @@ def import_data( file_path: str | Path | Sequence[str | Path], *, asset: Asset | str | None = None, - config: ImportConfig | None = None, + config: ImportConfig | Mapping[str | Path, ImportConfig] | None = None, data_type: DataTypeKey | None = None, show_progress: bool | None = None, leave_run_open: bool = False, @@ -178,7 +178,9 @@ def import_data( Pass every file in one call when they belong together. Two or more paths are uploaded as one zip. asset: Asset object or asset name to import data into. - config: Import configuration. Auto-detected if omitted. + config: Import configuration for one file, or a mapping of path + to configuration. A file with no entry is detected. + Auto-detected when omitted. data_type: Explicit data type key for ambiguous formats. show_progress: Display a progress spinner during upload. leave_run_open: If true, the import does not mark this run From ef7337b0f1c50986a9565be77974c4f41f379989 Mon Sep 17 00:00:00 2001 From: Ian Later Date: Fri, 13 Nov 2026 14:48:55 -0800 Subject: [PATCH 8/9] rust(chore): keep the CLI flag out of the Python release Python and the CLI release separately. The flag ships in a pull request stacked on this one. --- rust/crates/sift_cli/CHANGELOG.md | 6 +---- .../assets/docs/src/data/import-csv.md | 3 --- .../assets/docs/src/data/import-mcap.md | 3 --- .../assets/docs/src/data/import-tdms.md | 3 --- .../assets/docs/src/data/import-ulog.md | 3 --- .../assets/docs/src/data/importing.md | 24 ------------------- rust/crates/sift_cli/src/cli/mod.rs | 18 -------------- rust/crates/sift_cli/src/cmd/import/csv.rs | 6 ----- .../sift_cli/src/cmd/import/hdf5/import.rs | 9 ++----- .../sift_cli/src/cmd/import/hdf5/tests.rs | 1 - .../sift_cli/src/cmd/import/mcap/import.rs | 9 ++----- .../sift_cli/src/cmd/import/mcap/tests.rs | 1 - .../import/parquet/channel_per_row_dataset.rs | 1 - .../src/cmd/import/parquet/flat_dataset.rs | 1 - .../sift_cli/src/cmd/import/parquet/tests.rs | 2 -- .../src/cmd/import/tdms/detect_tdms_config.rs | 9 ++----- .../sift_cli/src/cmd/import/tdms/tests.rs | 1 - .../sift_cli/src/cmd/import/ulog/import.rs | 9 ++----- .../sift_cli/src/cmd/import/ulog/tests.rs | 1 - 19 files changed, 9 insertions(+), 101 deletions(-) diff --git a/rust/crates/sift_cli/CHANGELOG.md b/rust/crates/sift_cli/CHANGELOG.md index 1b9ef48672..44b8cd35bb 100644 --- a/rust/crates/sift_cli/CHANGELOG.md +++ b/rust/crates/sift_cli/CHANGELOG.md @@ -19,11 +19,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - MCP `create_declarative_chart` takes an optional `output` path. On any surface, it writes the delivered spec to that local file. Sending the spec just delivered again, to save it, repeats no request to Sift. - -- `import` accepts `--leave-run-open`. A finished import marks its run stopped - and fires run-stop automations unless the flag is set. Stop the run after - the last import. The flag does nothing when the import has no run. - + ## [v0.9.0] - November 12, 2026 ### What's New diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-csv.md b/rust/crates/sift_cli/assets/docs/src/data/import-csv.md index fbfe8a184f..e63171df6c 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-csv.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-csv.md @@ -32,9 +32,6 @@ sift-cli import csv ./telemetry.csv \ --wait ``` -If you have more than one file for that run, import them together. See -[Several files in one run](./importing.md#several-files-in-one-run). - ## Pointing at the right rows and time column If your file has metadata rows above the header, or the timestamp is not the diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md b/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md index 6dadaef198..c666949ae7 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-mcap.md @@ -27,9 +27,6 @@ sift-cli import mcap ./recording.mcap \ --wait ``` -If you have more than one file for that run, import them together. See -[Several files in one run](./importing.md#several-files-in-one-run). - ## Previewing the detected channels `--preview` (`-p`) parses the file locally and prints the channels the import diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md b/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md index a6bcc3f118..c1de6ca6a7 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-tdms.md @@ -23,9 +23,6 @@ sift-cli import tdms ./measurement.tdms \ --wait ``` -If you have more than one file for that run, import them together. See -[Several files in one run](./importing.md#several-files-in-one-run). - ## Channels with missing timing When a channel lacks timing metadata, choose how the CLI should respond with diff --git a/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md b/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md index b26e592f45..d313c8a17f 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md +++ b/rust/crates/sift_cli/assets/docs/src/data/import-ulog.md @@ -24,9 +24,6 @@ sift-cli import ulog ./flight.ulg \ --wait ``` -If you have more than one file for that run, import them together. See -[Several files in one run](./importing.md#several-files-in-one-run). - ## Anchoring timestamps ULog records boot-relative timestamps. The import anchors them to UTC using diff --git a/rust/crates/sift_cli/assets/docs/src/data/importing.md b/rust/crates/sift_cli/assets/docs/src/data/importing.md index 3122da8b82..9ef21c1e7b 100644 --- a/rust/crates/sift_cli/assets/docs/src/data/importing.md +++ b/rust/crates/sift_cli/assets/docs/src/data/importing.md @@ -53,30 +53,6 @@ queryable immediately afterward. sift-cli import csv ./telemetry.csv --asset rover-1 --wait ``` -### Several files in one run - -If the files are available together, import them in one call from the Python -client. Pass the paths to `client.data_import.import_from_path` or -`Run.import_data`. That upload is one import into one run. - -```python -client.data_import.import_from_path( - ["part-1.csv", "part-2.csv"], - asset="rover-1", - run_name="sortie-4", -) -``` - -The CLI imports one file per command. When a later file is not available yet, -pass `--leave-run-open`. A finished import then does not mark the run stopped -and does not fire run-stop automations. Omit the flag on the last import. -The flag does nothing when the import has no run. - -```sh -sift-cli import csv ./part-1.csv --asset rover-1 --run sortie-4 --leave-run-open -sift-cli import csv ./part-2.csv --asset rover-1 --run sortie-4 -``` - ### Time formats Imports that parse a time column accept a `--time-format` describing how diff --git a/rust/crates/sift_cli/src/cli/mod.rs b/rust/crates/sift_cli/src/cli/mod.rs index e1c3ffcbca..2387c9319a 100644 --- a/rust/crates/sift_cli/src/cli/mod.rs +++ b/rust/crates/sift_cli/src/cli/mod.rs @@ -486,15 +486,6 @@ pub struct ImportCsvArgs { /// Preview the parsed schema without uploading #[arg(short, long)] pub preview: bool, - - /// Leave the run open when the import finishes. - /// - /// By default a finished import marks its run stopped, which fires - /// run-stop automations. Set this when more files are coming to the run, - /// then stop the run after the last import. Ignored when the import has - /// no run. - #[arg(long)] - pub leave_run_open: bool, } #[derive(clap::Args)] @@ -553,15 +544,6 @@ pub struct CommonImportArgs { /// Preview the parsed schema without uploading #[arg(short, long)] pub preview: bool, - - /// Leave the run open when the import finishes. - /// - /// By default a finished import marks its run stopped, which fires - /// run-stop automations. Set this when more files are coming to the run, - /// then stop the run after the last import. Ignored when the import has - /// no run. - #[arg(long)] - pub leave_run_open: bool, } #[derive(Subcommand)] diff --git a/rust/crates/sift_cli/src/cmd/import/csv.rs b/rust/crates/sift_cli/src/cmd/import/csv.rs index 51d292b2c2..a6a882bbe2 100644 --- a/rust/crates/sift_cli/src/cmd/import/csv.rs +++ b/rust/crates/sift_cli/src/cmd/import/csv.rs @@ -345,7 +345,6 @@ fn create_data_import_request( }), ..Default::default() }), - leave_run_open: args.leave_run_open, ..Default::default() }) } @@ -391,12 +390,10 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, - leave_run_open: true, }, ) .expect("expected Result::Ok"); - assert!(req.leave_run_open); let csv_config = req.csv_config.expect("expected Option::Some"); assert_eq!(String::from("test_asset"), csv_config.asset_name); assert!(csv_config.run_id.is_empty()); @@ -443,7 +440,6 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, - leave_run_open: false, }, ) .expect("expected Result::Ok"); @@ -488,7 +484,6 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, - leave_run_open: false, }, ) .expect("expected Result::Ok"); @@ -549,7 +544,6 @@ mod test_create_data_import_request { relative_start_time: None, wait: false, preview: false, - leave_run_open: false, }, ) .expect("expected Result::Ok"); diff --git a/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs b/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs index d3e189431f..803883eefe 100644 --- a/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs +++ b/rust/crates/sift_cli/src/cmd/import/hdf5/import.rs @@ -98,8 +98,7 @@ pub async fn run(ctx: Context, args: ImportHdf5Args) -> Result { let file = File::open(&args.common.path).context("failed to open hdf5 file")?; - let create_data_import_req = - create_data_import_request(hdf5_config, args.common.leave_run_open); + let create_data_import_req = create_data_import_request(hdf5_config); let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -155,13 +154,9 @@ pub fn build_hdf5_config(args: &ImportHdf5Args) -> Result { }) } -fn create_data_import_request( - config: Hdf5Config, - leave_run_open: bool, -) -> CreateDataImportFromUploadRequest { +fn create_data_import_request(config: Hdf5Config) -> CreateDataImportFromUploadRequest { CreateDataImportFromUploadRequest { hdf5_config: Some(config), - leave_run_open, ..Default::default() } } diff --git a/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs b/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs index 54b5a49608..1f05fa45a4 100644 --- a/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/hdf5/tests.rs @@ -23,7 +23,6 @@ fn make_args() -> ImportHdf5Args { run_id: None, wait: false, preview: false, - leave_run_open: false, }, schema: Hdf5Schema::OneD, time_format: Some(TimeFormat::AbsoluteRfc3339), diff --git a/rust/crates/sift_cli/src/cmd/import/mcap/import.rs b/rust/crates/sift_cli/src/cmd/import/mcap/import.rs index 60da12a361..99096b4588 100644 --- a/rust/crates/sift_cli/src/cmd/import/mcap/import.rs +++ b/rust/crates/sift_cli/src/cmd/import/mcap/import.rs @@ -72,8 +72,7 @@ pub async fn run(ctx: Context, args: ImportMcapArgs) -> Result { let file = File::open(&args.common.path).context("failed to open mcap file")?; - let create_data_import_req = - create_data_import_request(mcap_config, args.common.leave_run_open); + let create_data_import_req = create_data_import_request(mcap_config); let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -180,13 +179,9 @@ pub fn channel_configs( configs } -fn create_data_import_request( - config: McapConfig, - leave_run_open: bool, -) -> CreateDataImportFromUploadRequest { +fn create_data_import_request(config: McapConfig) -> CreateDataImportFromUploadRequest { CreateDataImportFromUploadRequest { mcap_config: Some(config), - leave_run_open, ..Default::default() } } diff --git a/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs b/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs index 1469b2b622..bee76998b3 100644 --- a/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/mcap/tests.rs @@ -24,7 +24,6 @@ fn make_args() -> ImportMcapArgs { run_id: None, wait: false, preview: false, - leave_run_open: false, }, relative_start_time: None, metadata_record: Vec::new(), diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs b/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs index a4f0f8f8e4..5d7ccc738b 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/channel_per_row_dataset.rs @@ -155,7 +155,6 @@ fn create_data_import_request( .into(), config: config.config, }), - leave_run_open: args.common.leave_run_open, ..Default::default() }) } diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs b/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs index 8e7adc36c8..4fbba63017 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/flat_dataset.rs @@ -255,7 +255,6 @@ fn create_data_import_request( .into(), config: config.config, }), - leave_run_open: args.common.leave_run_open, ..Default::default() }; Ok(req) diff --git a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs index 39fc0572e4..53939c2234 100644 --- a/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/parquet/tests.rs @@ -28,7 +28,6 @@ fn make_test_args(time_path: Option<&str>, time_format: Option) -> F run_id: None, wait: false, preview: false, - leave_run_open: false, }, channel_path: vec![], data_type: vec![], @@ -552,7 +551,6 @@ fn make_channel_per_row_args( run_id: None, wait: false, preview: false, - leave_run_open: false, }, mode, time_path: time_path.map(|s| s.to_string()), diff --git a/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs b/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs index 9e4ae6ce4a..62d4cf54dd 100644 --- a/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs +++ b/rust/crates/sift_cli/src/cmd/import/tdms/detect_tdms_config.rs @@ -59,8 +59,7 @@ pub async fn run(ctx: Context, args: ImportTdmsArgs) -> Result { let file = File::open(&args.common.path).context("failed to open tdms file")?; let create_data_import_req = - create_data_import_request(tdms_config, args.common.leave_run_open) - .context("failed to create data import req")?; + create_data_import_request(tdms_config).context("failed to create data import req")?; let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -134,13 +133,9 @@ pub fn build_tdms_config(args: &ImportTdmsArgs) -> Result { }) } -fn create_data_import_request( - config: TdmsConfig, - leave_run_open: bool, -) -> Result { +fn create_data_import_request(config: TdmsConfig) -> Result { let req = CreateDataImportFromUploadRequest { tdms_config: Some(config), - leave_run_open, ..Default::default() }; Ok(req) diff --git a/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs b/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs index 31b5c7e6b6..2b90cff5f6 100644 --- a/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/tdms/tests.rs @@ -23,7 +23,6 @@ fn make_args() -> ImportTdmsArgs { run_id: None, wait: false, preview: false, - leave_run_open: false, }, start_time_override: None, fallback_method: TdmsFallbackMethod::FailOnError, diff --git a/rust/crates/sift_cli/src/cmd/import/ulog/import.rs b/rust/crates/sift_cli/src/cmd/import/ulog/import.rs index fa0d37ea28..ea80b46785 100644 --- a/rust/crates/sift_cli/src/cmd/import/ulog/import.rs +++ b/rust/crates/sift_cli/src/cmd/import/ulog/import.rs @@ -55,8 +55,7 @@ pub async fn run(ctx: Context, args: ImportUlogArgs) -> Result { let file = File::open(&args.common.path).context("failed to open ulog file")?; - let create_data_import_req = - create_data_import_request(ulog_config, args.common.leave_run_open); + let create_data_import_req = create_data_import_request(ulog_config); let CreateDataImportFromUploadResponse { upload_url, .. } = data_imports_client .create_data_import_from_upload(create_data_import_req) @@ -115,13 +114,9 @@ pub fn build_ulog_config(args: &ImportUlogArgs) -> Result { }) } -fn create_data_import_request( - config: UlogConfig, - leave_run_open: bool, -) -> CreateDataImportFromUploadRequest { +fn create_data_import_request(config: UlogConfig) -> CreateDataImportFromUploadRequest { CreateDataImportFromUploadRequest { ulog_config: Some(config), - leave_run_open, ..Default::default() } } diff --git a/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs b/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs index b16b21f60d..fa66a8d781 100644 --- a/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs +++ b/rust/crates/sift_cli/src/cmd/import/ulog/tests.rs @@ -19,7 +19,6 @@ fn make_args() -> ImportUlogArgs { run_id: None, wait: false, preview: false, - leave_run_open: false, }, relative_start_time: None, info_key: Vec::new(), From 1be40905fb40baa2e691c62849576dc72b3d79e5 Mon Sep 17 00:00:00 2001 From: Ian Later Date: Mon, 16 Nov 2026 11:48:54 -0800 Subject: [PATCH 9/9] python(chore): resolve one import, then choose the upload Config matching and the zip live with the other import helpers, so tests call them without a client. One file and several files share config, asset, and run resolution, and differ only at the upload. --- .../_internal/util/data_imports.py | 56 ++++- python/lib/sift_client/_internal/util/file.py | 7 + .../_internal/util/test_data_imports.py | 68 +++++ .../_tests/_internal/util/test_file.py | 24 +- .../lib/sift_client/resources/data_imports.py | 232 +++++++----------- 5 files changed, 242 insertions(+), 145 deletions(-) create mode 100644 python/lib/sift_client/_tests/_internal/util/test_data_imports.py diff --git a/python/lib/sift_client/_internal/util/data_imports.py b/python/lib/sift_client/_internal/util/data_imports.py index e5a5af4478..8e89cb60ee 100644 --- a/python/lib/sift_client/_internal/util/data_imports.py +++ b/python/lib/sift_client/_internal/util/data_imports.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Mapping +from pathlib import Path from typing import TYPE_CHECKING from sift_client._internal.time import to_timestamp_pb @@ -12,6 +14,7 @@ from sift_client.sift_types.data_import import ( EXTENSION_TO_DATA_TYPE_KEY, CsvImportConfig, + ImportConfigBase, ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig, ParquetTimeColumn, @@ -20,9 +23,8 @@ if TYPE_CHECKING: from datetime import datetime - from pathlib import Path - from sift_client.sift_types.data_import import DataTypeKey + from sift_client.sift_types.data_import import DataTypeKey, ImportConfig _RELATIVE_FORMAT_VALUES = {f.value for f in TimeFormat if f.is_relative} @@ -129,3 +131,53 @@ async def prepare_parquet_config( footer_bytes, footer_offset = await run_sync_function(lambda: extract_parquet_footer(path)) config.footer_offset = footer_offset config.footer_length = len(footer_bytes) + + +def _config_key_matches(path: Path, key: str | Path) -> bool: + """Whether a mapping key names this path. + + A key matches the path itself, or the file name when the key has no + directory. Names in one import are distinct, so a file name matches + one path. + """ + key_path = key if isinstance(key, Path) else Path(key) + if path == key_path: + return True + return key_path.parent == Path(".") and key_path.name == path.name + + +def _supplied_configs( + paths: list[Path], + config: ImportConfig | Mapping[str | Path, ImportConfig] | None, +) -> dict[Path, ImportConfig | None]: + """Return each path's config. None means detect that file. + + One import config applies to a single path. A mapping key is a path + from the list, or that file's name. A path with no entry is detected. + """ + if config is None: + return dict.fromkeys(paths) + if isinstance(config, ImportConfigBase): + if len(paths) != 1: + raise ValueError( + "'config' applies to one file. Pass a mapping of path to " + "config for a list, or omit it to detect each file." + ) + return {paths[0]: config} + if not isinstance(config, Mapping): + raise ValueError("'config' must be an import config or a mapping of path to config.") + + assigned: dict[Path, ImportConfig] = {} + for key, value in config.items(): + if not isinstance(value, ImportConfigBase): + raise ValueError(f"Config for {key!s} must be an import config.") + matches = [path for path in paths if _config_key_matches(path, key)] + if not matches: + raise ValueError(f"Config key {key!s} does not match a file in this import.") + if len(matches) > 1: + raise ValueError(f"Config key {key!s} matches more than one file.") + path = matches[0] + if path in assigned: + raise ValueError(f"More than one config matches {path}.") + assigned[path] = value + return {path: assigned.get(path) for path in paths} diff --git a/python/lib/sift_client/_internal/util/file.py b/python/lib/sift_client/_internal/util/file.py index b1606d1b54..eb8a0e2982 100644 --- a/python/lib/sift_client/_internal/util/file.py +++ b/python/lib/sift_client/_internal/util/file.py @@ -226,3 +226,10 @@ def extract_parquet_footer(path: Path) -> tuple[bytes, int]: f.seek(-(footer_len + 8), 2) footer_bytes = f.read(footer_len) return footer_bytes, os.path.getsize(path) - len(footer_bytes) - 8 + + +def _write_import_zip(archive: Path, paths: list[Path]) -> None: + """Pack each file into the archive under its own file name.""" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zip_file: + for path in paths: + zip_file.write(path, arcname=path.name) diff --git a/python/lib/sift_client/_tests/_internal/util/test_data_imports.py b/python/lib/sift_client/_tests/_internal/util/test_data_imports.py new file mode 100644 index 0000000000..e19d65f041 --- /dev/null +++ b/python/lib/sift_client/_tests/_internal/util/test_data_imports.py @@ -0,0 +1,68 @@ +"""Tests for data import helpers that do not need a client.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sift_client._internal.util.data_imports import _config_key_matches, _supplied_configs +from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.data_import import ( + CsvDataColumn, + CsvImportConfig, + CsvTimeColumn, + TimeFormat, +) + + +def _csv(name: str = "v") -> CsvImportConfig: + return CsvImportConfig( + asset_name="asset", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + data_columns=[CsvDataColumn(column=2, name=name, data_type=ChannelDataType.DOUBLE)], + ) + + +class TestConfigKeyMatches: + def test_matches_the_path_or_a_bare_file_name(self): + path = Path("dir/a.csv") + + assert _config_key_matches(path, path) + assert _config_key_matches(path, "a.csv") + assert not _config_key_matches(path, "other/a.csv") + assert not _config_key_matches(path, "b.csv") + + +class TestSuppliedConfigs: + def test_none_detects_every_path(self): + paths = [Path("a.csv"), Path("b.csv")] + + assert _supplied_configs(paths, None) == {paths[0]: None, paths[1]: None} + + def test_one_config_applies_to_one_path(self): + path = Path("a.csv") + config = _csv() + + assert _supplied_configs([path], config) == {path: config} + + def test_one_config_rejects_several_paths(self): + with pytest.raises(ValueError, match="one file"): + _supplied_configs([Path("a.csv"), Path("b.csv")], _csv()) + + def test_mapping_assigns_by_name_and_leaves_the_rest_to_detect(self): + first = Path("dir/a.csv") + second = Path("dir/b.csv") + config = _csv("supplied") + + assigned = _supplied_configs([first, second], {"a.csv": config}) + + assert assigned[first] is config + assert assigned[second] is None + + def test_unknown_key_and_two_keys_for_one_path_raise(self): + path = Path("a.csv") + with pytest.raises(ValueError, match="does not match"): + _supplied_configs([path], {"missing.csv": _csv()}) + with pytest.raises(ValueError, match="More than one config"): + _supplied_configs([path], {path: _csv(), "a.csv": _csv()}) diff --git a/python/lib/sift_client/_tests/_internal/util/test_file.py b/python/lib/sift_client/_tests/_internal/util/test_file.py index 5c4f1401f8..139cd8a092 100644 --- a/python/lib/sift_client/_tests/_internal/util/test_file.py +++ b/python/lib/sift_client/_tests/_internal/util/test_file.py @@ -1,11 +1,17 @@ from __future__ import annotations +import zipfile from unittest.mock import MagicMock import pytest from requests import HTTPError, Request, Response -from sift_client._internal.util.file import download_file, read_url_bytes, upload_file +from sift_client._internal.util.file import ( + _write_import_zip, + download_file, + read_url_bytes, + upload_file, +) @pytest.fixture @@ -146,3 +152,19 @@ def test_returns_body_without_session_auth(self): assert data == b"hello world" assert captured["url"] == "https://example.test/signed" assert captured["headers"] == {"Authorization": None} + + +class TestWriteImportZip: + def test_packs_each_file_under_its_name(self, tmp_path): + first = tmp_path / "a.csv" + second = tmp_path / "nested" / "b.csv" + second.parent.mkdir() + first.write_text("a") + second.write_text("b") + archive = tmp_path / "batch.zip" + + _write_import_zip(archive, [first, second]) + + with zipfile.ZipFile(archive) as zip_file: + assert zip_file.read("a.csv") == b"a" + assert zip_file.read("b.csv") == b"b" diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 7513b1e633..2c900c25aa 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,8 +1,6 @@ from __future__ import annotations import tempfile -import zipfile -from collections.abc import Mapping from pathlib import Path from typing import TYPE_CHECKING @@ -10,13 +8,14 @@ from sift_client._internal.time import to_timestamp_nanos from sift_client._internal.util.bit_field import check_bit_field_values from sift_client._internal.util.data_imports import ( + _supplied_configs, parse_csv_detect_response, parse_parquet_detect_response, prepare_parquet_config, resolve_data_type_key, ) from sift_client._internal.util.executor import run_sync_function -from sift_client._internal.util.file import extract_parquet_footer, upload_file +from sift_client._internal.util.file import _write_import_zip, extract_parquet_footer, upload_file from sift_client.resources._base import ResourceBase from sift_client.sift_types.asset import Asset from sift_client.sift_types.data_import import ( @@ -25,7 +24,6 @@ DataImport, DataTypeKey, ImportConfig, - ImportConfigBase, McapImportConfig, ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig, @@ -37,6 +35,7 @@ from sift_client.util import cel_utils as cel if TYPE_CHECKING: + from collections.abc import Mapping from datetime import datetime from typing import Sequence @@ -67,54 +66,39 @@ def _run_id_and_name( return None, default_name -def _config_key_matches(path: Path, key: str | Path) -> bool: - """Whether a mapping key names this path. - - A key matches the path itself, or the file name when the key has no - directory. Names in one import are distinct, so a file name matches - one path. - """ - key_path = key if isinstance(key, Path) else Path(key) - if path == key_path: - return True - return key_path.parent == Path(".") and key_path.name == path.name - - -def _supplied_configs( +def _import_target( paths: list[Path], - config: ImportConfig | Mapping[str | Path, ImportConfig] | None, -) -> dict[Path, ImportConfig | None]: - """Return each path's config. None means detect that file. + configs: list[ImportConfig], + asset: Asset | str | None, + run: Run | str | None, + run_name: str | None, +) -> tuple[str, str | None, str | None]: + """Return the asset name, run id, and run name for this import. - One import config applies to a single path. A mapping key is a path - from the list, or that file's name. A path with no entry is detected. + One file uses the asset on its config when ``asset`` is omitted. Several + files require ``asset``. The run comes from ``run`` or ``run_name``. When + neither is set, one file keeps the run on its config, and several files + use the first file's name. """ - if config is None: - return dict.fromkeys(paths) - if isinstance(config, ImportConfigBase): - if len(paths) != 1: - raise ValueError( - "'config' applies to one file. Pass a mapping of path to " - "config for a list, or omit it to detect each file." - ) - return {paths[0]: config} - if not isinstance(config, Mapping): - raise ValueError("'config' must be an import config or a mapping of path to config.") - - assigned: dict[Path, ImportConfig] = {} - for key, value in config.items(): - if not isinstance(value, ImportConfigBase): - raise ValueError(f"Config for {key!s} must be an import config.") - matches = [path for path in paths if _config_key_matches(path, key)] - if not matches: - raise ValueError(f"Config key {key!s} does not match a file in this import.") - if len(matches) > 1: - raise ValueError(f"Config key {key!s} matches more than one file.") - path = matches[0] - if path in assigned: - raise ValueError(f"More than one config matches {path}.") - assigned[path] = value - return {path: assigned.get(path) for path in paths} + if asset is None: + asset_name = None + elif isinstance(asset, Asset): + asset_name = asset.name + else: + asset_name = asset + if len(paths) == 1: + config = configs[0] + if asset_name is None: + if not config.asset_name: + raise ValueError("'asset' is required when not set on the config.") + asset_name = config.asset_name + default_name = None if config.run_name or config.run_id else paths[0].name + else: + if asset_name is None: + raise ValueError("'asset' is required to import more than one file.") + default_name = paths[0].name + resolved_run_id, resolved_run_name = _run_id_and_name(run, run_name, default_name=default_name) + return asset_name, resolved_run_id, resolved_run_name def _paths_from_argument(file_path: str | Path | Sequence[str | Path]) -> list[Path]: @@ -127,13 +111,6 @@ def _paths_from_argument(file_path: str | Path | Sequence[str | Path]) -> list[P return [Path(path) for path in file_path] -def _write_import_zip(archive: Path, paths: list[Path]) -> None: - """Pack each file into the archive under its own file name.""" - with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zip_file: - for path in paths: - zip_file.write(path, arcname=path.name) - - class DataImportAPIAsync(ResourceBase): """High-level API for importing data into Sift.""" @@ -302,113 +279,60 @@ async def import_from_path( if show_progress is None: show_progress = self._show_progress() supplied = _supplied_configs(paths, config) - if len(paths) > 1: - return await self._import_many( - paths, - asset=asset, - file_configs=supplied, - data_type=data_type, - time_format=time_format, - relative_start_time=relative_start_time, - run=run, - run_name=run_name, - show_progress=show_progress, - leave_run_open=leave_run_open, - ) - path = paths[0] - config = supplied[path] - if not path.is_file(): - raise FileNotFoundError(f"File not found: {path}") - - if config is None: - config = await self._detect_import_config( - path, - data_type=data_type, - time_format=time_format, - relative_start_time=relative_start_time, - ) - - if asset is not None: - config.asset_name = asset.name if isinstance(asset, Asset) else asset - elif not config.asset_name: - raise ValueError("'asset' is required when not set on the config.") - default_name = None if config.run_name or config.run_id else path.name - run_id, resolved_run_name = _run_id_and_name(run, run_name, default_name=default_name) - if run_id is not None: - config.run_id = run_id - elif resolved_run_name is not None: - config.run_name = resolved_run_name - - await self._prepare_import_file(config, path) - - data_import_id, upload_url = await self._low_level_client.create_from_upload( - config, leave_run_open=leave_run_open - ) - - return await self._job_for_upload( - path, data_import_id, upload_url, show_progress=show_progress - ) - - async def _import_many( - self, - paths: list[Path], - *, - asset: Asset | str | None, - file_configs: dict[Path, ImportConfig | None], - data_type: DataTypeKey | None, - time_format: TimeFormat | None, - relative_start_time: pd.Timestamp | datetime | str | None, - run: Run | str | None, - run_name: str | None, - show_progress: bool, - leave_run_open: bool, - ) -> Job: - """Zip the files and upload the archive as one import. - - A path with no config is detected. ``asset``, ``run``, and - ``run_name`` apply to the whole import. - """ names = [path.name for path in paths] if len(set(names)) != len(names): raise ValueError( f"Files in one import must have distinct names. Got {', '.join(names)}." ) - if asset is None: - raise ValueError("'asset' is required to import more than one file.") - asset_name = asset.name if isinstance(asset, Asset) else asset - resolved_run_id, resolved_run_name = _run_id_and_name( - run, run_name, default_name=paths[0].name - ) for path in paths: if not path.is_file(): raise FileNotFoundError(f"File not found: {path}") - - resolved: dict[str, ImportConfig] = {} + if len(paths) > 1 and asset is None: + raise ValueError("'asset' is required to import more than one file.") + prepared: list[ImportConfig] = [] for path in paths: - supplied = file_configs[path] - if supplied is None: - # A known extension selects its format, so data_type can describe - # the Parquet or HDF5 files in a mixed list. - file_data_type = ( - None if path.suffix.lower() in EXTENSION_TO_DATA_TYPE_KEY else data_type - ) - supplied = await self._detect_import_config( + # A known extension selects its format, so data_type can describe + # the Parquet or HDF5 files in a mixed list. One file keeps the + # override the caller passed. + file_data_type = data_type + if len(paths) > 1 and path.suffix.lower() in EXTENSION_TO_DATA_TYPE_KEY: + file_data_type = None + prepared.append( + await self._config_for_upload( path, + supplied[path], data_type=file_data_type, time_format=time_format, relative_start_time=relative_start_time, ) - await self._prepare_import_file(supplied, path) - resolved[path.name] = supplied + ) + asset_name, run_id, resolved_run_name = _import_target( + paths, prepared, asset, run, run_name + ) + if len(paths) == 1: + file_config = prepared[0] + file_config.asset_name = asset_name + if run_id is not None: + file_config.run_id = run_id + elif resolved_run_name is not None: + file_config.run_name = resolved_run_name + data_import_id, upload_url = await self._low_level_client.create_from_upload( + file_config, leave_run_open=leave_run_open + ) + return await self._job_for_upload( + paths[0], data_import_id, upload_url, show_progress=show_progress + ) + + named = {path.name: file_config for path, file_config in zip(paths, prepared)} with tempfile.TemporaryDirectory() as directory: archive = Path(directory) / "batch.zip" await run_sync_function(_write_import_zip, archive, paths) data_import_id, upload_url = await self._low_level_client.create_batch_from_upload( - resolved, + named, asset_name=asset_name, run_name=resolved_run_name, - run_id=resolved_run_id, + run_id=run_id, leave_run_open=leave_run_open, ) return await self._job_for_upload( @@ -440,6 +364,30 @@ async def _job_for_upload( job._set_data_import_id(data_import_id) return job + async def _config_for_upload( + self, + path: Path, + config: ImportConfig | None, + *, + data_type: DataTypeKey | None, + time_format: TimeFormat | None, + relative_start_time: pd.Timestamp | datetime | str | None, + ) -> ImportConfig: + """Use the supplied config, or detect one, then prepare the file. + + A detected ULog or MCAP config drops its channel list so the server + imports every channel. A supplied config keeps the channels you set. + """ + if config is None: + config = await self._detect_import_config( + path, + data_type=data_type, + time_format=time_format, + relative_start_time=relative_start_time, + ) + await self._prepare_import_file(config, path) + return config + async def _detect_import_config( self, path: Path,