diff --git a/python/changelog.d/+ingestion-backups-opt-in.breaking.md b/python/changelog.d/+ingestion-backups-opt-in.breaking.md deleted file mode 100644 index 35646a0a7..000000000 --- a/python/changelog.d/+ingestion-backups-opt-in.breaking.md +++ /dev/null @@ -1 +0,0 @@ -Ingestion streaming clients now default to `StreamingMode.LIVE_ONLY`; pass `streaming_mode=StreamingMode.LIVE_WITH_BACKUPS` to enable local telemetry backups. diff --git a/python/changelog.d/887.breaking.md b/python/changelog.d/887.breaking.md new file mode 100644 index 000000000..85eada84c --- /dev/null +++ b/python/changelog.d/887.breaking.md @@ -0,0 +1 @@ +`streaming_mode` is now a required argument of `create_ingestion_config_streaming_client` and `create_auto_register_streaming_client`. The previous default was `StreamingMode.LIVE_WITH_BACKUPS`: pass it to keep local telemetry backups, or pass `StreamingMode.LIVE_ONLY` to stream without them. diff --git a/python/docs/examples/ingestion.ipynb b/python/docs/examples/ingestion.ipynb index c2167e4d2..1ad3c5473 100644 --- a/python/docs/examples/ingestion.ipynb +++ b/python/docs/examples/ingestion.ipynb @@ -19,9 +19,10 @@ "\n", "## Streaming backups\n", "\n", - "Both streaming client factories default to `StreamingMode.LIVE_ONLY`: telemetry streams to Sift without local disk backups.\n", + "Both streaming client factories require `streaming_mode`. Import `StreamingMode` from `sift_client.resources.ingestion` and pass one of:\n", "\n", - "To enable local backups and replay after delivery failures, import `StreamingMode` from `sift_client.resources.ingestion` and pass `streaming_mode=StreamingMode.LIVE_WITH_BACKUPS` to `create_ingestion_config_streaming_client` or `create_auto_register_streaming_client`.\n", + "- `StreamingMode.LIVE_ONLY`: telemetry streams to Sift without local disk backups.\n", + "- `StreamingMode.LIVE_WITH_BACKUPS`: telemetry is also backed up to local disk and replayed after delivery failures.\n", "\n", "## Topics Covered\n", "\n", @@ -60,6 +61,7 @@ "from datetime import datetime, timezone\n", "\n", "from sift_client import SiftClient, SiftConnectionConfig\n", + "from sift_client.resources.ingestion import StreamingMode\n", "from sift_client.sift_types import (\n", " ChannelConfig,\n", " ChannelDataType,\n", @@ -100,6 +102,7 @@ " async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n", " ingestion_config=ingestion_config,\n", " run=run,\n", + " streaming_mode=StreamingMode.LIVE_ONLY,\n", " ) as ingest_client:\n", " # Send data in a loop\n", " for i in range(10):\n", @@ -181,6 +184,7 @@ " async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n", " ingestion_config=ingestion_config,\n", " run=run,\n", + " streaming_mode=StreamingMode.LIVE_ONLY,\n", " ) as ingest_client:\n", " # Get the flow descriptor and run ID from SiftStream\n", " descriptor = ingest_client.get_flow_descriptor(flow_name=\"onboard_sensors\")\n", @@ -304,6 +308,7 @@ " async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n", " ingestion_config=ingestion_config,\n", " run=run,\n", + " streaming_mode=StreamingMode.LIVE_ONLY,\n", " ) as ingest_client:\n", " # Get the flow descriptor and run ID\n", " descriptor = ingest_client.get_flow_descriptor(flow_name=\"onboard_sensors\")\n", @@ -429,6 +434,7 @@ " async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n", " ingestion_config=ingestion_config,\n", " run=run,\n", + " streaming_mode=StreamingMode.LIVE_ONLY,\n", " ) as ingest_client:\n", " # Queues for the pipeline\n", " queue1: asyncio.Queue[RawDataMessage] = asyncio.Queue()\n", @@ -665,6 +671,7 @@ " async with await client.async_.ingestion.create_auto_register_streaming_client(\n", " ingestion_config=ingestion_config,\n", " run=run,\n", + " streaming_mode=StreamingMode.LIVE_ONLY,\n", " staged_configs=[onboard_sensors], # staged\n", " ) as stream:\n", " for i in range(20):\n", diff --git a/python/examples/ingestion-tutorial/stream.py b/python/examples/ingestion-tutorial/stream.py index 2699ae303..8fc8561da 100644 --- a/python/examples/ingestion-tutorial/stream.py +++ b/python/examples/ingestion-tutorial/stream.py @@ -27,6 +27,9 @@ # Core Sift client and connection configuration. from sift_client import SiftClient, SiftConnectionConfig +# Selects whether streaming keeps local telemetry backups. +from sift_client.resources.ingestion import StreamingMode + # Sift ingestion types used to define telemetry structure and runs. from sift_client.sift_types import ( RunCreate, # Represents a telemetry collection session @@ -170,9 +173,8 @@ async def main() -> None: # All telemetry sent within this context will appear inside # this Run in Sift. # - # Streaming defaults to LIVE_ONLY (no local telemetry backups). - # To opt in, import StreamingMode from sift_client.resources.ingestion - # and pass streaming_mode=StreamingMode.LIVE_WITH_BACKUPS below. + # streaming_mode is required. LIVE_ONLY streams without local telemetry + # backups. Pass StreamingMode.LIVE_WITH_BACKUPS to keep them. # # WARNING: SiftStream should be reused for the entire program duration. # Do NOT recreate the streaming client per batch or per iteration — @@ -181,6 +183,7 @@ async def main() -> None: async with await client.async_.ingestion.create_ingestion_config_streaming_client( ingestion_config=ingestion_config, run=run, + streaming_mode=StreamingMode.LIVE_ONLY, ) as ingest_client: # Continue streaming until the user terminates the program while True: diff --git a/python/lib/sift_client/_internal/low_level_wrappers/ingestion.py b/python/lib/sift_client/_internal/low_level_wrappers/ingestion.py index ac3fefb76..345a5bd91 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/ingestion.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/ingestion.py @@ -245,11 +245,11 @@ async def _build_sift_stream_instance( api_key: str, grpc_uri: str, ingestion_config_form: IngestionConfigFormPy, + streaming_mode: StreamingMode, run_form: RunFormPy | None = None, run_id: str | None = None, asset_tags: list[str] | None = None, asset_metadata: list[MetadataPy] | None = None, - streaming_mode: StreamingMode = ..., # type: ignore[assignment] retry_policy: RetryPolicyPy | None = None, disk_backup_policy: DiskBackupPolicyPy | None = None, checkpoint_interval_seconds: int | None = None, @@ -277,9 +277,6 @@ async def _build_sift_stream_instance( _default_max_log_files = 7 _default_logfile_prefix = "sift_stream_bindings.log" - if streaming_mode is ...: # type: ignore[comparison-overlap] - streaming_mode = StreamingMode.LIVE_ONLY - # None disables metrics streaming, so it passes through to the builder as-is. Zero is # rejected here because SiftStream cannot schedule metrics on a zero interval. metrics_interval: DurationPy | None = None @@ -355,11 +352,11 @@ async def create_sift_stream_instance( api_key: str, grpc_uri: str, ingestion_config_form: IngestionConfigFormPy, + streaming_mode: StreamingMode, run_form: RunFormPy | None = None, run_id: str | None = None, asset_tags: list[str] | None = None, asset_metadata: list[MetadataPy] | None = None, - streaming_mode: StreamingMode = ..., # type: ignore[assignment] retry_policy: RetryPolicyPy | None = None, disk_backup_policy: DiskBackupPolicyPy | None = None, checkpoint_interval_seconds: int | None = None, diff --git a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_ingestion_stream_builder.py b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_ingestion_stream_builder.py index 9d59d0d3c..c07f71455 100644 --- a/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_ingestion_stream_builder.py +++ b/python/lib/sift_client/_tests/_internal/low_level_wrappers/test_ingestion_stream_builder.py @@ -43,8 +43,13 @@ async def _build(**kwargs): class TestStreamingMode: @pytest.mark.asyncio - async def test_defaults_to_live_only(self, config_builder): - await _build() + async def test_is_required(self, config_builder): + with pytest.raises(TypeError, match="streaming_mode"): + await _build() + + @pytest.mark.asyncio + async def test_live_only_skips_backups(self, config_builder): + await _build(streaming_mode=StreamingMode.LIVE_ONLY) config_builder.live_only.return_value.build.assert_awaited_once() config_builder.live_with_backups.assert_not_called() @@ -116,13 +121,13 @@ class TestRateLimitPolicy: async def test_sets_policy_on_config_builder(self, config_builder): policy = sift_stream_bindings.RateLimitPolicyPy.return_error() - await _build(rate_limit_policy=policy) + await _build(streaming_mode=StreamingMode.LIVE_ONLY, rate_limit_policy=policy) assert config_builder.rate_limit_policy is policy @pytest.mark.asyncio async def test_leaves_default_when_none(self, config_builder): - await _build() + await _build(streaming_mode=StreamingMode.LIVE_ONLY) assert not isinstance( config_builder.rate_limit_policy, sift_stream_bindings.RateLimitPolicyPy diff --git a/python/lib/sift_client/_tests/resources/test_exports.py b/python/lib/sift_client/_tests/resources/test_exports.py index 53e65b610..e9b86624e 100644 --- a/python/lib/sift_client/_tests/resources/test_exports.py +++ b/python/lib/sift_client/_tests/resources/test_exports.py @@ -24,7 +24,7 @@ from sift_client import SiftClient from sift_client.resources import DataExportAPI from sift_client.resources.exports import DataExportAPIAsync -from sift_client.resources.ingestion import TracingConfig +from sift_client.resources.ingestion import StreamingMode, TracingConfig from sift_client.sift_types.calculated_channel import ( CalculatedChannel, CalculatedChannelCreate, @@ -109,6 +109,7 @@ async def ingested_export_channel(sift_client, nostromo_asset): async with await sift_client.async_.ingestion.create_ingestion_config_streaming_client( ingestion_config=ingestion_config, + streaming_mode=StreamingMode.LIVE_ONLY, tracing_config=TracingConfig.disabled(), # suppresses ./logs ) as stream: await stream.send( diff --git a/python/lib/sift_client/_tests/resources/test_ingestion.py b/python/lib/sift_client/_tests/resources/test_ingestion.py index 7d1318aad..9b7c92865 100644 --- a/python/lib/sift_client/_tests/resources/test_ingestion.py +++ b/python/lib/sift_client/_tests/resources/test_ingestion.py @@ -15,13 +15,25 @@ sift_stream_bindings = pytest.importorskip("sift_stream_bindings") +FACTORY_NAMES = [ + "create_ingestion_config_streaming_client", + "create_auto_register_streaming_client", +] + + @pytest.mark.asyncio -@pytest.mark.parametrize( - "factory_name", - ["create_ingestion_config_streaming_client", "create_auto_register_streaming_client"], -) -@pytest.mark.parametrize("streaming_mode", [None, StreamingMode.LIVE_WITH_BACKUPS]) -async def test_streaming_mode_default_and_opt_in( +@pytest.mark.parametrize("factory_name", FACTORY_NAMES) +async def test_streaming_mode_is_required(mock_client, factory_name): + api = IngestionAPIAsync(mock_client) + + with pytest.raises(TypeError, match="streaming_mode"): + await getattr(api, factory_name)(ingestion_config=MagicMock()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("factory_name", FACTORY_NAMES) +@pytest.mark.parametrize("streaming_mode", list(StreamingMode)) +async def test_streaming_mode_is_passed_through( mock_client, monkeypatch, factory_name, streaming_mode ): build_stream = AsyncMock() @@ -36,14 +48,11 @@ async def test_streaming_mode_default_and_opt_in( MagicMock(from_stream=AsyncMock()), ) api = IngestionAPIAsync(mock_client) - kwargs = {} if streaming_mode is None else {"streaming_mode": streaming_mode} - await getattr(api, factory_name)(ingestion_config=MagicMock(), **kwargs) + await getattr(api, factory_name)(ingestion_config=MagicMock(), streaming_mode=streaming_mode) build_stream.assert_awaited_once() - assert build_stream.call_args.kwargs["streaming_mode"] == ( - StreamingMode.LIVE_ONLY if streaming_mode is None else streaming_mode - ) + assert build_stream.call_args.kwargs["streaming_mode"] == streaming_mode def _rate_limited_error(): diff --git a/python/lib/sift_client/resources/ingestion.py b/python/lib/sift_client/resources/ingestion.py index 06680062c..283a51250 100644 --- a/python/lib/sift_client/resources/ingestion.py +++ b/python/lib/sift_client/resources/ingestion.py @@ -156,10 +156,10 @@ async def create_ingestion_config_streaming_client( self, ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy, *, + streaming_mode: StreamingMode, run: RunCreate | dict | str | Run | None = None, asset_tags: list[str] | list[Tag] | None = None, asset_metadata: dict[str, str | float | bool] | None = None, - streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY, retry_policy: RetryPolicyPy | None = None, disk_backup_policy: DiskBackupPolicyPy | None = None, checkpoint_interval_seconds: int | None = None, @@ -172,11 +172,11 @@ async def create_ingestion_config_streaming_client( Args: ingestion_config: The ingestion config. Can be a IngestionConfig or IngestionConfigFormPy. + streaming_mode: Transport mode for the stream. Required. LIVE_ONLY streams without + disk backups. LIVE_WITH_BACKUPS also backs up to local disk. run: The run to associate with ingestion. Can be a Run, RunCreate, dict, or run ID string. asset_tags: Tags to associate with the asset. asset_metadata: Metadata to associate with the asset. - streaming_mode: Transport mode for the stream. Defaults to LIVE_ONLY, without disk - backups. Set to LIVE_WITH_BACKUPS to opt in to local disk backups. retry_policy: Retry policy for LIVE_WITH_BACKUPS mode. disk_backup_policy: Disk backup policy for LIVE_WITH_BACKUPS or FILE_BACKUP mode. checkpoint_interval_seconds: Checkpoint interval in seconds (LIVE_WITH_BACKUPS only). @@ -221,10 +221,10 @@ async def create_auto_register_streaming_client( self, ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy, *, + streaming_mode: StreamingMode, run: RunCreate | dict | str | Run | None = None, asset_tags: list[str] | list[Tag] | None = None, asset_metadata: dict[str, str | float | bool] | None = None, - streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY, retry_policy: RetryPolicyPy | None = None, disk_backup_policy: DiskBackupPolicyPy | None = None, checkpoint_interval_seconds: int | None = None, @@ -254,11 +254,11 @@ async def create_auto_register_streaming_client( ingestion_config: The ingestion config. Use `IngestionConfigCreate` with `flows=[]` for a fully schema-free start, or provide flow definitions to pre-register known flows and avoid any first-send latency for those flows. + streaming_mode: Transport mode for the stream. Required. LIVE_ONLY streams without + disk backups. LIVE_WITH_BACKUPS also backs up to local disk. run: The run to associate with ingestion. Can be a Run, RunCreate, dict, or run ID string. asset_tags: Tags to associate with the asset. asset_metadata: Metadata to associate with the asset. - streaming_mode: Transport mode for the stream. Defaults to LIVE_ONLY, without disk - backups. Set to LIVE_WITH_BACKUPS to opt in to local disk backups. retry_policy: Retry policy for LIVE_WITH_BACKUPS mode. disk_backup_policy: Disk backup policy for LIVE_WITH_BACKUPS or FILE_BACKUP mode. checkpoint_interval_seconds: Checkpoint interval in seconds (LIVE_WITH_BACKUPS only). @@ -326,10 +326,10 @@ async def _create( sift_client: SiftClient, ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy, *, + streaming_mode: StreamingMode, run: RunCreate | dict | str | Run | RunFormPy | None = None, asset_tags: list[str] | list[Tag] | None = None, asset_metadata: dict[str, str | float | bool] | None = None, - streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY, retry_policy: RetryPolicyPy | None = None, disk_backup_policy: DiskBackupPolicyPy | None = None, checkpoint_interval_seconds: int | None = None, @@ -343,11 +343,11 @@ async def _create( Args: sift_client: The Sift client to use. ingestion_config: The ingestion config to use for streaming. + streaming_mode: Transport mode for the stream. Required. LIVE_ONLY streams without + disk backups. LIVE_WITH_BACKUPS also backs up to local disk. run: The run to associate with ingestion. Can be a Run, RunCreate, dict, or run ID string. asset_tags: Tags to associate with the asset. asset_metadata: Metadata to associate with the asset. - streaming_mode: Transport mode for the stream. Defaults to LIVE_ONLY, without disk - backups. Set to LIVE_WITH_BACKUPS to opt in to local disk backups. retry_policy: Retry policy for LIVE_WITH_BACKUPS mode. disk_backup_policy: Disk backup policy for LIVE_WITH_BACKUPS or FILE_BACKUP mode. checkpoint_interval_seconds: Checkpoint interval in seconds (LIVE_WITH_BACKUPS only). @@ -686,10 +686,10 @@ async def _create( sift_client: SiftClient, ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy, *, + streaming_mode: StreamingMode, run: RunCreate | dict | str | Run | RunFormPy | None = None, asset_tags: list[str] | list[Tag] | None = None, asset_metadata: dict[str, str | float | bool] | None = None, - streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY, retry_policy: RetryPolicyPy | None = None, disk_backup_policy: DiskBackupPolicyPy | None = None, checkpoint_interval_seconds: int | None = None,