Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion python/changelog.d/+ingestion-backups-opt-in.breaking.md

This file was deleted.

1 change: 1 addition & 0 deletions python/changelog.d/887.breaking.md
Comment thread
alexluck-sift marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`streaming_mode` is now a required argument of `create_ingestion_config_streaming_client` and `create_auto_register_streaming_client`. The previous default was `StreamingMode.LIVE_WITH_BACKUPS`: pass it to keep local telemetry backups, or pass `StreamingMode.LIVE_ONLY` to stream without them.
11 changes: 9 additions & 2 deletions python/docs/examples/ingestion.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@
"\n",
"## Streaming backups\n",
"\n",
"Both streaming client factories default to `StreamingMode.LIVE_ONLY`: telemetry streams to Sift without local disk backups.\n",
"Both streaming client factories require `streaming_mode`. Import `StreamingMode` from `sift_client.resources.ingestion` and pass one of:\n",
"\n",
"To enable local backups and replay after delivery failures, import `StreamingMode` from `sift_client.resources.ingestion` and pass `streaming_mode=StreamingMode.LIVE_WITH_BACKUPS` to `create_ingestion_config_streaming_client` or `create_auto_register_streaming_client`.\n",
"- `StreamingMode.LIVE_ONLY`: telemetry streams to Sift without local disk backups.\n",
"- `StreamingMode.LIVE_WITH_BACKUPS`: telemetry is also backed up to local disk and replayed after delivery failures.\n",
"\n",
"## Topics Covered\n",
"\n",
Expand Down Expand Up @@ -60,6 +61,7 @@
"from datetime import datetime, timezone\n",
"\n",
"from sift_client import SiftClient, SiftConnectionConfig\n",
"from sift_client.resources.ingestion import StreamingMode\n",
"from sift_client.sift_types import (\n",
" ChannelConfig,\n",
" ChannelDataType,\n",
Expand Down Expand Up @@ -100,6 +102,7 @@
" async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n",
" ingestion_config=ingestion_config,\n",
" run=run,\n",
" streaming_mode=StreamingMode.LIVE_ONLY,\n",
" ) as ingest_client:\n",
" # Send data in a loop\n",
" for i in range(10):\n",
Expand Down Expand Up @@ -181,6 +184,7 @@
" async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n",
" ingestion_config=ingestion_config,\n",
" run=run,\n",
" streaming_mode=StreamingMode.LIVE_ONLY,\n",
" ) as ingest_client:\n",
" # Get the flow descriptor and run ID from SiftStream\n",
" descriptor = ingest_client.get_flow_descriptor(flow_name=\"onboard_sensors\")\n",
Expand Down Expand Up @@ -304,6 +308,7 @@
" async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n",
" ingestion_config=ingestion_config,\n",
" run=run,\n",
" streaming_mode=StreamingMode.LIVE_ONLY,\n",
" ) as ingest_client:\n",
" # Get the flow descriptor and run ID\n",
" descriptor = ingest_client.get_flow_descriptor(flow_name=\"onboard_sensors\")\n",
Expand Down Expand Up @@ -429,6 +434,7 @@
" async with await client.async_.ingestion.create_ingestion_config_streaming_client(\n",
" ingestion_config=ingestion_config,\n",
" run=run,\n",
" streaming_mode=StreamingMode.LIVE_ONLY,\n",
" ) as ingest_client:\n",
" # Queues for the pipeline\n",
" queue1: asyncio.Queue[RawDataMessage] = asyncio.Queue()\n",
Expand Down Expand Up @@ -665,6 +671,7 @@
" async with await client.async_.ingestion.create_auto_register_streaming_client(\n",
" ingestion_config=ingestion_config,\n",
" run=run,\n",
" streaming_mode=StreamingMode.LIVE_ONLY,\n",
" staged_configs=[onboard_sensors], # staged\n",
" ) as stream:\n",
" for i in range(20):\n",
Expand Down
9 changes: 6 additions & 3 deletions python/examples/ingestion-tutorial/stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@
# Core Sift client and connection configuration.
from sift_client import SiftClient, SiftConnectionConfig

# Selects whether streaming keeps local telemetry backups.
from sift_client.resources.ingestion import StreamingMode

# Sift ingestion types used to define telemetry structure and runs.
from sift_client.sift_types import (
RunCreate, # Represents a telemetry collection session
Expand Down Expand Up @@ -170,9 +173,8 @@ async def main() -> None:
# All telemetry sent within this context will appear inside
# this Run in Sift.
#
# Streaming defaults to LIVE_ONLY (no local telemetry backups).
# To opt in, import StreamingMode from sift_client.resources.ingestion
# and pass streaming_mode=StreamingMode.LIVE_WITH_BACKUPS below.
# streaming_mode is required. LIVE_ONLY streams without local telemetry
# backups. Pass StreamingMode.LIVE_WITH_BACKUPS to keep them.
#
# WARNING: SiftStream should be reused for the entire program duration.
# Do NOT recreate the streaming client per batch or per iteration —
Expand All @@ -181,6 +183,7 @@ async def main() -> None:
async with await client.async_.ingestion.create_ingestion_config_streaming_client(
ingestion_config=ingestion_config,
run=run,
streaming_mode=StreamingMode.LIVE_ONLY,
) as ingest_client:
# Continue streaming until the user terminates the program
while True:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,11 +245,11 @@ async def _build_sift_stream_instance(
api_key: str,
grpc_uri: str,
ingestion_config_form: IngestionConfigFormPy,
streaming_mode: StreamingMode,
run_form: RunFormPy | None = None,
run_id: str | None = None,
asset_tags: list[str] | None = None,
asset_metadata: list[MetadataPy] | None = None,
streaming_mode: StreamingMode = ..., # type: ignore[assignment]
retry_policy: RetryPolicyPy | None = None,
disk_backup_policy: DiskBackupPolicyPy | None = None,
checkpoint_interval_seconds: int | None = None,
Expand Down Expand Up @@ -277,9 +277,6 @@ async def _build_sift_stream_instance(
_default_max_log_files = 7
_default_logfile_prefix = "sift_stream_bindings.log"

if streaming_mode is ...: # type: ignore[comparison-overlap]
streaming_mode = StreamingMode.LIVE_ONLY

# None disables metrics streaming, so it passes through to the builder as-is. Zero is
# rejected here because SiftStream cannot schedule metrics on a zero interval.
metrics_interval: DurationPy | None = None
Expand Down Expand Up @@ -355,11 +352,11 @@ async def create_sift_stream_instance(
api_key: str,
grpc_uri: str,
ingestion_config_form: IngestionConfigFormPy,
streaming_mode: StreamingMode,
run_form: RunFormPy | None = None,
run_id: str | None = None,
asset_tags: list[str] | None = None,
asset_metadata: list[MetadataPy] | None = None,
streaming_mode: StreamingMode = ..., # type: ignore[assignment]
retry_policy: RetryPolicyPy | None = None,
disk_backup_policy: DiskBackupPolicyPy | None = None,
checkpoint_interval_seconds: int | None = None,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,13 @@ async def _build(**kwargs):

class TestStreamingMode:
@pytest.mark.asyncio
async def test_defaults_to_live_only(self, config_builder):
await _build()
async def test_is_required(self, config_builder):
with pytest.raises(TypeError, match="streaming_mode"):
await _build()

@pytest.mark.asyncio
async def test_live_only_skips_backups(self, config_builder):
await _build(streaming_mode=StreamingMode.LIVE_ONLY)

config_builder.live_only.return_value.build.assert_awaited_once()
config_builder.live_with_backups.assert_not_called()
Expand Down Expand Up @@ -116,13 +121,13 @@ class TestRateLimitPolicy:
async def test_sets_policy_on_config_builder(self, config_builder):
policy = sift_stream_bindings.RateLimitPolicyPy.return_error()

await _build(rate_limit_policy=policy)
await _build(streaming_mode=StreamingMode.LIVE_ONLY, rate_limit_policy=policy)

assert config_builder.rate_limit_policy is policy

@pytest.mark.asyncio
async def test_leaves_default_when_none(self, config_builder):
await _build()
await _build(streaming_mode=StreamingMode.LIVE_ONLY)

assert not isinstance(
config_builder.rate_limit_policy, sift_stream_bindings.RateLimitPolicyPy
Expand Down
3 changes: 2 additions & 1 deletion python/lib/sift_client/_tests/resources/test_exports.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from sift_client import SiftClient
from sift_client.resources import DataExportAPI
from sift_client.resources.exports import DataExportAPIAsync
from sift_client.resources.ingestion import TracingConfig
from sift_client.resources.ingestion import StreamingMode, TracingConfig
from sift_client.sift_types.calculated_channel import (
CalculatedChannel,
CalculatedChannelCreate,
Expand Down Expand Up @@ -109,6 +109,7 @@ async def ingested_export_channel(sift_client, nostromo_asset):

async with await sift_client.async_.ingestion.create_ingestion_config_streaming_client(
ingestion_config=ingestion_config,
streaming_mode=StreamingMode.LIVE_ONLY,
tracing_config=TracingConfig.disabled(), # suppresses ./logs
) as stream:
await stream.send(
Expand Down
31 changes: 20 additions & 11 deletions python/lib/sift_client/_tests/resources/test_ingestion.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,25 @@
sift_stream_bindings = pytest.importorskip("sift_stream_bindings")


FACTORY_NAMES = [
"create_ingestion_config_streaming_client",
"create_auto_register_streaming_client",
]


@pytest.mark.asyncio
@pytest.mark.parametrize(
"factory_name",
["create_ingestion_config_streaming_client", "create_auto_register_streaming_client"],
)
@pytest.mark.parametrize("streaming_mode", [None, StreamingMode.LIVE_WITH_BACKUPS])
async def test_streaming_mode_default_and_opt_in(
@pytest.mark.parametrize("factory_name", FACTORY_NAMES)
async def test_streaming_mode_is_required(mock_client, factory_name):
api = IngestionAPIAsync(mock_client)

with pytest.raises(TypeError, match="streaming_mode"):
await getattr(api, factory_name)(ingestion_config=MagicMock())


@pytest.mark.asyncio
@pytest.mark.parametrize("factory_name", FACTORY_NAMES)
@pytest.mark.parametrize("streaming_mode", list(StreamingMode))
async def test_streaming_mode_is_passed_through(
mock_client, monkeypatch, factory_name, streaming_mode
):
build_stream = AsyncMock()
Expand All @@ -36,14 +48,11 @@ async def test_streaming_mode_default_and_opt_in(
MagicMock(from_stream=AsyncMock()),
)
api = IngestionAPIAsync(mock_client)
kwargs = {} if streaming_mode is None else {"streaming_mode": streaming_mode}

await getattr(api, factory_name)(ingestion_config=MagicMock(), **kwargs)
await getattr(api, factory_name)(ingestion_config=MagicMock(), streaming_mode=streaming_mode)

build_stream.assert_awaited_once()
assert build_stream.call_args.kwargs["streaming_mode"] == (
StreamingMode.LIVE_ONLY if streaming_mode is None else streaming_mode
)
assert build_stream.call_args.kwargs["streaming_mode"] == streaming_mode


def _rate_limited_error():
Expand Down
20 changes: 10 additions & 10 deletions python/lib/sift_client/resources/ingestion.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,10 @@ async def create_ingestion_config_streaming_client(
self,
ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy,
*,
streaming_mode: StreamingMode,
run: RunCreate | dict | str | Run | None = None,
asset_tags: list[str] | list[Tag] | None = None,
asset_metadata: dict[str, str | float | bool] | None = None,
streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY,
retry_policy: RetryPolicyPy | None = None,
disk_backup_policy: DiskBackupPolicyPy | None = None,
checkpoint_interval_seconds: int | None = None,
Expand All @@ -172,11 +172,11 @@ async def create_ingestion_config_streaming_client(

Args:
ingestion_config: The ingestion config. Can be a IngestionConfig or IngestionConfigFormPy.
streaming_mode: Transport mode for the stream. Required. LIVE_ONLY streams without
disk backups. LIVE_WITH_BACKUPS also backs up to local disk.
run: The run to associate with ingestion. Can be a Run, RunCreate, dict, or run ID string.
asset_tags: Tags to associate with the asset.
asset_metadata: Metadata to associate with the asset.
streaming_mode: Transport mode for the stream. Defaults to LIVE_ONLY, without disk
backups. Set to LIVE_WITH_BACKUPS to opt in to local disk backups.
retry_policy: Retry policy for LIVE_WITH_BACKUPS mode.
disk_backup_policy: Disk backup policy for LIVE_WITH_BACKUPS or FILE_BACKUP mode.
checkpoint_interval_seconds: Checkpoint interval in seconds (LIVE_WITH_BACKUPS only).
Expand Down Expand Up @@ -221,10 +221,10 @@ async def create_auto_register_streaming_client(
self,
ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy,
*,
streaming_mode: StreamingMode,
run: RunCreate | dict | str | Run | None = None,
asset_tags: list[str] | list[Tag] | None = None,
asset_metadata: dict[str, str | float | bool] | None = None,
streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY,
retry_policy: RetryPolicyPy | None = None,
disk_backup_policy: DiskBackupPolicyPy | None = None,
checkpoint_interval_seconds: int | None = None,
Expand Down Expand Up @@ -254,11 +254,11 @@ async def create_auto_register_streaming_client(
ingestion_config: The ingestion config. Use `IngestionConfigCreate` with `flows=[]`
for a fully schema-free start, or provide flow definitions to pre-register known
flows and avoid any first-send latency for those flows.
streaming_mode: Transport mode for the stream. Required. LIVE_ONLY streams without
disk backups. LIVE_WITH_BACKUPS also backs up to local disk.
run: The run to associate with ingestion. Can be a Run, RunCreate, dict, or run ID string.
asset_tags: Tags to associate with the asset.
asset_metadata: Metadata to associate with the asset.
streaming_mode: Transport mode for the stream. Defaults to LIVE_ONLY, without disk
backups. Set to LIVE_WITH_BACKUPS to opt in to local disk backups.
retry_policy: Retry policy for LIVE_WITH_BACKUPS mode.
disk_backup_policy: Disk backup policy for LIVE_WITH_BACKUPS or FILE_BACKUP mode.
checkpoint_interval_seconds: Checkpoint interval in seconds (LIVE_WITH_BACKUPS only).
Expand Down Expand Up @@ -326,10 +326,10 @@ async def _create(
sift_client: SiftClient,
ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy,
*,
streaming_mode: StreamingMode,
run: RunCreate | dict | str | Run | RunFormPy | None = None,
asset_tags: list[str] | list[Tag] | None = None,
asset_metadata: dict[str, str | float | bool] | None = None,
streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY,
retry_policy: RetryPolicyPy | None = None,
disk_backup_policy: DiskBackupPolicyPy | None = None,
checkpoint_interval_seconds: int | None = None,
Expand All @@ -343,11 +343,11 @@ async def _create(
Args:
sift_client: The Sift client to use.
ingestion_config: The ingestion config to use for streaming.
streaming_mode: Transport mode for the stream. Required. LIVE_ONLY streams without
disk backups. LIVE_WITH_BACKUPS also backs up to local disk.
run: The run to associate with ingestion. Can be a Run, RunCreate, dict, or run ID string.
asset_tags: Tags to associate with the asset.
asset_metadata: Metadata to associate with the asset.
streaming_mode: Transport mode for the stream. Defaults to LIVE_ONLY, without disk
backups. Set to LIVE_WITH_BACKUPS to opt in to local disk backups.
retry_policy: Retry policy for LIVE_WITH_BACKUPS mode.
disk_backup_policy: Disk backup policy for LIVE_WITH_BACKUPS or FILE_BACKUP mode.
checkpoint_interval_seconds: Checkpoint interval in seconds (LIVE_WITH_BACKUPS only).
Expand Down Expand Up @@ -686,10 +686,10 @@ async def _create(
sift_client: SiftClient,
ingestion_config: IngestionConfig | IngestionConfigCreate | IngestionConfigFormPy,
*,
streaming_mode: StreamingMode,
run: RunCreate | dict | str | Run | RunFormPy | None = None,
asset_tags: list[str] | list[Tag] | None = None,
asset_metadata: dict[str, str | float | bool] | None = None,
streaming_mode: StreamingMode = StreamingMode.LIVE_ONLY,
retry_policy: RetryPolicyPy | None = None,
disk_backup_policy: DiskBackupPolicyPy | None = None,
checkpoint_interval_seconds: int | None = None,
Expand Down
Loading