From aabf00fb93935a35384dff8d4dbb429f2d2ea77c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:58:22 +0000 Subject: [PATCH 01/13] fix(location): keep a smaller statistics page after a failed statistics read A statistics read that timed out halved the shared page limit and rewound to catalog. Finishing catalog reset that limit to the maximum, so statistics retried at full size, timed out again and looped, keeping the warehouse busy. Statistics now has its own page limit: it survives the rewind and returns to the maximum once a statistics scan completes. --- .../src/native_commands/_helpers/execution.rs | 15 ++++- .../src/native_commands/models.rs | 6 +- .../sqlbuild-cli/src/native_commands/tests.rs | 2 + .../src/native_commands/tests/helpers.rs | 25 +++++++++ .../tests/test_location_metadata.rs | 55 +++++++++++++++++++ .../src/native_commands/tests/test_types.rs | 10 ++++ 6 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index a7a8b314c4..9d5ce63b7e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -346,6 +346,7 @@ pub(crate) fn metadata_sync( reconcile_at: Instant::now(), reconcile_seconds, limit: MAX_HISTORY_RECORDS, + statistics_limit: MAX_HISTORY_RECORDS, pending: None, next_at: Instant::now(), } @@ -389,7 +390,7 @@ pub(crate) async fn statistics_page( generation: state.generation, location: state.location().clone(), after, - limit: state.limit, + limit: state.statistics_limit, }, ) .await @@ -423,8 +424,13 @@ pub(crate) fn freeze_history_page( > BODY_BYTES { state.statistics = None; - if state.limit > 1 { - state.limit = (state.limit / 2).max(1); + let limit = if state.kind() == WarehouseHistoryKind::Statistics { + &mut state.statistics_limit + } else { + &mut state.limit + }; + if *limit > 1 { + *limit /= 2; } else { state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); drop(write_output(&[( @@ -450,6 +456,9 @@ pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> Loca state.statistics = None; } if state.source_after.is_none() { + if state.kind() == WarehouseHistoryKind::Statistics { + state.statistics_limit = MAX_HISTORY_RECORDS; + } state.index += 1; state.scan_id = Uuid::now_v7(); state.observed_since = None; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 61f4d42fd2..a1bbf08388 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -200,6 +200,8 @@ pub(crate) struct LocationMetadataSync { pub(crate) reconcile_at: tokio::time::Instant, pub(crate) reconcile_seconds: u32, pub(crate) limit: usize, + /// Survives a failed statistics read's rewind through catalog, which resets `limit`. + pub(crate) statistics_limit: usize, pub(crate) pending: Option, pub(crate) next_at: tokio::time::Instant, } @@ -207,14 +209,16 @@ pub(crate) struct LocationMetadataSync { impl LocationMetadataSync { /// Failed samples revisit catalog before retrying; no frozen delivery is discarded. pub(crate) fn retry_read(&mut self) { - self.limit = (self.limit / 2).max(1); self.statistics = None; if self.kind() == WarehouseHistoryKind::Statistics { + self.statistics_limit = (self.statistics_limit / 2).max(1); self.index -= 1; self.source_after = None; self.scan_id = Uuid::now_v7(); self.observed_since = None; self.watermark_loaded = false; + } else { + self.limit = (self.limit / 2).max(1); } self.next_at = tokio::time::Instant::now() + std::time::Duration::from_secs(super::constants::METADATA_POLL_SECONDS); diff --git a/crates/sqlbuild-cli/src/native_commands/tests.rs b/crates/sqlbuild-cli/src/native_commands/tests.rs index 36ed1676b5..6aecb77547 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests.rs @@ -6,6 +6,8 @@ mod decision; mod helpers; #[path = "tests/test_init.rs"] mod init; +#[path = "tests/test_location_metadata.rs"] +mod location_metadata; #[path = "tests/test_location_plan.rs"] mod location_plan; #[path = "tests/test_paths.rs"] diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index ec817e0ae9..ed08157d2e 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -1,8 +1,13 @@ use std::path::Path; +use sqlbuild_orchestration::warehouse_metadata::models::MetadataBatch; +use uuid::Uuid; + use crate::argv::main::parse_cli_arguments::parse_cli_arguments; use crate::argv::models::{ArgvOutcome, ArgvSettings}; use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::execution::acknowledge_metadata_page; +use crate::native_commands::models::{LocationMetadataPage, LocationMetadataSync}; /// Parse `argv` onto the `CliNamespace` defaults with the native host. pub(super) fn parse(argv: &[&str]) -> (ArgvOutcome, NativeArgvHost) { @@ -35,3 +40,23 @@ pub(super) fn plan_output(diff_line: &str, count: usize) -> Vec { serde_json::to_vec(&serde_json::json!({"selected_count": count, "models": models})) .expect("plan output") } + +/// Acknowledges `pages` complete, empty pages, each finishing the kind it was read for. +pub(super) fn acknowledge_complete_pages( + state: LocationMetadataSync, + pages: usize, +) -> LocationMetadataSync { + (0..pages).fold(state, |mut state, _| { + state.pending = Some(LocationMetadataPage { + batch: MetadataBatch { + kind: state.kind(), + session_id: Uuid::nil(), + generation: 1, + records: Vec::new(), + scan: None, + }, + next_source_after: None, + }); + acknowledge_metadata_page(state) + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs new file mode 100644 index 0000000000..a888efba54 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -0,0 +1,55 @@ +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseStateLocation, +}; + +use crate::native_commands::_helpers::execution::metadata_sync; +use crate::native_commands::tests::helpers::acknowledge_complete_pages; +use crate::native_commands::tests::test_types::MetadataRetryTestCase; + +#[test] +fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_scan_completes() { + let test_cases = [ + MetadataRetryTestCase { + description: "a timed-out statistics page is retried smaller after catalog is revisited", + failed_kind: WarehouseHistoryKind::Statistics, + acknowledged_pages: 1, + expected_statistics_limit: MAX_HISTORY_RECORDS / 2, + }, + MetadataRetryTestCase { + description: "a failed catalog page leaves the statistics page size alone", + failed_kind: WarehouseHistoryKind::Catalog, + acknowledged_pages: 1, + expected_statistics_limit: MAX_HISTORY_RECORDS, + }, + MetadataRetryTestCase { + description: "a completed statistics scan restores the full statistics page", + failed_kind: WarehouseHistoryKind::Statistics, + acknowledged_pages: 1 + WarehouseHistoryKind::ALL.len(), + expected_statistics_limit: MAX_HISTORY_RECORDS, + }, + ]; + for test_case in test_cases { + let location = WarehouseStateLocation { + database: Some("analytics".into()), + schema: "state".into(), + }; + let mut state = metadata_sync((vec![location], 1), 3600); + state.index = WarehouseHistoryKind::ALL + .iter() + .position(|kind| *kind == test_case.failed_kind) + .expect("kind is listed"); + state.retry_read(); + let state = acknowledge_complete_pages(state, test_case.acknowledged_pages); + assert_eq!( + (state.kind(), state.statistics_limit, state.limit), + ( + WarehouseHistoryKind::Statistics, + test_case.expected_statistics_limit, + MAX_HISTORY_RECORDS, + ), + "{description}", + description = test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 93270cf91a..580e0ee5e4 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -1,3 +1,5 @@ +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryKind; + use crate::argv::models::{TypeFailure, Value}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, NativeCommand, RulesAction}; @@ -89,3 +91,11 @@ pub(super) struct PlanDocumentTestCase { pub(super) planned: bool, pub(super) expected_first_model: serde_json::Value, } + +/// One failed metadata read and the statistics page size the sync uses afterwards. +pub(super) struct MetadataRetryTestCase { + pub(super) description: &'static str, + pub(super) failed_kind: WarehouseHistoryKind, + pub(super) acknowledged_pages: usize, + pub(super) expected_statistics_limit: usize, +} From 84691240e549ed83781404148820cf8ec522542f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:00:19 +0000 Subject: [PATCH 02/13] fix(snowflake): sync the catalog and check schemas without a running warehouse A code location's catalog sync listed relations and columns through information_schema, and every Snowflake connection checked its session schema through information_schema.schemata. Both need a running warehouse, so each sync cycle resumed it and kept it from suspending. The catalog page now uses the schema-scoped SHOW listings the build already uses when the adapter provides them, and schema_exists asks SHOW SCHEMAS, falling back to the information schema for names SHOW LIKE cannot express. --- .../_helpers/native_host/warehouse_catalog.rs | 106 ++++++++++++------ .../snowflake/classes/snowflake_adapter.py | 19 ++++ .../adapters/snowflake/_test_types.py | 9 ++ .../sqlbuild/adapters/snowflake/helpers.py | 8 ++ .../adapters/snowflake/test_client.py | 51 +++++++++ 5 files changed, 158 insertions(+), 35 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs index 112caea0a2..1fd2b10c0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ CatalogRead, WarehouseCatalogColumn, WarehouseCatalogRecord, WarehouseHistoryPage, @@ -49,23 +49,43 @@ pub(crate) fn read_catalog( "catalog listing is not authoritative for this namespace; scan incomplete", )); } - let relations = call_with( - &adapter.getattr("list_relations")?, - &[ - ("connection", connection.clone()), - ( - "database", - request.location.database.clone().into_bound_py_any(py)?, - ), - ( - "schemas", - (request.location.schema.clone(),).into_bound_py_any(py)?, - ), - ], + let schema_scoped = adapter.is_instance( + &py.import("sqlbuild.adapter.relations.types")? + .getattr("SchemaScopedMetadataReader")?, )?; - let mut selected: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); - for relation in relations.try_iter()? { - let relation = relation?; + let mut selected: Vec<(String, String, Bound<'_, PyAny>)> = Vec::new(); + let listed = if schema_scoped { + field( + &call_with( + &adapter.getattr("read_schema_relation_listing")?, + &[ + ("connection", connection.clone()), + namespace[0].clone(), + namespace[1].clone(), + ], + )?, + "entries", + )? + } else { + call_with( + &adapter.getattr("list_relations")?, + &[ + ("connection", connection.clone()), + namespace[0].clone(), + ( + "schemas", + (request.location.schema.clone(),).into_bound_py_any(py)?, + ), + ], + )? + }; + for item in listed.try_iter()? { + let item = item?; + let (relation, key) = if schema_scoped { + (field(&item, "relation")?, field(&item, "stored_name")?) + } else { + (item.clone(), field(&item, "name")?) + }; let name: String = field(&relation, "name")?.extract()?; if request .after @@ -73,7 +93,8 @@ pub(crate) fn read_catalog( .and_then(|cursor| cursor[0].as_deref()) .is_none_or(|after| name.as_str() > after) { - selected.push((name, relation)); + let relation_type: String = field(&relation, "relation_type")?.extract()?; + selected.push((name, relation_type, key)); } } selected.sort_by(|left, right| left.0.cmp(&right.0)); @@ -85,28 +106,43 @@ pub(crate) fn read_catalog( next_after: None, }); } - let names = PyTuple::new(py, selected.iter().map(|(name, _)| name))?; - let columns = call_with( - &adapter.getattr("get_all_columns")?, - &[ - ("connection", connection.clone()), - ("database", namespace[0].1.clone()), - ( - "schemas", - (request.location.schema.clone(),).into_bound_py_any(py)?, - ), - ("names", names.into_any()), - ], - )?; + let keys = selected.iter().map(|(_, _, key)| key); + let columns = if schema_scoped { + field( + &call_with( + &adapter.getattr("read_schema_column_listing")?, + &[ + ("connection", connection.clone()), + namespace[0].clone(), + namespace[1].clone(), + ("stored_names", PyFrozenSet::new(py, keys)?.into_any()), + ], + )?, + "columns_by_stored_name", + )? + } else { + call_with( + &adapter.getattr("get_all_columns")?, + &[ + ("connection", connection.clone()), + namespace[0].clone(), + ( + "schemas", + (request.location.schema.clone(),).into_bound_py_any(py)?, + ), + ("names", PyTuple::new(py, keys)?.into_any()), + ], + )? + }; let columns = columns.cast::()?; let next_after = if more { - selected.last().map(|(name, _)| vec![Some(name.clone())]) + selected.last().map(|(name, _, _)| vec![Some(name.clone())]) } else { None }; let mut records: Vec = Vec::new(); - for (name, relation) in selected { - let relation_columns = columns.get_item(name.as_str())?.ok_or_else(|| { + for (name, relation_type, key) in selected { + let relation_columns = columns.get_item(key)?.ok_or_else(|| { PyValueError::new_err("catalog relation columns are unavailable; scan incomplete") })?; let mut decoded: Vec = Vec::new(); @@ -121,7 +157,7 @@ pub(crate) fn read_catalog( state_database: request.location.database.clone(), state_schema: request.location.schema.clone(), name, - relation_type: field(&relation, "relation_type")?.extract()?, + relation_type, columns: decoded, })); } diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index ddf90d3385..1c19faa5bf 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -4165,6 +4165,25 @@ def schema_exists( database: str | None, schema: str, ) -> bool: + """Check with SHOW SCHEMAS, which needs no running warehouse, where the name allows.""" + + show_database: str | None = ( + database if database is not None else self._current_database(connection=connection) + ) + if show_database is not None and SHOW_LIKE_ESCAPE_CHARACTER not in schema: + try: + rows: list[dict[str, object]] = self._fetch_show_rows( + connection=connection, + query=( + f"SHOW SCHEMAS LIKE '{show_like_pattern(schema.upper())}' " + f"IN DATABASE {self.render_identifier(show_database)}" + ), + ) + except Exception as error: + if not is_missing_object_error(error): + raise + return False + return any(str(row.get("name")).upper() == schema.upper() for row in rows) query: str = ( "SELECT 1 FROM information_schema.schemata WHERE UPPER(schema_name) = UPPER(%s)" ) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 30ed59be03..e06ff737f7 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -417,3 +417,12 @@ class SnowflakePlatformDetectionTestCase: description: str environment: dict[str, str] expected_value: str + + +@dataclass(frozen=True) +class SnowflakeSchemaExistsTestCase: + description: str + schema: str + cursor: object + expected_exists: bool + expected_sql_fragment: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 4882cc8bc5..0c6d24b4f4 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -78,6 +78,14 @@ def fetchmany(self, size: int) -> list[tuple[object, ...]]: return self._rows[:size] +class FakeSnowflakeMissingObjectCursor(FakeSnowflakeQueryCursor): + """Cursor double whose statement fails as Snowflake reports a missing object.""" + + def execute(self, sql: str) -> None: + super().execute(sql) + raise RuntimeError("Database 'GONE' does not exist or not authorized.") + + class FakeSnowflakeDescribeConnection: """Connection double returning a preconfigured describe cursor.""" diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 6a9dcf2cdd..5c64c55bcf 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -47,6 +47,7 @@ SnowflakeRenderPythonFunctionTestCase, SnowflakeRenderTableFunctionTestCase, SnowflakeSchemaDiffTestCase, + SnowflakeSchemaExistsTestCase, SnowflakeTableFreshnessBatchTestCase, SnowflakeTableFreshnessMetadataErrorTestCase, SnowflakeTableFreshnessMetadataTestCase, @@ -59,6 +60,7 @@ FakeSnowflakeMetadataConnection, FakeSnowflakeMetadataCursor, FakeSnowflakeMetadataSequenceConnection, + FakeSnowflakeMissingObjectCursor, FakeSnowflakeQueryCursor, FakeSnowflakeRawConnection, build_bulk_columns_rows, @@ -1357,3 +1359,52 @@ def test_given_snowflake_relation_timestamps_when_listing_then_exposes_utc_age_m str(test_case.expected_created_at), str(test_case.expected_last_altered_at), ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeSchemaExistsTestCase( + description="SHOW SCHEMAS finds a schema stored in another case", + schema="staging", + cursor=FakeSnowflakeQueryCursor((("name",),), [("Staging",)]), + expected_exists=True, + expected_sql_fragment="SHOW SCHEMAS LIKE 'STAGING' IN DATABASE", + ), + SnowflakeSchemaExistsTestCase( + description="a neighbour matched by the LIKE wildcard is not the schema", + schema="stg_a", + cursor=FakeSnowflakeQueryCursor((("name",),), [("STGXA",)]), + expected_exists=False, + expected_sql_fragment="SHOW SCHEMAS LIKE 'STG_A' IN DATABASE", + ), + SnowflakeSchemaExistsTestCase( + description="a missing database holds no schema", + schema="staging", + cursor=FakeSnowflakeMissingObjectCursor((("name",),), []), + expected_exists=False, + expected_sql_fragment="SHOW SCHEMAS LIKE 'STAGING' IN DATABASE", + ), + SnowflakeSchemaExistsTestCase( + description="a name with a backslash reads the information schema", + schema="stag\\ing", + cursor=FakeSnowflakeMetadataCursor(row=(1,)), + expected_exists=True, + expected_sql_fragment="information_schema.schemata", + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_when_checking_existence_then_show_needs_no_running_warehouse( + test_case: SnowflakeSchemaExistsTestCase, +) -> None: + connection: FakeSnowflakeDescribeConnection = FakeSnowflakeDescribeConnection( + cast(Any, test_case.cursor) + ) + + exists: bool = SnowflakeAdapter().schema_exists( + connection=cast(Any, connection), database="ANALYTICS", schema=test_case.schema + ) + + assert exists is test_case.expected_exists + assert test_case.expected_sql_fragment in str(cast(Any, test_case.cursor).executed_sql) From ccba10450162af6e4b6265f5f92662b30d6c6bff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:43:11 +0000 Subject: [PATCH 03/13] feat(location): mirror column migrations, node results and consumption state The location sync now reads every state table SQLBuild writes. Column migrations, node results and consumption join the other kinds, ahead of catalog and statistics. Node result and consumption rows have no event ID, so the server derives one from their identity columns, and retried identical rows collapse to one. Each kind names its source state table, and a test fails when a state table has no kind. The janitor's built-in exclusions now use the same list. The warehouse_metadata main modules are grouped into ingest, location-read and person-read buckets. --- .../column_migration_history.json | 46 +++++++ .../orchestration/consumption_history.json | 63 +++++++++ .../orchestration/node_result_history.json | 52 ++++++++ clients/typescript/src/schema.d.ts | 2 +- .../src/janitor_planning/constants.rs | 20 +-- crates/sqlbuild-core/src/constants.rs | 21 +++ .../_helpers/resources/person_operations.rs | 10 +- .../api/_helpers/resources/person_reads.rs | 2 +- .../_helpers/resources/warehouse_metadata.rs | 30 +++-- .../src/platform/_helpers/retention.rs | 2 +- .../src/store/schema.sql | 36 +++++ .../src/warehouse_metadata/_helpers/events.rs | 9 +- .../main/{ => ingest}/ingest_audit_results.rs | 0 .../main/{ => ingest}/ingest_catalog.rs | 0 .../main/ingest/ingest_column_migrations.rs | 46 +++++++ .../main/ingest/ingest_consumption.rs | 57 ++++++++ .../main/{ => ingest}/ingest_fingerprints.rs | 0 .../{ => ingest}/ingest_janitor_events.rs | 0 .../main/{ => ingest}/ingest_microbatches.rs | 0 .../main/{ => ingest}/ingest_migrations.rs | 0 .../main/ingest/ingest_node_results.rs | 59 ++++++++ .../{ => ingest}/ingest_old_name_views.rs | 0 .../{ => ingest}/ingest_source_freshness.rs | 0 .../main/{ => ingest}/ingest_statistics.rs | 0 .../src/warehouse_metadata/main/ingest/mod.rs | 13 ++ .../main/{ => ingest}/prune_statistics.rs | 0 .../main/location_reads/mod.rs | 2 + .../read_statistics_due.rs | 0 .../{ => location_reads}/read_watermark.rs | 0 .../src/warehouse_metadata/main/mod.rs | 21 +-- .../list_person_catalog_nodes.rs | 0 .../main/person_reads/mod.rs | 6 + .../{ => person_reads}/read_person_catalog.rs | 0 .../read_person_column_lineage.rs | 0 .../read_person_fingerprint.rs | 0 .../read_person_node_status.rs | 0 .../read_person_statistics.rs | 0 .../src/warehouse_metadata/models.rs | 126 +++++++++++++++++- .../tests/api/fingerprint_reads/helpers.rs | 3 +- .../tests/api/node_status/helpers.rs | 8 +- .../api/statistics/eligibility/helpers.rs | 5 +- .../tests/api/statistics/helpers.rs | 3 +- .../tests/api/statistics/retention/helpers.rs | 5 +- .../tests/api/test_catalog_reads.rs | 8 +- .../tests/api/test_checks.rs | 2 +- .../tests/api/test_column_lineage.rs | 2 +- .../tests/api/test_maintenance.rs | 2 +- .../tests/api/test_microbatch_coverage.rs | 2 +- .../tests/api/test_types.rs | 7 + .../tests/api/test_warehouse_metadata.rs | 50 ++++++- .../api/warehouse_metadata/events/helpers.rs | 3 +- .../tests/credentials/roles/helpers.rs | 4 +- .../tests/sensors/test_sources.rs | 2 +- .../native_host/warehouse_metadata.rs | 77 ++++++++++- crates/sqlbuild-state/src/events/constants.rs | 15 +++ .../compiled_project/publication/helpers.py | 98 +++++++++++++- .../publication/test_warehouse_metadata.py | 67 ++++++++++ 57 files changed, 898 insertions(+), 88 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/column_migration_history.json create mode 100644 benchmarks/duckshop/edge_cases/orchestration/consumption_history.json create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_result_history.json rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_audit_results.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_catalog.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_fingerprints.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_janitor_events.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_microbatches.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_migrations.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_old_name_views.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_source_freshness.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/ingest_statistics.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => ingest}/prune_statistics.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => location_reads}/read_statistics_due.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => location_reads}/read_watermark.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => person_reads}/list_person_catalog_nodes.rs (100%) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => person_reads}/read_person_catalog.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => person_reads}/read_person_column_lineage.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => person_reads}/read_person_fingerprint.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => person_reads}/read_person_node_status.rs (100%) rename crates/sqlbuild-orchestration/src/warehouse_metadata/main/{ => person_reads}/read_person_statistics.rs (100%) diff --git a/benchmarks/duckshop/edge_cases/orchestration/column_migration_history.json b/benchmarks/duckshop/edge_cases/orchestration/column_migration_history.json new file mode 100644 index 0000000000..7b7cb83187 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/column_migration_history.json @@ -0,0 +1,46 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "rename-000", + "target_name": null, + "model_name": "orders", + "relation_database": null, + "relation_schema": "main", + "relation_name": "orders_🧾", + "origin_column": "customer_🧾", + "destination_column": "customer_id", + "discovery": "manual", + "decision": "rename", + "run_id": "external-rename", + "created_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "page_lengths": [ + 100, + 4 + ], + "later_index": 103, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id", + "retained_result": "rename-103", + "kind": "column_migrations", + "source_key": "event_id", + "timestamp_key": "created_at", + "id_prefix": "rename", + "later_record": { + "event_id": "rename-103", + "created_at": "2026-02-03T04:05:06.000001Z" + }, + "last_record": { + "event_id": "rename-103", + "created_at": "2026-02-03T04:05:06.000001Z" + }, + "record_overrides": { + "rename-103": { + "created_at": "2026-02-03T04:05:06.000001Z" + } + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json b/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json new file mode 100644 index 0000000000..90f8ba0007 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json @@ -0,0 +1,63 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "execution_id": "execution-000", + "phase": "succeeded", + "run_id": "external-run", + "node_type": "model", + "node_name": "orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders_🧾", + "version_hash": "orders-version", + "input_count": 1, + "input_kind": "source", + "input_name": "shop.orders_🧾", + "input_version": "source-version", + "recorded_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "page_lengths": [ + 100, + 7 + ], + "later_index": 106, + "source_count": 107, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id", + "retained_result": "execution-103", + "kind": "consumption", + "source_key": "execution_id", + "timestamp_key": "recorded_at", + "id_prefix": "execution", + "later_record": { + "execution_id": "execution-103", + "phase": "started", + "input_count": 0, + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.000001Z" + }, + "last_record": { + "execution_id": "execution-103", + "phase": "started", + "input_count": 0, + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.000001Z" + }, + "record_overrides": { + "execution-103": { + "phase": "started", + "input_count": 0, + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.000001Z" + } + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_result_history.json b/benchmarks/duckshop/edge_cases/orchestration/node_result_history.json new file mode 100644 index 0000000000..db3a8f2dae --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_result_history.json @@ -0,0 +1,52 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "node_type": "task", + "node_name": "result-000", + "target_database": null, + "target_schema": "main", + "target_name": null, + "run_id": "external-run", + "status": "success", + "payload_json_b64": "eyJub3RlIjoiUmV0YWluZWQgJ29yZGVycycgXHVkODNlXHVkZGZlIiwicm93cyI6N30=", + "metadata_json_b64": "e30=", + "error_message": null, + "materialized": "true", + "ts": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "page_lengths": [ + 100, + 7 + ], + "later_index": 106, + "source_count": 107, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id", + "retained_result": "result-103", + "kind": "node_results", + "source_key": "node_name", + "timestamp_key": "ts", + "id_prefix": "result", + "later_record": { + "node_name": "result-103", + "status": "failed", + "error_message": "Rejected 'orders' 🧾", + "ts": "2026-02-03T04:05:06.000001Z" + }, + "last_record": { + "node_name": "result-103", + "status": "failed", + "error_message": "Rejected 'orders' 🧾", + "ts": "2026-02-03T04:05:06.000001Z" + }, + "record_overrides": { + "result-103": { + "status": "failed", + "error_message": "Rejected 'orders' 🧾", + "ts": "2026-02-03T04:05:06.000001Z" + } + } +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 0a2c10dcbb..8a7d06b209 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -3039,7 +3039,7 @@ export interface components { * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "catalog" | "statistics"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "catalog" | "statistics"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 29ab3e1097..c68aa2bfc7 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -1,25 +1,9 @@ //! Janitor planning names, prefixes and skip reasons. -use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; -use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; -use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_core::constants::MIGRATION_TABLE_NAME; -use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; -use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::constants::STATE_TABLE_NAMES; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 10] = [ - FINGERPRINT_TABLE_NAME, - "_sqlbuild_microbatches", - SOURCE_FRESHNESS_TABLE_NAME, - CONSUMPTION_TABLE_NAME, - "_sqlbuild_node_results", - "_sqlbuild_audit_results", - "_sqlbuild_janitor_events", - MIGRATION_TABLE_NAME, - COLUMN_MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, -]; +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 10] = STATE_TABLE_NAMES; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; /// Separator between archive name segments. diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 7f5096db14..7eeb30d732 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -30,6 +30,27 @@ pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; pub const COLUMN_MIGRATION_TABLE_NAME: &str = "_sqlbuild_column_migrations"; /// State table of old-name view events. pub const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; +/// State table of each node's outcome in each run. +pub const NODE_RESULTS_TABLE_NAME: &str = "_sqlbuild_node_results"; +/// State table of microbatch requirements and completions. +pub const MICROBATCH_TABLE_NAME: &str = "_sqlbuild_microbatches"; +/// State table of audit outcomes. +pub const AUDIT_RESULTS_TABLE_NAME: &str = "_sqlbuild_audit_results"; +/// State table of janitor archive and drop events. +pub const JANITOR_EVENTS_TABLE_NAME: &str = "_sqlbuild_janitor_events"; +/// Every state table SQLBuild writes in a target's state schema. +pub const STATE_TABLE_NAMES: [&str; 10] = [ + FINGERPRINT_TABLE_NAME, + MICROBATCH_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, + CONSUMPTION_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, + AUDIT_RESULTS_TABLE_NAME, + JANITOR_EVENTS_TABLE_NAME, + MIGRATION_TABLE_NAME, + COLUMN_MIGRATION_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, +]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; /// Node key of a common table expression in the polyglot SQL AST. diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 602c4e4621..fd51e5d8cc 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -26,11 +26,11 @@ use crate::runs::models::{ use crate::runs::types::{CommandPlanner, RunsListing}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; -use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; -use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; -use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; -use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status; -use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; +use crate::warehouse_metadata::main::person_reads::read_person_catalog::read_person_catalog; +use crate::warehouse_metadata::main::person_reads::read_person_column_lineage::read_person_column_lineage; +use crate::warehouse_metadata::main::person_reads::read_person_fingerprint::read_person_fingerprint; +use crate::warehouse_metadata::main::person_reads::read_person_node_status::read_person_node_status; +use crate::warehouse_metadata::main::person_reads::read_person_statistics::read_person_statistics; use crate::warehouse_metadata::models::{ CatalogPage, CatalogQuery, ColumnLineageRead, FingerprintObservation, FingerprintQuery, NodeStatus, NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index 66d81a041a..1e2b6c3f05 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -41,7 +41,7 @@ use crate::warehouse_history::main::read_person_run_node_error::read_person_run_ use crate::warehouse_history::main::read_person_run_output::read_person_run_output; use crate::warehouse_history::main::read_person_run_statement_sql::read_person_run_statement_sql; use crate::warehouse_history::main::read_person_run_statements::read_person_run_statements; -use crate::warehouse_metadata::main::list_person_catalog_nodes::list_person_catalog_nodes; +use crate::warehouse_metadata::main::person_reads::list_person_catalog_nodes::list_person_catalog_nodes; use crate::warehouse_metadata::models::CatalogNodeRecord; #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", operation_id = "listPersonRunNodes", diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs index 9354aada8d..4632b8b76a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs @@ -8,17 +8,20 @@ use crate::api::_helpers::authentication::bearer::require_location; use crate::api::errors::ApiError; use crate::credentials::models::TokenIdentity; use crate::store::models::ControlStore; -use crate::warehouse_metadata::main::ingest_audit_results::ingest_audit_results; -use crate::warehouse_metadata::main::ingest_catalog::ingest_catalog; -use crate::warehouse_metadata::main::ingest_fingerprints::ingest_fingerprints; -use crate::warehouse_metadata::main::ingest_janitor_events::ingest_janitor_events; -use crate::warehouse_metadata::main::ingest_microbatches::ingest_microbatches; -use crate::warehouse_metadata::main::ingest_migrations::ingest_migrations; -use crate::warehouse_metadata::main::ingest_old_name_views::ingest_old_name_views; -use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; -use crate::warehouse_metadata::main::ingest_statistics::ingest_statistics; -use crate::warehouse_metadata::main::read_statistics_due::read_statistics_due; -use crate::warehouse_metadata::main::read_watermark::read_watermark; +use crate::warehouse_metadata::main::ingest::ingest_audit_results::ingest_audit_results; +use crate::warehouse_metadata::main::ingest::ingest_catalog::ingest_catalog; +use crate::warehouse_metadata::main::ingest::ingest_column_migrations::ingest_column_migrations; +use crate::warehouse_metadata::main::ingest::ingest_consumption::ingest_consumption; +use crate::warehouse_metadata::main::ingest::ingest_fingerprints::ingest_fingerprints; +use crate::warehouse_metadata::main::ingest::ingest_janitor_events::ingest_janitor_events; +use crate::warehouse_metadata::main::ingest::ingest_microbatches::ingest_microbatches; +use crate::warehouse_metadata::main::ingest::ingest_migrations::ingest_migrations; +use crate::warehouse_metadata::main::ingest::ingest_node_results::ingest_node_results; +use crate::warehouse_metadata::main::ingest::ingest_old_name_views::ingest_old_name_views; +use crate::warehouse_metadata::main::ingest::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::main::ingest::ingest_statistics::ingest_statistics; +use crate::warehouse_metadata::main::location_reads::read_statistics_due::read_statistics_due; +use crate::warehouse_metadata::main::location_reads::read_watermark::read_watermark; use crate::warehouse_metadata::models::{ MetadataBatch, StatisticsDueRead, StatisticsRead, WarehouseHistoryKind, WarehouseSyncResponse, WarehouseWatermarkRead, @@ -47,6 +50,11 @@ pub(crate) async fn report( WarehouseHistoryKind::Migrations => ingest_migrations(&store, scope, &batch).await, WarehouseHistoryKind::OldNameViews => ingest_old_name_views(&store, scope, &batch).await, WarehouseHistoryKind::Microbatches => ingest_microbatches(&store, scope, &batch).await, + WarehouseHistoryKind::ColumnMigrations => { + ingest_column_migrations(&store, scope, &batch).await + } + WarehouseHistoryKind::NodeResults => ingest_node_results(&store, scope, &batch).await, + WarehouseHistoryKind::Consumption => ingest_consumption(&store, scope, &batch).await, WarehouseHistoryKind::Catalog => ingest_catalog(&store, scope, &batch).await, WarehouseHistoryKind::Statistics => ingest_statistics(&store, scope, &batch).await, }?; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs index bacfb54d03..05bd7f95eb 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs @@ -14,7 +14,7 @@ use crate::store::errors::ControlError; use crate::store::main::{query::query, write::write}; use crate::store::models::{ControlStore, Parameter}; use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; -use crate::warehouse_metadata::main::prune_statistics::prune_statistics; +use crate::warehouse_metadata::main::ingest::prune_statistics::prune_statistics; use crate::warehouse_metadata::models::StatisticsRetention; /// Old evaluation phases go once a later attempt of the same sensor exists. diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index a7e4973c6d..f12fd87027 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -465,6 +465,42 @@ CREATE TABLE sqb_warehouse_microbatches ( ); CREATE INDEX sqb_microbatches_namespace ON sqb_warehouse_microbatches (location_id,target,state_database,state_schema,observed_at,event_id); CREATE INDEX sqb_microbatches_model ON sqb_warehouse_microbatches (location_id,target,model_name); +CREATE TABLE sqb_warehouse_column_migrations ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_column_migrations_namespace ON sqb_warehouse_column_migrations (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_node_results ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_node_results_namespace ON sqb_warehouse_node_results (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_consumption ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_consumption_namespace ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( location_id, target, (record_json ->> 'source_name'), (record_json ->> 'target_database'), (record_json ->> 'target_schema'), diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs index 8b1e5693c8..fd614ea6eb 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs @@ -27,7 +27,12 @@ pub(crate) async fn ingest_events( ), )?; let table = batch.kind.mirror_table(); - let source_table = format!("_sqlbuild_{}", batch.kind.as_str()); + let source_table = batch.kind.source_table().ok_or_else(|| { + ControlError::Invalid(format!( + "{} is not a state-table history", + batch.kind.as_str() + )) + })?; let mut transaction = begin_ingestion( store, (location_id, target), @@ -42,7 +47,7 @@ pub(crate) async fn ingest_events( ( record.location.database.as_deref(), &record.location.schema, - &source_table, + source_table, ), &record.source_id, )?; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_catalog.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs new file mode 100644 index 0000000000..0c2b149a9c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs @@ -0,0 +1,46 @@ +//! Mirror column rename decisions; the warehouse state table stays the owner of the history. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseColumnMigrationRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_column_migrations( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.created_at, + columns: Vec::new(), + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| [&record.event_id, &record.state_schema]) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs new file mode 100644 index 0000000000..7f8520483d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs @@ -0,0 +1,57 @@ +//! Mirror consumed input versions; rows carry no event identity, so their identity columns derive one. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseConsumptionRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_consumption( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: sha256_hex( + json!([ + record.execution_id, + record.phase, + record.input_kind, + record.input_name, + record.input_version, + record.recorded_at, + ]) + .to_string() + .as_bytes(), + ), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.recorded_at, + columns: Vec::new(), + record_json: json!(record), + }) + .collect(), + records + .iter() + .any(|record| record.state_schema.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_fingerprints.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_janitor_events.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_janitor_events.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_microbatches.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_microbatches.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_migrations.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_migrations.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs new file mode 100644 index 0000000000..b1f69c33d2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs @@ -0,0 +1,59 @@ +//! Mirror node outcomes; rows carry no event identity, so their identity columns derive one. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseNodeResultRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_node_results( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: sha256_hex( + json!([ + record.node_type, + record.node_name, + record.target_database, + record.target_schema, + record.target_name, + record.run_id, + record.status, + record.ts, + ]) + .to_string() + .as_bytes(), + ), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.ts, + columns: Vec::new(), + record_json: json!(record), + }) + .collect(), + records + .iter() + .any(|record| record.state_schema.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_old_name_views.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_old_name_views.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_source_freshness.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_statistics.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs new file mode 100644 index 0000000000..aa95f66a37 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs @@ -0,0 +1,13 @@ +pub mod ingest_audit_results; +pub mod ingest_catalog; +pub mod ingest_column_migrations; +pub mod ingest_consumption; +pub mod ingest_fingerprints; +pub mod ingest_janitor_events; +pub mod ingest_microbatches; +pub mod ingest_migrations; +pub mod ingest_node_results; +pub mod ingest_old_name_views; +pub mod ingest_source_freshness; +pub mod ingest_statistics; +pub(crate) mod prune_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/prune_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/prune_statistics.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/prune_statistics.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/prune_statistics.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs new file mode 100644 index 0000000000..af7a2a5d7a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs @@ -0,0 +1,2 @@ +pub mod read_statistics_due; +pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_statistics_due.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_statistics_due.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_watermark.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_watermark.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index f3f9f53dc5..d3bb217ec4 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,18 +1,3 @@ -pub mod ingest_audit_results; -pub mod ingest_catalog; -pub mod ingest_fingerprints; -pub mod ingest_janitor_events; -pub mod ingest_microbatches; -pub mod ingest_migrations; -pub mod ingest_old_name_views; -pub mod ingest_source_freshness; -pub mod ingest_statistics; -pub mod list_person_catalog_nodes; -pub(crate) mod prune_statistics; -pub mod read_person_catalog; -pub mod read_person_column_lineage; -pub mod read_person_fingerprint; -pub mod read_person_node_status; -pub mod read_person_statistics; -pub mod read_statistics_due; -pub mod read_watermark; +pub mod ingest; +pub mod location_reads; +pub mod person_reads; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/list_person_catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/list_person_catalog_nodes.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs new file mode 100644 index 0000000000..eb7fc45d0f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/mod.rs @@ -0,0 +1,6 @@ +pub mod list_person_catalog_nodes; +pub mod read_person_catalog; +pub mod read_person_column_lineage; +pub mod read_person_fingerprint; +pub mod read_person_node_status; +pub mod read_person_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_catalog.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_catalog.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_column_lineage.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_column_lineage.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_fingerprint.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_fingerprint.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_node_status.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_statistics.rs similarity index 100% rename from crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs rename to crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_statistics.rs diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index f0c0c2297e..74988e49c9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -4,6 +4,12 @@ use chrono::{DateTime, Duration, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use sqlbuild_core::constants::{ + AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, + FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, +}; + use crate::store::errors::ControlError; use crate::warehouse_metadata::constants::DEFAULT_STATISTICS_RETENTION_DAYS; @@ -202,6 +208,68 @@ pub struct WarehouseStateLocation { pub schema: String, } +/// Column renames a model migration decided; only the event identity and time are required. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseColumnMigrationRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub target_name: Option, + pub model_name: Option, + pub relation_database: Option, + pub relation_schema: Option, + pub relation_name: Option, + pub origin_column: Option, + pub destination_column: Option, + pub discovery: Option, + pub decision: Option, + pub run_id: Option, + pub created_at: DateTime, +} + +/// One node's recorded outcome in one run, with its encoded payloads kept as written. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseNodeResultRecord { + pub state_database: Option, + pub state_schema: String, + pub node_type: Option, + pub node_name: Option, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: Option, + pub status: Option, + pub payload_json_b64: Option, + pub metadata_json_b64: Option, + pub error_message: Option, + pub materialized: Option, + pub ts: DateTime, +} + +/// One input version an execution consumed, or the row marking that the execution started. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseConsumptionRecord { + pub state_database: Option, + pub state_schema: String, + pub execution_id: Option, + pub phase: Option, + pub run_id: Option, + pub node_type: Option, + pub node_name: Option, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub version_hash: Option, + pub input_count: Option, + pub input_kind: Option, + pub input_name: Option, + pub input_version: Option, + pub recorded_at: DateTime, +} + /// Catalog observations require full scans; only immutable history supports a watermark. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] @@ -213,12 +281,16 @@ pub enum WarehouseHistoryKind { Migrations, OldNameViews, Microbatches, + ColumnMigrations, + NodeResults, + Consumption, Catalog, Statistics, } impl WarehouseHistoryKind { - pub const ALL: [Self; 9] = [ + /// Catalog and statistics stay last: a failed statistics read rewinds one kind, to catalog. + pub const ALL: [Self; 12] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -226,6 +298,9 @@ impl WarehouseHistoryKind { Self::Migrations, Self::OldNameViews, Self::Microbatches, + Self::ColumnMigrations, + Self::NodeResults, + Self::Consumption, Self::Catalog, Self::Statistics, ]; @@ -240,12 +315,32 @@ impl WarehouseHistoryKind { Self::Migrations => "migrations", Self::OldNameViews => "old_name_views", Self::Microbatches => "microbatches", + Self::ColumnMigrations => "column_migrations", + Self::NodeResults => "node_results", + Self::Consumption => "consumption", Self::Catalog => "catalog", Self::Statistics => "statistics", } } #[must_use] + /// The state table a kind mirrors; catalog and statistics read the warehouse catalog instead. + pub const fn source_table(self) -> Option<&'static str> { + match self { + Self::Fingerprints => Some(FINGERPRINT_TABLE_NAME), + Self::SourceFreshness => Some(SOURCE_FRESHNESS_TABLE_NAME), + Self::AuditResults => Some(AUDIT_RESULTS_TABLE_NAME), + Self::JanitorEvents => Some(JANITOR_EVENTS_TABLE_NAME), + Self::Migrations => Some(MIGRATION_TABLE_NAME), + Self::OldNameViews => Some(OLD_NAME_VIEW_TABLE_NAME), + Self::Microbatches => Some(MICROBATCH_TABLE_NAME), + Self::ColumnMigrations => Some(COLUMN_MIGRATION_TABLE_NAME), + Self::NodeResults => Some(NODE_RESULTS_TABLE_NAME), + Self::Consumption => Some(CONSUMPTION_TABLE_NAME), + Self::Catalog | Self::Statistics => None, + } + } + pub const fn mirror_table(self) -> &'static str { match self { Self::Fingerprints => "sqb_warehouse_fingerprints", @@ -255,6 +350,9 @@ impl WarehouseHistoryKind { Self::Migrations => "sqb_warehouse_migrations", Self::OldNameViews => "sqb_warehouse_old_name_views", Self::Microbatches => "sqb_warehouse_microbatches", + Self::ColumnMigrations => "sqb_warehouse_column_migrations", + Self::NodeResults => "sqb_warehouse_node_results", + Self::Consumption => "sqb_warehouse_consumption", Self::Catalog => "sqb_warehouse_catalog", Self::Statistics => "sqb_warehouse_statistics", } @@ -346,6 +444,26 @@ pub struct FingerprintRead { pub limit: Option, } +/// Node results carry no event identity, so the cursor holds the row's identity columns. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeResultsRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 9]>, + pub observed_since: Option>, + pub limit: Option, +} + +/// Consumption rows carry no event identity, so the cursor holds the row's identity columns. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ConsumptionRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 7]>, + pub observed_since: Option>, + pub limit: Option, +} + /// Stable event identities and timestamps order observations without comparing their payloads. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -367,6 +485,9 @@ pub enum WarehouseHistoryRead { Migrations(StateEventsRead), OldNameViews(StateEventsRead), Microbatches(StateEventsRead), + ColumnMigrations(StateEventsRead), + NodeResults(NodeResultsRead), + Consumption(ConsumptionRead), Catalog(CatalogRead), Statistics(StatisticsRead), } @@ -381,6 +502,9 @@ pub enum WarehouseHistoryRecord { Migration(Box), OldNameView(Box), Microbatch(Box), + ColumnMigration(Box), + NodeResult(Box), + Consumption(Box), Catalog(WarehouseCatalogRecord), Statistics(WarehouseStatisticsRecord), } diff --git a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs index 3018d28bb1..45eef6062a 100644 --- a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs @@ -9,7 +9,8 @@ use crate::platform::models::{LocationHealth, LocationHealthReport}; use crate::store::errors::ControlError; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::{ - ingest_fingerprints::ingest_fingerprints, read_person_fingerprint::read_person_fingerprint, + ingest::ingest_fingerprints::ingest_fingerprints, + person_reads::read_person_fingerprint::read_person_fingerprint, }; use crate::warehouse_metadata::models::{FingerprintQuery, MetadataBatch}; diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs index 00d6459944..8287aa9c47 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -14,10 +14,10 @@ use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::warehouse_metadata::main::{ - ingest_catalog::ingest_catalog, ingest_fingerprints::ingest_fingerprints, - ingest_source_freshness::ingest_source_freshness, - list_person_catalog_nodes::list_person_catalog_nodes, - read_person_node_status::read_person_node_status, + ingest::ingest_catalog::ingest_catalog, ingest::ingest_fingerprints::ingest_fingerprints, + ingest::ingest_source_freshness::ingest_source_freshness, + person_reads::list_person_catalog_nodes::list_person_catalog_nodes, + person_reads::read_person_node_status::read_person_node_status, }; use crate::warehouse_metadata::models::{MetadataBatch, NodeStatusQuery}; diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs index 84a61e7b88..e9f5fb4264 100644 --- a/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs @@ -7,8 +7,9 @@ use uuid::Uuid; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::{ - ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, - read_statistics_due::read_statistics_due, + ingest::ingest_statistics::ingest_statistics, + location_reads::read_statistics_due::read_statistics_due, + person_reads::read_person_statistics::read_person_statistics, }; use crate::warehouse_metadata::models::{ MetadataBatch, StatisticsDueRead, StatisticsQuery, WarehouseStateLocation, diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs index eb74eba6b0..31b328458b 100644 --- a/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs @@ -9,7 +9,8 @@ use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, TransactionMode}; use crate::warehouse_metadata::main::{ - ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, + ingest::ingest_statistics::ingest_statistics, + person_reads::read_person_statistics::read_person_statistics, }; use crate::warehouse_metadata::models::{MetadataBatch, StatisticsQuery}; diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs index 3f63670636..179fd207a3 100644 --- a/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs @@ -6,9 +6,10 @@ use uuid::Uuid; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::store::models::ControlStore; -use crate::warehouse_metadata::main::prune_statistics::prune_statistics; +use crate::warehouse_metadata::main::ingest::prune_statistics::prune_statistics; use crate::warehouse_metadata::main::{ - ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, + ingest::ingest_statistics::ingest_statistics, + person_reads::read_person_statistics::read_person_statistics, }; use crate::warehouse_metadata::models::{MetadataBatch, StatisticsQuery, StatisticsRetention}; diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs index c46382999f..96bfd0d3d9 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -24,7 +24,7 @@ use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; use crate::warehouse_metadata::main::{ - ingest_catalog::ingest_catalog, read_person_catalog::read_person_catalog, + ingest::ingest_catalog::ingest_catalog, person_reads::read_person_catalog::read_person_catalog, }; use crate::warehouse_metadata::models::{CatalogQuery, MetadataBatch}; @@ -307,7 +307,7 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp .await .expect("revoke"); assert!(matches!( - crate::warehouse_metadata::main::read_person_statistics::read_person_statistics( + crate::warehouse_metadata::main::person_reads::read_person_statistics::read_person_statistics( store, credentials, ( @@ -328,7 +328,7 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp Err(ControlError::Denied { .. }) )); assert!(matches!( - crate::warehouse_metadata::main::read_person_node_status::read_person_node_status( + crate::warehouse_metadata::main::person_reads::read_person_node_status::read_person_node_status( store, credentials, ( @@ -351,7 +351,7 @@ fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namesp name: Some("orders".into()), }; assert!(matches!( - crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint( + crate::warehouse_metadata::main::person_reads::read_person_fingerprint::read_person_fingerprint( store, credentials, (location, "development", &evidence), diff --git a/crates/sqlbuild-orchestration/tests/api/test_checks.rs b/crates/sqlbuild-orchestration/tests/api/test_checks.rs index 46e3c3bf7a..2c1c86bbc8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_checks.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_checks.rs @@ -13,7 +13,7 @@ use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -use crate::warehouse_metadata::main::ingest_audit_results::ingest_audit_results; +use crate::warehouse_metadata::main::ingest::ingest_audit_results::ingest_audit_results; #[test] fn given_checks_sharing_an_audit_name_when_people_read_them_then_binding_keys_and_sql_permissions_hold() diff --git a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs index 68593a8c42..eb255b6e40 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs @@ -15,7 +15,7 @@ use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; +use crate::warehouse_metadata::main::person_reads::read_person_column_lineage::read_person_column_lineage; #[test] fn given_published_column_lineage_when_people_read_a_node_then_only_its_reachable_columns_return() { diff --git a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs index f395d47821..9fe6f671fa 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_maintenance.rs @@ -15,7 +15,7 @@ use crate::runs::tests::helpers::snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -use crate::warehouse_metadata::main::{ +use crate::warehouse_metadata::main::ingest::{ ingest_janitor_events::ingest_janitor_events, ingest_migrations::ingest_migrations, ingest_old_name_views::ingest_old_name_views, }; diff --git a/crates/sqlbuild-orchestration/tests/api/test_microbatch_coverage.rs b/crates/sqlbuild-orchestration/tests/api/test_microbatch_coverage.rs index 79bffe3bbc..734f5b50e3 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_microbatch_coverage.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_microbatch_coverage.rs @@ -10,7 +10,7 @@ use crate::api::tests::warehouse_history_helpers::{ }; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -use crate::warehouse_metadata::main::ingest_microbatches::ingest_microbatches; +use crate::warehouse_metadata::main::ingest::ingest_microbatches::ingest_microbatches; const HOUR_0: &str = "2026-01-01 00:00:00+00"; const HOUR_1: &str = "2026-01-01T09:00:00+08:00"; diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index e7c650905a..94f15bc1ef 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -1,6 +1,7 @@ use crate::runs::models::RunStatus; use crate::store::errors::ControlError; use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::models::WarehouseHistoryKind; pub(crate) struct ApiContractTestCase { pub(crate) description: &'static str, @@ -103,6 +104,12 @@ pub(crate) struct ColumnLineageTestCase { pub(crate) expected_leaf_edges: usize, } +pub(crate) struct StateTableClassificationTestCase { + pub(crate) description: &'static str, + pub(crate) expected_tables: Vec<&'static str>, + pub(crate) expected_last_kinds: [WarehouseHistoryKind; 2], +} + pub(crate) struct WarehouseMetadataTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index ef1a0ad714..9aae0418f6 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -3,11 +3,12 @@ use chrono::{Duration, Utc}; use reqwest::StatusCode; use serde_json::json; +use sqlbuild_core::constants::STATE_TABLE_NAMES; use uuid::Uuid; use crate::api::tests::changes_helpers::{connect, expect_topics}; use crate::api::tests::helpers::{start_api, stop_api}; -use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::api::tests::test_types::{StateTableClassificationTestCase, WarehouseMetadataTestCase}; use crate::credentials::main::tokens::issue_token::issue_token; use crate::credentials::models::{AuditContext, CredentialScope}; use crate::credentials::tests::location_helpers::create_location; @@ -26,6 +27,38 @@ use crate::store::tests::store_helpers::{close_test_store, open_test_store, run} use crate::store::tests::store_test_types::Backend; use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; +#[test] +fn given_state_tables_when_classified_then_each_is_mirrored_by_exactly_one_kind() { + let mut written: Vec<&str> = STATE_TABLE_NAMES.to_vec(); + written.sort_unstable(); + let test_cases = [StateTableClassificationTestCase { + description: "every state table SQLBuild writes, with statistics rewinding to catalog", + expected_tables: written, + expected_last_kinds: [ + WarehouseHistoryKind::Catalog, + WarehouseHistoryKind::Statistics, + ], + }]; + for test_case in test_cases { + let mut mirrored: Vec<&str> = WarehouseHistoryKind::ALL + .iter() + .filter_map(|kind| kind.source_table()) + .collect(); + mirrored.sort_unstable(); + assert_eq!( + mirrored, test_case.expected_tables, + "{}", + test_case.description + ); + assert_eq!( + WarehouseHistoryKind::ALL[WarehouseHistoryKind::ALL.len() - 2..], + test_case.expected_last_kinds, + "{}", + test_case.description + ); + } +} + #[test] fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicity_are_preserved() { let test_cases = [ @@ -348,6 +381,21 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit "microbatch_history", "event_id", ), + ( + WarehouseHistoryKind::ColumnMigrations, + "column_migration_history", + "event_id", + ), + ( + WarehouseHistoryKind::NodeResults, + "node_result_history", + "node_name", + ), + ( + WarehouseHistoryKind::Consumption, + "consumption_history", + "execution_id", + ), ] { crate::api::tests::warehouse_events_helpers::event_history( store, diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs index 3a437d2cb4..c8f58887a4 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs @@ -136,7 +136,8 @@ pub(crate) async fn event_history( description: "event page rolls back atomically", table: kind.mirror_table(), operation: "INSERT", - condition: "WHEN 'rejected-result' IN (NEW.record_json ->> 'event_id', NEW.record_json ->> 'result_id')", + condition: "WHEN 'rejected-result' IN (NEW.record_json ->> 'event_id', NEW.record_json ->> 'result_id', \ + NEW.record_json ->> 'node_name', NEW.record_json ->> 'execution_id')", sqlite_body: "SELECT RAISE(ABORT, 'injected audit failure')", postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; diff --git a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs index fa5ba26b5e..aeb4358e1c 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs @@ -29,8 +29,8 @@ use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, TransactionMode}; -use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; -use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; +use crate::warehouse_metadata::main::person_reads::read_person_catalog::read_person_catalog; +use crate::warehouse_metadata::main::person_reads::read_person_statistics::read_person_statistics; use crate::warehouse_metadata::models::{CatalogQuery, StatisticsQuery, StatisticsRetention}; pub(crate) const SUPPORT_ROLE: &str = "orders_support"; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 2f4831ba58..d16bcc357d 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -28,7 +28,7 @@ use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; -use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::main::ingest::ingest_source_freshness::ingest_source_freshness; use crate::warehouse_metadata::models::{ MetadataBatch, MetadataScan, WarehouseFreshnessRecord, WarehouseHistoryKind, WarehouseStateLocation, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index a5c265e81e..005fff9c83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -8,13 +8,19 @@ use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGER use sqlbuild_config::command_connection::constants::{ DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; -use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::constants::{ + COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, +}; use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, }; -use sqlbuild_state::events::constants::{MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS}; +use sqlbuild_state::consumption::constants::{CONSUMPTION_COLUMN_TYPES, RECORDED_AT_COLUMN}; +use sqlbuild_state::events::constants::{ + COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, NODE_RESULT_COLUMNS, OLD_NAME_VIEW_COLUMNS, +}; use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, }; @@ -50,7 +56,10 @@ pub(crate) fn read_history( | WarehouseHistoryRead::JanitorEvents(read) | WarehouseHistoryRead::Migrations(read) | WarehouseHistoryRead::OldNameViews(read) - | WarehouseHistoryRead::Microbatches(read) => &read.location, + | WarehouseHistoryRead::Microbatches(read) + | WarehouseHistoryRead::ColumnMigrations(read) => &read.location, + WarehouseHistoryRead::NodeResults(read) => &read.location, + WarehouseHistoryRead::Consumption(read) => &read.location, }; if location.schema.trim().is_empty() { return Err(PyValueError::new_err( @@ -110,6 +119,7 @@ fn read_page( request: &WarehouseHistoryRead, ) -> PyResult { let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let consumption_columns = CONSUMPTION_COLUMN_TYPES.map(|(name, _)| name); let event_columns: Vec = match request { WarehouseHistoryRead::AuditResults(_) => py .import("sqlbuild.executor.audit_results.constants")? @@ -196,6 +206,33 @@ fn read_page( event_columns.as_slice(), "created_at", ), + WarehouseHistoryRead::ColumnMigrations(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + COLUMN_MIGRATION_TABLE_NAME, + COLUMN_MIGRATION_COLUMNS.as_slice(), + "created_at", + ), + WarehouseHistoryRead::NodeResults(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + NODE_RESULTS_TABLE_NAME, + NODE_RESULT_COLUMNS.as_slice(), + "ts", + ), + WarehouseHistoryRead::Consumption(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + CONSUMPTION_TABLE_NAME, + consumption_columns.as_slice(), + RECORDED_AT_COLUMN, + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -268,7 +305,26 @@ fn read_page( WarehouseHistoryRead::JanitorEvents(_) => Some(&["event_id", "occurred_at"]), WarehouseHistoryRead::Migrations(_) | WarehouseHistoryRead::OldNameViews(_) - | WarehouseHistoryRead::Microbatches(_) => Some(&["event_id", "created_at"]), + | WarehouseHistoryRead::Microbatches(_) + | WarehouseHistoryRead::ColumnMigrations(_) => Some(&["event_id", "created_at"]), + WarehouseHistoryRead::NodeResults(_) => Some(&[ + "node_type", + "node_name", + "target_database", + "target_schema", + "target_name", + "run_id", + "status", + "ts", + ]), + WarehouseHistoryRead::Consumption(_) => Some(&[ + "execution_id", + "phase", + "input_kind", + "input_name", + "input_version", + RECORDED_AT_COLUMN, + ]), _ => None, }; let mut cursor_indexes: Vec = columns @@ -389,6 +445,9 @@ fn read_page( WarehouseHistoryRecord::Migration(record) => record.created_at, WarehouseHistoryRecord::OldNameView(record) => record.created_at, WarehouseHistoryRecord::Microbatch(record) => record.created_at, + WarehouseHistoryRecord::ColumnMigration(record) => record.created_at, + WarehouseHistoryRecord::NodeResult(record) => record.ts, + WarehouseHistoryRecord::Consumption(record) => record.recorded_at, }; if since.is_none_or(|since| observed >= since) { records.push(record); @@ -427,7 +486,15 @@ fn decode_record( qualified, )?, ), - WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::Migrations(read) => { + WarehouseHistoryRead::AuditResults(read) + | WarehouseHistoryRead::Migrations(read) + | WarehouseHistoryRead::ColumnMigrations(read) => { + return decode_event_record(&read.location, row, (columns, &[timestamp])); + } + WarehouseHistoryRead::NodeResults(read) => { + return decode_event_record(&read.location, row, (columns, &[timestamp])); + } + WarehouseHistoryRead::Consumption(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp])); } WarehouseHistoryRead::JanitorEvents(read) => { diff --git a/crates/sqlbuild-state/src/events/constants.rs b/crates/sqlbuild-state/src/events/constants.rs index c89814977d..51b8b09a1d 100644 --- a/crates/sqlbuild-state/src/events/constants.rs +++ b/crates/sqlbuild-state/src/events/constants.rs @@ -34,6 +34,21 @@ pub const MIGRATION_COLUMNS: [&str; 15] = [ "run_id", "created_at", ]; +/// Columns of `_sqlbuild_node_results`, in the order the executor writes them. +pub const NODE_RESULT_COLUMNS: [&str; 12] = [ + "node_type", + "node_name", + "target_database", + "target_schema", + "target_name", + "run_id", + "status", + "payload_json_b64", + "metadata_json_b64", + "error_message", + "materialized", + "ts", +]; pub const COLUMN_MIGRATION_COLUMNS: [&str; 12] = [ "event_id", "target_name", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index dff015df40..7d9fb1c271 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import hashlib import json import os @@ -28,6 +29,8 @@ import pytest from sqlbuild._native import ( + ColumnMigrationDecision, + ColumnMigrationEvent, Fingerprint, MigrationDecision, MigrationDiscovery, @@ -35,6 +38,9 @@ MigrationRelation, OldNameViewEvent, OldNameViewEventType, + StateSqlValueType, + build_consumption_create_table_sql, + render_state_sql_literal, write_fingerprint, write_migration_state_event, ) @@ -47,6 +53,8 @@ ) from sqlbuild.executor.janitor_events.models import JanitorEventRecord from sqlbuild.executor.janitor_events.types import JanitorEventType +from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore +from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.microbatches.classes.direct_store import DirectMicrobatchEventStore from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS @@ -233,6 +241,92 @@ def append_migration_history(adapter: Any, connection: Any, fixture: dict[str, A ) +def append_column_migration_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real column-rename facts, including idempotent duplicate attempts.""" + data: dict[str, Any] = migration_history_data(fixture, ("relation",)) + data["discovery"] = MigrationDiscovery(data["discovery"]) + data["decision"] = ColumnMigrationDecision(data["decision"]) + record: ColumnMigrationEvent = ColumnMigrationEvent(**data) + write_migration_history( + adapter, + connection, + fixture, + record, + "column", + adapter.render_create_column_migration_state_table_sql(database=None, schema="main"), + {}, + ) + + +def append_node_result_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real node-result writer; retried writes repeat rows, which carry no event identity.""" + store: DirectNodeResultStore = DirectNodeResultStore( + adapter=adapter, connection=connection, database=None, schema="main" + ) + rows: list[dict[str, Any]] = identified_rows(fixture) + for row in rows + [rows[0]] * fixture["duplicate_rows"]: + store.write( + NodeResultRecord( + node_type=row["node_type"], + node_name=row["node_name"], + target_database=row["target_database"], + target_schema=row["target_schema"], + target_name=row["target_name"], + run_id=row["run_id"], + status=row["status"], + payload=json.loads(base64.b64decode(row["payload_json_b64"])), + metadata=json.loads(base64.b64decode(row["metadata_json_b64"])), + error_message=row["error_message"], + materialized=row["materialized"] == "true", + ts=datetime.fromisoformat(row["ts"]), + ) + ) + + +def append_consumption_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Consumption rows in the real table layout; retried appends repeat identical rows.""" + _ = adapter.execute( + connection=connection, + sql=build_consumption_create_table_sql( + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ), + ) + columns: list[str] = list(fixture["record"])[2:] + types: dict[str, StateSqlValueType] = dict.fromkeys(columns, StateSqlValueType.STRING) | { + "input_count": StateSqlValueType.INTEGER, + "recorded_at": StateSqlValueType.TIMESTAMP, + } + rows: list[dict[str, Any]] = [ + row | {"recorded_at": datetime.fromisoformat(row["recorded_at"])} + for row in identified_rows(fixture) + ] + for row in rows + [rows[0]] * fixture["duplicate_rows"]: + values: list[str] = [ + render_state_sql_literal( + value=row[column], declared_type=types[column], timestamp_type="TIMESTAMP" + ) + for column in columns + ] + _ = adapter.execute( + connection=connection, + sql=f"INSERT INTO main._sqlbuild_consumption ({', '.join(columns)}) " + f"VALUES ({', '.join(values)})", + ) + + +def identified_rows(fixture: dict[str, Any]) -> list[dict[str, Any]]: + """The fixture record once per source identity, with its per-identity overrides.""" + return [ + fixture["record"] + | {fixture["source_key"]: f"{fixture['id_prefix']}-{index:03}"} + | fixture["record_overrides"].get(f"{fixture['id_prefix']}-{index:03}", {}) + for index in range(fixture["source_rows"]) + ] + + def append_old_name_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: """Real compatibility-view facts, including nullable and populated expiry.""" data: dict[str, Any] = migration_history_data(fixture, ("old", "new")) @@ -273,8 +367,8 @@ def write_migration_history( adapter: Any, connection: Any, fixture: dict[str, Any], - record: MigrationEvent | OldNameViewEvent, - kind: Literal["model", "old_name"], + record: MigrationEvent | ColumnMigrationEvent | OldNameViewEvent, + kind: Literal["model", "column", "old_name"], create_sql: str, last_values: dict[str, Any], ) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 4d466244e5..83f43139f4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -45,11 +45,14 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_audit_history, + append_column_migration_history, + append_consumption_history, append_contended_fingerprints, append_janitor_history, append_legacy_history, append_microbatch_history, append_migration_history, + append_node_result_history, append_old_name_history, append_retried_fingerprints, location_process, @@ -122,6 +125,38 @@ ), ("sqlite", "path", "migration_history", append_migration_history, "external-move", ()), ("sqlite", "path", "old_name_history", append_old_name_history, "external-move", ()), + ( + "duckdb", + "database", + "column_migration_history", + append_column_migration_history, + "external-rename", + (), + ), + ( + "sqlite", + "path", + "column_migration_history", + append_column_migration_history, + "external-rename", + (), + ), + ( + "duckdb", + "database", + "node_result_history", + append_node_result_history, + "external-run", + (), + ), + ( + "duckdb", + "database", + "consumption_history", + append_consumption_history, + "external-run", + (), + ), ( "duckdb", "database", @@ -259,6 +294,38 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and ), ("sqlite", "path", "migration_history", append_migration_history, "external-move", ()), ("sqlite", "path", "old_name_history", append_old_name_history, "external-move", ()), + ( + "duckdb", + "database", + "column_migration_history", + append_column_migration_history, + "external-rename", + (), + ), + ( + "sqlite", + "path", + "column_migration_history", + append_column_migration_history, + "external-rename", + (), + ), + ( + "duckdb", + "database", + "node_result_history", + append_node_result_history, + "external-run", + (), + ), + ( + "duckdb", + "database", + "consumption_history", + append_consumption_history, + "external-run", + (), + ), ] ], ids=lambda case: case.description, From af4a785b51df58f400ca50b3c1db6edd76371cfe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:51:50 +0000 Subject: [PATCH 04/13] fix(state): page state tables without event IDs on Snowflake and SQL Server A table without an event_id column pages with a NULL placeholder key. The history SQL wrapped that key in the adapter's binary conversion, and Snowflake rejects TO_BINARY(NULL, 'UTF-8'). The placeholder now stays a bare NULL and is left out of ORDER BY, where SQL Server rejects constants. --- .../src/state_tables/constants.rs | 2 + .../src/state_tables/main/read_history_sql.rs | 11 +++- .../src/state_tables/tests/test_sql.rs | 64 ++++++++++++++++++- .../src/state_tables/tests/test_types.rs | 8 +++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-state/src/state_tables/constants.rs b/crates/sqlbuild-state/src/state_tables/constants.rs index 8bf6191df2..1dd50fb1ef 100644 --- a/crates/sqlbuild-state/src/state_tables/constants.rs +++ b/crates/sqlbuild-state/src/state_tables/constants.rs @@ -1,3 +1,5 @@ //! Adapter decisions shared by append-only state history reads. pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; +/// Cursor key of a table without event IDs: always NULL, so it never orders or binds a value. +pub(crate) const MISSING_EVENT_ID_KEY: &str = "NULL"; diff --git a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs index 8a415a7352..8739ededca 100644 --- a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs +++ b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs @@ -1,6 +1,6 @@ //! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. -use crate::state_tables::constants::SQLSERVER_ADAPTER; +use crate::state_tables::constants::{MISSING_EVENT_ID_KEY, SQLSERVER_ADAPTER}; use crate::state_tables::models::HistoryRead; /// Cursor values are already rendered by the adapter; `None` sorts before every SQL value. @@ -25,11 +25,15 @@ pub fn read_history_sql( ", NULL AS event_id" }); let mut columns = key_columns.unwrap_or(columns).to_vec(); - columns.push(if has_event_id { "event_id" } else { "NULL" }); + columns.push(if has_event_id { + "event_id" + } else { + MISSING_EVENT_ID_KEY + }); let ordered = columns .iter() .map(|column| { - if *column == timestamp_column { + if *column == timestamp_column || *column == MISSING_EVENT_ID_KEY { Ok((*column).to_owned()) } else { binary(column, adapter) @@ -81,6 +85,7 @@ pub fn read_history_sql( let order: Vec = columns .iter() .zip(&ordered) + .filter(|(column, _)| **column != MISSING_EVENT_ID_KEY) .map(|(column, ordered)| format!("CASE WHEN {column} IS NULL THEN 0 ELSE 1 END, {ordered}")) .collect(); let top = adapter == SQLSERVER_ADAPTER; diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs index 5c9ef41669..adcb4e0b11 100644 --- a/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs +++ b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs @@ -1,7 +1,11 @@ use crate::state_tables::main::create_table_sql::create_table_sql; +use crate::state_tables::main::read_history_sql::read_history_sql; use crate::state_tables::main::values_insert_sql::values_insert_sql; +use crate::state_tables::models::HistoryRead; use crate::state_tables::tests::helpers::column; -use crate::state_tables::tests::test_types::{CreateTableTestCase, ValuesInsertTestCase}; +use crate::state_tables::tests::test_types::{ + CreateTableTestCase, HistoryReadTestCase, ValuesInsertTestCase, +}; #[test] fn given_columns_when_rendering_ddl_then_required_columns_are_not_null() { @@ -48,3 +52,61 @@ fn given_rows_when_rendering_insert_then_each_row_is_one_values_tuple() { ); } } + +#[test] +fn given_a_table_without_event_ids_when_paging_history_then_the_missing_key_never_orders_or_converts() + { + let test_cases = [ + HistoryReadTestCase { + description: "Snowflake rejects TO_BINARY of a bare NULL", + adapter: "snowflake", + has_event_id: false, + after: Some(vec![Some("'e'".to_owned()), Some("ts1".to_owned()), None]), + expected_sql: "SELECT k, ts, NULL AS event_id FROM s.t WHERE ((TO_BINARY(k, 'UTF-8') > \ + TO_BINARY('e', 'UTF-8')) OR (TO_BINARY(k, 'UTF-8') = TO_BINARY('e', 'UTF-8') AND ts > ts1) \ + OR (TO_BINARY(k, 'UTF-8') = TO_BINARY('e', 'UTF-8') AND ts = ts1 AND NULL IS NOT NULL)) \ + ORDER BY CASE WHEN k IS NULL THEN 0 ELSE 1 END, TO_BINARY(k, 'UTF-8'), \ + CASE WHEN ts IS NULL THEN 0 ELSE 1 END, ts LIMIT 5", + }, + HistoryReadTestCase { + description: "SQL Server rejects a constant in ORDER BY", + adapter: "sqlserver", + has_event_id: false, + after: None, + expected_sql: "SELECT TOP 5 k, ts, NULL AS event_id FROM s.t ORDER BY \ + CASE WHEN k IS NULL THEN 0 ELSE 1 END, CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), k)), \ + CASE WHEN ts IS NULL THEN 0 ELSE 1 END, ts", + }, + HistoryReadTestCase { + description: "a real event ID stays the last ordering key", + adapter: "snowflake", + has_event_id: true, + after: None, + expected_sql: "SELECT k, ts, event_id FROM s.t ORDER BY CASE WHEN k IS NULL THEN 0 ELSE 1 END, \ + TO_BINARY(k, 'UTF-8'), CASE WHEN ts IS NULL THEN 0 ELSE 1 END, ts, \ + CASE WHEN event_id IS NULL THEN 0 ELSE 1 END, TO_BINARY(event_id, 'UTF-8') LIMIT 5", + }, + ]; + + for test_case in test_cases { + assert_eq!( + read_history_sql( + "s.t", + test_case.after.as_deref(), + HistoryRead { + columns: &["k", "ts"], + key_columns: None, + timestamp_column: "ts", + observed_since: None, + adapter: test_case.adapter, + has_event_id: test_case.has_event_id, + limit: 5, + }, + ) + .expect("supported adapter"), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_types.rs b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs index 836ccf548d..6914593ee1 100644 --- a/crates/sqlbuild-state/src/state_tables/tests/test_types.rs +++ b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs @@ -11,3 +11,11 @@ pub(crate) struct ValuesInsertTestCase { pub(crate) rows: Vec>, pub(crate) expected_sql: &'static str, } + +pub(crate) struct HistoryReadTestCase { + pub(crate) description: &'static str, + pub(crate) adapter: &'static str, + pub(crate) has_event_id: bool, + pub(crate) after: Option>>, + pub(crate) expected_sql: &'static str, +} From 5ca9d1f6a881f460b8f2df43c0f1b752e0e1b43e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:04:05 +0000 Subject: [PATCH 05/13] chore(location): point duplicate-code exemptions at the moved ingest and person-read modules --- .../src/warehouse_metadata/models.rs | 3 ++- .../_helpers/native_host/warehouse_metadata.rs | 11 +++++------ fensu.toml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 74988e49c9..c79e16f910 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -323,8 +323,8 @@ impl WarehouseHistoryKind { } } - #[must_use] /// The state table a kind mirrors; catalog and statistics read the warehouse catalog instead. + #[must_use] pub const fn source_table(self) -> Option<&'static str> { match self { Self::Fingerprints => Some(FINGERPRINT_TABLE_NAME), @@ -341,6 +341,7 @@ impl WarehouseHistoryKind { } } + #[must_use] pub const fn mirror_table(self) -> &'static str { match self { Self::Fingerprints => "sqb_warehouse_fingerprints", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 005fff9c83..b00d15d6b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -15,7 +15,8 @@ use sqlbuild_core::constants::{ use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, + ConsumptionRead, NodeResultsRead, WarehouseHistoryPage, WarehouseHistoryRead, + WarehouseHistoryRecord, WarehouseStateLocation, }; use sqlbuild_state::consumption::constants::{CONSUMPTION_COLUMN_TYPES, RECORDED_AT_COLUMN}; use sqlbuild_state::events::constants::{ @@ -491,11 +492,9 @@ fn decode_record( | WarehouseHistoryRead::ColumnMigrations(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp])); } - WarehouseHistoryRead::NodeResults(read) => { - return decode_event_record(&read.location, row, (columns, &[timestamp])); - } - WarehouseHistoryRead::Consumption(read) => { - return decode_event_record(&read.location, row, (columns, &[timestamp])); + WarehouseHistoryRead::NodeResults(NodeResultsRead { location, .. }) + | WarehouseHistoryRead::Consumption(ConsumptionRead { location, .. }) => { + return decode_event_record(location, row, (columns, &[timestamp])); } WarehouseHistoryRead::JanitorEvents(read) => { return decode_event_record( diff --git a/fensu.toml b/fensu.toml index b2ee2457c4..7a8eac2005 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1381,7 +1381,7 @@ paths = ["crates/sqlbuild-orchestration/src/api/_helpers/resources/administrator reason = "Each function is one utoipa-annotated axum route: OpenAPI needs one annotated handler per operation, and each body only checks the caller's scope and calls one store operation. Scope checks and store work live in their shared owners." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/authorization/main/list_role_definitions.rs", "crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs", "crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs", "crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_jobs.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_trigger_changes.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs", "crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs", "crates/sqlbuild-orchestration/src/platform/main/read_person_location_health.rs", "crates/sqlbuild-orchestration/src/platform/main/location_health.rs", "crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_runs.rs", "crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_person_ticks.rs", "crates/sqlbuild-orchestration/src/store/main/read_records.rs", "crates/sqlbuild-orchestration/src/platform/main/location_work.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_maintenance.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_microbatch_coverage.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_runs.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_events.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_statements.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/node_runs.rs"] +paths = ["crates/sqlbuild-orchestration/src/authorization/main/list_role_definitions.rs", "crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs", "crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs", "crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_jobs.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_trigger_changes.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs", "crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs", "crates/sqlbuild-orchestration/src/platform/main/read_person_location_health.rs", "crates/sqlbuild-orchestration/src/platform/main/location_health.rs", "crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_runs.rs", "crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_person_ticks.rs", "crates/sqlbuild-orchestration/src/store/main/read_records.rs", "crates/sqlbuild-orchestration/src/platform/main/location_work.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_catalog.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_maintenance.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_microbatch_coverage.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_statistics.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_runs.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_events.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_statements.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/node_runs.rs"] reason = "Each is one control-store read with its own SQL and record type. Transactions, person authorization and decoding go through the shared begin_authorized, row_in_target, query_records, read_records and query_bodies helpers; what remains alike is the call shape." [[dupes.allowlist]] @@ -1823,7 +1823,7 @@ paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerpr reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_column_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_consumption.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_node_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_old_name_views.rs"] reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." [[targets.python.rule_exceptions]] From c9efef0527dcd12f73dd679275167f1ab7731fb8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:16:01 +0000 Subject: [PATCH 06/13] refactor(location): keep each sync kind's names in one table --- .../src/warehouse_metadata/models.rs | 102 +++++++++++------- 1 file changed, 61 insertions(+), 41 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index c79e16f910..d678e6d29a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -305,58 +305,78 @@ impl WarehouseHistoryKind { Self::Statistics, ]; - #[must_use] - pub const fn as_str(self) -> &'static str { + /// Wire name, control-store mirror table and warehouse state table of each kind. + const fn names(self) -> (&'static str, &'static str, Option<&'static str>) { match self { - Self::Fingerprints => "fingerprints", - Self::SourceFreshness => "source_freshness", - Self::AuditResults => "audit_results", - Self::JanitorEvents => "janitor_events", - Self::Migrations => "migrations", - Self::OldNameViews => "old_name_views", - Self::Microbatches => "microbatches", - Self::ColumnMigrations => "column_migrations", - Self::NodeResults => "node_results", - Self::Consumption => "consumption", - Self::Catalog => "catalog", - Self::Statistics => "statistics", + Self::Fingerprints => ( + "fingerprints", + "sqb_warehouse_fingerprints", + Some(FINGERPRINT_TABLE_NAME), + ), + Self::SourceFreshness => ( + "source_freshness", + "sqb_warehouse_freshness", + Some(SOURCE_FRESHNESS_TABLE_NAME), + ), + Self::AuditResults => ( + "audit_results", + "sqb_warehouse_audit_results", + Some(AUDIT_RESULTS_TABLE_NAME), + ), + Self::JanitorEvents => ( + "janitor_events", + "sqb_warehouse_janitor_events", + Some(JANITOR_EVENTS_TABLE_NAME), + ), + Self::Migrations => ( + "migrations", + "sqb_warehouse_migrations", + Some(MIGRATION_TABLE_NAME), + ), + Self::OldNameViews => ( + "old_name_views", + "sqb_warehouse_old_name_views", + Some(OLD_NAME_VIEW_TABLE_NAME), + ), + Self::Microbatches => ( + "microbatches", + "sqb_warehouse_microbatches", + Some(MICROBATCH_TABLE_NAME), + ), + Self::ColumnMigrations => ( + "column_migrations", + "sqb_warehouse_column_migrations", + Some(COLUMN_MIGRATION_TABLE_NAME), + ), + Self::NodeResults => ( + "node_results", + "sqb_warehouse_node_results", + Some(NODE_RESULTS_TABLE_NAME), + ), + Self::Consumption => ( + "consumption", + "sqb_warehouse_consumption", + Some(CONSUMPTION_TABLE_NAME), + ), + Self::Catalog => ("catalog", "sqb_warehouse_catalog", None), + Self::Statistics => ("statistics", "sqb_warehouse_statistics", None), } } + #[must_use] + pub const fn as_str(self) -> &'static str { + self.names().0 + } + /// The state table a kind mirrors; catalog and statistics read the warehouse catalog instead. #[must_use] pub const fn source_table(self) -> Option<&'static str> { - match self { - Self::Fingerprints => Some(FINGERPRINT_TABLE_NAME), - Self::SourceFreshness => Some(SOURCE_FRESHNESS_TABLE_NAME), - Self::AuditResults => Some(AUDIT_RESULTS_TABLE_NAME), - Self::JanitorEvents => Some(JANITOR_EVENTS_TABLE_NAME), - Self::Migrations => Some(MIGRATION_TABLE_NAME), - Self::OldNameViews => Some(OLD_NAME_VIEW_TABLE_NAME), - Self::Microbatches => Some(MICROBATCH_TABLE_NAME), - Self::ColumnMigrations => Some(COLUMN_MIGRATION_TABLE_NAME), - Self::NodeResults => Some(NODE_RESULTS_TABLE_NAME), - Self::Consumption => Some(CONSUMPTION_TABLE_NAME), - Self::Catalog | Self::Statistics => None, - } + self.names().2 } #[must_use] pub const fn mirror_table(self) -> &'static str { - match self { - Self::Fingerprints => "sqb_warehouse_fingerprints", - Self::SourceFreshness => "sqb_warehouse_freshness", - Self::AuditResults => "sqb_warehouse_audit_results", - Self::JanitorEvents => "sqb_warehouse_janitor_events", - Self::Migrations => "sqb_warehouse_migrations", - Self::OldNameViews => "sqb_warehouse_old_name_views", - Self::Microbatches => "sqb_warehouse_microbatches", - Self::ColumnMigrations => "sqb_warehouse_column_migrations", - Self::NodeResults => "sqb_warehouse_node_results", - Self::Consumption => "sqb_warehouse_consumption", - Self::Catalog => "sqb_warehouse_catalog", - Self::Statistics => "sqb_warehouse_statistics", - } + self.names().1 } } From 2de4fe3e1acac5e155422c43abfd0e3dc0ac77fb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:19:20 +0000 Subject: [PATCH 07/13] test(location): accept one relation prefix in migration history fixtures --- .../sqlbuild/compiler/compiled_project/publication/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 7d9fb1c271..aa7f70188d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -348,7 +348,7 @@ def append_old_name_history(adapter: Any, connection: Any, fixture: dict[str, An ) -def migration_history_data(fixture: dict[str, Any], relations: tuple[str, str]) -> dict[str, Any]: +def migration_history_data(fixture: dict[str, Any], relations: tuple[str, ...]) -> dict[str, Any]: """Reconstruct native relation values from the warehouse-shaped fixture.""" data: dict[str, Any] = fixture["record"].copy() del data["state_database"] From 9d53f179a461ec6b130fd2536e229c2079de945c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:42:56 +0000 Subject: [PATCH 08/13] feat(location): read the warehouse only after a run ends An idle location no longer polls the warehouse every 30 seconds. It asks the control server every 5 seconds whether a run for its target ended since its last synchronization began, and starts one only then. The first synchronization still runs at startup, and full reconciliation still runs on the first synchronization after its interval. --- clients/typescript/src/schema.d.ts | 85 +++++++++++++++++++ .../src/native_commands/_helpers/execution.rs | 38 ++++++++- .../src/native_commands/constants.rs | 4 +- .../main/_location_metadata.rs | 16 +++- .../src/native_commands/models.rs | 4 + .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 23 ++++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/_helpers/work_client.rs | 12 ++- .../src/store/schema.sql | 1 + .../main/location_reads/mod.rs | 1 + .../main/location_reads/read_sync_due.rs | 29 +++++++ .../src/warehouse_metadata/models.rs | 16 ++++ .../publication/catalog/helpers.py | 19 +++++ .../publication/test_warehouse_metadata.py | 19 ++++- 15 files changed, 261 insertions(+), 11 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 8a7d06b209..78da36b52f 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1010,6 +1010,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/sync-due": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["sync_due"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/watermark": { parameters: { query?: never; @@ -3107,6 +3123,20 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description Both times come from the server clock, so the reader compares them without clock skew. */ + WarehouseSyncDue: { + /** Format: date-time */ + now: string; + /** Format: date-time */ + runs_ended_at?: string | null; + }; + /** @description A registered reader asks whether a run ended since its last synchronization began. */ + WarehouseSyncDueRead: { + /** Format: int64 */ + generation: number; + /** Format: uuid */ + session_id: string; + }; /** * @description Closed failure stages keep warehouse errors and credentials out of health observations. * @enum {string} @@ -5836,6 +5866,61 @@ export interface operations { }; }; }; + sync_due: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WarehouseSyncDueRead"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncDue"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; watermark: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 9d5ce63b7e..a7859224da 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -37,7 +37,7 @@ use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, StatisticsRetention, - WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, + WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, WarehouseSyncDueRead, }; use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; use sqlbuild_orchestration::webhooks::main::{ @@ -70,7 +70,8 @@ use crate::native_commands::_helpers::terminal::{ }; use crate::native_commands::constants::{ ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_POLL_SECONDS, - METADATA_RECONCILE_SECONDS, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, + METADATA_RECONCILE_SECONDS, METADATA_SYNC_CHECK_SECONDS, SCHEDULER_COMMAND, + UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; @@ -349,6 +350,8 @@ pub(crate) fn metadata_sync( statistics_limit: MAX_HISTORY_RECORDS, pending: None, next_at: Instant::now(), + idle: true, + cycle_started_at: None, } } @@ -447,6 +450,34 @@ pub(crate) fn freeze_history_page( Ok(state) } +/// An idle location asks the server, never the warehouse, whether a run ended since it last began. +pub(crate) async fn begin_after_run( + mut state: LocationMetadataSync, + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + target: &str, +) -> (LocationMetadataSync, Result) { + state.next_at = Instant::now() + Duration::from_secs(METADATA_SYNC_CHECK_SECONDS); + let request = WarehouseSyncDueRead { + session_id: poll.session_id, + generation: state.generation, + }; + let due = match client.metadata_sync_due(target, &request).await { + Ok(due) => due, + Err(error) => return (state, Err(error)), + }; + let ran = match (state.cycle_started_at, due.runs_ended_at) { + (None, _) => true, + (Some(started_at), Some(ended_at)) => ended_at >= started_at, + (Some(_), None) => false, + }; + if ran { + state.idle = false; + state.cycle_started_at = Some(due.now); + state.next_at = Instant::now(); + } + (state, Ok(ran)) +} + pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> LocationMetadataSync { let Some(pending) = state.pending.take() else { return state; @@ -470,7 +501,8 @@ pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> Loca state.reconcile_at = Instant::now() + Duration::from_secs(u64::from(state.reconcile_seconds)); } - state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + state.idle = true; + state.next_at = Instant::now(); drop(write_output(&[( Stream::Stderr, "Warehouse history synchronization completed.\n", diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 6406a3a58b..ab853320e3 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -93,8 +93,10 @@ pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; /// Reserve half the location work claim for acknowledgement retries after killing a callback. pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; -/// Independent read timeout and delay between automatic history polls. +/// Read timeout, and the delay before retrying a failed page of an unfinished synchronization. pub(crate) const METADATA_POLL_SECONDS: u64 = 30; +/// How often an idle location asks the server, never the warehouse, whether a run ended. +pub(crate) const METADATA_SYNC_CHECK_SECONDS: u64 = 5; /// Full key-set reconciliation is independent of the incremental poll and overlap window. pub(crate) const METADATA_RECONCILE_SECONDS: u32 = 300; pub(crate) const METADATA_OVERLAP_SECONDS: i64 = 300; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index 7ad9339b76..90bbbf4164 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -16,8 +16,8 @@ use tokio::time::Instant; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - acknowledge_metadata_page, freeze_history_page, history_request, location_result, - metadata_health, statistics_page, + acknowledge_metadata_page, begin_after_run, freeze_history_page, history_request, + location_result, metadata_health, statistics_page, }; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; @@ -41,6 +41,18 @@ pub(super) async fn sync_location_metadata( } let started = Instant::now(); tokio::pin!(stopped); + if state.idle { + let (next, due) = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = begin_after_run(state, (client, poll), &options.target) => result, + }; + state = next; + match due { + Ok(true) => {} + Ok(false) => return Ok(Some(state)), + Err(error) => return retry_metadata(state, error), + } + } if state.pending.is_none() { if state.kind() == WarehouseHistoryKind::Statistics && state.statistics.is_none() { state.statistics = Some(tokio::select! { diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index a1bbf08388..4e5f7958a2 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -204,6 +204,10 @@ pub(crate) struct LocationMetadataSync { pub(crate) statistics_limit: usize, pub(crate) pending: Option, pub(crate) next_at: tokio::time::Instant, + /// Between synchronizations; the warehouse is read again only after a run ends. + pub(crate) idle: bool, + /// Server time when the current or last synchronization began; `None` before the first. + pub(crate) cycle_started_at: Option>, } impl LocationMetadataSync { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 7613b7fada..5df6d1402d 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -39,6 +39,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(request): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + read_sync_due(&store, (location_id, &target), &request).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index c52635f5df..967b3dae1c 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -96,6 +96,7 @@ use crate::api::_helpers::resources::{ trigger_reads::ticks, warehouse_metadata::report, warehouse_metadata::statistics_due, + warehouse_metadata::sync_due, warehouse_metadata::watermark ))] pub(crate) struct ControlApi; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index 6dee6aa9dd..eaf407d5aa 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,7 +10,8 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - MetadataBatch, StatisticsDueRead, StatisticsRead, WarehouseSyncResponse, WarehouseWatermarkRead, + MetadataBatch, StatisticsDueRead, StatisticsRead, WarehouseSyncDue, WarehouseSyncDueRead, + WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -44,6 +45,15 @@ impl LocationWorkClient { .await } + pub async fn metadata_sync_due( + &self, + target: &str, + request: &WarehouseSyncDueRead, + ) -> Result { + self.post_metadata((target, Some("sync-due")), request) + .await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_metadata( &self, diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index f12fd87027..215b451ebb 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -162,6 +162,7 @@ CREATE TABLE sqb_runs ( last_heartbeat_at {time} ); CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); +CREATE INDEX sqb_runs_location_ended ON sqb_runs(location_id, target, ended_at); CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); CREATE TABLE sqb_run_nodes ( diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs index af7a2a5d7a..05bf3ab152 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/mod.rs @@ -1,2 +1,3 @@ pub mod read_statistics_due; +pub mod read_sync_due; pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs new file mode 100644 index 0000000000..573db4f6a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/location_reads/read_sync_due.rs @@ -0,0 +1,29 @@ +//! Runs that ended, not a stored request, decide when a location synchronizes again. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; +use crate::warehouse_metadata::models::{WarehouseSyncDue, WarehouseSyncDueRead}; + +pub async fn read_sync_due( + store: &ControlStore, + scope: (Uuid, &str), + request: &WarehouseSyncDueRead, +) -> Result { + let transaction = + begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT MAX(ended_at) AS ended_at FROM sqb_runs WHERE location_id=$1 AND target=$2", + vec![Parameter::Id(scope.0), Parameter::Text(scope.1.into())], + ) + .await?; + let runs_ended_at = serde_json::from_value(rows[0]["ended_at"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + commit(transaction).await?; + Ok(WarehouseSyncDue { runs_ended_at, now }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index d678e6d29a..4c44e2d64e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -390,6 +390,22 @@ pub struct WarehouseWatermarkRead { pub kind: WarehouseHistoryKind, } +/// A registered reader asks whether a run ended since its last synchronization began. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseSyncDueRead { + pub session_id: Uuid, + pub generation: i64, +} + +/// Both times come from the server clock, so the reader compares them without clock skew. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseSyncDue { + pub runs_ended_at: Option>, + pub now: DateTime, +} + /// Private interpreter input; cursor values are data and receive adapter literal rendering. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 3a2de107cd..1aa1aeb223 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -4,15 +4,34 @@ import sqlite3 import time from contextlib import closing +from datetime import UTC, datetime from pathlib import Path from subprocess import Popen from typing import Any +from uuid import uuid4 + + +def record_finished_run(control: Path) -> None: + """Record a run that just ended for every published target, which makes its location resync.""" + ended_at: str = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + run_id: str = str(uuid4()) + with closing(sqlite3.connect(control)) as database: + _ = database.execute( + "INSERT INTO sqb_runs (run_id,location_id,target,snapshot_id,kind,origin,root_run_id," + "requested_by,selection_json,request_json,write_set_digest,status,queued_at,ended_at," + "node_counts_json,last_event_storage_id) SELECT ?,location_id,target,snapshot_id," + "'adhoc','manual',?,'test-admin','[]','{}','','success',?,?,'{}',0 " + "FROM sqb_current_snapshots", + (run_id, run_id, ended_at, ended_at), + ) + database.commit() def wait_for_catalog( control: Path, expected: list[dict[str, Any]], process: Popen[bytes], log: Path ) -> None: """Wait for the exact synthetic relation, not a count that unrelated rows can satisfy.""" + record_finished_run(control) deadline: float = time.monotonic() + 45 observed: list[dict[str, Any]] = [{}] while observed != expected: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 83f43139f4..5ca8ad1315 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -43,6 +43,9 @@ StateEventsTestCase, WarehouseHistoryTestCase, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + record_finished_run, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_audit_history, append_column_migration_history, @@ -396,7 +399,8 @@ def test_given_a_served_location_when_state_event_history_is_rebuilt_and_retaine mirrored = database.execute( f"SELECT event_id,record_json FROM {mirror_table} ORDER BY event_id" ).fetchall() - time.sleep(0.05) + record_finished_run(control) + time.sleep(0.5) assert {json.loads(row)[source_key] for _, row in mirrored} == expected_ids for event_id, text in mirrored: record: dict[str, Any] = json.loads(text) @@ -415,6 +419,7 @@ def test_given_a_served_location_when_state_event_history_is_rebuilt_and_retaine ) _ = adapter.execute(connection=connection, sql=warehouse_sql) adapter.close(connection) + record_finished_run(control) @pytest.mark.parametrize( @@ -945,7 +950,8 @@ def test_given_a_served_location_when_warehouse_history_changes_then_periodic_re "SELECT (SELECT COUNT(*) FROM sqb_warehouse_fingerprints)," "(SELECT COUNT(*) FROM sqb_warehouse_freshness)" ).fetchone() - time.sleep(0.05) + record_finished_run(control) + time.sleep(0.5) with closing(sqlite3.connect(control)) as database: bodies: list[tuple[bytes]] = database.execute( "SELECT t.body FROM sqb_warehouse_fingerprints f " @@ -971,6 +977,7 @@ def test_given_a_served_location_when_warehouse_history_changes_then_periodic_re connection=connection, sql=warehouse_sql.format(table=table) ) adapter.close(connection) + record_finished_run(control) @pytest.mark.parametrize( @@ -1040,6 +1047,7 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre for table in ("sqb_warehouse_fingerprints", "sqb_warehouse_freshness"): _ = database.execute(stage["mirror_sql"].format(table=table)) database.commit() + record_finished_run(control) expected: set[str] = set(stage["expected_runs"]) | { f"middle-{index:03}" for index in range(stage.get("expected_overlap_count", stage["overlap_count"])) @@ -1080,8 +1088,15 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre observed_health_at = max( datetime.fromisoformat(report["received_at"]) for report in reports ) + time.sleep(6) + completed: int = log.read_text().count("Warehouse history synchronization completed.") + time.sleep(12) + assert ( + log.read_text().count("Warehouse history synchronization completed.") == completed + ), "an idle location read the warehouse without a run ending" for availability in fixture["availability"]: _ = (project / availability["source"]).rename(project / availability["destination"]) + record_finished_run(control) reports = wait_for_warehouse_sync_health( api, availability, location, log, after=observed_health_at ) From 48ed85f75dc29b72da16a8cc7c6dd12e191e17a2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:32:58 +0000 Subject: [PATCH 09/13] feat(location): sync a location from the warehouse on request People with connection-test access can ask a location to read SQLBuild's state and catalog from the warehouse now, with Sync now on the location page or sqb control sync. The request is audited location work that the location acknowledges and follows with a sync, or one more sync after the current one. --- clients/typescript/src/schema.d.ts | 86 +++++++++++- .../sqlbuild-cli/src/argv/command_table.json | 6 +- .../native_commands/main/_location_work.rs | 82 ++++++----- .../src/native_commands/main/_run_location.rs | 5 +- .../main/_run_warehouse_sync.rs | 46 +++++++ .../src/native_commands/main/mod.rs | 1 + .../main/run_person_invocation.rs | 4 + .../src/native_commands/models.rs | 8 ++ .../src/person_login/_helpers/transport.rs | 130 +++++++++++++----- .../sqlbuild-cli/src/person_login/main/mod.rs | 1 + .../src/person_login/main/read_node_sql.rs | 20 +-- .../main/request_warehouse_sync.rs | 16 +++ .../authentication/password_sessions.rs | 4 + .../resources/person_location_work.rs | 52 ++++++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/main/claim_location_work.rs | 2 +- .../platform/main/complete_location_work.rs | 3 +- .../platform/main/request_location_work.rs | 2 +- .../src/platform/models.rs | 7 +- .../tests/api/test_location_work.rs | 34 +++++ .../tests/api/test_person_location_work.rs | 36 ++++- .../sqlbuild/references/docs/cli/server.md | 21 +++ .../publication/catalog/helpers.py | 25 +++- .../publication/test_person_sql.py | 28 ++++ .../publication/test_warehouse_metadata.py | 13 ++ ui/src/lib/api/client.ts | 2 + ui/src/lib/api/http-client.test.ts | 29 ++++ ui/src/lib/api/http-client.ts | 11 ++ ui/src/lib/api/mock/mock-client.ts | 10 ++ ui/src/lib/api/types.ts | 2 +- .../platform/warehouse-sync-now.svelte | 71 ++++++++++ .../shell/warehouse-sync-status.svelte | 2 +- ui/src/routes/locations/[id]/+page.svelte | 2 + website/src/content/docs/docs/cli/server.mdx | 20 +++ 34 files changed, 674 insertions(+), 108 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_run_warehouse_sync.rs create mode 100644 crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs create mode 100644 ui/src/lib/components/platform/warehouse-sync-now.svelte diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 78da36b52f..041ea0880a 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -818,6 +818,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/warehouse-syncs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestWarehouseSync"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/platform/events": { parameters: { query?: never; @@ -1540,11 +1556,6 @@ export interface components { rejected_sha256: string; table: components["schemas"]["Table"]; }; - ConnectionTestRequest: { - reason: string; - /** Format: uuid */ - request_id: string; - }; /** @description What the UI needs first: how people sign in, and what this person may see and do. */ ControlBootstrap: { access: components["schemas"]["PersonAccess"]; @@ -1792,6 +1803,9 @@ export interface components { LocationWork: { /** @enum {string} */ kind: "check_connection"; + } | { + /** @enum {string} */ + kind: "sync_warehouse"; } | { evaluation: Record; /** @enum {string} */ @@ -1826,6 +1840,9 @@ export interface components { document: Record; /** @enum {string} */ kind: "plan"; + } | { + /** @enum {string} */ + kind: "sync_started"; } | { /** @enum {string} */ kind: "superseded"; @@ -2209,6 +2226,12 @@ export interface components { /** @description A running schedule's next due times from `next_due_at` on, at most five; empty otherwise. */ upcoming: string[]; }; + /** @description A person asks a location for one named read-only operation, such as a connection test. */ + PersonWorkRequest: { + reason: string; + /** Format: uuid */ + request_id: string; + }; /** @description Observed physical evidence is not a live warehouse-existence guarantee. */ PhysicalEvidence: { observation: components["schemas"]["CatalogObservation"]; @@ -4134,7 +4157,7 @@ export interface operations { }; requestBody: { content: { - "application/json": components["schemas"]["ConnectionTestRequest"]; + "application/json": components["schemas"]["PersonWorkRequest"]; }; }; responses: { @@ -5338,6 +5361,57 @@ export interface operations { }; }; }; + requestWarehouseSync: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PersonWorkRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonPlatformEvents: { parameters: { query?: { diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index c433d00395..bc15cbc49d 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -82,12 +82,12 @@ "name": "control", "parser": { "prog": "sqb control", - "description": "Read one exact node's current published SQL using your saved sqb login session. Results are JSON; progress goes to stderr. No project or warehouse connection is opened.", + "description": "Use your saved sqb login session to read one exact node's current published SQL, or to ask a location to sync from the warehouse now. Results are JSON; progress goes to stderr. No project or warehouse connection is opened.", "groups": [{"title": "positional arguments"}, {"title": "options"}], "actions": [ {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, - {"kind": "store", "dest": "person_action", "choices": ["sql"], "help": "inspect published raw and compiled SQL", "group": 0}, - {"kind": "store", "dest": "person_node", "help": "exact published node key, for example model:orders (not a selector)", "group": 0}, + {"kind": "store", "dest": "person_action", "choices": ["sql", "sync"], "help": "sql: inspect published raw and compiled SQL; sync: ask the location to read SQLBuild's state and catalog from the warehouse now", "group": 0}, + {"kind": "store", "dest": "person_node", "nargs": "?", "help": "for sql: exact published node key, for example model:orders (not a selector)", "group": 0}, {"kind": "store", "options": ["--server"], "dest": "login_server", "required": true, "help": "HTTPS SQLBuild server origin used by sqb login", "group": 1}, {"kind": "store", "options": ["--ca-cert"], "dest": "login_ca", "help": "additional trusted PEM CA certificate", "group": 1}, {"kind": "store", "options": ["--location-id"], "dest": "person_location", "required": true, "help": "published location UUID", "group": 1}, diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index d67613c7a2..41b65faff0 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -20,13 +20,13 @@ use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::main::_location_sensor::evaluate_location_sensor; use crate::native_commands::models::{LocationServe, LocationWatch}; -/// Retry the exact completed result after uncertain delivery; never rerun it to obtain a new result. +/// Retry the exact completed result without rerunning it; `true` asks the metadata loop to sync. pub(super) async fn run_location_work( (options, script): (&LocationServe, &Path), (client, poll, watch): (&LocationWorkClient, &LocationWorkPoll, &LocationWatch), generation: Option, stopped: impl Future>, -) -> Result>, ControlError> { +) -> Result, bool)>, ControlError> { tokio::pin!(stopped); let claim = tokio::select! { result = &mut stopped => { result?; return Ok(None); }, @@ -34,12 +34,12 @@ pub(super) async fn run_location_work( Ok(claim) => claim, Err(error) => { work_error(&error)?; - return Ok(Some(Vec::new())); + return Ok(Some((Vec::new(), false))); }, }, }; let Some(claim) = claim else { - return Ok(Some(Vec::new())); + return Ok(Some((Vec::new(), false))); }; let deadline = Instant::now() + (claim.lease_expires_at - Utc::now()) @@ -49,12 +49,6 @@ pub(super) async fn run_location_work( let outcome = if generation == Some(claim.generation) && location_inputs(&options.project_dir, &watch.rules, &watch.local_files) == watch.observed { - let message = match &claim.request.work { - LocationWork::CheckConnection {} => "Checking requested location connection...\n", - LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", - LocationWork::Plan { .. } => "Planning requested command...\n", - }; - drop(write_output(&[(Stream::Stderr, message)])); let mut shutting_down = false; let expires = async { tokio::select! { @@ -62,34 +56,13 @@ pub(super) async fn run_location_work( () = tokio::time::sleep_until(deadline) => Ok(()), } }; - let checked = match claim.request.work { - LocationWork::CheckConnection {} => check_location_connection(options, script, expires) - .await? - .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), - LocationWork::EvaluateSensor { evaluation } => { - evaluate_location_sensor(options, script, &evaluation, expires) - .await? - .map(|outcome| { - ( - LocationWorkOutcome::SensorEvaluation { outcome }, - Vec::new(), - ) - }) - } - LocationWork::Plan { - selection, - settings, - .. - } => plan_location((options, script), (&selection, &settings), expires) - .await? - .map(|document| (LocationWorkOutcome::Plan { document }, Vec::new())), - }; + let checked = perform_work((options, script), claim.request.work, expires).await?; let Some((outcome, local_files)) = checked else { drop(write_output(&[( Stream::Stderr, "Location work stopped before completion; no result reported.\n", )])); - return Ok((!shutting_down).then_some(files)); + return Ok((!shutting_down).then_some((files, false))); }; files = local_files; let mut expected = watch.observed.clone(); @@ -106,6 +79,7 @@ pub(super) async fn run_location_work( } else { LocationWorkOutcome::Superseded {} }; + let sync = matches!(outcome, LocationWorkOutcome::SyncStarted {}); let result = LocationWorkResult { claim_id: claim.claim_id, session_id: poll.session_id, @@ -123,7 +97,7 @@ pub(super) async fn run_location_work( Stream::Stderr, "Location work completed.\n", )])); - return Ok(Some(files)); + return Ok(Some((files, sync))); } Err(error) => { work_error(&error)?; @@ -141,7 +115,45 @@ pub(super) async fn run_location_work( Stream::Stderr, "Location work acknowledgement expired; check the server result before retrying.\n", )])); - Ok(Some(files)) + Ok(Some((files, sync))) +} + +/// One named operation; `None` means it stopped at shutdown or its lease ended. +async fn perform_work( + (options, script): (&LocationServe, &Path), + work: LocationWork, + expires: impl Future>, +) -> Result)>, ControlError> { + let message = match &work { + LocationWork::CheckConnection {} => "Checking requested location connection...\n", + LocationWork::SyncWarehouse {} => "Synchronizing warehouse history on request...\n", + LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", + LocationWork::Plan { .. } => "Planning requested command...\n", + }; + drop(write_output(&[(Stream::Stderr, message)])); + Ok(match work { + LocationWork::SyncWarehouse {} => Some((LocationWorkOutcome::SyncStarted {}, Vec::new())), + LocationWork::CheckConnection {} => check_location_connection(options, script, expires) + .await? + .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), + LocationWork::EvaluateSensor { evaluation } => { + evaluate_location_sensor(options, script, &evaluation, expires) + .await? + .map(|outcome| { + ( + LocationWorkOutcome::SensorEvaluation { outcome }, + Vec::new(), + ) + }) + } + LocationWork::Plan { + selection, + settings, + .. + } => plan_location((options, script), (&selection, &settings), expires) + .await? + .map(|document| (LocationWorkOutcome::Plan { document }, Vec::new())), + }) } fn work_error(error: &ClientError) -> Result<(), ControlError> { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index 43e3a3f2c0..9968de6124 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -155,7 +155,7 @@ async fn poll_location( tokio::pin!(stopped); if generation.is_some() && watch.attempted.as_ref() == Some(&watch.observed) { if Instant::now() >= work_at { - let Some(files) = run_location_work( + let Some((files, sync)) = run_location_work( (options, script), (client, poll, &watch), generation, @@ -167,6 +167,9 @@ async fn poll_location( }; watch = exclude_location_databases(&options.project_dir, files, watch); work_at = Instant::now() + Duration::from_secs(2); + if sync && let Some(state) = metadata.as_mut() { + state.request_sync(); + } } if let Some(state) = metadata.take() { metadata = sync_location_metadata( diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_warehouse_sync.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_warehouse_sync.rs new file mode 100644 index 0000000000..41a8a2a011 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_warehouse_sync.rs @@ -0,0 +1,46 @@ +//! `sqb control sync` asks the location to read the warehouse; the location runs the sync itself. + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::text; +use crate::native_commands::_helpers::terminal::write_output; +use crate::person_login::main::request_warehouse_sync::request_warehouse_sync; + +pub(super) fn run_warehouse_sync(namespace: &Namespace) -> i32 { + drop(write_output(&[( + Stream::Stderr, + "Requesting a warehouse sync…\n", + )])); + let result = text(namespace, "person_location") + .unwrap_or_default() + .parse() + .map_err(|_| std::io::Error::other("--location-id must be a UUID")) + .and_then(|location| { + let ca = text(namespace, "login_ca").map(std::path::PathBuf::from); + request_warehouse_sync( + ( + &text(namespace, "login_server").unwrap_or_default(), + ca.as_deref(), + ), + (location, &text(namespace, "target").unwrap_or_default()), + ) + }); + match result { + Ok(applied) => { + drop(write_output(&[ + (Stream::Stdout, &format!("{{\"applied\": {applied}}}\n")), + ( + Stream::Stderr, + "Warehouse sync requested; the location starts it within a few seconds.\n", + ), + ])); + 0 + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Warehouse sync request failed: {error}\n"), + )])); + 1 + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 813d58cd45..c5ee3267d2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -10,6 +10,7 @@ pub(super) mod _run_control; pub(super) mod _run_cost; pub(super) mod _run_dev; pub(super) mod _run_location; +pub(super) mod _run_warehouse_sync; pub mod effective_target; pub(crate) mod parse_native_invocation; pub(crate) mod run_control_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs index 036e68a8d5..dc18d23dfd 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs @@ -3,12 +3,16 @@ use crate::argv::models::{Namespace, Stream}; use crate::native_commands::_helpers::decision::text; use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_run_warehouse_sync::run_warehouse_sync; use crate::person_login::main::read_node_sql::read_node_sql; pub(crate) fn run_person_invocation(namespace: &Namespace) -> Option { if text(namespace, "command").as_deref() != Some("control") { return None; } + if text(namespace, "person_action").as_deref() == Some("sync") { + return Some(run_warehouse_sync(namespace)); + } drop(write_output(&[( Stream::Stderr, "Reading published SQL…\n", diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 4e5f7958a2..99f7eb26df 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -211,6 +211,14 @@ pub(crate) struct LocationMetadataSync { } impl LocationMetadataSync { + /// A person asked for a sync: start one now when idle, or one more after the current one. + pub(crate) fn request_sync(&mut self) { + self.cycle_started_at = None; + if self.idle { + self.next_at = tokio::time::Instant::now(); + } + } + /// Failed samples revisit catalog before retrying; no frozen delivery is discarded. pub(crate) fn retry_read(&mut self) { self.statistics = None; diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs index f44ef2f0ce..7703fe34ff 100644 --- a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs +++ b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs @@ -4,11 +4,14 @@ use std::path::Path; use std::time::Duration; use reqwest::header::{AUTHORIZATION, HeaderValue}; -use sqlbuild_orchestration::api::models::ApiErrorBody; +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; +use sqlbuild_orchestration::api::models::{ApiErrorBody, MutationResponse}; use sqlbuild_orchestration::credentials::models::SessionIdentity; +use sqlbuild_orchestration::platform::models::PersonWorkRequest; use sqlbuild_orchestration::snapshots::models::NodeSql; use uuid::Uuid; +use crate::person_login::_helpers::credentials; use crate::person_login::constants::REQUEST_SECONDS; use crate::person_login::models::{PersonClient, PersonCredential}; @@ -38,6 +41,21 @@ pub(crate) fn connect(origin: &str, ca: Option<&Path>) -> Result( + (server, ca): (&str, Option<&Path>), + request: impl AsyncFnOnce(&PersonClient, &PersonCredential) -> Result, +) -> Result { + let origin = parse_origin(server, "CLI").map_err(std::io::Error::other)?; + let credential = credentials::load(&origin)? + .ok_or_else(|| std::io::Error::other("not signed in; run sqb login"))?; + let client = connect(&origin, ca)?; + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()? + .block_on(request(&client, &credential)) +} + pub(crate) fn authorized( client: &PersonClient, method: reqwest::Method, @@ -104,6 +122,65 @@ pub(crate) async fn node_sql( credential: &PersonCredential, (location, target, node): (Uuid, &str, &str), ) -> Result, std::io::Error> { + let path = target_path(client, (location, target), &["nodes", node, "sql"])?; + send_json( + authorized(client, reqwest::Method::GET, &path, credential)?, + ( + "read published SQL from", + "SQL inspection", + "SQL inspection result", + ), + ) + .await +} + +/// Whether the server queued the request; `false` means this request ID was already queued. +pub(crate) async fn warehouse_sync( + client: &PersonClient, + credential: &PersonCredential, + (location, target): (Uuid, &str), +) -> Result { + let path = target_path(client, (location, target), &["warehouse-syncs"])?; + let request = PersonWorkRequest { + request_id: Uuid::now_v7(), + reason: "Requested from the CLI".into(), + }; + send_json::( + authorized(client, reqwest::Method::POST, &path, credential)?.json(&request), + ( + "request a warehouse sync from", + "Warehouse sync request", + "warehouse sync response", + ), + ) + .await + .map(|response| response.applied) +} + +/// Each failure names the action, e.g. "could not read published SQL from the server". +async fn send_json( + request: reqwest::RequestBuilder, + (unreachable, action, result): (&str, &str, &str), +) -> Result { + let response = request + .send() + .await + .map_err(|_| std::io::Error::other(format!("could not {unreachable} the server")))?; + if !response.status().is_success() { + return Err(rejection(response, action).await); + } + response + .json() + .await + .map_err(|_| std::io::Error::other(format!("server returned an invalid {result}"))) +} + +/// Each segment is percent-encoded, so a target or node key cannot change the route. +fn target_path( + client: &PersonClient, + (location, target): (Uuid, &str), + rest: &[&str], +) -> Result { let mut url = reqwest::Url::parse(&client.origin).map_err(std::io::Error::other)?; url.path_segments_mut() .map_err(|()| std::io::Error::other("invalid server origin"))? @@ -115,39 +192,26 @@ pub(crate) async fn node_sql( &location.to_string(), "targets", target, - "nodes", - node, - "sql", - ]); - let response = authorized(client, reqwest::Method::GET, url.path(), credential)? - .send() - .await - .map_err(|_| std::io::Error::other("could not read published SQL from the server"))?; + ]) + .extend(rest); + Ok(url.path().to_owned()) +} + +async fn rejection(response: reqwest::Response, action: &str) -> std::io::Error { let status = response.status(); - if !status.is_success() { - if status == reqwest::StatusCode::UNAUTHORIZED { - return Err(std::io::Error::other( - "CLI session is unavailable or revoked; run sqb login again", - )); - } - let decision = response - .json::() - .await - .map_err(|_| { - std::io::Error::other(format!( - "SQL inspection rejected (HTTP {}); invalid error response", - status.as_u16() - )) - })? - .decision_id; - return Err(std::io::Error::other(format!( - "SQL inspection rejected (HTTP {}{})", + if status == reqwest::StatusCode::UNAUTHORIZED { + return std::io::Error::other("CLI session is unavailable or revoked; run sqb login again"); + } + match response.json::().await { + Ok(body) => std::io::Error::other(format!( + "{action} rejected (HTTP {}{})", status.as_u16(), - decision.map_or_else(String::new, |id| format!("; decision {id}")) - ))); + body.decision_id + .map_or_else(String::new, |id| format!("; decision {id}")) + )), + Err(_) => std::io::Error::other(format!( + "{action} rejected (HTTP {}); invalid error response", + status.as_u16() + )), } - response - .json() - .await - .map_err(|_| std::io::Error::other("server returned an invalid SQL inspection result")) } diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs index bf1d2af998..db72101ec9 100644 --- a/crates/sqlbuild-cli/src/person_login/main/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod read_node_sql; +pub(crate) mod request_warehouse_sync; pub(crate) mod run_login; diff --git a/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs b/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs index 99970998c7..b751802dc6 100644 --- a/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs +++ b/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs @@ -2,31 +2,21 @@ use std::path::Path; -use sqlbuild_orchestration::api::main::parse_origin::parse_origin; use sqlbuild_orchestration::snapshots::models::NodeSql; use uuid::Uuid; -use crate::person_login::_helpers::{credentials, transport}; +use crate::person_login::_helpers::transport; pub(crate) fn read_node_sql( - (server, ca): (&str, Option<&Path>), + server: (&str, Option<&Path>), (location, target, node): (Uuid, &str, &str), ) -> Result, std::io::Error> { - let origin = parse_origin(server, "CLI").map_err(std::io::Error::other)?; if target.trim().is_empty() || node.trim().is_empty() { return Err(std::io::Error::other( "target and exact node key must not be blank", )); } - let credential = credentials::load(&origin)? - .ok_or_else(|| std::io::Error::other("not signed in; run sqb login"))?; - let client = transport::connect(&origin, ca)?; - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()?; - runtime.block_on(transport::node_sql( - &client, - &credential, - (location, target, node), - )) + transport::with_person_session(server, async |client, credential| { + transport::node_sql(client, credential, (location, target, node)).await + }) } diff --git a/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs b/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs new file mode 100644 index 0000000000..28f04804a5 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/request_warehouse_sync.rs @@ -0,0 +1,16 @@ +//! A saved person session asks a location to read the warehouse now; the server checks the target. + +use std::path::Path; + +use uuid::Uuid; + +use crate::person_login::_helpers::transport; + +pub(crate) fn request_warehouse_sync( + server: (&str, Option<&Path>), + (location, target): (Uuid, &str), +) -> Result { + transport::with_person_session(server, async |client, credential| { + transport::warehouse_sync(client, credential, (location, target)).await + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 7788726caf..f048bc2f1b 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -110,6 +110,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/connection-tests", post(crate::api::_helpers::resources::person_location_work::request), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/warehouse-syncs", + post(crate::api::_helpers::resources::person_location_work::request_sync), + ) .route( "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}", get(crate::api::_helpers::resources::person_location_work::result), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs index f3040a94db..142f7425d3 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs @@ -1,4 +1,4 @@ -//! Person connection tests reuse the location's named outbound work, not arbitrary commands. +//! Person connection tests and warehouse syncs reuse the location's named outbound work, not arbitrary commands. use axum::extract::{Path, State}; use axum::{Extension, Json}; @@ -11,14 +11,14 @@ use crate::credentials::models::AuthenticatedPerson; use crate::platform::main::read_person_connection_test::read_person_connection_test; use crate::platform::main::request_location_work::request_location_work; use crate::platform::models::{ - ConnectionTestRequest, LocationWork, LocationWorkRecord, LocationWorkRequest, + LocationWork, LocationWorkRecord, LocationWorkRequest, PersonWorkRequest, }; use crate::store::models::ControlStore; #[utoipa::path(post, path = "/api/control/locations/{location_id}/targets/{target}/connection-tests", operation_id = "requestConnectionTest", params(("location_id" = Uuid, Path), ("target" = String, Path), ("x-sqb-csrf" = String, Header)), - request_body = ConnectionTestRequest, + request_body = PersonWorkRequest, responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] @@ -26,18 +26,56 @@ pub(crate) async fn request( (State(store), Extension(context)): (State, Extension), Path((location_id, target)): Path<(Uuid, String)>, Extension(AuthenticatedPerson(principal, _)): Extension, - Json(request): Json, + Json(request): Json, +) -> Result, ApiError> { + request_work( + (&store, &context, principal), + (location_id, target), + request, + LocationWork::CheckConnection {}, + ) + .await +} + +#[utoipa::path(post, path = "/api/control/locations/{location_id}/targets/{target}/warehouse-syncs", + operation_id = "requestWarehouseSync", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("x-sqb-csrf" = String, Header)), + request_body = PersonWorkRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn request_sync( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target)): Path<(Uuid, String)>, + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(request): Json, +) -> Result, ApiError> { + request_work( + (&store, &context, principal), + (location_id, target), + request, + LocationWork::SyncWarehouse {}, + ) + .await +} + +/// Both requests need connection-test authority: each makes the location read the warehouse, never write it. +async fn request_work( + (store, context, principal): (&ControlStore, &AuditContext, Uuid), + (location_id, target): (Uuid, String), + request: PersonWorkRequest, + work: LocationWork, ) -> Result, ApiError> { Ok(Json(MutationResponse { applied: request_location_work( - &store, - (&context, Some(principal)), + store, + (context, Some(principal)), &LocationWorkRequest { request_id: request.request_id, location_id, target, reason: request.reason, - work: LocationWork::CheckConnection {}, + work, }, ) .await?, diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 967b3dae1c..54d45356d9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -59,6 +59,7 @@ use crate::api::_helpers::resources::{ person_bootstrap::read, person_maintenance::coverage, person_location_work::request, + person_location_work::request_sync, person_location_work::result, person_plans::request, person_plans::read, diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs index 0337aecc42..ffc4910264 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -28,7 +28,7 @@ pub async fn claim_location_work( "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ AND w.expires_at>$3 AND w.completed_at IS NULL \ AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ - AND (w.request_json->'work'->>'kind' IN ('check_connection','plan') OR EXISTS ( \ + AND (w.request_json->'work'->>'kind' IN ('check_connection','sync_warehouse','plan') OR EXISTS ( \ SELECT 1 FROM sqb_sensor_evaluation_events e JOIN sqb_triggers t ON t.trigger_id=e.trigger_id \ JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id JOIN sqb_scheduler_lease l ON l.name='scheduler' \ WHERE e.evaluation_id=w.request_id AND e.phase='requested' AND s.status='running' \ diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs index cc19fc50ad..8edbec28d0 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -93,7 +93,8 @@ fn redacted(result: &LocationWorkResult) -> LocationWorkResult { fn valid_outcome(work: &LocationWork, outcome: &LocationWorkOutcome) -> bool { match (work, outcome) { (_, LocationWorkOutcome::Superseded {}) - | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) => true, + | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) + | (LocationWork::SyncWarehouse {}, LocationWorkOutcome::SyncStarted {}) => true, (LocationWork::Plan { .. }, LocationWorkOutcome::Plan { document }) => { document .as_object() diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index 0e6ca1c7ed..fa1e4427f7 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -29,7 +29,7 @@ pub async fn request_location_work( "plans must be requested by a person on the current publication".into(), )); } - LocationWork::CheckConnection {} => {} + LocationWork::CheckConnection {} | LocationWork::SyncWarehouse {} => {} } if request.target.trim().is_empty() || request.reason.trim().is_empty() diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 928081c82e..d1e91f47d8 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -69,6 +69,8 @@ pub struct LocationHealthObservation { #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationWork { CheckConnection {}, + /// Read SQLBuild's state and catalog back from the warehouse now, not only after a run ends. + SyncWarehouse {}, EvaluateSensor { #[schema(value_type = Object)] evaluation: Box, @@ -100,9 +102,10 @@ pub struct PlanRequest { pub command: Vec, } +/// A person asks a location for one named read-only operation, such as a connection test. #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] -pub struct ConnectionTestRequest { +pub struct PersonWorkRequest { pub request_id: Uuid, pub reason: String, } @@ -138,6 +141,8 @@ pub enum LocationWorkOutcome { #[schema(value_type = Object)] document: serde_json::Value, }, + /// The location took the request; its next metadata sync reads the warehouse. + SyncStarted {}, Superseded {}, } diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs index ac5bff2d86..5752d74e7f 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -290,6 +290,40 @@ fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable .expect("expired excluded") .is_none() ); + let sync = LocationWorkRequest { + request_id: Uuid::now_v7(), + work: LocationWork::SyncWarehouse {}, + ..next + }; + request_location_work(store, (&audit, None), &sync) + .await + .expect("sync request"); + let claim = worker.poll(&poll).await.expect("poll").expect("sync claim"); + assert_eq!(claim.request.work, LocationWork::SyncWarehouse {}); + let answer = |outcome| LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome, + }; + assert!( + worker + .complete( + sync.request_id, + &answer(LocationWorkOutcome::Connection { healthy: true }) + ) + .await + .is_err(), + "a sync request is not answered by a connection result" + ); + assert!( + worker + .complete( + sync.request_id, + &answer(LocationWorkOutcome::SyncStarted {}) + ) + .await + .expect("sync started") + ); stop_api(api).await; close_test_store(fixture).await; }); diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs index 552171a323..ef1f3164d5 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -1,4 +1,4 @@ -//! Person connection tests use live built-ins and the same atomic work queue on both stores. +//! Person connection tests and warehouse syncs use live built-ins and the same atomic work queue on both stores. use uuid::Uuid; @@ -130,6 +130,28 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced ); assert_eq!(result.requested_by, format!("principal:{person}")); assert!(result.result_json.is_none()); + let sync = LocationWorkRequest { + request_id: Uuid::now_v7(), + reason: "Sync orders state now".into(), + work: LocationWork::SyncWarehouse {}, + ..request.clone() + }; + assert!( + request_location_work(store, (&context, Some(credentials)), &sync) + .await + .expect("connection-test authority requests a sync") + ); + assert!( + read_person_connection_test( + store, + (&context, credentials), + (location, "development", sync.request_id) + ) + .await + .expect("authorized read") + .is_none(), + "a sync request never reads back as a connection test" + ); for (other, target) in [(Uuid::now_v7(), "development"), (location, "preview")] { assert!(matches!( read_person_connection_test( @@ -165,6 +187,18 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced request_location_work(store, (&context, Some(credentials)), &request).await, Err(ControlError::Denied { .. }) )); + assert!(matches!( + request_location_work( + store, + (&context, Some(credentials)), + &LocationWorkRequest { + request_id: Uuid::now_v7(), + ..sync + } + ) + .await, + Err(ControlError::Denied { .. }) + )); assert!(matches!( read_person_connection_test( store, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index a37f7ae52b..51d0f0119e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/cli/server/ - Set retention - Grant roles to OIDC groups (beta) - Inspect published SQL +- Sync from the warehouse now - Run ad-hoc commands - Plan builds - Define project roles @@ -159,6 +160,26 @@ data. Every successful inspection records an audit entry without copying SQL int audit cannot be saved, the request fails without returning SQL. Revocation takes effect on the next request, including sessions created by [`sqb login`](login.md). +## Sync from the warehouse now + +A code location reads what has been built, checks and freshness back from the warehouse when it +starts and after each run the server starts has ended. It does not see a `sqb build` you run on +your own machine until then. To sync now, select **Sync now** on the location's page, or run: + +```bash +sqb control sync \ + --server https://control.example.com \ + --location-id "$LOCATION_ID" \ + --target development +``` + +The command prints `{"applied": true}` when the server has queued the request. The location starts +the sync within a few seconds, and the top bar shows when it last synced. A sync reads the +warehouse and never writes to it. + +Requesting a sync needs the same access as a connection test: Developer or Admin on the location +and target. Viewer and Operator cannot request one. Every request is recorded in the audit log. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 1aa1aeb223..65f57034c2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -4,7 +4,7 @@ import sqlite3 import time from contextlib import closing -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from subprocess import Popen from typing import Any @@ -27,6 +27,29 @@ def record_finished_run(control: Path) -> None: database.commit() +def request_warehouse_sync(control: Path) -> str: + """Queue the location work a person's Sync now creates, for the one published target.""" + request_id: str = str(uuid4()) + now: datetime = datetime.now(UTC) + with closing(sqlite3.connect(control)) as database: + _ = database.execute( + "INSERT INTO sqb_location_work (request_id,location_id,target,snapshot_id,generation," + "request_json,requested_by,requested_at,expires_at) SELECT ?,c.location_id,c.target," + "c.snapshot_id,s.generation,json_object('request_id',?,'location_id',c.location_id," + "'target',c.target,'reason','Sync now','work',json_object('kind','sync_warehouse'))," + "'test-admin',?,? FROM sqb_current_snapshots c JOIN sqb_snapshots s " + "ON s.snapshot_id=c.snapshot_id", + ( + request_id, + request_id, + now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + (now + timedelta(minutes=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + ), + ) + database.commit() + return request_id + + def wait_for_catalog( control: Path, expected: list[dict[str, Any]], process: Popen[bytes], log: Path ) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py index d7e12791d8..b267a969a4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -1,8 +1,10 @@ """An installed CLI inspects actual published SQL through a bounded live person session.""" import json +import sqlite3 import subprocess import sys +from contextlib import closing from functools import partial from pathlib import Path from typing import Any @@ -103,6 +105,16 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori "--target", "development", ] + sync: list[str] = [ + binary, + "control", + "sync", + *common, + "--location-id", + worker.api.location_id, + "--target", + "development", + ] absent: subprocess.CompletedProcess[str] = invoke(command) assert absent.returncode != 0 and not absent.stdout and "not signed in" in absent.stderr with pending_login( @@ -126,6 +138,12 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori assert ( denied.returncode != 0 and not denied.stdout and "HTTP 403; decision " in denied.stderr ) + refused: subprocess.CompletedProcess[str] = invoke(sync) + assert ( + refused.returncode != 0 + and not refused.stdout + and "Warehouse sync request rejected (HTTP 403; decision " in refused.stderr + ) administrator, _ = browser(origin, certificate) admin_csrf: str = sign_in( administrator, origin, "password", admin["name"], "orders-person-password-2026" @@ -140,6 +158,15 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori request(administrator, origin + "/api/admin/roles", assignment, csrf=admin_csrf)[0] == 200 ) + requested: subprocess.CompletedProcess[str] = invoke(sync) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"applied": True} + with closing(sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3")) as store: + queued: list[tuple[str]] = store.execute( + "SELECT requested_by FROM sqb_location_work " + "WHERE json_extract(request_json,'$.work.kind')='sync_warehouse'" + ).fetchall() + assert queued == [("principal:" + person["principal_id"],)] inspected: subprocess.CompletedProcess[str] = invoke(command) assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr sql: dict[str, Any] = json.loads(inspected.stdout) @@ -182,6 +209,7 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori and not revoked.stdout and "HTTP 403; decision " in revoked.stderr ) + assert invoke(sync).returncode != 0 changed: subprocess.CompletedProcess[str] = administer( (Path(binary), tmp_path / "control"), ["set-password", person["principal_id"]], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 5ca8ad1315..30d62cfc26 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -45,6 +45,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( record_finished_run, + request_warehouse_sync, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_audit_history, @@ -63,6 +64,7 @@ snapshot_api, sync_warehouse_history, wait_for_generation, + wait_for_location_work, wait_for_warehouse_sync_health, warehouse_history_page, warehouse_history_pages, @@ -1094,6 +1096,17 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre assert ( log.read_text().count("Warehouse history synchronization completed.") == completed ), "an idle location read the warehouse without a run ending" + work: dict[str, Any] = wait_for_location_work( + api, request_warehouse_sync(control), location, log + ) + assert work["result_json"]["outcome"] == {"kind": "sync_started"} + deadline = time.monotonic() + 45 + while ( + log.read_text().count("Warehouse history synchronization completed.") == completed + ): + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.05) for availability in fixture["availability"]: _ = (project / availability["source"]).rename(project / availability["destination"]) record_finished_run(control) diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index 3bb3b5a18a..9a87552387 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -136,6 +136,8 @@ export interface ApiClient extends PlatformApi { /** Null when the location has reported no recent sync. */ getWarehouseSync(scope: Scope): Promise; + /** Asks the location to read SQLBuild's state back from the warehouse now. `connection.test` */ + requestWarehouseSync(scope: Scope): Promise; /** Live "changed" signals (SSE in the server client); consumers refetch. */ changes: ChangeSubscription; diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index b9979ba996..c9bb4078ae 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -551,6 +551,35 @@ describe('administration writes', () => { }); }); +describe('requestWarehouseSync', () => { + function server(status: number) { + const calls: Request[] = []; + vi.stubGlobal('document', { cookie: '__Host-sqb-csrf=token-1' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + calls.push(request); + return status === 200 ? Response.json({ applied: true }) : Response.json({ error: 'You may not test this location', decision_id: 'd-1' }, { status }); + }) + ); + return { calls, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('asks the location to sync with one CSRF-protected request and does not wait for the sync', async () => { + const { calls, client } = server(200); + await client.requestWarehouseSync(ORDERS); + expect(calls.map((c) => [c.method, new URL(c.url).pathname, c.headers.get('x-sqb-csrf')])).toEqual([ + ['POST', '/api/control/locations/0191f0aa-orders/targets/development/warehouse-syncs', 'token-1'] + ]); + expect(await calls[0].json()).toMatchObject({ reason: 'Requested from the web UI' }); + }); + + it('reports a refused request with its status', async () => { + const { client } = server(403); + await expect(client.requestWarehouseSync(ORDERS)).rejects.toMatchObject({ status: 403 }); + }); +}); + describe('testConnection', () => { const record = (completed: string | null, outcome: unknown = null) => ({ generation: 1, diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 9f3fa8d2de..56cd7f6cac 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -80,6 +80,7 @@ const SUPPORTED: ReadonlySet = new Set([ 'listRoles', 'getRole', 'getWarehouseSync', + 'requestWarehouseSync', 'maintenanceHistory', 'testConnection', 'listWarehouseConnections', @@ -549,6 +550,16 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!data) throw new ApiError(errorText(error, response), response.status); return warehouseSyncOf(data); }, + /** The location takes the request within seconds; the sync's result arrives as a `warehouse_sync` change. */ + async requestWarehouseSync(scope: Scope): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { error, response } = await http.POST('/api/control/locations/{location_id}/targets/{target}/warehouse-syncs', { + body: { request_id: crypto.randomUUID(), reason: 'Requested from the web UI' }, + params: { path: { location_id: locationId, target }, header: { 'x-sqb-csrf': csrfToken(document.cookie) } } + }); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + }, /** Asks the location to connect, then waits for its answer until the request expires. */ async testConnection(scope: Scope): Promise { const { locationId, target } = parseLocationKey(scope.locationId); diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index b2f1f64d8f..c1a1d11812 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -334,6 +334,16 @@ export class MockApiClient implements ApiClient { return this.#reply({ ...this.#sync(scope) }); } + async requestWarehouseSync(scope: Scope): Promise { + this.#require('connection.test'); + const sync = this.#sync(scope); + setTimeout(() => { + Object.assign(sync, { lastSyncedAt: new Date().toISOString(), lastError: null }); + this.changes.emit(`warehouse_sync:${scope.locationId}`); + }, 1500); + return this.#reply(undefined); + } + // ---- platform and admin: delegated to MockPlatformApi (mock/platform.ts) ---- platformHealth(...args: Parameters) { diff --git a/ui/src/lib/api/types.ts b/ui/src/lib/api/types.ts index 3349a87cc6..1156d197a2 100644 --- a/ui/src/lib/api/types.ts +++ b/ui/src/lib/api/types.ts @@ -61,7 +61,7 @@ export type Snapshot = { sqbVersion: string; }; -/** The location reads SQLBuild's own tables back from the warehouse on its own schedule. */ +/** The location reads SQLBuild's own tables back from the warehouse after each run ends, and on request. */ export type WarehouseSync = { lastSyncedAt: Iso | null; /** Why the latest sync failed; null when it succeeded. */ diff --git a/ui/src/lib/components/platform/warehouse-sync-now.svelte b/ui/src/lib/components/platform/warehouse-sync-now.svelte new file mode 100644 index 0000000000..288c4f85ca --- /dev/null +++ b/ui/src/lib/components/platform/warehouse-sync-now.svelte @@ -0,0 +1,71 @@ + + +
+ Warehouse sync + {#if pending} + + {:else if last?.lastError} + + {:else if last?.lastSyncedAt} + + {:else if last || sync.error} + + {/if} + {#if failure}{failure}{/if} + Sync now +
+ + diff --git a/ui/src/lib/components/shell/warehouse-sync-status.svelte b/ui/src/lib/components/shell/warehouse-sync-status.svelte index 859abdd183..8951d8d87c 100644 --- a/ui/src/lib/components/shell/warehouse-sync-status.svelte +++ b/ui/src/lib/components/shell/warehouse-sync-status.svelte @@ -2,7 +2,7 @@ import { formatAgo, formatStamp } from '$lib/format/time'; import { app } from '$lib/state/app.svelte'; - // The location syncs SQLBuild's own tables from the warehouse on its own schedule; + // The location syncs SQLBuild's own tables from the warehouse after each run ends and on request; // the shell only says when that last happened. const sync = $derived(app.warehouseSync); diff --git a/ui/src/routes/locations/[id]/+page.svelte b/ui/src/routes/locations/[id]/+page.svelte index 3060ab8f1e..7876dc3cdc 100644 --- a/ui/src/routes/locations/[id]/+page.svelte +++ b/ui/src/routes/locations/[id]/+page.svelte @@ -9,6 +9,7 @@ import SegmentedFilter from '$lib/components/list/segmented-filter.svelte'; import StatusLabel from '$lib/components/list/status-label.svelte'; import ConnectionTest from '$lib/components/platform/connection-test.svelte'; + import WarehouseSyncNow from '$lib/components/platform/warehouse-sync-now.svelte'; import KeyValues from '$lib/components/platform/key-values.svelte'; import LocationTokens from '$lib/components/platform/location-tokens.svelte'; import Panel from '$lib/components/platform/panel.svelte'; @@ -95,6 +96,7 @@ + diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 0518129d53..a580028186 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -148,6 +148,26 @@ data. Every successful inspection records an audit entry without copying SQL int audit cannot be saved, the request fails without returning SQL. Revocation takes effect on the next request, including sessions created by [`sqb login`](/docs/cli/login/). +## Sync from the warehouse now + +A code location reads what has been built, checks and freshness back from the warehouse when it +starts and after each run the server starts has ended. It does not see a `sqb build` you run on +your own machine until then. To sync now, select **Sync now** on the location's page, or run: + +```bash +sqb control sync \ + --server https://control.example.com \ + --location-id "$LOCATION_ID" \ + --target development +``` + +The command prints `{"applied": true}` when the server has queued the request. The location starts +the sync within a few seconds, and the top bar shows when it last synced. A sync reads the +warehouse and never writes to it. + +Requesting a sync needs the same access as a connection test: Developer or Admin on the location +and target. Viewer and Operator cannot request one. Every request is recorded in the audit log. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a From 89d4023c49e6648051d58de635ed89a11b280208 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:32:58 +0000 Subject: [PATCH 10/13] chore: count only shipped code in GitHub's language statistics Test suites, development scripts and benchmarks are marked as documentation for Linguist. --- .gitattributes | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..3dcbe083f9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# GitHub's language bar counts shipped code only: test suites, development scripts and +# benchmarks are documentation to Linguist, in every language. +tests/** linguist-documentation +scripts/** linguist-documentation +benchmarks/** linguist-documentation +crates/**/tests/** linguist-documentation +crates/**/tests.rs linguist-documentation +ui/src/**/*.test.ts linguist-documentation +ui/scripts/** linguist-documentation +clients/typescript/tests/** linguist-documentation +clients/typescript/scripts/** linguist-documentation From 9a2c2cd9a3f35e3cab5175499744618125ac255d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:00:02 +0000 Subject: [PATCH 11/13] test(location): refuse sync requests outside their location, target and role The sync-due read refuses another location, another target's session and a stale generation, and answers only two times. A sync request is refused outside the Developer's location and target and for Viewer and Operator. --- .../tests/api/test_person_location_work.rs | 61 ++++++++++++- .../tests/api/test_warehouse_metadata.rs | 6 ++ .../tests/api/warehouse_metadata/helpers.rs | 86 ++++++++++++++++++- 3 files changed, 150 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs index ef1f3164d5..0e5058430d 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -152,6 +152,25 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced .is_none(), "a sync request never reads back as a connection test" ); + for (other, target) in [(Uuid::now_v7(), "development"), (location, "preview")] { + assert!( + matches!( + request_location_work( + store, + (&context, Some(credentials)), + &LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id: other, + target: target.into(), + ..sync.clone() + } + ) + .await, + Err(ControlError::Denied { .. }) + ), + "a sync outside the Developer's location and target is denied" + ); + } for (other, target) in [(Uuid::now_v7(), "development"), (location, "preview")] { assert!(matches!( read_person_connection_test( @@ -178,7 +197,7 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced (&context, sessions[0].identity.principal_id), &RoleAssignmentRequest { revoke: true, - ..assignment + ..assignment.clone() }, ) .await @@ -193,7 +212,7 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced (&context, Some(credentials)), &LocationWorkRequest { request_id: Uuid::now_v7(), - ..sync + ..sync.clone() } ) .await, @@ -208,6 +227,44 @@ fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_preced .await, Err(ControlError::Denied { .. }) )); + for role in [BuiltinRole::Viewer, BuiltinRole::Operator] { + let lesser = RoleAssignmentRequest { + role: role.into(), + ..assignment.clone() + }; + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &lesser, + ) + .await + .expect("assign lesser role"); + assert!( + matches!( + request_location_work( + store, + (&context, Some(credentials)), + &LocationWorkRequest { + request_id: Uuid::now_v7(), + ..sync.clone() + } + ) + .await, + Err(ControlError::Denied { .. }) + ), + "{role:?} cannot request a sync" + ); + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &RoleAssignmentRequest { + revoke: true, + ..lesser + }, + ) + .await + .expect("revoke lesser role"); + } close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 9aae0418f6..56536e0d3f 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -349,6 +349,12 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit (&client, &url, &token.bearer), ) .await; + crate::api::tests::warehouse_metadata_helpers::sync_due_scope( + store, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; crate::api::tests::warehouse_reconciliation_helpers::reconcile_history( store, (&client, &url, &token.bearer), diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs index 3e9aa5a83f..103e9001b4 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -9,7 +9,7 @@ use crate::store::main::decompress::decompress; use crate::store::main::{ begin::begin, commit::commit, migrate_store::migrate_store, query::query, }; -use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; use crate::store::tests::fault_helpers::{install_fault, remove_fault}; use crate::store::tests::fault_types::TransactionFault; use crate::warehouse_metadata::models::{MetadataBatch, WarehouseWatermarkRead}; @@ -515,6 +515,90 @@ pub(crate) async fn fingerprint_history( } } +/// Sync-due answers only the fenced reader's own location and target, with nothing but two times. +pub(crate) async fn sync_due_scope( + store: &ControlStore, + (session_id, preview_session_id): (Uuid, Uuid), + (client, url, bearer): (&Client, &str, &str), +) { + let ended_at = "2026-01-01T09:00:00Z"; + let run_id = Uuid::now_v7(); + let (transaction, _) = query(begin(store, TransactionMode::Write).await.expect("write"), + "INSERT INTO sqb_runs (run_id,location_id,target,snapshot_id,kind,origin,root_run_id,requested_by,\ + selection_json,request_json,write_set_digest,status,queued_at,ended_at,node_counts_json,last_event_storage_id) \ + SELECT $1,location_id,target,snapshot_id,'adhoc','manual',$2,'test-admin',$3,$4,'','success',$5,$6,$7,0 \ + FROM sqb_current_snapshots WHERE target='preview'", + vec![Parameter::Id(run_id), Parameter::Id(run_id), Parameter::Json(json!([])), Parameter::Json(json!({})), + Parameter::Time(ended_at.parse().expect("time")), Parameter::Time(ended_at.parse().expect("time")), + Parameter::Json(json!({}))]).await.expect("preview run"); + commit(transaction).await.expect("close write"); + let due_url = format!("{url}/sync-due"); + let preview_url = due_url.replace("/development/", "/preview/"); + let body = + |session: Uuid, generation: i64| json!({"session_id": session, "generation": generation}); + for (endpoint, session, expected) in [ + (&due_url, session_id, Value::Null), + (&preview_url, preview_session_id, json!(ended_at)), + ] { + let reply: Value = client + .post(endpoint) + .bearer_auth(bearer) + .json(&body(session, 1)) + .send() + .await + .expect("sync due") + .json() + .await + .expect("sync due reply"); + let fields = reply.as_object().expect("object"); + let mut keys: Vec<_> = fields.keys().collect(); + keys.sort(); + assert_eq!(keys, ["now", "runs_ended_at"]); + let ended = fields["runs_ended_at"] + .as_str() + .map(|time| time.parse::>().expect("time")); + let wanted = expected.as_str().map(|time| time.parse().expect("time")); + assert_eq!(ended, wanted, "another target's runs never count"); + } + let other_location = due_url.replace( + url.split('/').nth(5).expect("location id"), + &Uuid::now_v7().to_string(), + ); + for (endpoint, request, status) in [ + (&due_url, body(session_id, 0), StatusCode::CONFLICT), + ( + &due_url, + body(preview_session_id, 1), + StatusCode::BAD_REQUEST, + ), + (&preview_url, body(session_id, 1), StatusCode::BAD_REQUEST), + (&other_location, body(session_id, 1), StatusCode::FORBIDDEN), + ] { + assert_eq!( + client + .post(endpoint) + .bearer_auth(bearer) + .json(&request) + .send() + .await + .expect("refused sync due") + .status(), + status, + "{endpoint} {request}" + ); + } + assert_eq!( + client + .post(&due_url) + .json(&body(session_id, 1)) + .send() + .await + .expect("anonymous sync due") + .status(), + StatusCode::UNAUTHORIZED + ); +} + async fn mirrored_fingerprints(store: &ControlStore) -> Vec { let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), "SELECT f.event_id,f.source_event_id,f.definition_text_hash,f.record_json,t.body FROM sqb_warehouse_fingerprints f \ From f88f15b9480b241590be8689821b0bd3e9da56ab Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:00:03 +0000 Subject: [PATCH 12/13] fix(location): keep a quiet location's last sync visible Locations now sync only after runs, so location health reads every retained day instead of the last two. Sync now waits for a changed sync report instead of comparing browser and server clocks, and stops waiting when the request expires. sqb control sql without a node and sync with a node are usage errors. --- .../main/run_person_invocation.rs | 15 ++++++- .../src/platform/_helpers/recent_health.rs | 43 ++++++++++++------- .../tests/api/test_location_health.rs | 26 ++++++++++- .../publication/test_person_sql.py | 7 +++ .../platform/warehouse-sync-now.svelte | 14 +++--- 5 files changed, 82 insertions(+), 23 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs index dc18d23dfd..558af309fd 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs @@ -10,7 +10,20 @@ pub(crate) fn run_person_invocation(namespace: &Namespace) -> Option { if text(namespace, "command").as_deref() != Some("control") { return None; } - if text(namespace, "person_action").as_deref() == Some("sync") { + let action = text(namespace, "person_action"); + let misuse = match (action.as_deref(), text(namespace, "person_node").is_some()) { + (Some("sync"), true) => Some("sync takes no node; it syncs the whole target"), + (Some("sql"), false) => Some("sql needs an exact node key, for example model:orders"), + _ => None, + }; + if let Some(message) = misuse { + drop(write_output(&[( + Stream::Stderr, + &format!("sqb control: error: {message}\n"), + )])); + return Some(2); + } + if action.as_deref() == Some("sync") { return Some(run_warehouse_sync(namespace)); } drop(write_output(&[( diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs index 40e68baa26..80808f3d8f 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs @@ -2,42 +2,53 @@ use uuid::Uuid; +use crate::platform::constants::LOCATION_RETENTION_DAYS; use crate::platform::models::LocationHealthObservation; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -/// At most 100, newest first, from today's and yesterday's reports plus each instance's last check-in, of one target when given. +/// At most 100, newest first, from every retained day (a sync follows only runs) plus each instance's last check-in. pub(crate) async fn recent_health( transaction: ControlTransaction, location_id: Uuid, target: Option<&str>, ) -> Result<(ControlTransaction, Vec), ControlError> { let today = transaction.now.date_naive(); - let yesterday = today - chrono::TimeDelta::days(1); + let days = 0..=LOCATION_RETENTION_DAYS; + let target_placeholder = days.clone().count() + 1; let (event_target, instance_target) = match target { - Some(_) => (" AND payload->'report'->>'target'=$3", " AND target=$2"), - None => ("", ""), + Some(_) => ( + format!(" AND payload->'report'->>'target'=${target_placeholder}"), + " AND target=$2", + ), + None => (String::new(), ""), }; let target_parameter: Vec = target .map(|target| Parameter::Text(target.to_owned())) .into_iter() .collect(); + let daily = days + .clone() + .map(|day| { + format!("SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=${} AND event_type='location_health'{event_target} ORDER BY invocation_sequence DESC LIMIT 100) day_{day}", day + 1) + }) + .collect::>() + .join(" UNION ALL "); let (transaction, rows) = query( transaction, - &format!("SELECT received_at, payload FROM (\ - SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ - WHERE invocation_id=$1 AND event_type='location_health'{event_target} ORDER BY invocation_sequence DESC LIMIT 100) current_day \ - UNION ALL \ - SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ - WHERE invocation_id=$2 AND event_type='location_health'{event_target} ORDER BY invocation_sequence DESC LIMIT 100) previous_day\ - ) observations ORDER BY received_at DESC, invocation_sequence DESC LIMIT 100"), - [ - Parameter::Text(format!("location/{location_id}/{today}")), - Parameter::Text(format!("location/{location_id}/{yesterday}")), - ] - .into_iter() + &format!( + "SELECT received_at, payload FROM ({daily}) observations \ + ORDER BY received_at DESC, invocation_sequence DESC LIMIT 100" + ), + days.map(|day| { + Parameter::Text(format!( + "location/{location_id}/{}", + today - chrono::TimeDelta::days(day) + )) + }) .chain(target_parameter.clone()) .collect(), ) diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs index 928b2759ca..14a932e7cc 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -6,8 +6,10 @@ use crate::platform::main::report_remote_location_health::report_remote_location use crate::platform::models::{LocationHealth, LocationHealthObservation, LocationHealthReport}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::migrate_store::migrate_store; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Parameter, TransactionMode}; use crate::store::tests::helpers::read_recall_events; -use chrono::Utc; +use chrono::{TimeDelta, Utc}; use reqwest::StatusCode; use serde_json::json; use uuid::Uuid; @@ -202,6 +204,28 @@ fn given_scoped_location_observations_when_retried_or_read_then_history_and_auth .is_empty(), "another target's read leaves this target's observations out" ); + // A location syncs only after runs, so its newest report can be days old. + for (age, kept) in [(7, true), (8, false), (0, true)] { + let transaction = begin(store, TransactionMode::Write).await.expect("age"); + let day = (transaction.now - TimeDelta::days(age)).date_naive(); + let (transaction, _) = query( + transaction, + "UPDATE recall_events SET invocation_id=$1 WHERE event_type='location_health'", + vec![Parameter::Text(format!("location/{location_id}/{day}"))], + ) + .await + .expect("aged report"); + commit(transaction).await.expect("aged report committed"); + let reports = person_read("preview #1").await.expect("aged health"); + assert_eq!( + reports + .iter() + .any(|observation| observation.report == connection), + kept, + "{}: a report {age} days old is read within location health retention", + test_case.description + ); + } let events = read_recall_events(store, 0, 100) .await .expect("read durable events"); diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py index b267a969a4..d73478e099 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -115,6 +115,13 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori "--target", "development", ] + for misused in ([*command[:3], *command[4:]], [*sync[:3], "model:orders", *sync[3:]]): + usage: subprocess.CompletedProcess[str] = invoke(misused) + assert (usage.returncode, usage.stdout, usage.stderr[:20]) == ( + 2, + "", + "sqb control: error: ", + ) absent: subprocess.CompletedProcess[str] = invoke(command) assert absent.returncode != 0 and not absent.stdout and "not signed in" in absent.stderr with pending_login( diff --git a/ui/src/lib/components/platform/warehouse-sync-now.svelte b/ui/src/lib/components/platform/warehouse-sync-now.svelte index 288c4f85ca..85496aaa7d 100644 --- a/ui/src/lib/components/platform/warehouse-sync-now.svelte +++ b/ui/src/lib/components/platform/warehouse-sync-now.svelte @@ -13,19 +13,23 @@ const sync = createResource((client) => client.getWarehouseSync(scope), ['warehouse_sync']); let requesting = $state(false); - /** When this page asked; cleared once a sync newer than the request is reported. */ - let requestedAt = $state(null); + /** + * The sync report this page saw when it asked, compared by value rather than by clock, since browser and + * server clocks differ. Any newer report ends the wait; so does the request's five-minute expiry. + */ + let requested = $state<{ at: number; seen: string } | null>(null); let failure = $state(null); const last = $derived(sync.data ?? null); - const pending = $derived(requestedAt !== null && !(last?.lastSyncedAt && Date.parse(last.lastSyncedAt) >= requestedAt)); + const reportOf = (report: typeof last) => `${report?.lastSyncedAt ?? ''}|${report?.lastError ?? ''}`; + const pending = $derived(requested !== null && reportOf(last) === requested.seen && app.now - requested.at < 5 * 60_000); async function request() { requesting = true; failure = null; try { - const asked = Date.now(); + const seen = { at: Date.now(), seen: reportOf(last) }; await app.client.requestWarehouseSync(scope); - requestedAt = asked; + requested = seen; } catch (error) { failure = errorMessage(error); } finally { From eaf7f4101eb35fd962de50e2a56f733f12a9edff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:04:49 +0000 Subject: [PATCH 13/13] fix(ui): end the Sync now wait on any new sync report A requested sync that fails the same way as before now ends the wait too. --- .../src/platform/_helpers/recent_health.rs | 2 +- ui/src/lib/api/http-locations.test.ts | 2 +- ui/src/lib/api/http-locations.ts | 3 ++- ui/src/lib/api/mock/mock-client.ts | 3 ++- ui/src/lib/api/types.ts | 2 ++ ui/src/lib/components/platform/warehouse-sync-now.svelte | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs index 80808f3d8f..7800fbbf9c 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/recent_health.rs @@ -9,7 +9,7 @@ use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -/// At most 100, newest first, from every retained day (a sync follows only runs) plus each instance's last check-in. +/// At most 100, newest first, from the health retention window (a sync follows only runs) plus instance check-ins. pub(crate) async fn recent_health( transaction: ControlTransaction, location_id: Uuid, diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 5521f5162f..50e6d53950 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -73,7 +73,7 @@ describe('warehouseSyncOf', () => { sync('2026-11-08T12:01:00Z', 'catalog', null), sync('2026-11-08T12:00:00Z', 'statistics', 'delivery') ]); - expect(result).toEqual({ lastSyncedAt: '2026-11-08T12:02:00Z', lastError: 'catalog: failed while reading the warehouse' }); + expect(result).toEqual({ lastSyncedAt: '2026-11-08T12:02:00Z', lastError: 'catalog: failed while reading the warehouse', lastReportedAt: '2026-11-08T12:03:00Z' }); }); it("gives each table its newest sync, the time it last synced cleanly, and where the state tables are read from", () => { diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index 32a21cd1c4..436cb4d6ad 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -167,6 +167,7 @@ export function warehouseSyncOf(observations: Wire['LocationHealthObservation'][ const failing = [...newest.values()].flatMap((s) => (s.report.health.failure ? [`${s.report.health.table}: failed while ${SYNC_STAGE[s.report.health.failure]}`] : [])); return { lastSyncedAt: syncs.find((s) => !s.report.health.failure)?.received_at ?? null, - lastError: failing.length ? failing.sort().join('\n') : null + lastError: failing.length ? failing.sort().join('\n') : null, + lastReportedAt: syncs[0].received_at }; } diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index c1a1d11812..5e3d7f4561 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -338,7 +338,8 @@ export class MockApiClient implements ApiClient { this.#require('connection.test'); const sync = this.#sync(scope); setTimeout(() => { - Object.assign(sync, { lastSyncedAt: new Date().toISOString(), lastError: null }); + const now = new Date().toISOString(); + Object.assign(sync, { lastSyncedAt: now, lastError: null, lastReportedAt: now }); this.changes.emit(`warehouse_sync:${scope.locationId}`); }, 1500); return this.#reply(undefined); diff --git a/ui/src/lib/api/types.ts b/ui/src/lib/api/types.ts index 1156d197a2..895d746c92 100644 --- a/ui/src/lib/api/types.ts +++ b/ui/src/lib/api/types.ts @@ -66,6 +66,8 @@ export type WarehouseSync = { lastSyncedAt: Iso | null; /** Why the latest sync failed; null when it succeeded. */ lastError: string | null; + /** When the newest sync report arrived, failed or not; a new one means a requested sync has finished. */ + lastReportedAt?: Iso | null; }; /** The one target a location is deployed against, e.g. `prod` on Snowflake. */ diff --git a/ui/src/lib/components/platform/warehouse-sync-now.svelte b/ui/src/lib/components/platform/warehouse-sync-now.svelte index 85496aaa7d..84872ef1c3 100644 --- a/ui/src/lib/components/platform/warehouse-sync-now.svelte +++ b/ui/src/lib/components/platform/warehouse-sync-now.svelte @@ -20,7 +20,7 @@ let requested = $state<{ at: number; seen: string } | null>(null); let failure = $state(null); const last = $derived(sync.data ?? null); - const reportOf = (report: typeof last) => `${report?.lastSyncedAt ?? ''}|${report?.lastError ?? ''}`; + const reportOf = (report: typeof last) => `${report?.lastReportedAt ?? ''}|${report?.lastSyncedAt ?? ''}|${report?.lastError ?? ''}`; const pending = $derived(requested !== null && reportOf(last) === requested.seen && app.now - requested.at < 5 * 60_000); async function request() {