From d411fdb4a990aacfa58a329c914787a1c7080b85 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:12:09 +0000 Subject: [PATCH 01/49] feat(orchestration): jobs, groups, sensors and calendars join the global unique-name check Publication now rejects a job, job group, sensor or calendar schedule whose name is already used by any other project resource or orchestration declaration, with the same D007 conflict discovery raises for models, sources and the rest. The narrower duplicate-job and duplicate-trigger checks are removed; the object-identity check stays. build_orchestration_snapshot and compile_snapshot take the discovered project inputs so publication can build the full name set. Fixtures whose jobs reused model names are renamed, and DuckShop gains a name-collision edge case. --- .../orchestration/calendar_orders.py | 2 +- .../orchestration/conflicting_settings.py | 2 +- .../orchestration/invalid_settings.py | 2 +- .../orchestration/name_collision.py | 5 + .../scheduler_crash_recovery.json | 6 +- .../_helpers/resource_names.rs | 9 +- .../src/project_validation/main/mod.rs | 1 + .../main/validate_orchestration_names.rs | 9 ++ .../src/project_validation/models.rs | 10 ++ .../src/project_validation/tests.rs | 2 + .../src/project_validation/tests/helpers.rs | 27 ++++- .../test_validate_orchestration_names.rs | 85 +++++++++++++ .../compiled_project/snapshot_declarations.rs | 16 +-- .../_helpers/discovery/project_facts.rs | 25 ++++ .../_helpers/discovery/project_validation.rs | 14 ++- .../_helpers/location_command/command.rs | 19 +-- .../_helpers/location_command/snapshot.rs | 63 ++++++++-- .../_helpers/native_host/constants.rs | 8 ++ .../native_host/orchestration_declarations.rs | 76 +++++++++++- .../native_host/orchestration_sensors.rs | 2 +- .../native_host/orchestration_triggers.rs | 10 -- deploy/project/orchestration/orders.py | 2 +- src/sqlbuild/_native.pyi | 1 + .../main/compile_snapshot.py | 13 +- .../commands/main/test/test_select_by_name.py | 2 +- .../publication/test_backfill.py | 2 +- .../publication/test_calendar.py | 4 +- .../publication/test_publication.py | 14 ++- .../publication/test_rerun.py | 2 +- .../publication/test_warehouse_metadata.py | 5 +- .../publication/test_worker.py | 21 ++-- .../test_orchestration_snapshot.py | 114 ++++++++++++++---- .../sqlbuild/compiler/discovery/test_main.py | 2 +- .../docs/concepts/python-nodes/overview.mdx | 2 +- 34 files changed, 474 insertions(+), 103 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/name_collision.py create mode 100644 crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py index 644b92c5d6..80d336fbbb 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -2,7 +2,7 @@ from sqlbuild.orchestration import calendar_schedule, job -orders = job("orders", description="Build order reporting.", select="orders") +orders = job("order_reporting", description="Build order reporting.", select="orders") @calendar_schedule( diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py index ae5a4c9dec..e864cf753d 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py +++ b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py @@ -3,7 +3,7 @@ from sqlbuild.orchestration import job orders = job( - "orders", + "refresh_orders", description="Refresh orders.", select="orders", settings={"load_sources": False, "reload": True}, diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py index 248b1b4ce1..621d038079 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py +++ b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py @@ -3,7 +3,7 @@ from sqlbuild.orchestration import job orders = job( - "orders", + "refresh_orders", description="Refresh orders.", select="orders", settings={"select": "customers"}, diff --git a/benchmarks/duckshop/edge_cases/orchestration/name_collision.py b/benchmarks/duckshop/edge_cases/orchestration/name_collision.py new file mode 100644 index 0000000000..fb0e024e9c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/name_collision.py @@ -0,0 +1,5 @@ +"""A job may not reuse a model's name: jobs share the project's one resource namespace.""" + +from sqlbuild.orchestration import job + +orders = job("orders", description="Build orders.", select="orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json index 12eb4f25d5..ab9c620d90 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json +++ b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json @@ -1,9 +1,9 @@ { "description": "Three minutely cron jobs start ten hours back with the default latest catch-up. Two installed schedulers catch up while being SIGKILLed at seeded random moments; every lapse of a killed leader's lease is simulated. After a final uninterrupted scheduler, each trigger has ticked its latest due minute exactly once and nothing off schedule, each tick requests or is covered by exactly one run, and every requested run has one request event.", "jobs": [ - "orders", - "customers", - "inventory" + "refresh_orders", + "refresh_customers", + "refresh_inventory" ], "backlog_minutes": 600, "rounds": 16, diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs index 7f2032bafa..c9b65c04f6 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -1,4 +1,4 @@ -//! Python's `_validate_unique_project_resource_names`: one namespace across resource kinds. +//! One namespace across resource kinds, orchestration declarations included. use std::collections::{HashMap, HashSet}; @@ -16,8 +16,8 @@ pub(crate) fn validate_unique_project_resource_names( return Err(conflict(format!( "Project resource name '{name}' is declared as both {existing_kind} in \ {existing_path} and {kind} in {path}; model, source, seed, function, loader, \ - task, asset, check, provider, hook, unit test, and scenario names must be \ - globally unique" + task, asset, check, provider, hook, unit test, scenario, job, job group, sensor, \ + and calendar schedule names must be globally unique" ))); } seen.insert(name.to_owned(), (kind.to_owned(), path.to_owned())); @@ -69,5 +69,8 @@ pub(crate) fn validate_unique_project_resource_names( declare(name, "unit test", &test.file.path)?; } } + for declaration in &facts.orchestration { + declare(&declaration.name, declaration.kind, &declaration.path)?; + } Ok(()) } diff --git a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs index 2c19ca312a..9eaf797a34 100644 --- a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs +++ b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs @@ -1,2 +1,3 @@ pub mod at_most_one_csv_data_row; +pub mod validate_orchestration_names; pub mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs b/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs new file mode 100644 index 0000000000..ac3a340b34 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs @@ -0,0 +1,9 @@ +//! Orchestration declarations share the project's one resource namespace. + +use crate::project_validation::_helpers::resource_names::validate_unique_project_resource_names; +use crate::project_validation::models::{ProjectFacts, ValidationFailure}; + +/// Reject the first name a job, group, sensor or calendar schedule shares with anything else. +pub fn validate_orchestration_names(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + validate_unique_project_resource_names(facts) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 58c65bb869..a583cc66ce 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -178,6 +178,16 @@ pub struct ProjectFacts { pub project_model_paths: Option>, /// Python's `csv.field_size_limit()`, or `None` for its default. pub csv_field_limit: Option, + /// Jobs, job groups, sensors and calendar schedules, known only when a location publishes. + pub orchestration: Vec, +} + +/// A named orchestration declaration: its name, its kind as conflicts word it, and its module. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OrchestrationNameFact { + pub name: String, + pub kind: &'static str, + pub path: String, } /// Why discovered inputs are rejected: a discovery error, or a seed CSV Python cannot read. diff --git a/crates/sqlbuild-discovery/src/project_validation/tests.rs b/crates/sqlbuild-discovery/src/project_validation/tests.rs index d7aebec514..ba01477136 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests.rs @@ -6,5 +6,7 @@ mod csv_header; mod helpers; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_validate_orchestration_names.rs"] +mod validate_orchestration_names; #[path = "tests/test_validate_project_inputs.rs"] mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs index dc141fbc95..601bc978ea 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs @@ -1,9 +1,11 @@ +use crate::models::FailureKind; use crate::project_validation::_helpers::csv_header::csv_header; use crate::project_validation::main::validate_project_inputs::validate_project_inputs; use crate::project_validation::models::{ - CsvSettings, DependencyFact, ModelFact, NamedPath, NodeFact, NodeKind, ProjectFacts, - SchemaFileFact, SeedEntryFact, SeedFileFact, ValidationFailure, + CsvSettings, DependencyFact, ModelFact, NamedPath, NodeFact, NodeKind, OrchestrationNameFact, + ProjectFacts, SchemaFileFact, SeedEntryFact, SeedFileFact, ValidationFailure, }; +use crate::project_validation::tests::test_types::failure; /// The header the reader returns, or its error's message. pub(super) fn read_header( @@ -88,3 +90,24 @@ pub(super) fn path_default_facts(paths: &[&str], keys: &[&str]) -> ProjectFacts ..ProjectFacts::default() } } + +/// An orchestration declaration named `name`, of `kind`, in one module. +pub(super) fn declared(name: &str, kind: &'static str) -> OrchestrationNameFact { + OrchestrationNameFact { + name: name.to_owned(), + kind, + path: "orchestration/daily.py".to_owned(), + } +} + +/// A global unique-name conflict that `message` begins. +pub(super) fn collision(message: &str) -> Result<(), String> { + failure( + FailureKind::Conflict, + &format!( + "{message}; model, source, seed, function, loader, task, asset, check, provider, \ + hook, unit test, scenario, job, job group, sensor, and calendar schedule names must \ + be globally unique" + ), + ) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs new file mode 100644 index 0000000000..beb6d48ffa --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs @@ -0,0 +1,85 @@ +use crate::project_validation::main::validate_orchestration_names::validate_orchestration_names; +use crate::project_validation::models::{ + ProjectFacts, TestBlockFact, TestFileFact, ValidationFailure, +}; +use crate::project_validation::tests::helpers::{collision, declared, model, named}; +use crate::project_validation::tests::test_types::ValidationTestCase; + +#[test] +fn given_orchestration_names_when_validating_then_they_share_the_project_namespace() { + let test_cases = [ + ValidationTestCase { + description: "distinct names pass", + facts: ProjectFacts { + models: vec![model("orders")], + orchestration: vec![ + declared("refresh_orders", "job"), + declared("nightly", "job group"), + declared("orders_arrived", "sensor"), + declared("fulfillment_calendar", "calendar schedule"), + ], + ..ProjectFacts::default() + }, + expected_failure: Ok(()), + }, + ValidationTestCase { + description: "a job named like a model", + facts: ProjectFacts { + models: vec![model("orders")], + orchestration: vec![declared("orders", "job")], + ..ProjectFacts::default() + }, + expected_failure: collision( + "Project resource name 'orders' is declared as both model in models/orders.sql \ + and job in orchestration/daily.py", + ), + }, + ValidationTestCase { + description: "a sensor named like a unit test", + facts: ProjectFacts { + tests: vec![TestFileFact { + file: named("orders_arrived", "tests/orders_arrived.sql"), + blocks: vec![TestBlockFact::default()], + }], + orchestration: vec![declared("orders_arrived", "sensor")], + ..ProjectFacts::default() + }, + expected_failure: collision( + "Project resource name 'orders_arrived' is declared as both unit test in \ + tests/orders_arrived.sql and sensor in orchestration/daily.py", + ), + }, + ValidationTestCase { + description: "a calendar schedule named like a provider", + facts: ProjectFacts { + providers: vec![named("fulfillment", "providers/fulfillment.py")], + orchestration: vec![declared("fulfillment", "calendar schedule")], + ..ProjectFacts::default() + }, + expected_failure: collision( + "Project resource name 'fulfillment' is declared as both provider in \ + providers/fulfillment.py and calendar schedule in orchestration/daily.py", + ), + }, + ValidationTestCase { + description: "a job group named like a job", + facts: ProjectFacts { + orchestration: vec![declared("daily", "job"), declared("daily", "job group")], + ..ProjectFacts::default() + }, + expected_failure: collision( + "Project resource name 'daily' is declared as both job in orchestration/daily.py \ + and job group in orchestration/daily.py", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + validate_orchestration_names(&test_case.facts) + .map_err(|failure: ValidationFailure| failure.to_string()), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 0c2ac4c08b..882e3e8ba4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -31,7 +31,7 @@ use crate::bindings::_helpers::native_host::orchestration_triggers::{ pub(crate) fn discover_declarations<'py>( py: Python<'py>, project_dir: &Path, -) -> PyResult>> { +) -> PyResult, String)>> { let tree = ProjectTree::new(project_dir); let files = py .detach(|| { @@ -44,12 +44,12 @@ pub(crate) fn discover_declarations<'py>( .map_err(|error| { PyValueError::new_err(format!("orchestration discovery failed: {error:?}")) })?; - let mut declarations: Vec> = Vec::new(); + let mut declarations: Vec<(Bound<'py, PyAny>, String)> = Vec::new(); let mut seen: BTreeSet = BTreeSet::new(); for file in files { for job in module_jobs(py, &tree.absolute(&file), project_dir)? { if seen.insert(job.as_ptr() as usize) { - declarations.push(job); + declarations.push((job, file.clone())); } } } @@ -87,11 +87,7 @@ pub(crate) fn compile_declarations( } } let mut triggers: BTreeMap = BTreeMap::new(); - let mut names: BTreeSet = BTreeSet::new(); for job in authored_jobs { - if !names.insert(job.name.clone()) { - return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); - } if !job.targets.is_empty() && !job.targets.contains(&input.target) { continue; } @@ -140,12 +136,6 @@ pub(crate) fn compile_declarations( } let mut groups: BTreeMap = BTreeMap::new(); for group in authored_groups { - if !names.insert(group.name.clone()) { - return Err(PyValueError::new_err(format!( - "duplicate job or group {}", - group.name - ))); - } if !group.targets.is_empty() && !group.targets.contains(&input.target) { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index 5bfcaeb0d1..4e7d9f6e52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -17,6 +17,13 @@ use sqlbuild_discovery::project_validation::models::{ const CSV_MODULE: &str = "csv"; /// `DeclarationKind.AUDIT`: a generic audit file. const GENERIC_AUDIT_KIND: &str = "audit"; +/// Each Python node kind and the discovered inputs field that lists it, in discovery order. +const PYTHON_NODE_FIELDS: [(&str, &str); 4] = [ + ("loader", "loader_functions"), + ("task", "task_functions"), + ("asset", "asset_functions"), + ("check", "check_functions"), +]; /// `ScopeKind.PRIVATE`. const PRIVATE_SCOPE: &str = "private"; @@ -72,9 +79,27 @@ pub(crate) fn project_facts( .collect::>()?, project_model_paths: injection().map(|injection| injection.project_model_paths.clone()), csv_field_limit: Some(csv_field_limit(py)?), + orchestration: Vec::new(), }) } +/// The discovered Python nodes by kind, name and path only, as the resource-name check reads them. +pub(crate) fn named_node_rows(inputs: &Bound<'_, PyAny>) -> PyResult> { + let mut rows: Vec = Vec::new(); + for (kind, name) in PYTHON_NODE_FIELDS { + rows.extend(each(inputs, name, |node| { + Ok(NodeRow { + kind: kind.to_owned(), + name: native_field(node, "name")?.extract()?, + path: text(node, "relative_path")?, + decorator_config: false, + depends_on: Vec::new(), + }) + })?); + } + Ok(rows) +} + /// `read` applied to each item of the iterable attribute `name` of `owner`. pub(crate) fn each( owner: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs index 92d7a38846..b3b1a096e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs @@ -33,10 +33,12 @@ fn validate_discovered_project_inputs( nodes: Vec, ) -> Result<(), PyErr> { let facts: ProjectFacts = project_facts(py, &inputs, nodes)?; - let Err(failure) = validate_project_inputs(&facts) else { - return Ok(()); - }; - Err(match failure { + validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) +} + +/// The Python exception a failed project validation raises. +pub(crate) fn validation_error(py: Python<'_>, failure: ValidationFailure) -> PyErr { + match failure { ValidationFailure::Discovery(failure) => failure_error(py, failure), ValidationFailure::Unreadable { absolute_path, @@ -56,7 +58,7 @@ fn validate_discovered_project_inputs( ), ValidationFailure::CsvDialect(error) => { if !CSV_DIALECT_ERRORS.contains(&error.class.as_str()) { - return Err(PyKeyError::new_err(error.class.to_string())); + return PyKeyError::new_err(error.class.to_string()); } raised( py.import(BUILTINS_MODULE) @@ -64,7 +66,7 @@ fn validate_discovered_project_inputs( .and_then(|class| class.call1((error.message,))), ) } - }) + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index ccb2e600ba..92cc25b235 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -22,7 +22,9 @@ use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSn use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; -use crate::bindings::_helpers::location_command::snapshot::build_orchestration_snapshot; +use crate::bindings::_helpers::location_command::snapshot::{ + build_orchestration_snapshot, orchestration_snapshot, +}; use crate::bindings::_helpers::location_command::worker::execute_worker; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ @@ -136,15 +138,18 @@ fn compile_location<'py>( )?; let project = field(&compiled, "project")?; let view = project.cast::()?.borrow(); - let snapshot = build_orchestration_snapshot( + let snapshot = orchestration_snapshot( view, &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, project_dir.extract()?, - arguments - .value("action")? - .extract::>()? - .is_some_and(|action| action == EXECUTE_ACTION) - .then(Vec::new), + ( + discovered.clone(), + arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action == EXECUTE_ACTION) + .then(Vec::new), + ), )?; status(py, "Location snapshot compiled.")?; Ok((snapshot, discovered)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 6e2353b734..a56c2c2e99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -9,8 +9,10 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{IntoPyObject, pyfunction}; use serde_json::json; use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ compile_declarations, discover_declarations, @@ -19,21 +21,55 @@ use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; +use crate::bindings::_helpers::discovery::project_validation::validation_error; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; -use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + collect_declarations, orchestration_names, +}; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::records::fields::field; +/// Where declarations passed to the snapshot directly are reported as declared. +const DECLARATIONS_ARGUMENT: &str = "declarations argument"; + /// Compile catalog nodes and identities in the location, never in the server. #[pyfunction] -#[pyo3(signature = (project, metadata_json, project_dir, *, declarations=None))] -pub(crate) fn build_orchestration_snapshot( - project: PyRef<'_, NativeProjectView>, +#[pyo3(signature = (project, metadata_json, project_dir, **options))] +pub(crate) fn build_orchestration_snapshot<'py>( + project: PyRef<'py, NativeProjectView>, + metadata_json: &str, + project_dir: PathBuf, + options: Option>, +) -> PyResult { + let arguments = KeywordArguments::read( + project.py(), + "build_orchestration_snapshot", + options, + (&["discovered_inputs"], &["declarations"]), + )?; + let declarations = arguments.value("declarations")?; + orchestration_snapshot( + project, + metadata_json, + project_dir, + ( + arguments.value("discovered_inputs")?, + (!declarations.is_none()) + .then(|| declarations.extract()) + .transpose()?, + ), + ) +} + +/// Compile the snapshot from the project, its discovery, and the declarations passed directly, if any. +pub(crate) fn orchestration_snapshot<'py>( + project: PyRef<'py, NativeProjectView>, metadata_json: &str, project_dir: PathBuf, - declarations: Option>>, + (discovered_inputs, declarations): (Bound<'py, PyAny>, Option>>), ) -> PyResult { let py = project.py(); let mut input: SnapshotInput = serde_json::from_str(metadata_json) @@ -66,12 +102,25 @@ pub(crate) fn build_orchestration_snapshot( input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); let declarations = match declarations { - Some(declarations) => declarations, + Some(declarations) => declarations + .into_iter() + .map(|declaration| (declaration, DECLARATIONS_ARGUMENT.to_owned())) + .collect(), None => discover_declarations(py, &project_dir)?, }; let graph = project.lineage_graph(py)?; if !declarations.is_empty() { - let authored = collect_declarations(py, declarations)?; + let mut facts = + project_facts(py, &discovered_inputs, named_node_rows(&discovered_inputs)?)?; + facts.orchestration = orchestration_names(py, &declarations)?; + validate_orchestration_names(&facts).map_err(|failure| validation_error(py, failure))?; + let authored = collect_declarations( + py, + declarations + .into_iter() + .map(|(declaration, _)| declaration) + .collect(), + )?; input = compile_declarations( py, input, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index db99dfcfe9..c5354c4914 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -11,3 +11,11 @@ pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; /// The sensor context is supplied by the host, not a provider. pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; + +pub(crate) const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.models.authoring"; + +/// Kinds named in the global unique-name check. +pub(crate) const JOB_LABEL: &str = "job"; +pub(crate) const GROUP_LABEL: &str = "job group"; +pub(crate) const SENSOR_LABEL: &str = "sensor"; +pub(crate) const CALENDAR_LABEL: &str = "calendar schedule"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 4bfe0de411..87651dcd21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -8,9 +8,12 @@ use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use serde_json::Value; +use sqlbuild_discovery::project_validation::models::OrchestrationNameFact; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::native_host::constants::RUN_EVENT_CONDITION; +use crate::bindings::_helpers::native_host::constants::{ + AUTHORING_MODULE, CALENDAR_LABEL, GROUP_LABEL, JOB_LABEL, RUN_EVENT_CONDITION, SENSOR_LABEL, +}; use crate::bindings::_helpers::native_host::host::entry; #[derive(FromPyObject)] @@ -231,3 +234,74 @@ pub(crate) fn collect_declarations<'py>( } Ok(collected) } + +/// Every named declaration reachable from `roots`, labelled with the module of the root that reaches it first. +pub(crate) fn orchestration_names( + py: Python<'_>, + roots: &[(Bound<'_, PyAny>, String)], +) -> PyResult> { + let authoring = py.import(AUTHORING_MODULE)?; + let classes = [ + (authoring.getattr("Job")?, JOB_LABEL), + (authoring.getattr("JobGroup")?, GROUP_LABEL), + (authoring.getattr("Sensor")?, SENSOR_LABEL), + (authoring.getattr("CalendarSchedule")?, CALENDAR_LABEL), + ]; + let mut seen: HashSet = HashSet::new(); + let mut names: Vec = Vec::new(); + for (root, path) in roots { + let mut pending: Vec> = vec![root.clone()]; + while let Some(declaration) = pending.pop() { + if !seen.insert(declaration.as_ptr() as usize) { + continue; + } + let Some(kind) = classes + .iter() + .find(|(class, _)| declaration.is_instance(class).unwrap_or(false)) + .map(|(_, kind)| *kind) + else { + continue; + }; + pending.extend(nested_declarations(&declaration, kind)?); + let name: Option = if kind == CALENDAR_LABEL { + let function = declaration.getattr("function")?; + (!function.is_none()) + .then(|| function.getattr("__name__")?.extract()) + .transpose()? + } else { + Some(declaration.getattr("name")?.extract()?) + }; + names.extend(name.map(|name| OrchestrationNameFact { + name, + kind, + path: path.clone(), + })); + } + } + Ok(names) +} + +/// The declarations one declaration refers to: a group's jobs, a calendar's job, a sensor's job and run-event job. +fn nested_declarations<'py>( + declaration: &Bound<'py, PyAny>, + kind: &str, +) -> PyResult>> { + Ok(match kind { + GROUP_LABEL => declaration + .getattr("jobs")? + .try_iter()? + .collect::>>()?, + CALENDAR_LABEL => vec![declaration.getattr("job")?], + SENSOR_LABEL => { + let condition = declaration.getattr("when")?; + let mut nested = vec![declaration.getattr("run")?]; + if !condition.is_none() + && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION + { + nested.push(condition.getattr("job")?); + } + nested + } + _ => Vec::new(), + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 434c71c2fc..8a01736e93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -35,7 +35,7 @@ pub(crate) fn evaluate_sensor( let authoring = py.import("sqlbuild.orchestration_declarations.models.authoring")?; let sensor_class = authoring.getattr("Sensor")?; let mut matches: Vec> = Vec::new(); - for declaration in discover_declarations(py, project_dir)? { + for (declaration, _) in discover_declarations(py, project_dir)? { if declaration.is_instance(&sensor_class)? && declaration.getattr("name")?.extract::()? == *name { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 532b5e0723..583ae015e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -233,9 +233,6 @@ pub(crate) fn compile_calendars( "calendar {name} needs a description and a published job" ))); } - if definitions.triggers.contains_key(&name) { - return Err(PyValueError::new_err(format!("duplicate trigger {name}"))); - } let schedule = calendar_times(calendar, &session, &by_name, start)?; let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; trigger.starts_at.get_or_insert(start); @@ -256,14 +253,7 @@ pub(crate) fn compile_sensors( } else { String::new() }; - let mut names: BTreeSet = BTreeSet::new(); for sensor in authored { - if !names.insert(sensor.name.clone()) || definitions.triggers.contains_key(&sensor.name) { - return Err(PyValueError::new_err(format!( - "duplicate trigger {}", - sensor.name - ))); - } if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == &input.target) { continue; diff --git a/deploy/project/orchestration/orders.py b/deploy/project/orchestration/orders.py index 47aeb4c1f3..8023794654 100644 --- a/deploy/project/orchestration/orders.py +++ b/deploy/project/orchestration/orders.py @@ -3,7 +3,7 @@ from sqlbuild.orchestration import job, schedule orders = job( - "orders", + "refresh_orders", select="orders", description="Refresh orders every minute.", schedule=schedule(cron="* * * * *", timezone="UTC"), diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d9dfe2d850..0e5f0ccaf2 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1768,6 +1768,7 @@ def build_orchestration_snapshot( metadata_json: str, project_dir: Path, *, + discovered_inputs: DiscoveredProjectInputs, declarations: Sequence[object] | None = None, ) -> NativeOrchestrationSnapshot: ... def run_location_command( diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index 0f9fc8b052..fb42de5d94 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -5,7 +5,11 @@ from collections.abc import Sequence from pathlib import Path -from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot +from sqlbuild._native import ( + DiscoveredProjectInputs, + NativeOrchestrationSnapshot, + build_orchestration_snapshot, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule, @@ -20,9 +24,14 @@ def compile_snapshot( project: CompiledProjectView, metadata_json: str, project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, declarations: Sequence[Job | JobGroup | CalendarSchedule | Sensor] | None = None, ) -> NativeOrchestrationSnapshot: """Compile authoring records and retained project facts into a data-only snapshot.""" return build_orchestration_snapshot( - project, metadata_json, project_dir, declarations=declarations + project, + metadata_json, + project_dir, + discovered_inputs=discovered_inputs, + declarations=declarations, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py index baf6b94d53..cecb5fdc91 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py @@ -152,7 +152,7 @@ def test_given_test_names_when_selecting_then_exactly_those_tests_run( expected_output_fragments=( "Project resource name 'orders' is declared as both model in models/orders.sql " "and unit test in tests/unit/orders.sql", - "unit test, and scenario names must be globally unique", + "and calendar schedule names must be globally unique", ), ), ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py index 9d54256fc6..512db91881 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -75,7 +75,7 @@ def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_ "sources/raw_orders.yml": "sources:\n - name: raw_orders\n description: Synthetic order windows.\n schema: main\n table: raw_orders\n", "orchestration/orders.py": ( "from sqlbuild.orchestration import job, job_group\n" - "orders = job('orders', description='Windowed orders.', select='orders', " + "orders = job('windowed_orders', description='Windowed orders.', select='orders', " "settings={'start_cursor': '0', 'end_cursor': '1'})\n" "annual_orders = job_group('annual_orders', description='Windowed group.', jobs=[orders])\n" ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 9f3a98f241..f7d26b84fa 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -74,7 +74,7 @@ def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_val "orchestration/orders.py": ( "from datetime import datetime, timedelta\nfrom itertools import repeat\n" "from sqlbuild.orchestration import calendar_schedule, job\n" - "orders = job('orders', description='Order reporting.', select='orders')\n" + "orders = job('order_reporting', description='Order reporting.', select='orders')\n" f"@calendar_schedule(job=orders, description='Delivery times.', horizon={test_case.horizon!r}, timezone={test_case.timezone!r})\n" f"def deliveries(window):\n return {test_case.expression}\n" ), @@ -275,7 +275,7 @@ def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not | { "orchestration/orders.py": ( "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" - "orders = job('orders', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "orders = job('order_reporting', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" "def deliveries(window):\n" f" marker = Path({str(evaluated)!r})\n" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index a0a60c70c7..49b4a75a47 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -13,6 +13,7 @@ from _pytest.capture import CaptureResult from sqlbuild._native import ( + DiscoveredProjectInputs, NativeOrchestrationSnapshot, build_compiled_project, build_orchestration_snapshot, @@ -67,8 +68,9 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical ), } write_files(project_dir=project_dir, files=files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=project_dir), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) @@ -79,7 +81,7 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical "target": test_case.target, } first: NativeOrchestrationSnapshot = build_orchestration_snapshot( - project, json.dumps(metadata), project_dir + project, json.dumps(metadata), project_dir, discovered_inputs=discovered ) record: dict[str, Any] = json.loads(first.publish(api.url, api.bearer)) assert record["generation"] == 1 @@ -93,13 +95,17 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical ) } write_files(project_dir=project_dir, files=edited) + discovered = discover_project_inputs(project_dir=project_dir) changed: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=project_dir), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) next_snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( - changed, json.dumps(metadata | {"generation": 2}), project_dir + changed, + json.dumps(metadata | {"generation": 2}), + project_dir, + discovered_inputs=discovered, ) updated: dict[str, Any] = json.loads(next_snapshot.publish(api.url, api.bearer)) assert updated["generation"] == test_case.expected_generation diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py index 1d1193a3a9..4682b6bb22 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -63,7 +63,7 @@ def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_ "from sqlbuild.orchestration import job, job_group, schedule\n" "staging = job('staging', description='Stage orders.', select='orders broken_orders')\n" "marts = job('marts', description='Build totals.', select='totals')\n" - "inventory = job('inventory', description='Build inventory.', select='inventory')\n" + "inventory = job('stock', description='Build inventory.', select='inventory')\n" "annual_orders = job_group('annual_orders', description='Retry boundaries.', " "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 * * *', " "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 1ebf21518f..4d466244e5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -18,6 +18,7 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, Fingerprint, SourceFreshnessRecord, SourceFreshnessRenderers, @@ -670,8 +671,9 @@ def test_given_external_warehouse_retention_when_a_complete_scan_arrives_then_mi adapter.close(connection) declarations: Path = tmp_path / "declarations" write_files(project_dir=declarations, files=TYPE_PROOF_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=declarations) compiled: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=declarations), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) @@ -680,6 +682,7 @@ def test_given_external_warehouse_retention_when_a_complete_scan_arrives_then_mi compiled, json.dumps(METADATA | {"location_id": api.location_id, "target": "development"}), declarations, + discovered_inputs=discovered, ).publish(api.url, api.bearer) session_id: str = str(uuid4()) heartbeat: Request = Request( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index ae32ee21e4..0fb2cbabee 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -16,6 +16,7 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, NativeOrchestrationSnapshot, build_compiled_project, build_orchestration_snapshot, @@ -286,15 +287,16 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ ) -> None: project_dir: Path = tmp_path / "project" write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=project_dir), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) with snapshot_api(tmp_path / "control") as api: metadata: dict[str, Any] = METADATA | {"location_id": api.location_id} baseline: NativeOrchestrationSnapshot = build_orchestration_snapshot( - project, json.dumps(metadata), project_dir + project, json.dumps(metadata), project_dir, discovered_inputs=discovered ) publication: dict[str, Any] = json.loads(baseline.publish(api.url, api.bearer)) claim: ClaimedWorker = claimed_worker(tmp_path / "control", api, publication) @@ -305,13 +307,14 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ ) } write_files(project_dir=project_dir, files=changed) + discovered = discover_project_inputs(project_dir=project_dir) drifted: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=project_dir), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) wrong: NativeOrchestrationSnapshot = build_orchestration_snapshot( - drifted, json.dumps(metadata), project_dir + drifted, json.dumps(metadata), project_dir, discovered_inputs=discovered ) with pytest.raises(RuntimeError, match=test_case.expected_error): _ = wrong.start_worker(api.url, claim.bearer, identity) @@ -846,8 +849,8 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s ), "orchestration/orders.py": ( "from sqlbuild.orchestration import job, job_group, schedule\n" - "orders = job('orders', description='Long query.', select='orders')\n" - "totals = job('totals', description='Waiting query.', select='totals')\n" + "orders = job('long_orders', description='Long query.', select='orders')\n" + "totals = job('waiting_totals', description='Waiting query.', select='totals')\n" "annual_orders = job_group('annual_orders', description='Cancellable group.', " "jobs=[totals, orders], schedule=schedule(cron='0 0 * * *', timezone='UTC', " "starts_at='2000-01-01T00:00:00Z'))\n" @@ -869,7 +872,7 @@ def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_b job_key="annual_orders", ) assert parent["status"] == "running", worker.log.read_text() - cancel_running_statement(worker, job_key="orders", run_id=parent["run_id"]) + cancel_running_statement(worker, job_key="long_orders", run_id=parent["run_id"]) completed: dict[str, Any] = wait_for_run( worker.api, worker.process, worker.log, job_key="annual_orders" ) @@ -879,8 +882,8 @@ def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_b "SELECT job_key, status FROM sqb_runs ORDER BY job_key" ).fetchall() == [ ("annual_orders", "cancelled"), - ("orders", "cancelled"), - ("totals", "cancelled"), + ("long_orders", "cancelled"), + ("waiting_totals", "cancelled"), ] assert control.execute( "SELECT count(*) FROM sqb_runs WHERE parent_run_id=? AND ended_at>?", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 9824426203..972081f9a0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from dataclasses import replace from pathlib import Path from typing import Any @@ -10,6 +11,8 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, + DiscoveryConflictError, NativeOrchestrationSnapshot, PythonNodeDiscoveryError, build_compiled_project, @@ -47,6 +50,11 @@ ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +GLOBALLY_UNIQUE: str = ( + "; model, source, seed, function, loader, task, asset, check, provider, hook, unit test, " + "scenario, job, job group, sensor, and calendar schedule names must be globally unique" +) + @pytest.mark.parametrize( "test_case", @@ -172,14 +180,15 @@ def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_an metadata: str = json.dumps(METADATA) for directory in (tmp_path / "original", tmp_path / "relocated"): write_files(project_dir=directory, files=test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=directory) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=directory), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) assert not any(item.severity == "error" for item in project.diagnostics) snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( - project, metadata, directory + project, metadata, directory, discovered_inputs=discovered ) publication: dict[str, Any] = json.loads(snapshot.to_json()) publications.append(publication) @@ -251,14 +260,17 @@ def test_given_semantic_edits_when_publishing_then_local_identities_keep_planner for label, inputs in (("original", files), ("changed", changed)): directory: Path = tmp_path / label write_files(project_dir=directory, files=inputs) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=directory) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=directory), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) assert not any(item.severity == "error" for item in project.diagnostics) snapshot: dict[str, Any] = json.loads( - build_orchestration_snapshot(project, json.dumps(METADATA), directory).to_json() + build_orchestration_snapshot( + project, json.dumps(METADATA), directory, discovered_inputs=discovered + ).to_json() ) hashes: dict[str, str] = {} for key in ( @@ -310,8 +322,9 @@ def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snaps tmp_path: Path, ) -> None: write_files(project_dir=tmp_path, files=test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) @@ -324,6 +337,7 @@ def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snaps project, json.dumps(METADATA | {"nodes": test_case.nodes}), tmp_path, + discovered_inputs=discovered, ) @@ -365,13 +379,16 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s tmp_path: Path, ) -> None: write_files(project_dir=tmp_path, files=test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) publication: dict[str, Any] = json.loads( - build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() ) edges: list[list[str]] = publication["documents"]["lineage"]["edges"] assert sorted(edges) == test_case.expected_edges @@ -397,21 +414,28 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang tmp_path: Path, ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) installed: dict[str, Any] = json.loads( - build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() ) (tmp_path / "uv.lock").write_text(test_case.first_lockfile) locked: dict[str, Any] = json.loads( - build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() ) (tmp_path / "uv.lock").write_text(test_case.second_lockfile) changed: dict[str, Any] = json.loads( - build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() ) assert ( len( @@ -421,7 +445,9 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang ) assert installed["nodes"] == locked["nodes"] == changed["nodes"] assert changed == json.loads( - build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() ) @@ -459,8 +485,9 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select tmp_path: Path, test_case: SnapshotScheduleTestCase ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) @@ -482,6 +509,7 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select metadata_json=json.dumps(METADATA), project_dir=tmp_path, declarations=[nightly, excluded], + discovered_inputs=discovered, ).to_json() ) assert first["documents"]["jobs"] == { @@ -512,6 +540,7 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select metadata_json=json.dumps(METADATA), project_dir=tmp_path, declarations=[edited], + discovered_inputs=discovered, ).to_json() ) assert second["documents"]["triggers"].keys() == first["documents"]["triggers"].keys() @@ -547,7 +576,7 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select expected_message="starts_at", ), SnapshotDeclarationRejectionTestCase( - description="duplicate names", copies=2, expected_message="duplicate job orders" + description="duplicate names", copies=2, expected_message="duplicate job daily_build" ), SnapshotDeclarationRejectionTestCase( description="unknown setting", @@ -607,14 +636,15 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject tmp_path: Path, test_case: SnapshotDeclarationRejectionTestCase ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) declarations: list[Job] = [ job( - "orders", + "daily_build", description=test_case.job_description, select=test_case.select, settings=test_case.settings, @@ -629,6 +659,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject metadata_json=json.dumps(METADATA), project_dir=tmp_path, declarations=declarations, + discovered_inputs=discovered, ) @@ -921,8 +952,9 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects tmp_path: Path, test_case: SnapshotDiscoveryTestCase ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) @@ -932,6 +964,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects metadata_json=json.dumps(METADATA), project_dir=tmp_path, declarations=test_case.declarations, + discovered_inputs=discovered, ).to_json() ) assert tuple(publication["documents"].get("jobs", {})) == test_case.expected_jobs @@ -967,7 +1000,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects files={ "orchestration/orders.py": ( "from sqlbuild.orchestration import job, sensor\n" - "orders = job('orders', description='Build orders.', select='orders')\n" + "orders = job('refresh_orders', description='Build orders.', select='orders')\n" "poll = sensor('poll', description='Poll orders.', run=orders)\n" ), }, @@ -979,7 +1012,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects files={ "orchestration/orders.py": ( "from sqlbuild.orchestration import job, sensor, platform_event, PlatformEventKind\n" - "orders = job('orders', description='Build orders.', select='orders')\n" + "orders = job('refresh_orders', description='Build orders.', select='orders')\n" "@sensor('poll', description='Poll orders.', run=orders, " "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]))\n" "def poll(ctx):\n return []\n" @@ -1024,8 +1057,39 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects "jobs = [job('daily', description='Orders.', select='orders') for _ in range(2)]\n" ), }, - expected_error=ValueError, - expected_message="duplicate job daily", + expected_error=DiscoveryConflictError, + expected_message=re.escape( + "Project resource name 'daily' is declared as both job in orchestration/jobs.py " + "and job in orchestration/jobs.py" + GLOBALLY_UNIQUE + ), + ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop jobs cannot reuse a model name", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/name_collision.py" + ).read_text() + }, + expected_error=DiscoveryConflictError, + expected_message=re.escape( + "Project resource name 'orders' is declared as both model in models/orders.sql " + "and job in orchestration/jobs.py" + GLOBALLY_UNIQUE + ), + ), + SnapshotDiscoveryRejectionTestCase( + description="a sensor cannot reuse its job's name", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, sensor\n" + "daily = job('daily', description='Orders.', select='orders')\n" + "poll = sensor('daily', description='Poll orders.', run=daily)\n" + ), + }, + expected_error=DiscoveryConflictError, + expected_message=re.escape( + "Project resource name 'daily' is declared as both sensor in orchestration/jobs.py " + "and job in orchestration/jobs.py" + GLOBALLY_UNIQUE + ), ), SnapshotDiscoveryRejectionTestCase( description="broken imports fail on every retry", @@ -1289,13 +1353,17 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho tmp_path: Path, test_case: SnapshotDiscoveryRejectionTestCase ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), + discovered_inputs=discovered, adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) for _ in range(2): with pytest.raises(test_case.expected_error, match=test_case.expected_message): compile_snapshot( - project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + discovered_inputs=discovered, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 20a56de1a2..79ec931f6c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -1059,7 +1059,7 @@ def test_given_parameterized_test_when_discovering_inputs_then_cases_are_not_dup expected_error_fragment=( "Project resource name 'orders' is declared as both model in " "models/marts/orders.sql and unit test in tests/unit/orders.sql; .*unit test, " - "and scenario names must be globally unique" + "scenario, job, job group, sensor, and calendar schedule names must be globally unique" ), ), DiscoverProjectInputsErrorTestCase( diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 7348bbf664..3f20e3ab26 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -146,7 +146,7 @@ sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, and scenarios. +Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendar schedules. ## Lifecycle: run, build, check From 5f31bb481f3339a28e1acda338469ee9ccabd350 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:26:17 +0000 Subject: [PATCH 02/49] docs: sync the docs skill with the python nodes page --- .../sqlbuild/references/docs/concepts/python-nodes/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index d2343d6641..17cdf8438d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -162,7 +162,7 @@ sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, and scenarios. +Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendar schedules. ## Lifecycle: run, build, check From 7986d70c1de9e7ce60a18b89382f5b21c7da3b34 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:34:12 +0000 Subject: [PATCH 03/49] feat(orchestration): say how to rename a colliding declaration A name conflict already names both declarations and their files. It now ends with how to resolve it, and when a job, job group, sensor or calendar schedule is involved, says where that name comes from: a job, group or sensor takes it from its first argument, a calendar schedule from the function it decorates. --- .../_helpers/resource_names.rs | 90 ++++++++++++++----- .../src/project_validation/constants.rs | 4 + .../src/project_validation/tests/helpers.rs | 4 +- .../test_validate_orchestration_names.rs | 23 +++++ .../test_orchestration_snapshot.py | 12 +-- 5 files changed, 102 insertions(+), 31 deletions(-) diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs index c9b65c04f6..3f9c07638b 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -4,27 +4,20 @@ use std::collections::{HashMap, HashSet}; use crate::project_validation::_helpers::conflicts::{conflict, schema_seeds}; use crate::project_validation::_helpers::identities::block_or_file_name; +use crate::project_validation::constants::ORCHESTRATION_RENAME_ADVICE; use crate::project_validation::models::{NodeKind, ProjectFacts, ValidationFailure}; /// Reject the first resource name declared twice across models, sources, seeds and the rest. pub(crate) fn validate_unique_project_resource_names( facts: &ProjectFacts, ) -> Result<(), ValidationFailure> { - let mut seen: HashMap = HashMap::new(); - let mut declare = |name: &str, kind: &str, path: &str| -> Result<(), ValidationFailure> { - if let Some((existing_kind, existing_path)) = seen.get(name) { - return Err(conflict(format!( - "Project resource name '{name}' is declared as both {existing_kind} in \ - {existing_path} and {kind} in {path}; model, source, seed, function, loader, \ - task, asset, check, provider, hook, unit test, scenario, job, job group, sensor, \ - and calendar schedule names must be globally unique" - ))); - } - seen.insert(name.to_owned(), (kind.to_owned(), path.to_owned())); - Ok(()) - }; + let mut seen: HashMap = HashMap::new(); for model in &facts.models { - declare(&model.file.name, "model", &model.file.path)?; + declare( + &mut seen, + (&model.file.name, "model", &model.file.path), + false, + )?; } let mut managed_sources: HashSet<&str> = HashSet::new(); for file in &facts.sources { @@ -32,32 +25,48 @@ pub(crate) fn validate_unique_project_resource_names( if entry.managed { managed_sources.insert(&entry.name); } - declare(&entry.name, "source", &file.path)?; + declare(&mut seen, (&entry.name, "source", &file.path), false)?; } } for seed in schema_seeds(facts) { - declare(&seed.name, "seed", &seed.path)?; + declare(&mut seen, (&seed.name, "seed", &seed.path), false)?; } for function in facts.sql_functions.iter().chain(&facts.python_functions) { - declare(&function.name, "function", &function.path)?; + declare( + &mut seen, + (&function.name, "function", &function.path), + false, + )?; } for node in &facts.nodes { if node.kind == NodeKind::Loader && managed_sources.contains(node.name.as_str()) { continue; } - declare(&node.name, node.kind.label(), &node.path)?; + declare( + &mut seen, + (&node.name, node.kind.label(), &node.path), + false, + )?; } for provider in &facts.providers { - declare(&provider.name, "provider", &provider.path)?; + declare( + &mut seen, + (&provider.name, "provider", &provider.path), + false, + )?; } for hook in &facts.sql_hooks { - declare(&hook.name, "sql hook", &hook.path)?; + declare(&mut seen, (&hook.name, "sql hook", &hook.path), false)?; } for hook in &facts.hook_functions { - declare(&hook.name, "python hook", &hook.path)?; + declare(&mut seen, (&hook.name, "python hook", &hook.path), false)?; } for scenario in &facts.scenarios { - declare(&scenario.name, "scenario", &scenario.path)?; + declare( + &mut seen, + (&scenario.name, "scenario", &scenario.path), + false, + )?; } let mut test_names: HashSet<&str> = HashSet::new(); for test in &facts.tests { @@ -66,11 +75,44 @@ pub(crate) fn validate_unique_project_resource_names( if !test_names.insert(name) { continue; } - declare(name, "unit test", &test.file.path)?; + declare(&mut seen, (name, "unit test", &test.file.path), false)?; } } for declaration in &facts.orchestration { - declare(&declaration.name, declaration.kind, &declaration.path)?; + declare( + &mut seen, + (&declaration.name, declaration.kind, &declaration.path), + true, + )?; + } + Ok(()) +} + +/// A name's first declaration: its kind, its file and whether it is an orchestration declaration. +type Declared = (String, String, bool); + +/// Record one declaration, or name both declarations and say how to rename one of them. +fn declare( + seen: &mut HashMap, + (name, kind, path): (&str, &str, &str), + orchestration: bool, +) -> Result<(), ValidationFailure> { + if let Some((existing_kind, existing_path, existing_orchestration)) = seen.get(name) { + let advice: &str = if orchestration || *existing_orchestration { + ORCHESTRATION_RENAME_ADVICE + } else { + "" + }; + return Err(conflict(format!( + "Project resource name '{name}' is declared as both {existing_kind} in \ + {existing_path} and {kind} in {path}; model, source, seed, function, loader, task, \ + asset, check, provider, hook, unit test, scenario, job, job group, sensor, and \ + calendar schedule names must be globally unique; rename one of them{advice}" + ))); } + seen.insert( + name.to_owned(), + (kind.to_owned(), path.to_owned(), orchestration), + ); Ok(()) } diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs index e03d600d5a..e2dcb6a95a 100644 --- a/crates/sqlbuild-discovery/src/project_validation/constants.rs +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -12,3 +12,7 @@ pub(crate) const RESERVED_MODEL_NAMES: [&str; 1] = ["_chain_"]; pub(crate) const INITIAL_SEED_READ_BYTES: u64 = 64 * 1024; /// A header and two data rows: enough to know a seed holds more than one data row. pub(crate) const RECORDS_FOR_MANY_ROWS: usize = 3; + +/// How a colliding orchestration declaration is renamed, added to its name conflict. +pub(crate) const ORCHESTRATION_RENAME_ADVICE: &str = " (a job, job group or sensor takes its name \ + from its first argument, and a calendar schedule from the function it decorates)"; diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs index 601bc978ea..64c9bc7474 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs @@ -101,13 +101,13 @@ pub(super) fn declared(name: &str, kind: &'static str) -> OrchestrationNameFact } /// A global unique-name conflict that `message` begins. -pub(super) fn collision(message: &str) -> Result<(), String> { +pub(super) fn collision(message: &str, advice: &str) -> Result<(), String> { failure( FailureKind::Conflict, &format!( "{message}; model, source, seed, function, loader, task, asset, check, provider, \ hook, unit test, scenario, job, job group, sensor, and calendar schedule names must \ - be globally unique" + be globally unique; rename one of them{advice}" ), ) } diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs index beb6d48ffa..eb5505f446 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs @@ -5,6 +5,9 @@ use crate::project_validation::models::{ use crate::project_validation::tests::helpers::{collision, declared, model, named}; use crate::project_validation::tests::test_types::ValidationTestCase; +const RENAME_DECLARATION: &str = " (a job, job group or sensor takes its name from its first \ + argument, and a calendar schedule from the function it decorates)"; + #[test] fn given_orchestration_names_when_validating_then_they_share_the_project_namespace() { let test_cases = [ @@ -32,6 +35,7 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa expected_failure: collision( "Project resource name 'orders' is declared as both model in models/orders.sql \ and job in orchestration/daily.py", + RENAME_DECLARATION, ), }, ValidationTestCase { @@ -47,6 +51,7 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa expected_failure: collision( "Project resource name 'orders_arrived' is declared as both unit test in \ tests/orders_arrived.sql and sensor in orchestration/daily.py", + RENAME_DECLARATION, ), }, ValidationTestCase { @@ -59,6 +64,23 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa expected_failure: collision( "Project resource name 'fulfillment' is declared as both provider in \ providers/fulfillment.py and calendar schedule in orchestration/daily.py", + RENAME_DECLARATION, + ), + }, + ValidationTestCase { + description: "a unit test named like a model gets no orchestration advice", + facts: ProjectFacts { + models: vec![model("orders")], + tests: vec![TestFileFact { + file: named("orders", "tests/orders.sql"), + blocks: vec![TestBlockFact::default()], + }], + ..ProjectFacts::default() + }, + expected_failure: collision( + "Project resource name 'orders' is declared as both model in models/orders.sql \ + and unit test in tests/orders.sql", + "", ), }, ValidationTestCase { @@ -70,6 +92,7 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa expected_failure: collision( "Project resource name 'daily' is declared as both job in orchestration/daily.py \ and job group in orchestration/daily.py", + RENAME_DECLARATION, ), }, ]; diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 972081f9a0..9b1ba8d315 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -50,9 +50,11 @@ ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files -GLOBALLY_UNIQUE: str = ( +ORCHESTRATION_NAME_CONFLICT: str = ( "; model, source, seed, function, loader, task, asset, check, provider, hook, unit test, " - "scenario, job, job group, sensor, and calendar schedule names must be globally unique" + "scenario, job, job group, sensor, and calendar schedule names must be globally unique; " + "rename one of them (a job, job group or sensor takes its name from its first argument, and " + "a calendar schedule from the function it decorates)" ) @@ -1060,7 +1062,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=DiscoveryConflictError, expected_message=re.escape( "Project resource name 'daily' is declared as both job in orchestration/jobs.py " - "and job in orchestration/jobs.py" + GLOBALLY_UNIQUE + "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT ), ), SnapshotDiscoveryRejectionTestCase( @@ -1073,7 +1075,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=DiscoveryConflictError, expected_message=re.escape( "Project resource name 'orders' is declared as both model in models/orders.sql " - "and job in orchestration/jobs.py" + GLOBALLY_UNIQUE + "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT ), ), SnapshotDiscoveryRejectionTestCase( @@ -1088,7 +1090,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=DiscoveryConflictError, expected_message=re.escape( "Project resource name 'daily' is declared as both sensor in orchestration/jobs.py " - "and job in orchestration/jobs.py" + GLOBALLY_UNIQUE + "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT ), ), SnapshotDiscoveryRejectionTestCase( From 67e90e0c9c7ede0e32f85d80cc8def0335a96ddf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:42:47 +0000 Subject: [PATCH 04/49] refactor(orchestration): collect declared names before scanning for conflicts The name check now gathers every declaration as a named record and then scans them once, instead of threading a mutable map through a helper. The first conflict found is the same. --- .../_helpers/resource_names.rs | 167 +++++++++++------- .../src/project_validation/models.rs | 10 ++ 2 files changed, 109 insertions(+), 68 deletions(-) diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs index 3f9c07638b..2d10e91f6a 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -5,19 +5,35 @@ use std::collections::{HashMap, HashSet}; use crate::project_validation::_helpers::conflicts::{conflict, schema_seeds}; use crate::project_validation::_helpers::identities::block_or_file_name; use crate::project_validation::constants::ORCHESTRATION_RENAME_ADVICE; -use crate::project_validation::models::{NodeKind, ProjectFacts, ValidationFailure}; +use crate::project_validation::models::{ + DeclaredName, NamedPath, NodeKind, ProjectFacts, ValidationFailure, +}; /// Reject the first resource name declared twice across models, sources, seeds and the rest. pub(crate) fn validate_unique_project_resource_names( facts: &ProjectFacts, ) -> Result<(), ValidationFailure> { - let mut seen: HashMap = HashMap::new(); + let seeds: Vec = schema_seeds(facts); + let mut seen: HashMap<&str, DeclaredName<'_>> = HashMap::new(); + for declared in declarations(facts, &seeds) { + if let Some(first) = seen.get(declared.name) { + return Err(name_conflict(first, &declared)); + } + seen.insert(declared.name, declared); + } + Ok(()) +} + +/// Every declared name in the order conflicts are reported. +fn declarations<'a>(facts: &'a ProjectFacts, seeds: &'a [NamedPath]) -> Vec> { + let mut declared: Vec> = Vec::new(); for model in &facts.models { - declare( - &mut seen, - (&model.file.name, "model", &model.file.path), - false, - )?; + declared.push(DeclaredName { + name: &model.file.name, + kind: "model", + path: &model.file.path, + orchestration: false, + }); } let mut managed_sources: HashSet<&str> = HashSet::new(); for file in &facts.sources { @@ -25,94 +41,109 @@ pub(crate) fn validate_unique_project_resource_names( if entry.managed { managed_sources.insert(&entry.name); } - declare(&mut seen, (&entry.name, "source", &file.path), false)?; + declared.push(DeclaredName { + name: &entry.name, + kind: "source", + path: &file.path, + orchestration: false, + }); } } - for seed in schema_seeds(facts) { - declare(&mut seen, (&seed.name, "seed", &seed.path), false)?; + for seed in seeds { + declared.push(DeclaredName { + name: &seed.name, + kind: "seed", + path: &seed.path, + orchestration: false, + }); } for function in facts.sql_functions.iter().chain(&facts.python_functions) { - declare( - &mut seen, - (&function.name, "function", &function.path), - false, - )?; + declared.push(DeclaredName { + name: &function.name, + kind: "function", + path: &function.path, + orchestration: false, + }); } for node in &facts.nodes { if node.kind == NodeKind::Loader && managed_sources.contains(node.name.as_str()) { continue; } - declare( - &mut seen, - (&node.name, node.kind.label(), &node.path), - false, - )?; + declared.push(DeclaredName { + name: &node.name, + kind: node.kind.label(), + path: &node.path, + orchestration: false, + }); } for provider in &facts.providers { - declare( - &mut seen, - (&provider.name, "provider", &provider.path), - false, - )?; + declared.push(DeclaredName { + name: &provider.name, + kind: "provider", + path: &provider.path, + orchestration: false, + }); } for hook in &facts.sql_hooks { - declare(&mut seen, (&hook.name, "sql hook", &hook.path), false)?; + declared.push(DeclaredName { + name: &hook.name, + kind: "sql hook", + path: &hook.path, + orchestration: false, + }); } for hook in &facts.hook_functions { - declare(&mut seen, (&hook.name, "python hook", &hook.path), false)?; + declared.push(DeclaredName { + name: &hook.name, + kind: "python hook", + path: &hook.path, + orchestration: false, + }); } for scenario in &facts.scenarios { - declare( - &mut seen, - (&scenario.name, "scenario", &scenario.path), - false, - )?; + declared.push(DeclaredName { + name: &scenario.name, + kind: "scenario", + path: &scenario.path, + orchestration: false, + }); } let mut test_names: HashSet<&str> = HashSet::new(); for test in &facts.tests { for block in &test.blocks { let name: &str = block_or_file_name(block.name.as_deref(), &test.file.name); - if !test_names.insert(name) { - continue; + if test_names.insert(name) { + declared.push(DeclaredName { + name, + kind: "unit test", + path: &test.file.path, + orchestration: false, + }); } - declare(&mut seen, (name, "unit test", &test.file.path), false)?; } } for declaration in &facts.orchestration { - declare( - &mut seen, - (&declaration.name, declaration.kind, &declaration.path), - true, - )?; + declared.push(DeclaredName { + name: &declaration.name, + kind: declaration.kind, + path: &declaration.path, + orchestration: true, + }); } - Ok(()) + declared } -/// A name's first declaration: its kind, its file and whether it is an orchestration declaration. -type Declared = (String, String, bool); - -/// Record one declaration, or name both declarations and say how to rename one of them. -fn declare( - seen: &mut HashMap, - (name, kind, path): (&str, &str, &str), - orchestration: bool, -) -> Result<(), ValidationFailure> { - if let Some((existing_kind, existing_path, existing_orchestration)) = seen.get(name) { - let advice: &str = if orchestration || *existing_orchestration { - ORCHESTRATION_RENAME_ADVICE - } else { - "" - }; - return Err(conflict(format!( - "Project resource name '{name}' is declared as both {existing_kind} in \ - {existing_path} and {kind} in {path}; model, source, seed, function, loader, task, \ - asset, check, provider, hook, unit test, scenario, job, job group, sensor, and \ - calendar schedule names must be globally unique; rename one of them{advice}" - ))); - } - seen.insert( - name.to_owned(), - (kind.to_owned(), path.to_owned(), orchestration), - ); - Ok(()) +/// Name both declarations and say how to rename one of them. +fn name_conflict(first: &DeclaredName<'_>, second: &DeclaredName<'_>) -> ValidationFailure { + let advice: &str = if first.orchestration || second.orchestration { + ORCHESTRATION_RENAME_ADVICE + } else { + "" + }; + conflict(format!( + "Project resource name '{}' is declared as both {} in {} and {} in {}; model, source, seed, function, loader, task, asset, check, provider, \ + hook, unit test, scenario, job, job group, sensor, and calendar schedule names must be \ + globally unique; rename one of them{advice}", + first.name, first.kind, first.path, second.kind, second.path + )) } diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index a583cc66ce..2b185d25ab 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -182,6 +182,16 @@ pub struct ProjectFacts { pub orchestration: Vec, } +/// One name in the project's resource namespace, as a name conflict reports it. +#[derive(Debug)] +pub(crate) struct DeclaredName<'a> { + pub(crate) name: &'a str, + pub(crate) kind: &'a str, + pub(crate) path: &'a str, + /// A job, job group, sensor or calendar schedule, whose conflicts say how it is named. + pub(crate) orchestration: bool, +} + /// A named orchestration declaration: its name, its kind as conflicts word it, and its module. #[derive(Clone, Debug, PartialEq, Eq)] pub struct OrchestrationNameFact { From 13acf62143a8bb6e4808ff3fcbb33deaa5d48229 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:08:45 +0000 Subject: [PATCH 05/49] feat(server): plan a build command on the location A person with view on every node of a target can ask the location to run sqb plan --json on a build command. The server freezes the command as it would an ad-hoc run and queues it as location work, so a plan never becomes a run, fires a sensor or takes a build slot. The location runs the plan as a child process on the frozen node keys and posts the document back; the server redacts warehouse text in it. Only the requester reads it, and Python identity diffs need view.sql. --- clients/typescript/src/schema.d.ts | 137 +++++++ .../sqlbuild-cli/src/argv/_helpers/table.rs | 11 + .../argv/main/accepted_settings_arguments.rs | 26 ++ .../argv/main/command_settings_arguments.rs | 10 +- crates/sqlbuild-cli/src/argv/main/mod.rs | 1 + .../src/native_commands/_helpers/execution.rs | 2 +- .../src/native_commands/constants.rs | 3 + .../native_commands/main/_location_plan.rs | 89 +++++ .../native_commands/main/_location_work.rs | 9 + .../src/native_commands/main/mod.rs | 1 + .../authentication/password_sessions.rs | 8 + .../src/api/_helpers/resources/mod.rs | 1 + .../api/_helpers/resources/person_plans.rs | 63 +++ .../sqlbuild-orchestration/src/api/models.rs | 4 +- .../src/platform/_helpers/liveness.rs | 5 +- .../src/platform/_helpers/problems.rs | 3 +- .../src/platform/constants.rs | 15 + .../src/platform/main/claim_location_work.rs | 16 +- .../platform/main/complete_location_work.rs | 64 ++- .../platform/main/enqueue_location_work.rs | 56 +++ .../src/platform/main/mod.rs | 2 + .../src/platform/main/read_person_plan.rs | 88 ++++ .../platform/main/request_location_work.rs | 62 +-- .../src/platform/models.rs | 28 +- .../src/runs/_helpers/requests/mod.rs | 1 + .../src/runs/_helpers/requests/plan.rs | 112 ++++++ .../src/runs/constants.rs | 3 + .../src/runs/main/request_person_run.rs | 14 +- .../sqlbuild-orchestration/src/runs/models.rs | 5 + .../tests/api/plans/helpers.rs | 112 ++++++ .../tests/api/test_contracts.rs | 4 + .../tests/api/test_person_plans.rs | 377 ++++++++++++++++++ .../tests/api/test_types.rs | 7 + 33 files changed, 1259 insertions(+), 80 deletions(-) create mode 100644 crates/sqlbuild-cli/src/argv/main/accepted_settings_arguments.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/enqueue_location_work.rs create mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_person_plan.rs create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/plans/helpers.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_person_plans.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 91a87c7d96..00540ebab7 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -546,6 +546,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/plans": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestPlan"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPlan"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -1646,6 +1678,12 @@ export interface components { evaluation: Record; /** @enum {string} */ kind: "evaluate_sensor"; + } | { + command: string[]; + /** @enum {string} */ + kind: "plan"; + selection: string[]; + settings: Record; }; LocationWorkClaim: { /** Format: uuid */ @@ -1666,6 +1704,10 @@ export interface components { /** @enum {string} */ kind: "sensor_evaluation"; outcome: components["schemas"]["CustomSensorOutcome"]; + } | { + document: Record; + /** @enum {string} */ + kind: "plan"; } | { /** @enum {string} */ kind: "superseded"; @@ -1972,6 +2014,12 @@ export interface components { /** @enum {string} */ status: "unavailable"; }; + /** @description A person's build command to plan on the location; the server freezes it like an ad-hoc run. */ + PlanRequest: { + command: string[]; + /** Format: uuid */ + request_id: string; + }; /** @enum {string} */ PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; /** @description Problems derived from live state when read; nothing here is stored. */ @@ -4090,6 +4138,95 @@ export interface operations { }; }; }; + requestPlan: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PlanRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPlan: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkRecord"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { diff --git a/crates/sqlbuild-cli/src/argv/_helpers/table.rs b/crates/sqlbuild-cli/src/argv/_helpers/table.rs index 12bfc5b45b..c094f1210f 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/table.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/table.rs @@ -23,6 +23,17 @@ pub(crate) fn root_parser() -> &'static ParserSpec { }) } +/// One `sqb` subcommand's parser, such as `build` or `plan`. +pub(crate) fn command_parser(command: &str) -> Result<&'static ParserSpec, String> { + root_parser() + .actions + .iter() + .flat_map(|action| &action.parsers) + .find(|(name, _)| name == command) + .map(|(_, parser)| parser) + .ok_or_else(|| format!("{command} command is unavailable")) +} + /// The `sqb dbt ` execution parser with the command named in its texts. pub(crate) fn dbt_execution_parser(command: &str) -> ParserSpec { let mut parser: ParserSpec = DBT_EXECUTION diff --git a/crates/sqlbuild-cli/src/argv/main/accepted_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/accepted_settings_arguments.rs new file mode 100644 index 0000000000..3128f38f7b --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/accepted_settings_arguments.rs @@ -0,0 +1,26 @@ +//! Turn frozen settings into one command's flags, leaving out settings that command does not define. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value as Json; + +use crate::argv::_helpers::table::command_parser; +use crate::argv::main::command_settings_arguments::command_settings_arguments; + +/// The flags `command` takes from `settings`; a setting it lacks is dropped rather than refused. +pub fn accepted_settings_arguments( + command: &str, + settings: &BTreeMap, +) -> Result, String> { + let defined: BTreeSet<&str> = command_parser(command)? + .actions + .iter() + .filter_map(|action| action.dest.as_deref()) + .collect(); + let accepted: BTreeMap = settings + .iter() + .filter(|(name, _)| defined.contains(name.as_str())) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(); + command_settings_arguments(command, &accepted) +} diff --git a/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs index 5087b6ff53..02c148f1d5 100644 --- a/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs @@ -5,20 +5,14 @@ use std::collections::BTreeMap; use serde_json::Value as Json; use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; -use crate::argv::_helpers::table::root_parser; +use crate::argv::_helpers::table::command_parser; /// Produce argument tokens for `command`, never shell text; a setting the command lacks is refused. pub fn command_settings_arguments( command: &str, settings: &BTreeMap, ) -> Result, String> { - let parser = root_parser() - .actions - .iter() - .flat_map(|action| &action.parsers) - .find(|(name, _)| name == command) - .map(|(_, parser)| parser) - .ok_or_else(|| format!("{command} command is unavailable"))?; + let parser = command_parser(command)?; let mut arguments: Vec = Vec::new(); for (name, value) in settings { if let Some(argument) = setting_argument(parser, name, value)? { diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 10bbe8475b..10cdc0c9bf 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,3 +1,4 @@ +pub mod accepted_settings_arguments; pub mod cli_namespace_defaults; pub mod command_settings_arguments; pub mod job_settings_arguments; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 0b9f3ae3e4..3aa53ed604 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -700,7 +700,7 @@ pub(crate) async fn check_location_connection( ))) } -fn location_child(options: &LocationServe, script: &Path) -> Command { +pub(crate) fn location_child(options: &LocationServe, script: &Path) -> Command { let mut command = Command::new(script); command.arg("--project-dir").arg(&options.project_dir); if terminal_columns().is_err() { diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index a3c0d46a69..bd2eb9b498 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -103,3 +103,6 @@ pub(crate) const METADATA_REPLY_BYTES: u64 = 3 * sqlbuild_orchestration::api::constants::BODY_BYTES as u64; /// Bound piped terminal input independently of the credential owner's password policy. pub(crate) const PASSWORD_INPUT_BYTES: u64 = 4096; + +/// The command a location runs to answer a plan request. +pub(crate) const PLAN_COMMAND: &str = "plan"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs new file mode 100644 index 0000000000..aee88ba2d4 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -0,0 +1,89 @@ +//! A requested plan runs as an ordinary `sqb plan --json` child on the server-frozen node keys. + +use std::collections::{BTreeMap, BTreeSet}; +use std::future::Future; +use std::path::Path; +use std::process::Stdio; + +use serde_json::{Value, json}; +use sqlbuild_orchestration::platform::constants::PLAN_DOCUMENT_BYTES; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::AsyncReadExt; +use tokio::process::Child; + +use crate::argv::main::accepted_settings_arguments::accepted_settings_arguments; +use crate::native_commands::_helpers::execution::location_child; +use crate::native_commands::constants::PLAN_COMMAND; +use crate::native_commands::models::LocationServe; + +/// The plan document, an `{"error": ...}` report when planning failed, or `None` when stopped first. +pub(super) async fn plan_location( + (options, script): (&LocationServe, &Path), + (selection, settings): (&BTreeSet, &BTreeMap), + stopped: impl Future>, +) -> Result, ControlError> { + let mut child = match plan_child((options, script), (selection, settings)) { + Ok(child) => child?, + Err(message) => return Ok(Some(failure(&message))), + }; + let stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("plan output is unavailable".into()))?; + let mut output: Vec = Vec::new(); + let limit = u64::try_from(PLAN_DOCUMENT_BYTES).unwrap_or(u64::MAX) + 1; + tokio::pin!(stopped); + let status = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = async { + let mut reader = stdout.take(limit); + reader.read_to_end(&mut output).await?; + drop(reader); + child.wait().await + } => result?, + }; + if output.len() > PLAN_DOCUMENT_BYTES { + return Ok(Some(failure("the plan is too large to show"))); + } + Ok(Some(match serde_json::from_slice::(&output) { + Ok(document) if document.is_object() && matches!(status.code(), Some(0 | 1)) => document, + _ => failure("sqb plan did not produce a plan; see the location's log"), + })) +} + +/// The started child, or why the frozen request cannot become plan arguments. +fn plan_child( + (options, script): (&LocationServe, &Path), + (selection, settings): (&BTreeSet, &BTreeMap), +) -> Result, String> { + let flags = accepted_settings_arguments(PLAN_COMMAND, settings)?; + let mut command = location_child(options, script); + command + .args([ + PLAN_COMMAND, + "--target", + &options.target, + "--json", + "--no-color", + ]) + .args(flags); + for key in selection { + let (_, name) = key + .split_once(':') + .ok_or_else(|| "the plan request names an invalid node".to_owned())?; + command.args(["--select", name]); + } + Ok(command + .env_remove("SQLBUILD_LOCATION_TOKEN") + .env_remove("SQLBUILD_RUN_TOKEN") + .stdout(Stdio::piped()) + .spawn()) +} + +fn failure(message: &str) -> Value { + json!({"error": {"code": "plan_failed", "message": message, "help": null}}) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index abf575eb77..d67613c7a2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -16,6 +16,7 @@ use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::check_location_connection; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::main::_location_sensor::evaluate_location_sensor; use crate::native_commands::models::{LocationServe, LocationWatch}; @@ -51,6 +52,7 @@ pub(super) async fn run_location_work( let message = match &claim.request.work { LocationWork::CheckConnection {} => "Checking requested location connection...\n", LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", + LocationWork::Plan { .. } => "Planning requested command...\n", }; drop(write_output(&[(Stream::Stderr, message)])); let mut shutting_down = false; @@ -74,6 +76,13 @@ pub(super) async fn run_location_work( ) }) } + LocationWork::Plan { + selection, + settings, + .. + } => plan_location((options, script), (&selection, &settings), expires) + .await? + .map(|document| (LocationWorkOutcome::Plan { document }, Vec::new())), }; let Some((outcome, local_files)) = checked else { drop(write_output(&[( diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index c5fb741b92..a92114c039 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,6 +1,7 @@ pub(super) mod _cost_report_options; pub(super) mod _location_compiler; pub(super) mod _location_metadata; +pub(super) mod _location_plan; pub(super) mod _location_sensor; pub(super) mod _location_work; pub(super) mod _run_admin; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 929078663a..74dea56542 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -110,6 +110,14 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}", get(crate::api::_helpers::resources::person_location_work::result), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/plans", + post(crate::api::_helpers::resources::person_plans::request), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}", + get(crate::api::_helpers::resources::person_plans::read), + ) .route( "/api/control/locations/{location_id}/targets/{target}/runs", get(crate::api::_helpers::resources::person_operations::recent_runs), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index 0eddfb9c89..043cd0ad73 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod person_location_work; pub(crate) mod person_maintenance; pub(crate) mod person_node_sql; pub(crate) mod person_operations; +pub(crate) mod person_plans; pub(crate) mod person_reads; pub(crate) mod request_json; pub(crate) mod router; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs new file mode 100644 index 0000000000..219cd6a4a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs @@ -0,0 +1,63 @@ +//! A person plans a build command on the location; the plan never becomes a run. + +use std::sync::Arc; + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::{AuditContext, AuthenticatedPerson}; +use crate::platform::main::read_person_plan::read_person_plan; +use crate::platform::models::{LocationWorkRecord, PlanRequest}; +use crate::runs::main::request_person_run::request_person_run; +use crate::runs::models::PersonRunRequest; +use crate::runs::types::CommandPlanner; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/control/locations/{location_id}/targets/{target}/plans", + operation_id = "requestPlan", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("x-sqb-csrf" = String, Header)), + request_body = PlanRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn request( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target)): Path<(Uuid, String)>, + (Extension(AuthenticatedPerson(principal, _)), Extension(planner)): ( + Extension, + Extension>, + ), + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_person_run( + &store, + (&context, principal), + PersonRunRequest::Plan(planner.as_ref(), (location_id, &target, &request)), + ) + .await?, + })) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}", + operation_id = "readPlan", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("request_id" = Uuid, Path)), + responses((status = 200, body = Option), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn read( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target, request_id)): Path<(Uuid, String, Uuid)>, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_person_plan( + &store, + (&context, principal), + (location_id, &target, request_id), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index cb27eb28ea..f224f5e552 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -7,7 +7,7 @@ use crate::api::_helpers::authentication::{cli_login, oidc}; use crate::api::_helpers::resources::{ administrators, audit, changes, location_health, location_work, person_bootstrap, person_checks, person_location_work, person_maintenance, person_node_sql, person_operations, - person_reads, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, + person_plans, person_reads, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] @@ -55,6 +55,8 @@ use crate::api::_helpers::resources::{ person_maintenance::coverage, person_location_work::request, person_location_work::result, + person_plans::request, + person_plans::read, person_node_sql::read, password_sessions::login, password_sessions::current, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs index 51c5fd6959..d0298eee0d 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs @@ -4,6 +4,7 @@ use chrono::TimeDelta; use uuid::Uuid; use crate::platform::_helpers::observations::observe; +use crate::platform::constants::LOCATION_SILENCE_SECONDS; use crate::platform::models::{PlatformEvent, WatchdogLocation}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -19,7 +20,9 @@ pub(crate) async fn scan_location_page( after: Option<(Uuid, String)>, ) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Write).await?; - let mut parameters = vec![Parameter::Time(transaction.now - TimeDelta::seconds(90))]; + let mut parameters = vec![Parameter::Time( + transaction.now - TimeDelta::seconds(LOCATION_SILENCE_SECONDS), + )]; let cursor = if let Some((location_id, target)) = after { parameters.extend([Parameter::Id(location_id), Parameter::Text(target)]); "AND (i.location_id>$2 OR (i.location_id=$2 AND i.target>$3))" diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs index f8e57a9574..d177f3e4f0 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs @@ -3,6 +3,7 @@ use chrono::{DateTime, TimeDelta, Utc}; use uuid::Uuid; +use crate::platform::constants::LOCATION_SILENCE_SECONDS; use crate::platform::models::{LateTarget, OverdueTick, ProblemScan}; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; @@ -25,7 +26,7 @@ pub(crate) async fn scan_problems(store: &ControlStore) -> Result$3 AND w.completed_at IS NULL \ AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ - AND (w.request_json->'work'->>'kind'='check_connection' OR EXISTS ( \ + AND (w.request_json->'work'->>'kind' IN ('check_connection','plan') OR EXISTS ( \ SELECT 1 FROM sqb_sensor_evaluation_events e JOIN sqb_triggers t ON t.trigger_id=e.trigger_id \ JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id JOIN sqb_scheduler_lease l ON l.name='scheduler' \ WHERE e.evaluation_id=w.request_id AND e.phase='requested' AND s.status='running' \ @@ -35,7 +36,7 @@ pub async fn claim_location_work( AND l.epoch=CAST(e.fact_json->'request'->>'epoch' AS BIGINT) \ AND NOT EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events f \ WHERE f.evaluation_id=e.evaluation_id AND f.phase<>'requested'))) \ - ORDER BY w.requested_at, w.request_id LIMIT 1", + ORDER BY CASE WHEN w.request_json->'work'->>'kind'='plan' THEN 1 ELSE 0 END, w.requested_at, w.request_id LIMIT 1", vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; let Some(row) = rows.first() else { commit(transaction).await?; @@ -45,6 +46,11 @@ pub async fn claim_location_work( .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; + let lease = if matches!(stored.record.request_json.work, LocationWork::Plan { .. }) { + PLAN_LEASE_SECONDS + } else { + 60 + }; let retained = stored.session_id == Some(poll.session_id) && stored.claimed_until.is_some_and(|until| until > now); let (claim_id, lease_expires_at) = if retained { @@ -55,7 +61,7 @@ pub async fn claim_location_work( } else { ( Uuid::now_v7(), - (now + Duration::seconds(60)).min(stored.record.expires_at), + (now + Duration::seconds(lease)).min(stored.record.expires_at), ) }; let (transaction, _) = query(transaction, diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs index 8af79d1995..cc19fc50ad 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -1,8 +1,10 @@ -//! A completion is immutable; only the current unexpired claim may publish its result. +//! A completion is immutable; only the current unexpired claim may publish its result, redacted. -use serde_json::json; +use serde_json::{Value, json}; +use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; use uuid::Uuid; +use crate::platform::constants::PLAN_DOCUMENT_BYTES; use crate::platform::main::read_location_work::read_location_work; use crate::platform::main::require_instance::require_instance; use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; @@ -19,6 +21,7 @@ pub async fn complete_location_work( (location_id, request_id): (Uuid, Uuid), result: &LocationWorkResult, ) -> Result { + let result = &redacted(result); let transaction = begin(store, TransactionMode::Write).await?; let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored @@ -51,9 +54,53 @@ pub async fn complete_location_work( { return Err(ControlError::Fenced); } - let valid = match (&stored.record.request_json.work, &result.outcome) { + let valid = valid_outcome(&stored.record.request_json.work, &result.outcome); + if !valid { + return Err(ControlError::Invalid( + "location result does not match its work".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "UPDATE sqb_location_work SET result_json=$2, completed_at=$3 WHERE request_id=$1 AND completed_at IS NULL", + vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(true) +} + +/// Warehouse text in a plan's error and warnings is redacted before it is compared or stored. +fn redacted(result: &LocationWorkResult) -> LocationWorkResult { + let mut result = result.clone(); + if let LocationWorkOutcome::Plan { document } = &mut result.outcome { + let texts = ["/error/message", "/error/help"] + .into_iter() + .map(str::to_owned) + .chain( + (0..document["warnings"].as_array().map_or(0, Vec::len)) + .map(|index| format!("/warnings/{index}/message")), + ) + .collect::>(); + for pointer in texts { + if let Some(Value::String(text)) = document.pointer_mut(&pointer) { + *text = redact_error_text(text); + } + } + } + result +} + +/// Whether the outcome answers the work it claims to, within the bounds the server accepts. +fn valid_outcome(work: &LocationWork, outcome: &LocationWorkOutcome) -> bool { + match (work, outcome) { (_, LocationWorkOutcome::Superseded {}) | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) => true, + (LocationWork::Plan { .. }, LocationWorkOutcome::Plan { document }) => { + document + .as_object() + .is_some_and(|plan| plan.contains_key("models") || plan.contains_key("error")) + && serde_json::to_vec(document) + .is_ok_and(|bytes| bytes.len() <= PLAN_DOCUMENT_BYTES) + } ( LocationWork::EvaluateSensor { .. }, LocationWorkOutcome::SensorEvaluation { outcome }, @@ -66,16 +113,5 @@ pub async fn complete_location_work( CustomSensorOutcome::Failed | CustomSensorOutcome::TimedOut => true, }, _ => false, - }; - if !valid { - return Err(ControlError::Invalid( - "location result does not match its work".into(), - )); } - let now = transaction.now; - let (transaction, _) = query(transaction, - "UPDATE sqb_location_work SET result_json=$2, completed_at=$3 WHERE request_id=$1 AND completed_at IS NULL", - vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; - commit(transaction).await?; - Ok(true) } diff --git a/crates/sqlbuild-orchestration/src/platform/main/enqueue_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/enqueue_location_work.rs new file mode 100644 index 0000000000..39444d4b20 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/enqueue_location_work.rs @@ -0,0 +1,56 @@ +//! Pin new location work to the current publication and audit the request in the caller's transaction. + +use chrono::Duration; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::platform::models::LocationWorkRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// The caller has already authorised the request and ruled out a retry of the same request ID. +pub(crate) async fn enqueue_location_work( + transaction: ControlTransaction, + context: &AuditContext, + request: &LocationWorkRequest, + lifetime: Duration, +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ + JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ + WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; + let snapshot = rows + .first() + .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; + let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; + let generation = snapshot["generation"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), + Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + lifetime)]).await?; + append_audit( + transaction, + context, + ( + Uuid::new_v5(&request.request_id, b"location-work-requested"), + "location.work_requested", + ), + ( + json!({"location_id": request.location_id, "target": request.target}), + json!(null), + json!(request), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index d1c062ba19..830600e840 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1,11 +1,13 @@ pub mod claim_location_work; pub mod complete_location_work; pub mod connect_location_work; +pub(crate) mod enqueue_location_work; pub mod location_health; pub mod location_work; pub(crate) mod read_location_work; pub mod read_person_connection_test; pub mod read_person_location_health; +pub mod read_person_plan; pub mod read_person_problems; pub(crate) mod read_platform_events; pub mod report_location_health; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_person_plan.rs b/crates/sqlbuild-orchestration/src/platform/main/read_person_plan.rs new file mode 100644 index 0000000000..ad91412e38 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_person_plan.rs @@ -0,0 +1,88 @@ +//! A plan is read only by the person who asked for it, with view still held on every current node. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::authorization::main::decide_person_scopes::decide_person_scopes; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRecord}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; + +/// Python source and dependency diffs stay out of the plan unless the reader may also read SQL on every current node. +pub async fn read_person_plan( + store: &ControlStore, + (context, principal_id): (&AuditContext, Uuid), + (location_id, target, request_id): (Uuid, &str, Uuid), +) -> Result, ControlError> { + let (transaction, context) = authorize_listing( + begin(store, TransactionMode::Write).await?, + (context, principal_id), + (location_id, target), + DefaultRole::None, + ) + .await?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; + let Some(mut record) = stored.map(|stored| stored.record).filter(|record| { + record.requested_by == context.actor + && record.request_json.location_id == location_id + && record.request_json.target == target + && matches!(record.request_json.work, LocationWork::Plan { .. }) + }) else { + commit(transaction).await?; + return Ok(None); + }; + let (transaction, sql) = may_read_sql(transaction, principal_id, (location_id, target)).await?; + commit(transaction).await?; + if !sql { + without_identity_diffs(&mut record); + } + Ok(Some(record)) +} + +/// Whether the person holds `view.sql` on every current node, decided without recording a denial. +async fn may_read_sql( + transaction: ControlTransaction, + principal_id: Uuid, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, current) = query(transaction, + "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let scope = PermissionScope { + action: Action::ViewSql, + location_id: Some(location_id), + target: Some(target.into()), + nodes: current + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect::>(), + }; + let (transaction, refusals) = + decide_person_scopes(transaction, principal_id, (target, &[scope])).await?; + Ok((transaction, refusals.as_deref() == Some(&[None]))) +} + +fn without_identity_diffs(record: &mut LocationWorkRecord) { + let Some(LocationWorkOutcome::Plan { document }) = record + .result_json + .as_mut() + .map(|result| &mut result.outcome) + else { + return; + }; + let nodes = document + .get_mut("python_nodes") + .and_then(serde_json::Value::as_array_mut); + for node in nodes.into_iter().flatten() { + if let Some(node) = node.as_object_mut() { + node.remove("identity_diff"); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index 502f521cf8..0e6ca1c7ed 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -1,29 +1,35 @@ //! Named work requires a trusted owner or live person authority; retries preserve the original pin. use chrono::Duration; -use serde_json::json; use uuid::Uuid; -use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::platform::_helpers::connection_tests::authorize_connection_test; +use crate::platform::main::enqueue_location_work::enqueue_location_work; use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; pub async fn request_location_work( store: &ControlStore, (context, credentials): (&AuditContext, Option), request: &LocationWorkRequest, ) -> Result { - if matches!(request.work, LocationWork::EvaluateSensor { .. }) { - return Err(ControlError::Invalid( - "sensor work must originate from a fenced evaluation".into(), - )); + match request.work { + LocationWork::EvaluateSensor { .. } => { + return Err(ControlError::Invalid( + "sensor work must originate from a fenced evaluation".into(), + )); + } + LocationWork::Plan { .. } => { + return Err(ControlError::Invalid( + "plans must be requested by a person on the current publication".into(), + )); + } + LocationWork::CheckConnection {} => {} } if request.target.trim().is_empty() || request.reason.trim().is_empty() @@ -50,43 +56,7 @@ pub async fn request_location_work( commit(transaction).await?; return Ok(false); } - let (transaction, rows) = query(transaction, - "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ - JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ - WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", - vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; - let snapshot = rows - .first() - .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; - let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) - .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; - let generation = snapshot["generation"] - .as_i64() - .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; - let now = transaction.now; - let (transaction, _) = query(transaction, - "INSERT INTO sqb_location_work \ - (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", - vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), - Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), - Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + Duration::minutes(5))]).await?; - commit( - append_audit( - transaction, - &context, - ( - Uuid::new_v5(&request.request_id, b"location-work-requested"), - "location.work_requested", - ), - ( - json!({"location_id": request.location_id, "target": request.target}), - json!(null), - json!(request), - ), - ) - .await?, - ) - .await?; + commit(enqueue_location_work(transaction, &context, request, Duration::minutes(5)).await?) + .await?; Ok(true) } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 1bb5c963c8..cf3a2de965 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -73,6 +73,13 @@ pub enum LocationWork { #[schema(value_type = Object)] evaluation: Box, }, + /// `sqb plan --json` on exact node keys the server froze from a person's build command. + Plan { + command: Vec, + selection: std::collections::BTreeSet, + #[schema(value_type = Object)] + settings: std::collections::BTreeMap, + }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -85,6 +92,14 @@ pub struct LocationWorkRequest { pub work: LocationWork, } +/// A person's build command to plan on the location; the server freezes it like an ad-hoc run. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PlanRequest { + pub request_id: Uuid, + pub command: Vec, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct ConnectionTestRequest { @@ -112,8 +127,17 @@ pub struct LocationWorkClaim { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationWorkOutcome { - Connection { healthy: bool }, - SensorEvaluation { outcome: CustomSensorOutcome }, + Connection { + healthy: bool, + }, + SensorEvaluation { + outcome: CustomSensorOutcome, + }, + /// The plan document `sqb plan --json` printed, or its `{"error": ...}` report. + Plan { + #[schema(value_type = Object)] + document: serde_json::Value, + }, Superseded {}, } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs index 2231ab538f..85c7ebfc6d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -4,5 +4,6 @@ pub(in crate::runs) mod job; pub(in crate::runs) mod manual_job; pub(in crate::runs) mod manual_rerun; pub(in crate::runs) mod permissions; +pub(in crate::runs) mod plan; pub(in crate::runs) mod publication; pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs new file mode 100644 index 0000000000..d8cf137c68 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs @@ -0,0 +1,112 @@ +//! A plan is frozen like an ad-hoc build but queued as location work, so it never becomes a run. + +use chrono::Duration; +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::platform::constants::{LOCATION_SILENCE_SECONDS, PLAN_EXPIRY_SECONDS, PLAN_QUEUE_LIMIT}; +use crate::platform::main::enqueue_location_work::enqueue_location_work; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRequest, PlanRequest}; +use crate::runs::_helpers::requests::adhoc::frozen_adhoc_request; +use crate::runs::constants::PLAN_REASON; +use crate::runs::models::{AdhocCheckRequest, RunCommand}; +use crate::runs::types::CommandPlanner; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Needs view on every current node; a refused command, a silent location or a full queue queues nothing. +pub(in crate::runs) async fn request_plan_in( + transaction: ControlTransaction, + (context, principal_id, planner): (&AuditContext, Uuid, &dyn CommandPlanner), + (location_id, target, request): (Uuid, &str, &PlanRequest), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, context) = authorize_listing( + transaction, + (context, principal_id), + (location_id, target), + DefaultRole::None, + ) + .await?; + let (transaction, previous) = read_location_work(transaction, request.request_id).await?; + if let Some(previous) = previous { + let same = previous.record.requested_by == context.actor + && previous.record.request_json.location_id == location_id + && previous.record.request_json.target == target + && matches!(&previous.record.request_json.work, + LocationWork::Plan { command, .. } if *command == request.command); + if !same { + return Err(ControlError::Invalid( + "location work ID already has different content or actor".into(), + )); + } + return Ok((transaction, false)); + } + let check = AdhocCheckRequest { + location_id, + target: target.to_owned(), + command: request.command.clone(), + priority: 0, + }; + let (transaction, frozen) = frozen_adhoc_request( + transaction, + (&context, planner), + (&check, request.request_id), + ) + .await?; + let (verb, frozen) = frozen.map_err(ControlError::Invalid)?; + if frozen.command != RunCommand::Build { + return Err(ControlError::Invalid(format!( + "sqb plan previews builds; sqb {verb} cannot be planned" + ))); + } + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT \ + (SELECT COUNT(*) FROM sqb_location_instances i WHERE i.location_id=$1 AND i.target=$2 AND i.last_seen_at>=$3) AS live, \ + (SELECT COUNT(*) FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.completed_at IS NULL AND w.expires_at>$4 AND w.request_json->'work'->>'kind'='plan') AS queued, \ + (SELECT COUNT(*) FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.completed_at IS NULL AND w.expires_at>$4 AND w.request_json->'work'->>'kind'='plan' \ + AND w.requested_by=$5) AS mine", + vec![Parameter::Id(location_id), Parameter::Text(target.to_owned()), + Parameter::Time(now - Duration::seconds(LOCATION_SILENCE_SECONDS)), Parameter::Time(now), + Parameter::Text(context.actor.clone())]).await?; + let count = |name: &str| rows.first().and_then(|row| row[name].as_i64()).unwrap_or(0); + if count("live") == 0 { + return Err(ControlError::Invalid(format!( + "no sqb location serve has reported for {target} in the last {LOCATION_SILENCE_SECONDS} seconds; start it and plan again" + ))); + } + if count("mine") > 0 { + return Err(ControlError::Invalid( + "your previous plan for this target is still running".into(), + )); + } + if count("queued") >= PLAN_QUEUE_LIMIT { + return Err(ControlError::Invalid(format!( + "{PLAN_QUEUE_LIMIT} plans are already waiting for this target; try again shortly" + ))); + } + let work = LocationWorkRequest { + request_id: request.request_id, + location_id, + target: target.to_owned(), + reason: PLAN_REASON.into(), + work: LocationWork::Plan { + command: request.command.clone(), + selection: frozen.selection, + settings: frozen.settings, + }, + }; + let transaction = enqueue_location_work( + transaction, + &context, + &work, + Duration::seconds(PLAN_EXPIRY_SECONDS), + ) + .await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 9a7bea5f08..523399ac8e 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -20,3 +20,6 @@ pub(crate) const ADHOC_AUDIT_TAG: &[u8] = b"adhoc-run-request"; /// The build setting a full-refresh launch adds to the frozen request and each group child. pub(crate) const FULL_REFRESH_SETTING: &str = "full_refresh"; + +/// The reason recorded on a person's plan request; plans carry no reason of their own. +pub(crate) const PLAN_REASON: &str = "plan"; diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs index c88868b7e0..61e20ded53 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs @@ -6,12 +6,13 @@ use crate::credentials::models::AuditContext; use crate::runs::_helpers::requests::adhoc::request_adhoc_run_in; use crate::runs::_helpers::requests::cancel::cancel_person_run; use crate::runs::_helpers::requests::manual_job::request_job_run_in; +use crate::runs::_helpers::requests::plan::request_plan_in; use crate::runs::models::PersonRunRequest; use crate::store::errors::ControlError; use crate::store::main::write::write; use crate::store::models::ControlStore; -/// A person queues a published job or an ad-hoc command, or cancels a run; each is authorised on the nodes it covers. +/// A person queues a published job, an ad-hoc command or a plan, or cancels a run; each is authorised on the nodes it covers. pub async fn request_person_run( store: &ControlStore, (context, credentials): (&AuditContext, Uuid), @@ -27,6 +28,17 @@ pub async fn request_person_run( }) .await } + PersonRunRequest::Plan(planner, (location_id, target, request)) => { + write(store, |transaction| async move { + request_plan_in( + transaction, + (context, credentials, planner), + (location_id, target, request), + ) + .await + }) + .await + } PersonRunRequest::Adhoc(planner, request) => { write(store, |transaction| async move { request_adhoc_run_in(transaction, (context, credentials), (planner, request)).await diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 50649157a9..42e4c46e08 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -684,4 +684,9 @@ pub enum PersonRunRequest<'a> { Job(&'a JobRunRequest), Adhoc(&'a dyn CommandPlanner, &'a AdhocRunRequest), Cancel(&'a CancelRunRequest), + /// A build command planned on the location for `(location_id, target)`; it never becomes a run. + Plan( + &'a dyn CommandPlanner, + (Uuid, &'a str, &'a crate::platform::models::PlanRequest), + ), } diff --git a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs new file mode 100644 index 0000000000..adc0e0fa7c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs @@ -0,0 +1,112 @@ +//! People, requests and counts the plan contract builds on. + +use uuid::Uuid; + +use crate::authorization::main::assign_builtin_role::assign_builtin_role; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::main::sessions::set_password::set_password; +use crate::credentials::models::{ + AuditContext, BuiltinRole, PasswordAuthority, RoleAssignmentRequest, +}; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWorkPoll, PlanRequest, +}; +use crate::runs::main::request_person_run::request_person_run; +use crate::runs::models::PersonRunRequest; +use crate::runs::tests::test_types::ExactKeyPlanner; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) fn plan_request(tokens: &[&str]) -> PlanRequest { + PlanRequest { + request_id: Uuid::now_v7(), + command: tokens.iter().map(|token| (*token).to_owned()).collect(), + } +} + +/// Request a plan on the `development` target as `person`. +pub(crate) async fn ask( + store: &ControlStore, + (context, person): (&AuditContext, Uuid), + location: Uuid, + request: &PlanRequest, +) -> Result { + request_person_run( + store, + (context, person), + PersonRunRequest::Plan(&ExactKeyPlanner, (location, "development", request)), + ) + .await +} + +/// A principal with a password and, when given, a built-in role on the location's `development` target. +pub(crate) async fn person( + store: &ControlStore, + (context, admin): (&AuditContext, Uuid), + (name, location): (&str, Uuid), + roles: &[BuiltinRole], +) -> Uuid { + let person = create_principal(store, context, name) + .await + .expect("person"); + set_password( + store, + context, + PasswordAuthority::Operator(person), + "orders-password-2026", + ) + .await + .expect("password"); + for role in roles { + assign_builtin_role( + store, + (context, admin), + &RoleAssignmentRequest { + principal_id: person, + role: (*role).into(), + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }, + ) + .await + .expect("assign"); + } + person +} + +pub(crate) async fn register_instance(store: &ControlStore, location: Uuid) -> LocationWorkPoll { + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: "development".into(), + }; + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register instance"); + poll +} + +/// Run and run-node rows together; a plan must add none. +pub(crate) async fn run_rows(store: &ControlStore) -> i64 { + let transaction = begin(store, TransactionMode::Read).await.expect("begin"); + let (transaction, rows) = query( + transaction, + "SELECT (SELECT COUNT(*) FROM sqb_runs) + (SELECT COUNT(*) FROM sqb_run_nodes) AS n", + Vec::new(), + ) + .await + .expect("count"); + commit(transaction).await.expect("commit"); + rows[0]["n"].as_i64().expect("count") +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index a1b7ab1c0d..39db9b74da 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -22,6 +22,8 @@ mod output_helpers; mod person_sessions_helpers; #[path = "person_sessions/test_types.rs"] mod person_sessions_test_types; +#[path = "plans/helpers.rs"] +mod plans_helpers; #[path = "recall/helpers.rs"] mod recall_helpers; #[path = "statistics/eligibility/helpers.rs"] @@ -68,6 +70,8 @@ mod test_passwords; mod test_person_live; #[path = "test_person_location_work.rs"] mod test_person_location_work; +#[path = "test_person_plans.rs"] +mod test_person_plans; #[path = "test_person_sessions.rs"] mod test_person_sessions; #[path = "test_proxy.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs new file mode 100644 index 0000000000..cd952becb7 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -0,0 +1,377 @@ +//! A person's plan is frozen like an ad-hoc build, queued as location work and read only by them. + +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::plans_helpers::{ask, person, plan_request, register_instance, run_rows}; +use crate::api::tests::test_types::PlanTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::models::{AuditContext, BuiltinRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::claim_location_work::claim_location_work; +use crate::platform::main::complete_location_work::complete_location_work; +use crate::platform::main::read_person_plan::read_person_plan; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ + LocationWork, LocationWorkOutcome, LocationWorkRequest, LocationWorkResult, PlanRequest, +}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_work_for_their_requester() + { + let test_cases = [ + PlanTestCase { + description: "SQLite plans", + backend: Backend::Sqlite, + expected_selection: &["customers", "orders"], + expected_warning: "connect failed: db_password=[REDACTED]", + }, + PlanTestCase { + description: "PostgreSQL plans", + backend: Backend::Postgres, + expected_selection: &["customers", "orders"], + expected_warning: "connect failed: db_password=[REDACTED]", + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some(std::net::Ipv4Addr::LOCALHOST.into()), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let owner = (&context, admin); + let viewer = person( + store, + owner, + ("orders-viewer", location), + &[BuiltinRole::Viewer], + ) + .await; + let developer = person( + store, + owner, + ("orders-developer", location), + &[BuiltinRole::Developer], + ) + .await; + let analyst = person( + store, + owner, + ("orders-analyst", location), + &[BuiltinRole::Viewer], + ) + .await; + let auditor = person( + store, + owner, + ("orders-auditor", location), + &[BuiltinRole::Viewer], + ) + .await; + let outsider = person(store, owner, ("orders-outsider", location), &[]).await; + let first = plan_request(&[ + "build", + "customers", + "orders", + "--full-refresh", + "--warehouse", + "WH_PLAN", + ]); + + let silent = ask(store, (&context, viewer), location, &first).await; + assert!( + matches!(&silent, Err(ControlError::Invalid(message)) if message.contains("sqb location serve")), + "{description}: no live location, got {silent:?}", + description = test_case.description + ); + let poll = register_instance(store, location).await; + assert!( + matches!( + ask(store, (&context, outsider), location, &first).await, + Err(ControlError::Denied { .. }) + ), + "{description}: view is required", + description = test_case.description + ); + assert!( + matches!( + ask(store, (&context, viewer), location, &plan_request(&["seed", "customers"])).await, + Err(ControlError::Invalid(message)) if message.contains("cannot be planned") + ), + "{description}: only builds are planned", + description = test_case.description + ); + assert!( + ask(store, (&context, viewer), location, &first) + .await + .expect("viewer plans"), + "{description}: a viewer may plan", + description = test_case.description + ); + assert!( + !ask(store, (&context, viewer), location, &first) + .await + .expect("idempotent retry"), + "{description}: a retry queues nothing", + description = test_case.description + ); + let altered = PlanRequest { + command: vec!["build".into(), "inventory".into()], + ..first.clone() + }; + assert!( + ask(store, (&context, viewer), location, &altered) + .await + .is_err(), + "{description}: a request ID cannot carry another command", + description = test_case.description + ); + assert!( + ask(store, (&context, developer), location, &first) + .await + .is_err(), + "{description}: a request ID cannot carry another actor", + description = test_case.description + ); + assert!( + matches!( + ask(store, (&context, viewer), location, &plan_request(&["build", "inventory"])).await, + Err(ControlError::Invalid(message)) if message.contains("still running") + ), + "{description}: one unfinished plan per person", + description = test_case.description + ); + for queued in [developer, analyst, auditor] { + assert!( + ask( + store, + (&context, queued), + location, + &plan_request(&["build", "products"]) + ) + .await + .expect("queue has room"), + "{description}: queue has room", + description = test_case.description + ); + } + assert!( + matches!( + ask(store, (&context, admin), location, &plan_request(&["build", "products"])).await, + Err(ControlError::Invalid(message)) if message.contains("already waiting") + ), + "{description}: four plans per target", + description = test_case.description + ); + let direct = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + reason: "plan".into(), + work: LocationWork::Plan { + command: vec!["build".into(), "orders".into()], + selection: ["orders".to_owned()].into(), + settings: Default::default(), + }, + }; + assert!( + request_location_work(store, (&context, None), &direct) + .await + .is_err(), + "{description}: plans only come from a person's frozen command", + description = test_case.description + ); + let connection = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + request_location_work(store, (&context, None), &connection) + .await + .expect("connection test"); + let claimed = claim_location_work(store, location, &poll) + .await + .expect("claim") + .expect("work"); + assert_eq!( + claimed.request.request_id, + connection.request_id, + "{description}: other location work goes before plans", + description = test_case.description + ); + complete_location_work( + store, + (location, connection.request_id), + &LocationWorkResult { + claim_id: claimed.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: true }, + }, + ) + .await + .expect("complete connection"); + let claimed = claim_location_work(store, location, &poll) + .await + .expect("claim") + .expect("plan"); + assert_eq!( + claimed.request.request_id, + first.request_id, + "{description}", + description = test_case.description + ); + let LocationWork::Plan { + selection, + settings, + .. + } = &claimed.request.work + else { + panic!( + "{description}: plan work", + description = test_case.description + ) + }; + assert_eq!( + selection.iter().map(String::as_str).collect::>(), + test_case.expected_selection, + "{description}", + description = test_case.description + ); + assert_eq!( + settings["full_refresh"], + json!(true), + "{description}", + description = test_case.description + ); + assert_eq!( + settings["warehouse"], + json!("WH_PLAN"), + "{description}", + description = test_case.description + ); + let lease = (claimed.lease_expires_at - chrono::Utc::now()).num_seconds(); + assert!( + (240..=300).contains(&lease), + "{description}: plan lease {lease}", + description = test_case.description + ); + assert_eq!( + run_rows(store).await, + 0, + "{description}: a plan is never a run", + description = test_case.description + ); + + let complete = |document: serde_json::Value| LocationWorkResult { + claim_id: claimed.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Plan { document }, + }; + for invalid in [ + json!(["models"]), + json!({"selected_count": 1}), + json!({"models": "x".repeat(9 * 1024 * 1024)}), + ] { + assert!( + complete_location_work(store, (location, first.request_id), &complete(invalid)) + .await + .is_err(), + "{description}: shape and size are checked", + description = test_case.description + ); + } + let document = json!({ + "models": [{"name": "orders", "action": "create_table", "reason": "first_run"}], + "warnings": [{"severity": "warning", "message": "connect failed: db_password=orders-pass"}], + "python_nodes": [{"name": "orders_export", "identity_diff": {"source_diff": ["-a", "+b"]}}], + }); + assert!( + complete_location_work( + store, + (location, first.request_id), + &complete(document.clone()) + ) + .await + .expect("complete plan"), + "{description}", + description = test_case.description + ); + assert!( + !complete_location_work(store, (location, first.request_id), &complete(document)) + .await + .expect("same result retried after redaction"), + "{description}", + description = test_case.description + ); + + let read = |reader: Uuid| { + read_person_plan( + store, + (&context, reader), + (location, "development", first.request_id), + ) + }; + let record = read(viewer).await.expect("read").expect("own plan"); + let Some(LocationWorkOutcome::Plan { document }) = + record.result_json.map(|result| result.outcome) + else { + panic!( + "{description}: plan result", + description = test_case.description + ) + }; + assert_eq!( + document["warnings"][0]["message"], + json!(test_case.expected_warning), + "{description}", + description = test_case.description + ); + assert_eq!( + document["models"][0]["reason"], + json!("first_run"), + "{description}", + description = test_case.description + ); + assert!( + document["python_nodes"][0].get("identity_diff").is_none(), + "{description}: viewers lack view.sql", + description = test_case.description + ); + assert!( + read(developer).await.expect("read").is_none(), + "{description}: only the requester reads a plan", + description = test_case.description + ); + assert!( + matches!(read(outsider).await, Err(ControlError::Denied { .. })), + "{description}: view is required to read", + description = test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 0b878cb812..b3f09f5fd2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -213,3 +213,10 @@ pub(crate) struct OlderNodeFinished { error_code: Option, skipped_because_of: Option, } + +pub(crate) struct PlanTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_selection: &'static [&'static str], + pub(crate) expected_warning: &'static str, +} From 5f7315468f98b374f613f62287a7740e1e96a8c7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:43:45 +0000 Subject: [PATCH 06/49] feat(ui): show the location's real plan on the plan page A build command on the plan page is now planned by the location with sqb plan against the warehouse: each model's action and reason, backfills, migrations, sources, seeds and warnings, behind a planner spinner. Viewers can see plans; Run still needs run.adhoc and a plan of the exact command. An error stays under the command over the last good plan instead of replacing the page. --- ui/src/lib/api/client.ts | 3 + ui/src/lib/api/http-client.ts | 35 ++- ui/src/lib/api/http-runs.test.ts | 15 +- ui/src/lib/api/http-runs.ts | 27 +- ui/src/lib/api/mock/mock-client.ts | 6 + ui/src/lib/api/mock/operate-store.ts | 22 ++ ui/src/lib/api/types/operate.ts | 34 +++ ui/src/routes/plan/+page.svelte | 306 ++++++++++++++----- website/src/content/docs/docs/cli/server.mdx | 33 ++ 9 files changed, 402 insertions(+), 79 deletions(-) diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index ad23a18dbc..a7df56b9e7 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -1,5 +1,6 @@ import type { AdhocRunCheck, + PlanResult, AdhocRunRequest, Bootstrap, BulkRunResult, @@ -112,6 +113,8 @@ export interface ApiClient extends PlatformApi { /** The server's pre-launch check: resolution, write set and authorization. */ checkAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; launchAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; + /** Asks the location to run `sqb plan` on a build command against the warehouse, and waits for it. */ + planBuild(scope: Scope, command: string[]): Promise; listJobs(scope: Scope): Promise; getJob(scope: Scope, key: string): Promise; diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 35e0d1a326..f6483f5b1a 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -12,12 +12,12 @@ import { overviewOf } from './http-overview'; import { toPlatformEvents, toPlatformHealth } from './http-problems'; import { roleDetailOf, toRoleDefinition } from './http-roles'; import { jobDetailOf, runWindowOf, toJob, toTick, toTrigger, toTriggerStateEvent, triggerDetailOf } from './http-jobs'; -import { RUN_WINDOW, groupRunNodes, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; +import { RUN_WINDOW, groupRunNodes, planResultOf, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, PlatformEvent, Scope, WarehouseSync } from './types'; import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, LocationSummary, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; -import type { AdhocRunCheck, AdhocRunRequest, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; +import type { AdhocRunCheck, AdhocRunRequest, PlanResult, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; type Wire = components['schemas']; @@ -74,13 +74,17 @@ const SUPPORTED: ReadonlySet = new Set([ 'testConnection', 'listSnapshotNodes', 'checkAdhocRun', - 'launchAdhocRun' + 'launchAdhocRun', + 'planBuild' ]); /** How often a connection test asks for the location's answer, and the longest it waits. */ const CONNECTION_TEST_POLL_MS = 1000; const CONNECTION_TEST_WAIT_MS = 60_000; +/** How often the plan page asks for the location's plan; the server expires a plan after ten minutes. */ +const PLAN_POLL_MS = 750; + /** The server's largest audit page, and how many the audit log reads before it stops looking. */ const AUDIT_PAGE = 100; const AUDIT_PAGES = 10; @@ -376,6 +380,31 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!response.ok) throw new ApiError(errorText(error, response), response.status); return requireRun(scope, body.run_id); }, + /** Queues the plan, then waits for the location's document; a plan the location could not make is a result, not a throw. */ + async planBuild(scope: Scope, command: string[]): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const path = { location_id: locationId, target }; + const requestId = crypto.randomUUID(); + const asked = await http.POST('/api/control/locations/{location_id}/targets/{target}/plans', { + body: { request_id: requestId, command }, + params: { path, header: { 'x-sqb-csrf': csrfToken(document.cookie) } } + }); + if (asked.response.status === 401) onUnauthorized(); + if (asked.response.status === 400) return { plan: null, error: errorText(asked.error, asked.response) }; + if (!asked.response.ok) throw new ApiError(errorText(asked.error, asked.response), asked.response.status); + for (;;) { + await new Promise((resolve) => setTimeout(resolve, PLAN_POLL_MS)); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/plans/{request_id}', { + params: { path: { ...path, request_id: requestId } } + }); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + if (!data) throw new ApiError('The plan request disappeared', 404); + const outcome = data.result_json?.outcome; + if (outcome) return planResultOf(outcome); + if (Date.parse(data.expires_at) <= Date.now()) return { plan: null, error: 'The location did not plan the command in time; check that sqb location serve is running' }; + } + }, async listSnapshotChanges(scope: Scope, query: { limit?: number } = {}): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const [snapshots, runs] = await Promise.all([ diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index 97ec36118b..c72d850889 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { groupRunNodes, runDetailOf, runRowPage, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; +import { groupRunNodes, planResultOf, runDetailOf, runRowPage, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; type RunRecord = components['schemas']['RunRecord']; type W = components['schemas']; @@ -189,3 +189,16 @@ describe('group run nodes', () => { ]); }); }); + +describe('planResultOf', () => { + it.each([ + { description: 'a plan keeps its models and fills missing sections', outcome: { kind: 'plan', document: { selected_count: 1, models: [{ name: 'orders', reason: 'first_run' }] } }, expected: { plan: 'orders', error: null } }, + { description: "sqb plan's own error is the result", outcome: { kind: 'plan', document: { error: { code: 'S007', message: "unknown selector name 'ordrs'" } } }, expected: { plan: null, error: "S007: unknown selector name 'ordrs'" } }, + { description: 'a superseded plan asks to plan again', outcome: { kind: 'superseded' }, expected: { plan: null, error: 'The project changed on the location while it was planning; plan again' } } + ])('$description', ({ outcome, expected }) => { + const result = planResultOf(outcome as never); + expect(result.error).toBe(expected.error); + expect(result.plan?.models[0]?.name ?? null).toBe(expected.plan); + if (result.plan) expect(result.plan.warnings).toEqual([]); + }); +}); diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index a5e2c0375b..8b127ffe0c 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -1,7 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { isActiveRun } from '$lib/status'; import type { Capability, Principal } from './types'; -import type { AdhocRunCheck, LaunchVerb, NodeCounts, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement } from './types/operate'; +import type { AdhocRunCheck, LaunchVerb, NodeCounts, PlanDocument, PlanResult, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement } from './types/operate'; import { RECALL_EVENT_TYPES, SCHEDULER_FACTS } from './types/operate'; type Wire = components['schemas']; @@ -11,6 +11,31 @@ export const RUN_WINDOW = 1000; const PERSON = 'principal:'; +/** The location's answer to a plan request: the plan, `sqb plan`'s own error, or a superseded plan. */ +export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult { + if (outcome.kind === 'superseded') return { plan: null, error: 'The project changed on the location while it was planning; plan again' }; + if (outcome.kind !== 'plan') return { plan: null, error: 'The location answered with something other than a plan' }; + const document = outcome.document as Partial & { error?: { code?: string; message?: string } }; + if (document.error) return { plan: null, error: [document.error.code, document.error.message].filter(Boolean).join(': ') }; + const list = (value: T[] | undefined): T[] => value ?? []; + return { + plan: { + selected_count: document.selected_count ?? 0, + models: list(document.models), + seeds: list(document.seeds), + source_loads: list(document.source_loads), + functions: list(document.functions), + python_nodes: list(document.python_nodes), + warnings: list(document.warnings), + migrations: list(document.migrations), + column_migrations: list(document.column_migrations), + table_type_conversions: list(document.table_type_conversions), + retention: list(document.retention) + }, + error: null + }; +} + /** The server's dry run of an ad-hoc command. */ export function toAdhocCheck(wire: Wire['AdhocCheck']): AdhocRunCheck { return { diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index f84884dfcc..308c8821fa 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -3,6 +3,7 @@ import { ApiError } from '../client'; import { LocalChangeBus, type ConnectionState } from '../signals'; import type { AdhocRunCheck, + PlanResult, AdhocRunRequest, Bootstrap, BuiltInRole, @@ -266,6 +267,11 @@ export class MockApiClient implements ApiClient { return this.#operate.launchAdhocRun(scope, request); } + async planBuild(scope: Scope, command: string[]): Promise { + this.#location(scope); + return this.#operate.planBuild(scope, command); + } + async listJobs(scope: Scope): Promise { this.#location(scope); return this.#operate.listJobs(scope); diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 4723e9ddc0..9e68d16751 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -9,6 +9,7 @@ import type { LocalChangeBus } from '../signals'; import type { AdhocPermission, AdhocRunCheck, + PlanResult, AdhocRunRequest, BuiltInRole, BulkRunResult, @@ -522,6 +523,27 @@ export class MockOperate { return this.#ctx.reply(this.#view(rec, scope)); } + /** Every selected model is planned as unchanged unless the command asks for a full refresh. */ + async planBuild(scope: Scope, command: string[]): Promise { + const check = this.#check(scope, { command, priority: 0 }); + await new Promise((resolve) => setTimeout(resolve, 1200)); + if (check.error) return this.#ctx.reply({ plan: null, error: check.error }); + if (check.verb !== 'build') return this.#ctx.reply({ plan: null, error: `sqb plan previews builds; sqb ${check.verb} cannot be planned` }); + const reason = command.includes('--full-refresh') ? 'full_refresh' : 'no_change'; + const models = check.selection.filter((key) => key.startsWith('model:')).map((key) => ({ + name: key.slice('model:'.length), + materialization_type: 'table', + action: 'create_table', + reason, + backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null }, + qualified_name: `main.${key.slice('model:'.length)}` + })); + return this.#ctx.reply({ + plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [] }, + error: null + }); + } + // ---- jobs ---- listJobs(scope: Scope): Promise { diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index c81c6679dd..3a7dc1adfd 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -485,6 +485,40 @@ export type AdhocRunCheck = { launchable: boolean; }; +/** One model in `sqb plan --json`: what the build would do to it and why. */ +export type PlanModel = { + name: string; + materialization_type: string; + action: string; + reason: string; + incremental_strategy?: string; + cursor_column?: string; + cursor_bounds?: { start: string | null; end: string | null } | null; + backfill: { action: string; duration: string | null }; + qualified_name?: string; + changed_functions?: string[]; +}; + +export type PlanWarning = { severity: string; message: string; model_name?: string; code?: string }; + +/** `sqb plan --json` as the location printed it; sections the page doesn't show are kept as they are. */ +export type PlanDocument = { + selected_count: number; + models: PlanModel[]; + seeds: { name: string; reason: string; qualified_name?: string }[]; + source_loads: { name: string; kind: string; write_strategy?: string; cursor_column?: string; is_reload: boolean }[]; + functions: { name: string; language: string; reason?: string }[]; + python_nodes: { name: string; kind: string; phase: string; identity_status: string }[]; + warnings: PlanWarning[]; + migrations: { model: string; origin_model?: string; decision: string; completed_at: string | null }[]; + column_migrations: { model: string; origin_column: string; destination_column: string; decision: string }[]; + table_type_conversions: { model: string; actual_type?: string; desired_type?: string }[]; + retention: { model: string; action?: string }[]; +}; + +/** The location's plan of a build command, or why it could not plan it. */ +export type PlanResult = { plan: PlanDocument; error: null } | { plan: null; error: string }; + /** Cursor window for a launch, typed from the selected nodes' cursor type. */ export type CursorWindow = { type: 'timestamp' | 'integer'; diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index b8b8f66bd3..299e30629a 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -1,11 +1,11 @@ - {#if picked.length && canRunPicked} + {#if picked.length && def?.kind === 'job'} + + goto(planHref([], 'build', { job: name, exclude: picked }))}>▷ Run all but {picked.length} + goto(planHref(picked, 'build', { job: name }))}>▷ Run {picked.length} selected + {:else if picked.length && canRunPicked} goto(planHref(pickedNodes.map((n) => n.key)))}>▷ Run {picked.length} selected {:else} @@ -204,7 +208,7 @@ {#each runs as r (r.id)} goto(`/runs/${r.id}`)}> {runStatusLabel[r.status]} - {shortRunId(r.id)} + {shortRunId(r.id)}{#if r.narrowing}part{/if} {runOriginLabel[r.origin]}{r.parentRunId ? ` in group run ${shortRunId(r.parentRunId)}` : r.triggerName ? ` · ${r.triggerName}` : ''} {formatNodeCounts(r.nodeCounts)} {r.startedAt ? formatDuration(r.startedAt, r.endedAt, app.now) : '—'} @@ -263,6 +267,11 @@ } .chip i, dd i, + .runs td .part { + margin-left: 8px; + color: var(--muted-foreground); + font-size: 11.5px; + } .runs td i { display: inline-block; width: 7px; diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 299e30629a..cca55884e7 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -1,6 +1,7 @@ @@ -77,11 +79,16 @@
- {#if issues.length}{:else}{/if} + {#if loaded && issues.length}{:else}{/if}
- {issues.length ? `${app.location?.name} needs attention` : `${app.location?.name} is healthy`} - {issues.length ? `${issues.length} ${issues.length === 1 ? 'item' : 'items'} · ${latestText}` : latestText} + {#if !loaded} + {app.location?.name} + {overview.error ? 'Couldn’t check its health' : 'Checking…'} + {:else} + {issues.length ? `${app.location?.name} needs attention` : `${app.location?.name} is healthy`} + {issues.length ? `${issues.length} ${issues.length === 1 ? 'item' : 'items'} · ${latestText}` : latestText} + {/if}
{#if m}
diff --git a/ui/src/routes/connections/[id]/+page.svelte b/ui/src/routes/connections/[id]/+page.svelte deleted file mode 100644 index 4c73c23cf7..0000000000 --- a/ui/src/routes/connections/[id]/+page.svelte +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - -
-
- ← Connections - {name} - {#if warehouse} - Warehouse · {warehouse.adapter} · target {warehouse.target} - {/if} -
- - {#if connections.error} -
Couldn't load connections: {connections.error}
- {:else if connections.data && !warehouse} -
- {app.location?.name} has no warehouse connection named {name}. Looking for an API or storage connection? See Providers. -
- {:else if warehouse} -
-
-
-
- -
-
- - Read-only. Set in {warehouse.sourceFile}; secrets show only where they come from. -
- -
-
- - -
- {:else} -
Loading…
- {/if} -
From 66d5dc55d1c704911810db0477cf165f3310bfdf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:35:40 +0000 Subject: [PATCH 38/49] fix(ui): switch pages at once instead of holding them behind a loading bar A click kept the current page for up to 1.5 seconds while the next one loaded, with a bar along the top. Pages now switch after at most 200 ms and show remembered data or their own loading state, and the bar is gone. --- ui/src/lib/state/prefetch.ts | 11 +++++++---- ui/src/routes/+layout.svelte | 32 +------------------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/ui/src/lib/state/prefetch.ts b/ui/src/lib/state/prefetch.ts index 543592a8e4..b4ff520435 100644 --- a/ui/src/lib/state/prefetch.ts +++ b/ui/src/lib/state/prefetch.ts @@ -1,8 +1,11 @@ import { browser } from '$app/environment'; import type { ApiClient, Scope } from '$lib/api'; -/** How long a click waits on the current page for the next page's data before switching anyway. */ -const HOLD_MS = 1500; +/** + * How long a click waits on the current page for the next page's data, so a quick answer shows + * whole instead of empty for a moment; longer than this, the page switches and shows its own loading state. + */ +const HOLD_MS = 200; /** A prefetched read older than this, such as from a hover with no click, is read again instead. */ const KEEP_MS = 5000; @@ -37,8 +40,8 @@ export function withPrefetch(client: ApiClient): ApiClient { } /** - * Starts the reads the page at `pathname` makes first and waits for them, up to a limit, so a click - * stays on the current page until the next one has its data. Errors are left for the page to show. + * Starts the reads the page at `pathname` makes first and waits for them briefly, so a fast answer + * arrives with the page. Errors are left for the page to show. */ export async function prefetch(scope: Scope | null, pathname: string, search: URLSearchParams, reads: [RegExp, Read][]): Promise { if (!browser || !base || !scope) return; diff --git a/ui/src/routes/+layout.svelte b/ui/src/routes/+layout.svelte index 6fa65c2e53..5eb9983ea2 100644 --- a/ui/src/routes/+layout.svelte +++ b/ui/src/routes/+layout.svelte @@ -1,7 +1,7 @@ diff --git a/ui/src/routes/+layout.ts b/ui/src/routes/+layout.ts index 980c38269f..e080d95af7 100644 --- a/ui/src/routes/+layout.ts +++ b/ui/src/routes/+layout.ts @@ -38,7 +38,7 @@ const PAGE_READS: [RegExp, Read][] = [ [/^\/maintenance$/, (c, s) => [c.maintenanceHistory(s)]], [/^\/locations$/, (c) => [c.listLocations(), c.platformHealth()]], [/^\/locations\/([^/]+)$/, (c, _, m) => [c.getLocation(part(m))]], - [/^\/connections(\/[^/]+)?$/, (c, s) => [c.listWarehouseConnections(s)]], + [/^\/connections$/, (c) => [c.listWarehouseConnections()]], [/^\/providers(\/[^/]+)?$/, (c, s) => [c.listProviders(s)]] ]; diff --git a/ui/src/routes/connections/+page.svelte b/ui/src/routes/connections/+page.svelte index 2a760a63fe..d76ee8dfc3 100644 --- a/ui/src/routes/connections/+page.svelte +++ b/ui/src/routes/connections/+page.svelte @@ -4,53 +4,62 @@ import Button from '$lib/components/ui/button/button.svelte'; import ConnectionTest from '$lib/components/platform/connection-test.svelte'; import Panel from '$lib/components/platform/panel.svelte'; - import type { SettingField, WarehouseConnection } from '$lib/api'; + import StatusLabel from '$lib/components/list/status-label.svelte'; + import { formatMs, formatNumber } from '$lib/build-area/labels'; + import { formatAgo, formatStamp } from '$lib/format/time'; import { app } from '$lib/state/app.svelte'; import { createResource } from '$lib/state/resource.svelte'; - const connections = createResource((client, scope) => client.listWarehouseConnections(scope)); + const connections = createResource((client) => client.listWarehouseConnections(), ['warehouse_sync']); + const list = $derived(connections.data ?? []); - const warehouse = $derived(connections.data ?? []); - - function shownValue(c: WarehouseConnection, field: SettingField): string { - if (field.type === 'secret') return field.secretSource ?? 'not set'; - const value = c.values[field.key] ?? field.default; - return value === undefined ? '—' : String(value); - } + const where = (s: { database: string | null; schema: string }) => (s.database ? `${s.database}.${s.schema}` : s.schema); -
+
{#if connections.error}

Couldn't load connections: {connections.error}

{/if} - - {#each warehouse as c (c.name)} -
-
+ {#each list as c (c.id)} + +
+
- {c.name} - {c.adapter} · {c.purpose === 'catalog' ? 'used to read SQLBuild’s own tables back (warehouse sync)' : `builds target ${c.target}`} · {c.sourceFile} + Warehouse +
connections.refresh()} />
-
- {#each c.settings as f (f.key)} -
-
{f.key}
-
{shownValue(c, f)}
-
- {/each} -
- + {#if c.tables.length} + + + + {#each c.tables as t (t.table)} + + + + + + + + {/each} + +
Synced tableNewest syncRecordsTookLast clean sync
{t.table} + + {#if t.failure}{:else}{/if} + + {formatNumber(t.recordsSent)}{formatMs(t.elapsedMs)}{t.lastSyncedAt ? formatAgo(t.lastSyncedAt, app.now) : 'never'}
+ {:else} +

No warehouse sync reported yet.

+ {/if}
- {:else} -

{connections.data ? 'This location’s target has no warehouse connection. Add one to the target in the project config.' : connections.error ? '' : 'Loading…'}

- {/each} - - + + {:else} +

{connections.data ? 'No location you can see in full has reported yet.' : connections.slow ? 'Loading…' : ''}

+ {/each}
From e785371b2eda332173bbb83f5cbe4cd2946c3964 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:42:58 +0000 Subject: [PATCH 40/49] fix(ui): ask once per page load for a target whose health is refused Each refused health read is audited, so Connections no longer asks again on every reload, signal or manual test for a target the person may not list whole. --- ui/src/lib/api/http-client.test.ts | 31 ++++++++++++++++++++++++++++++ ui/src/lib/api/http-client.ts | 5 ++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index a4cdfe8d1d..b9979ba996 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -795,3 +795,34 @@ describe('getCatalogNode statistics', () => { expect([refreshed?.lastRows, refreshed?.warehouse, reads.length - first]).toEqual([1500, null, 1]); }); }); + +describe('listWarehouseConnections', () => { + const access = (target: string) => ({ location_id: '0191f0aa-orders', name: 'Orders', target, actions: [] }); + + /** Development's health is readable; preview's is refused, as for a partial viewer. */ + function server() { + const health: string[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/bootstrap')) return Response.json({ login_mode: 'password', access: { name: 'ada', principal_id: 'p', actions: [], locations: [access('development'), access('preview')] } }); + if (path.endsWith('/health')) { + health.push(path); + return path.includes('/targets/preview/') ? Response.json({ error: 'refused' }, { status: 403 }) : Response.json([]); + } + return Response.json([]); + }) + ); + return { health, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('asks for a refused target once per page load, since each refusal is audited', async () => { + const { health, client } = server(); + expect((await client.listWarehouseConnections()).map((c) => c.target)).toEqual(['development']); + expect((await client.listWarehouseConnections()).map((c) => c.target)).toEqual(['development']); + expect(health.filter((p) => p.includes('/preview/'))).toHaveLength(1); + expect(health.filter((p) => p.includes('/development/'))).toHaveLength(2); + }); +}); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 1bdfd3dac6..3d695857fd 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -265,6 +265,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const outputRefused = new Set(); // A node's error text needs view.evidence on that node alone. const errorRefused = new Set(); + // Targets whose health the person may not read; each refusal is audited, so Connections asks once per page load. + const healthRefused = new Set(); // Statistics samples per relation, oldest first, with the cursor after the newest; a refresh reads only newer samples. const samples = new Map(); async function relationStatistics(scope: Scope, relation: { database: string | null; schema: string; name: string }): Promise { @@ -608,12 +610,13 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt }, /** Targets whose health the person may not read are left out, as their connection checks are part of it. */ async listWarehouseConnections(): Promise { - const locations = await bootstrapLocations(); + const locations = (await bootstrapLocations()).filter((l) => !healthRefused.has(l.id)); const read = await Promise.all( locations.map(async (location) => { const { locationId, target } = parseLocationKey(location.id); const { data, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/health', { params: { path: { location_id: locationId, target } } }); if (response.status === 401) onUnauthorized(); + if (response.status === 403) healthRefused.add(location.id); return data ? [toWarehouseConnection(location, data)] : []; }) ); From 558b1d14cea57de57220e2a5425fa70c043f6c38 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:56:06 +0000 Subject: [PATCH 41/49] feat(platform): show administrators the server's settings GET /api/admin/server reports the server version, storage, when the installation and its schema were created, the sign-in mode, whether this server runs the scheduler, and how long each kind of record is kept. It needs user.manage everywhere, like the people list. The Settings page shows it to administrators and only the person's own access to others. --- clients/typescript/src/schema.d.ts | 94 +++++++++++++++++++ .../src/native_commands/_helpers/decision.rs | 21 +++-- .../src/native_commands/_helpers/execution.rs | 10 +- .../src/native_commands/main/_run_control.rs | 10 +- .../authentication/password_sessions.rs | 4 + .../api/_helpers/resources/administrators.rs | 35 ++++++- .../src/api/main/serve_api.rs | 12 +-- .../sqlbuild-orchestration/src/api/models.rs | 36 +++++++ .../src/credentials/models.rs | 4 + .../src/platform/main/mod.rs | 1 + .../src/platform/main/read_installation.rs | 45 +++++++++ .../src/platform/models.rs | 26 +++++ .../src/snapshots/models.rs | 4 + .../src/warehouse_metadata/models.rs | 4 + .../tests/platform/test_contracts.rs | 2 + .../tests/platform/test_installation.rs | 72 ++++++++++++++ .../tests/platform/test_types.rs | 8 +- ui/src/lib/api/http-client.ts | 11 ++- ui/src/lib/api/http-people.test.ts | 24 ++++- ui/src/lib/api/http-people.ts | 27 +++++- ui/src/lib/api/mock/platform.ts | 6 +- ui/src/lib/api/types/platform.ts | 8 +- ui/src/routes/settings/+page.svelte | 35 +++---- 23 files changed, 446 insertions(+), 53 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/platform/main/read_installation.rs create mode 100644 crates/sqlbuild-orchestration/tests/platform/test_installation.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 33483d3b6c..c9808d2b6c 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -64,6 +64,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/admin/server": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readServerSettings"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/auth/cli/approve": { parameters: { query?: never; @@ -1597,6 +1613,16 @@ export interface components { failed: components["schemas"]["Table"][]; failure?: string | null; }; + /** @description The control store's installation and schema, as recorded when it was created. */ + InstallationRecord: { + /** Format: date-time */ + installed_at: string; + /** Format: date-time */ + schema_applied_at: string; + /** @description Digest of the schema; a server starts only on the exact schema it creates. */ + schema_checksum: string; + storage: components["schemas"]["StorageKind"]; + }; /** @description One job or group as declared, for a person who may list the target; free-form setting values are withheld. */ JobDetailRecord: { concurrency_groups: string[]; @@ -2724,6 +2750,35 @@ export interface components { minimum_interval_seconds: number; when: components["schemas"]["SensorCondition"]; }; + /** @description How long the server keeps each kind of record. */ + ServerRetention: { + /** Format: int64 */ + audit_days: number; + /** Format: int64 */ + heartbeat_days: number; + /** Format: int64 */ + location_health_days: number; + /** Format: int64 */ + platform_alert_days: number; + /** Format: int64 */ + sensor_evaluation_days: number; + /** + * Format: int64 + * @description Publications kept per location target. + */ + snapshots_kept: number; + /** Format: int64 */ + statistics_days: number; + }; + /** @description The server's version, storage, sign-in and retention, for administrators. */ + ServerSettings: { + installation: components["schemas"]["InstallationRecord"]; + login_mode: components["schemas"]["LoginMode"]; + retention: components["schemas"]["ServerRetention"]; + /** @description Whether this server runs the scheduler; another server may hold the lease. */ + scheduler: boolean; + version: string; + }; SessionIdentity: { /** Format: date-time */ expires_at: string; @@ -2817,6 +2872,11 @@ export interface components { /** Format: uuid */ next_after?: string | null; }; + /** + * @description Where the control store keeps its data. + * @enum {string} + */ + StorageKind: "sqlite" | "postgresql"; /** * @description The tables Recall writes. * @enum {string} @@ -3200,6 +3260,40 @@ export interface operations { }; }; }; + readServerSettings: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ServerSettings"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; approveCliLogin: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 75db8c5df3..75fbb92064 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use serde_json::json; -use sqlbuild_orchestration::api::models::{ApiAuthentication, HumanLogin}; +use sqlbuild_orchestration::api::models::{ApiAuthentication, HumanLogin, ServerFacts}; use sqlbuild_orchestration::credentials::models::{AuditRetention, CredentialScope}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; @@ -263,16 +263,11 @@ pub(crate) fn control_address( Ok(Some(SocketAddr::new(host, port))) } -/// API settings are validated together before any store is opened. +/// API settings are validated together before any store is opened; `standalone` always runs the scheduler. pub(crate) fn control_api_settings( namespace: &Namespace, -) -> Result< - ( - ApiAuthentication, - (StatisticsRetention, AuditRetention, SnapshotRetention), - ), - ControlError, -> { + standalone: bool, +) -> Result<(ApiAuthentication, ServerFacts), ControlError> { let statistics = match retention_setting(namespace, "statistics-retention-days")? { Some(days) => StatisticsRetention::days(days)?, None => StatisticsRetention::default(), @@ -287,7 +282,13 @@ pub(crate) fn control_api_settings( }; Ok(( control_authentication(namespace)?, - (statistics, audit, snapshots), + ServerFacts { + version: env!("CARGO_PKG_VERSION").into(), + scheduler: standalone || !flag(namespace, "control_no_scheduler"), + audit, + statistics, + snapshots, + }, )) } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 3aa53ed604..b7ea9fa268 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -13,7 +13,9 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use crate::adhoc::main::adhoc_planner::adhoc_planner; -use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, HumanLogin}; +use sqlbuild_orchestration::api::models::{ + ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts, +}; use sqlbuild_orchestration::credentials::models::AuditRetention; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; @@ -112,7 +114,11 @@ pub(crate) async fn run_development_roles( (ApiAuthentication { login: HumanLogin::Development(scope), ..ApiAuthentication::default() - }, (Vec::new(), adhoc_planner()), Default::default()), + }, (Vec::new(), adhoc_planner()), ServerFacts { + version: env!("CARGO_PKG_VERSION").into(), + scheduler, + ..ServerFacts::default() + }), (move || { drop(write_output(&[(Stream::Stderr, &format!( "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 673283718f..60753b43f3 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -8,7 +8,7 @@ use tokio::sync::watch; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, control_api_settings, flag}; +use crate::native_commands::_helpers::decision::{control_address, control_api_settings}; use crate::native_commands::_helpers::execution::{ open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; @@ -27,7 +27,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; - let (authentication, (statistics, audit, snapshots)) = control_api_settings(namespace)?; + let (authentication, facts) = control_api_settings(namespace, standalone)?; let store = open_control_store(namespace).await?; let (workers, handoff) = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); @@ -42,7 +42,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - (authentication, handoff, statistics), + (authentication, handoff, facts.clone()), ( move || { drop(write_output(&[( @@ -56,10 +56,10 @@ pub(crate) async fn run_control( start_platform_roles( &mut tasks, (&store, &shutdown), - (audit, statistics, snapshots), + (facts.audit, facts.statistics, facts.snapshots), )?; } - if standalone || !flag(namespace, "control_no_scheduler") { + if facts.scheduler { drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); let scheduler_store = store.clone(); tasks.spawn(async move { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 0814484ec7..916a5810d0 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -69,6 +69,10 @@ pub(crate) fn person_routes() -> Router { "/api/admin/people", get(crate::api::_helpers::resources::administrators::people), ) + .route( + "/api/admin/server", + get(crate::api::_helpers::resources::administrators::server), + ) .route( "/api/admin/locations/{location_id}/targets/{target}/roles", get(crate::api::_helpers::resources::administrators::definitions), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs index 72c7bc9589..f06b608797 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -5,7 +5,9 @@ use axum::{Extension, Json}; use uuid::Uuid; use crate::api::errors::ApiError; -use crate::api::models::MutationResponse; +use crate::api::models::{ + LoginMode, MutationResponse, ServerFacts, ServerRetention, ServerSettings, +}; use crate::authorization::main::assign_builtin_role::assign_builtin_role; use crate::authorization::main::change_administrator::change_administrator; use crate::authorization::main::list_role_definitions::list_role_definitions; @@ -16,6 +18,11 @@ use crate::credentials::models::{ AdministratorChange, AuditContext, PersonSummary, RoleAssignmentRequest, RoleHoldings, RoleOverview, }; +use crate::platform::constants::{ + HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, PLATFORM_ALERT_RETENTION_DAYS, + SENSOR_EVALUATION_RETENTION_DAYS, +}; +use crate::platform::main::read_installation::read_installation; use crate::store::models::ControlStore; #[utoipa::path(post, path = "/api/admin/people/{principal_id}", operation_id = "changeAdministrator", @@ -87,3 +94,29 @@ pub(crate) async fn definitions( list_role_definitions(&store, (&context, principal), (location_id, &target)).await?, )) } + +#[utoipa::path(get, path = "/api/admin/server", operation_id = "readServerSettings", + responses((status = 200, body = ServerSettings), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn server( + (State(store), Extension(context)): (State, Extension), + (Extension(facts), Extension(login_mode)): (Extension, Extension), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + let installation = read_installation(&store, (&context, principal)).await?; + Ok(Json(ServerSettings { + version: facts.version, + login_mode, + scheduler: facts.scheduler, + installation, + retention: ServerRetention { + audit_days: i64::from(facts.audit.kept_days()), + statistics_days: i64::from(facts.statistics.kept_days()), + snapshots_kept: i64::from(facts.snapshots.kept_count()), + platform_alert_days: PLATFORM_ALERT_RETENTION_DAYS, + heartbeat_days: HEARTBEAT_RETENTION_DAYS, + location_health_days: LOCATION_RETENTION_DAYS, + sensor_evaluation_days: SENSOR_EVALUATION_RETENTION_DAYS, + }, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 1397e69ed2..87447e8f93 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -8,23 +8,18 @@ use crate::api::_helpers::resources::server_errors::generic_server_errors; use crate::api::_helpers::resources::ui_assets::{UI_ASSETS, serve_ui}; use crate::api::_helpers::resources::{changes, router::router}; use crate::api::main::openapi_document::openapi_document; -use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope, GroupRole}; +use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope, GroupRole, ServerFacts}; use crate::authorization::main::publish_group_roles::publish_group_roles; use crate::runs::types::RunHandoff; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::subscribe_changes::subscribe_changes; use crate::store::models::ControlStore; -use crate::warehouse_metadata::models::StatisticsRetention; pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, - (authentication, (images, planner), retention): ( - ApiAuthentication, - RunHandoff, - StatisticsRetention, - ), + (authentication, (images, planner), facts): (ApiAuthentication, RunHandoff, ServerFacts), (ready, shutdown): ( impl FnOnce() + Send + 'static, impl Future + Send + 'static, @@ -44,7 +39,8 @@ pub async fn serve_api( .merge(authentication_routes.layer(Extension(feed))) .layer(Extension(std::sync::Arc::new(images))) .layer(Extension(planner)) - .layer(Extension(retention)) + .layer(Extension(facts.statistics)) + .layer(Extension(facts)) .route( "/api/openapi.json", get(|| async { Json(openapi_document()) }), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 782c3255b6..dd7cd5a91b 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -17,6 +17,7 @@ use crate::api::_helpers::resources::{ administrators::people, administrators::holdings, administrators::definitions, + administrators::server, audit::read, person_operations::request_job, person_operations::request_adhoc, @@ -500,3 +501,38 @@ pub(crate) struct BoundedReader { pub(crate) remaining: usize, pub(crate) exceeded: bool, } + +/// What the server was started with, reported by the settings read. +#[derive(Clone, Debug, Default)] +pub struct ServerFacts { + pub version: String, + /// Whether this process runs the scheduler loop. + pub scheduler: bool, + pub audit: crate::credentials::models::AuditRetention, + pub statistics: crate::warehouse_metadata::models::StatisticsRetention, + pub snapshots: crate::snapshots::models::SnapshotRetention, +} + +/// The server's version, storage, sign-in and retention, for administrators. +#[derive(Clone, Debug, Serialize, utoipa::ToSchema)] +pub struct ServerSettings { + pub version: String, + pub login_mode: LoginMode, + /// Whether this server runs the scheduler; another server may hold the lease. + pub scheduler: bool, + pub installation: crate::platform::models::InstallationRecord, + pub retention: ServerRetention, +} + +/// How long the server keeps each kind of record. +#[derive(Clone, Debug, Serialize, utoipa::ToSchema)] +pub struct ServerRetention { + pub audit_days: i64, + pub statistics_days: i64, + /// Publications kept per location target. + pub snapshots_kept: i64, + pub platform_alert_days: i64, + pub heartbeat_days: i64, + pub location_health_days: i64, + pub sensor_evaluation_days: i64, +} diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index f2027c65c2..d965dae987 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -126,6 +126,10 @@ impl AuditRetention { pub(crate) fn cutoff(self, now: DateTime) -> DateTime { now - chrono::Duration::days(i64::from(self.days)) } + + pub(crate) fn kept_days(self) -> u16 { + self.days + } } impl Default for AuditRetention { diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index 20d6bc6038..f075ce27f5 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod enqueue_location_work; pub mod list_person_platform_events; pub mod location_health; pub mod location_work; +pub mod read_installation; pub(crate) mod read_location_work; pub mod read_person_connection_test; pub mod read_person_location_health; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs new file mode 100644 index 0000000000..6e20822bba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs @@ -0,0 +1,45 @@ +//! Administrators read when the control store was created and which schema it holds. + +use uuid::Uuid; + +use crate::authorization::main::begin_authorized::begin_authorized; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::platform::models::{InstallationRecord, StorageKind, StoredInstallation}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query_records::query_records}; +use crate::store::models::{ControlStore, ControlTransaction}; + +/// Requires `user.manage` everywhere, as the people list does. +pub async fn read_installation( + store: &ControlStore, + credentials: (&AuditContext, Uuid), +) -> Result { + let (transaction, _) = begin_authorized( + store, + credentials, + &PermissionScope::action(Action::UserManage, None), + DefaultRole::None, + ) + .await?; + let (transaction, rows): (ControlTransaction, Vec) = query_records( + transaction, + "SELECT i.created_at AS installed_at, v.checksum AS schema_checksum, v.applied_at AS schema_applied_at \ + FROM sqb_installation i JOIN sqb_schema_version v ON v.singleton=i.singleton WHERE i.singleton=1", + Vec::new(), + ) + .await?; + commit(transaction).await?; + let stored = rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("the control store has no installation record".into()) + })?; + let storage: StorageKind = match store { + ControlStore::Sqlite(_) => StorageKind::Sqlite, + ControlStore::Postgres(_) => StorageKind::Postgresql, + }; + Ok(InstallationRecord { + storage, + installed_at: stored.installed_at, + schema_checksum: stored.schema_checksum, + schema_applied_at: stored.schema_applied_at, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 8e5bff885c..bc7006b5c9 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -351,3 +351,29 @@ pub(crate) struct ProblemScan { pub(crate) broken: Vec, pub(crate) late_targets: std::collections::BTreeSet<(Uuid, String)>, } + +/// Where the control store keeps its data. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum StorageKind { + Sqlite, + Postgresql, +} + +/// The control store's installation and schema, as recorded when it was created. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct InstallationRecord { + pub storage: StorageKind, + pub installed_at: DateTime, + /// Digest of the schema; a server starts only on the exact schema it creates. + pub schema_checksum: String, + pub schema_applied_at: DateTime, +} + +/// The stored installation and schema rows. +#[derive(Debug, Deserialize)] +pub(crate) struct StoredInstallation { + pub(crate) installed_at: DateTime, + pub(crate) schema_checksum: String, + pub(crate) schema_applied_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 82c6f9d5a3..77c3558329 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -113,6 +113,10 @@ impl SnapshotRetention { pub(crate) fn kept(self) -> i64 { i64::from(self.count) } + + pub(crate) fn kept_count(self) -> u16 { + self.count + } } impl Default for SnapshotRetention { diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index e71dcc67ad..f0c0c2297e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -600,6 +600,10 @@ impl StatisticsRetention { pub(crate) fn cutoff(self, now: DateTime) -> DateTime { now - Duration::days(i64::from(self.days)) } + + pub(crate) fn kept_days(self) -> u16 { + self.days + } } impl Default for StatisticsRetention { diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs index eb9b98bffe..894a2240a1 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "helpers.rs"] mod helpers; +#[path = "test_installation.rs"] +mod test_installation; #[path = "test_problems.rs"] mod test_problems; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/platform/test_installation.rs b/crates/sqlbuild-orchestration/tests/platform/test_installation.rs new file mode 100644 index 0000000000..63ac1fed38 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_installation.rs @@ -0,0 +1,72 @@ +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::AuditContext; +use crate::platform::main::read_installation::read_installation; +use crate::platform::models::StorageKind; +use crate::platform::tests::test_types::InstallationTestCase; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use uuid::Uuid; + +#[test] +fn given_a_migrated_store_when_people_read_the_installation_then_only_administrators_see_it() { + let test_cases = [ + InstallationTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_storage: StorageKind::Sqlite, + }, + InstallationTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_storage: StorageKind::Postgresql, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store) + .await + .expect("migrate installation store"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("administrator"); + let developer = create_principal(store, &context, "orders-developer") + .await + .expect("developer"); + let installation = read_installation(store, (&context, admin)) + .await + .expect("administrator reads the installation"); + assert_eq!( + ( + installation.storage, + installation.schema_checksum.len(), + installation.schema_applied_at == installation.installed_at + ), + (test_case.expected_storage, 64, true), + "{}: the storage, a SHA-256 schema digest, and a schema created with the installation", + test_case.description + ); + assert!( + matches!( + read_installation(store, (&context, developer)).await, + Err(ControlError::Denied { .. }) + ), + "{}: a person without user.manage everywhere is refused", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 8fa6dd4a47..9b0dcf7bb7 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -1,6 +1,6 @@ use std::sync::{Arc, Mutex}; -use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope, StorageKind}; use chrono::{DateTime, Utc}; use serde::Deserialize; @@ -58,3 +58,9 @@ pub(crate) struct ProblemsTestCase { pub(crate) leaders: usize, pub(crate) expected_leader: bool, } + +pub(crate) struct InstallationTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_storage: StorageKind, +} diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 3d695857fd..8e82b6ad32 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -8,7 +8,7 @@ import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toMicrobatchCoverag import { auditMatches, toAuditEntry } from './http-audit'; import { toSnapshotChanges } from './http-changes'; import { manualConnectionTest, toLocationDetail, toLocationSummary, toWarehouseConnection, warehouseSyncOf } from './http-locations'; -import { toAccount, toRoleAssignments } from './http-people'; +import { toAccount, toRoleAssignments, toServerSettings } from './http-people'; import { toMaintenanceHistory } from './http-maintenance'; import { overviewOf } from './http-overview'; import { toPlatformEventRecords, toPlatformEvents, toPlatformHealth } from './http-problems'; @@ -18,7 +18,7 @@ import { timelineOf } from './http-timeline'; import { RUN_WINDOW, groupRunNodes, parseRunCursor, planResultOf, queueEntriesOf, toAdhocCheck, runDetailOf, runRowPageOver, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, NavCounts, PlatformEvent, Scope, WarehouseSync } from './types'; -import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; +import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, ServerSettings, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; @@ -82,6 +82,7 @@ const SUPPORTED: ReadonlySet = new Set([ 'maintenanceHistory', 'testConnection', 'listWarehouseConnections', + 'serverSettings', 'listSnapshotNodes', 'checkAdhocRun', 'launchAdhocRun', @@ -631,6 +632,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return toLocationDetail(location, health.data, snapshot, Date.now()); }, listAccounts: async (): Promise => (await people()).map(toAccount), + async serverSettings(): Promise { + const { data, error, response } = await http.GET('/api/admin/server'); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toServerSettings(data); + }, listRoles: async (locationId: string): Promise => (await roleOverviews(locationId)).map(toRoleDefinition), async getRole(locationId: string, name: string): Promise { const [roles, assignments] = await Promise.all([roleOverviews(locationId), implemented.listRoleAssignments()]); diff --git a/ui/src/lib/api/http-people.test.ts b/ui/src/lib/api/http-people.test.ts index 518daeb696..c70d6ed5fc 100644 --- a/ui/src/lib/api/http-people.test.ts +++ b/ui/src/lib/api/http-people.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { toAccount, toRoleAssignments } from './http-people'; +import { toAccount, toRoleAssignments, toServerSettings } from './http-people'; const person = (name: string, groups: string[], has_password = false) => ({ principal_id: `p-${name}`, @@ -36,3 +36,25 @@ describe('people', () => { ]); }); }); + +describe('toServerSettings', () => { + it('names each kept record with its own retention', () => { + const settings = toServerSettings({ + version: '0.205.1', + login_mode: 'password', + scheduler: false, + installation: { storage: 'sqlite', installed_at: '2026-11-01T00:00:00Z', schema_checksum: 'abc', schema_applied_at: '2026-11-01T00:00:00Z' }, + retention: { audit_days: 400, snapshots_kept: 10, statistics_days: 90, platform_alert_days: 30, location_health_days: 7, sensor_evaluation_days: 6, heartbeat_days: 1 } + }); + expect([settings.authMode, settings.schedulerEnabled, settings.storage]).toEqual(['password', false, 'sqlite']); + expect(settings.retention.map((r) => [r.name, r.value])).toEqual([ + ['Security audit log', '400 days'], + ['Published versions', '10'], + ['Warehouse row counts', '90 days'], + ['Platform problems', '30 days'], + ['Location check-ins', '7 days'], + ['Sensor checks', '6 days'], + ['Server heartbeats', '1 day'] + ]); + }); +}); diff --git a/ui/src/lib/api/http-people.ts b/ui/src/lib/api/http-people.ts index 9f3af49421..5ce06fac92 100644 --- a/ui/src/lib/api/http-people.ts +++ b/ui/src/lib/api/http-people.ts @@ -1,5 +1,5 @@ import type { components } from '$control/schema.d.ts'; -import type { Account, RoleAssignment } from './types/platform'; +import type { Account, RoleAssignment, ServerSettings } from './types/platform'; type Wire = components['schemas']; @@ -56,3 +56,28 @@ export function toRoleAssignments(holdings: Wire['RoleHoldings'], people: Wire[' ); return [...manual, ...viaGroups]; } + +const days = (n: number) => `${n} ${n === 1 ? 'day' : 'days'}`; + +/** The server's own facts, with each kind of record and how long the server keeps it. */ +export function toServerSettings(wire: Wire['ServerSettings']): ServerSettings { + const r = wire.retention; + return { + serverVersion: wire.version, + storage: wire.installation.storage, + installedAt: wire.installation.installed_at, + schemaChecksum: wire.installation.schema_checksum, + schemaAppliedAt: wire.installation.schema_applied_at, + authMode: wire.login_mode, + schedulerEnabled: wire.scheduler, + retention: [ + { name: 'Security audit log', value: days(r.audit_days), description: 'Who did what, and every refusal.' }, + { name: 'Published versions', value: String(r.snapshots_kept), description: 'Kept for each location target; older versions are removed.' }, + { name: 'Warehouse row counts', value: days(r.statistics_days), description: 'Row counts and sizes sampled from the warehouse.' }, + { name: 'Platform problems', value: days(r.platform_alert_days), description: 'Late schedules, failed launches and locations that stopped checking in.' }, + { name: 'Location check-ins', value: days(r.location_health_days), description: 'Health reports from location processes.' }, + { name: 'Sensor checks', value: days(r.sensor_evaluation_days), description: 'Each sensor check. Where each sensor is up to is always kept.' }, + { name: 'Server heartbeats', value: days(r.heartbeat_days), description: 'Check-ins from the servers themselves.' } + ] + }; +} diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 671f791c77..944f0e0cbb 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -903,9 +903,9 @@ export class MockPlatformApi implements PlatformApi { return this.#ctx.reply({ serverVersion: this.#ctx.serverVersion, storage: 'postgresql', - schemaVersion: 14, - latestSchemaVersion: 14, - schemaMigratedAt: new Date(this.#ctx.now - 5 * DAY).toISOString(), + installedAt: new Date(this.#ctx.now - 120 * DAY).toISOString(), + schemaChecksum: '6f1c2a9e4b7d0c3f8a5e2b1d9c4f7a0e3b6d8c1f5a2e9b4d7c0f3a6e1b8d5c2f', + schemaAppliedAt: new Date(this.#ctx.now - 5 * DAY).toISOString(), authMode: 'oidc', schedulerEnabled: true, retention: [ diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index 81b227be12..b374e8b8b0 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -526,10 +526,12 @@ export type RetentionSetting = { name: string; value: string; description: strin export type ServerSettings = { serverVersion: string; storage: 'postgresql' | 'sqlite'; - schemaVersion: number; - latestSchemaVersion: number; - schemaMigratedAt: Iso; + installedAt: Iso; + /** A server starts only on the exact schema it creates, so a running server's schema is always current. */ + schemaChecksum: string; + schemaAppliedAt: Iso; authMode: AuthMode; + /** Whether this server runs the scheduler; another server may hold the lease. */ schedulerEnabled: boolean; retention: RetentionSetting[]; }; diff --git a/ui/src/routes/settings/+page.svelte b/ui/src/routes/settings/+page.svelte index 70c94682f2..75948b1f44 100644 --- a/ui/src/routes/settings/+page.svelte +++ b/ui/src/routes/settings/+page.svelte @@ -10,7 +10,9 @@ import { app } from '$lib/state/app.svelte'; import { createResource } from '$lib/state/resource.svelte'; - const settings = createResource((client) => client.serverSettings()); + // Server facts are for administrators; others aren't asked, since each refusal is audited. + const admin = $derived(app.can('user.manage')); + const settings = createResource((client) => (admin ? client.serverSettings() : Promise.resolve(null))); const s = $derived(settings.data ?? null); const authLabel: Record = { @@ -24,8 +26,8 @@
-
-
+
+ {#if admin}
{#if settings.error}

Couldn't load: {settings.error}

@@ -34,16 +36,12 @@ rows={[ { label: 'Version', value: s.serverVersion }, { label: 'Storage', value: s.storage === 'postgresql' ? 'PostgreSQL' : 'SQLite' }, - { - label: 'Database', - value: s.schemaVersion === s.latestSchemaVersion ? `Up to date (schema ${s.schemaVersion}), updated ${formatAgo(s.schemaMigratedAt, app.now)}` : `Needs upgrading: schema ${s.schemaVersion} of ${s.latestSchemaVersion}`, - title: formatStamp(s.schemaMigratedAt), - tone: s.schemaVersion === s.latestSchemaVersion ? undefined : 'var(--sb-warning)' - }, + { label: 'Installed', value: formatAgo(s.installedAt, app.now), title: formatStamp(s.installedAt) }, + { label: 'Schema', value: `${s.schemaChecksum.slice(0, 12)} · set up ${formatAgo(s.schemaAppliedAt, app.now)}`, title: `${s.schemaChecksum}\n${formatStamp(s.schemaAppliedAt)}`, mono: true }, { label: 'Authentication', value: authLabel[s.authMode] }, { label: 'Scheduler', - value: s.schedulerEnabled ? 'Runs on this server' : 'Not on this server (started with --no-scheduler)', + value: s.schedulerEnabled ? 'Runs on this server' : 'Runs elsewhere (this server started with --no-scheduler)', tone: s.schedulerEnabled ? undefined : 'var(--sb-warning)' } ]} @@ -68,7 +66,7 @@ -
+
{/if}
@@ -76,8 +74,8 @@ !!row)} /> @@ -95,9 +93,11 @@

On the command line, sign in with sqb login: it opens this browser and keeps a short-lived session, so you don't need an API token.

{/if} - - - + {#if app.client.supports('explainAccess')} + + + + {/if} @@ -118,6 +118,9 @@ gap: 0 40px; max-width: 1280px; } + .columns.single { + grid-template-columns: minmax(0, 640px); + } .err { color: var(--sb-error); font-size: 12.5px; From 185c78facd42b7301f29788612b68e4a36c3d7b0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:02:31 +0000 Subject: [PATCH 42/49] test(platform): check the server settings route answers administrators only --- crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index 0fd7138491..f5558bc4cb 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -335,6 +335,7 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ "/api/admin/people".to_owned(), "/api/admin/roles".to_owned(), format!("/api/admin/locations/{location}/targets/development/roles"), + "/api/admin/server".to_owned(), ] { let listed = request( &provider.client, From 670e1b07af6631f1b74ba7fb319b6d122cb52e30 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:05:34 +0000 Subject: [PATCH 43/49] feat(platform): report calendars running out of dates as problems The people's problems read now lists running calendar schedules whose last date is at most a day away, by the watchdog's own threshold, for targets the person lists. The Locations problems show them per location. --- clients/typescript/src/schema.d.ts | 22 +++++++++ .../src/platform/_helpers/problems.rs | 46 ++++++++++++++++++- .../src/platform/_helpers/watchdog.rs | 10 ++-- .../src/platform/main/read_person_problems.rs | 12 ++++- .../src/platform/models.rs | 29 ++++++++++++ .../tests/platform/test_problems.rs | 31 ++++++++++++- ui/src/lib/api/http-problems.test.ts | 25 ++++++++++ ui/src/lib/api/http-problems.ts | 26 ++++++++++- 8 files changed, 191 insertions(+), 10 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index c9808d2b6c..fc28b737d9 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1830,6 +1830,26 @@ export interface components { * @enum {string} */ LoginMode: "password" | "trusted_proxy" | "oidc"; + /** @description A running calendar schedule whose last date is at most a day away, or which has none left. */ + LowCalendar: { + /** + * Format: date-time + * @description The calendar's last date; absent when none are left. + */ + horizon_end?: string | null; + /** Format: uuid */ + location_id: string; + location_name: string; + name: string; + /** + * Format: date-time + * @description When the location next works out the calendar's dates. + */ + refresh_at: string; + target: string; + /** Format: uuid */ + trigger_id: string; + }; /** @description Recorded moves, their old-name view steps and janitor cleanups, newest moves first. */ MaintenanceHistory: { /** @description Empty when the history is for one model; cleanups are not tied to models. */ @@ -2209,6 +2229,8 @@ export interface components { /** @description Problems derived from live state when read; nothing here is stored. */ PlatformProblems: { broken_locations: components["schemas"]["BrokenLocation"][]; + /** @description Running calendar schedules whose dates run out within a day. */ + low_calendars?: components["schemas"]["LowCalendar"][]; overdue_ticks: components["schemas"]["OverdueTick"][]; scheduler_leader: boolean; /** diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs index d177f3e4f0..ded3ac32a8 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs @@ -3,8 +3,9 @@ use chrono::{DateTime, TimeDelta, Utc}; use uuid::Uuid; +use crate::declarations::models::Schedule; use crate::platform::constants::LOCATION_SILENCE_SECONDS; -use crate::platform::models::{LateTarget, OverdueTick, ProblemScan}; +use crate::platform::models::{LateTarget, LowCalendar, OverdueTick, ProblemScan, RunningSchedule}; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; @@ -39,8 +40,38 @@ pub(crate) async fn scan_problems(store: &ControlStore) -> Result = schedules + .into_iter() + .map(decode_row) + .collect::>()?; + let low_calendars: Vec = schedules + .into_iter() + .filter_map(|schedule| { + let (refresh_at, horizon_end) = calendar_low(&schedule.definition_json.schedule, now)?; + Some(LowCalendar { + trigger_id: schedule.trigger_id, + name: schedule.name, + location_id: schedule.location_id, + location_name: schedule.location_name, + target: schedule.target, + refresh_at, + horizon_end, + }) + }) + .collect(); Ok(ProblemScan { + low_calendars, now, lease: leases.into_iter().next().map(decode_row).transpose()?, broken: locations @@ -96,3 +127,16 @@ pub(crate) async fn late_ticks( fn overdue_before(now: DateTime) -> DateTime { now - TimeDelta::seconds(30) } + +/// A calendar is low when its last date is at most a day away; gives when it refreshes and its last date. +pub(crate) fn calendar_low( + schedule: &Schedule, + now: DateTime, +) -> Option<(DateTime, Option>)> { + let Schedule::Calendar { times, refresh_at } = schedule else { + return None; + }; + let last: Option> = times.last().copied(); + last.is_none_or(|last| last <= now + TimeDelta::days(1)) + .then_some((*refresh_at, last)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index 2bfb02d6df..a85f056c46 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -4,9 +4,9 @@ use chrono::TimeDelta; use tokio::sync::watch; use uuid::Uuid; -use crate::declarations::models::Schedule; use crate::platform::_helpers::liveness::scan_location_page; use crate::platform::_helpers::observations::observe; +use crate::platform::_helpers::problems::calendar_low; use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -113,10 +113,8 @@ async fn scan_trigger_page( ) .await?; } - if let Schedule::Calendar { times, refresh_at } = trigger.definition_json.schedule - && times - .last() - .is_none_or(|last| *last <= now + TimeDelta::days(1)) + if let Some((refresh_at, horizon_end)) = + calendar_low(&trigger.definition_json.schedule, now) { transaction = observe( transaction, @@ -125,7 +123,7 @@ async fn scan_trigger_page( PlatformEvent::CalendarHorizonLow { trigger_id: trigger.trigger_id, refresh_at, - horizon_end: times.last().copied(), + horizon_end, }, None, ), diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs b/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs index 59d3cd19b9..28b10e4c8c 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs @@ -1,4 +1,4 @@ -//! People see the scheduler's state, and broken locations and late ticks only for targets they list. +//! People see the scheduler's state, and broken locations, late ticks and low calendars only for targets they list. use std::collections::BTreeSet; @@ -24,6 +24,11 @@ pub async fn read_person_problems( .iter() .map(|location| (location.location_id, location.target.clone())) .chain(scan.late_targets.iter().cloned()) + .chain( + scan.low_calendars + .iter() + .map(|calendar| (calendar.location_id, calendar.target.clone())), + ) .collect::>() { if person_holds_target( @@ -49,5 +54,10 @@ pub async fn read_person_problems( .into_iter() .filter(|location| listed.contains(&(location.location_id, location.target.clone()))) .collect(), + low_calendars: scan + .low_calendars + .into_iter() + .filter(|calendar| listed.contains(&(calendar.location_id, calendar.target.clone()))) + .collect(), }) } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index bc7006b5c9..928081c82e 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -310,6 +310,34 @@ pub struct PlatformProblems { pub scheduler_lease_expires_at: Option>, pub broken_locations: Vec, pub overdue_ticks: Vec, + /// Running calendar schedules whose dates run out within a day. + #[serde(default)] + pub low_calendars: Vec, +} + +/// A running calendar schedule whose last date is at most a day away, or which has none left. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LowCalendar { + pub trigger_id: Uuid, + pub name: String, + pub location_id: Uuid, + pub location_name: String, + pub target: String, + /// When the location next works out the calendar's dates. + pub refresh_at: DateTime, + /// The calendar's last date; absent when none are left. + pub horizon_end: Option>, +} + +/// A running schedule of a live location, with its declaration. +#[derive(Debug, Deserialize)] +pub(crate) struct RunningSchedule { + pub(crate) trigger_id: Uuid, + pub(crate) name: String, + pub(crate) location_id: Uuid, + pub(crate) location_name: String, + pub(crate) target: String, + pub(crate) definition_json: TriggerDefinition, } /// Every instance of this location target has missed two check-ins. @@ -350,6 +378,7 @@ pub(crate) struct ProblemScan { pub(crate) lease: Option, pub(crate) broken: Vec, pub(crate) late_targets: std::collections::BTreeSet<(Uuid, String)>, + pub(crate) low_calendars: Vec, } /// Where the control store keeps its data. diff --git a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs index 2d95047ac6..6e2ed8a3a8 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs @@ -143,6 +143,22 @@ fn given_broken_locations_and_late_ticks_when_people_read_problems_then_only_lis 200, "each listed target keeps its own 100 late ticks" ); + assert_eq!( + everything + .low_calendars + .iter() + .map(|calendar| ( + calendar.location_name.as_str(), + calendar.name.as_str(), + calendar.horizon_end.is_some() + )) + .collect::>(), + vec![ + ("inventory", "expired_orders", true), + ("orders", "expired_orders", true) + ], + "only running calendars whose dates run out within a day, not paused or ten-day ones" + ); let scoped = read(developer).await.expect("developer problems"); assert_eq!(scoped.scheduler_leader, everything.scheduler_leader); assert_eq!( @@ -155,6 +171,15 @@ fn given_broken_locations_and_late_ticks_when_people_read_problems_then_only_lis "only the listed location is reported broken" ); assert_eq!(scoped.overdue_ticks.len(), 100); + assert_eq!( + scoped + .low_calendars + .iter() + .map(|calendar| calendar.location_id) + .collect::>(), + vec![locations[1]], + "another location's low calendar stays hidden" + ); assert!( scoped .overdue_ticks @@ -165,7 +190,11 @@ fn given_broken_locations_and_late_ticks_when_people_read_problems_then_only_lis ); let hidden = read(outsider).await.expect("outsider problems"); assert_eq!(hidden.scheduler_leader, everything.scheduler_leader); - assert!(hidden.broken_locations.is_empty() && hidden.overdue_ticks.is_empty()); + assert!( + hidden.broken_locations.is_empty() + && hidden.overdue_ticks.is_empty() + && hidden.low_calendars.is_empty() + ); let transaction = begin(store, TransactionMode::Write) .await .expect("observe events"); diff --git a/ui/src/lib/api/http-problems.test.ts b/ui/src/lib/api/http-problems.test.ts index 337dff1856..523adcf75d 100644 --- a/ui/src/lib/api/http-problems.test.ts +++ b/ui/src/lib/api/http-problems.test.ts @@ -31,6 +31,31 @@ describe('toPlatformEvents', () => { expect(broken).toMatchObject({ kind: 'location_broken', locationId: 'l2:development', occurredAt: '2026-11-08T11:55:00Z', subject: 'inventory · development' }); }); + it('groups low calendars per location, dated and named by the one that runs out first', () => { + const calendar = (name: string, location: string, horizon_end: string | null) => ({ + trigger_id: `c-${name}`, + name, + location_id: location, + location_name: location === 'l1' ? 'orders' : 'inventory', + target: 'development', + refresh_at: '2026-11-08T13:00:00Z', + horizon_end + }); + const events = toPlatformEvents( + { + scheduler_leader: true, + broken_locations: [], + overdue_ticks: [], + low_calendars: [calendar('month_end', 'l1', '2026-11-09T06:00:00Z'), calendar('quarter_end', 'l1', '2026-11-08T18:00:00Z'), calendar('stock_counts', 'l2', null)] + }, + NOW + ); + expect(events.map((e) => [e.kind, e.locationId, e.occurredAt, e.message])).toEqual([ + ['calendar_horizon_low', 'l1:development', '2026-11-08T18:00:00Z', '2 calendars in orders · development are running out of dates, starting with quarter_end.'], + ['calendar_horizon_low', 'l2:development', '2026-11-08T13:00:00Z', 'stock_counts in inventory · development has no dates left; the location works out more at its next refresh.'] + ]); + }); + it('groups late ticks per location, dated and named by the oldest', () => { const events = toPlatformEvents( { diff --git a/ui/src/lib/api/http-problems.ts b/ui/src/lib/api/http-problems.ts index 3ec89f5449..456997bde2 100644 --- a/ui/src/lib/api/http-problems.ts +++ b/ui/src/lib/api/http-problems.ts @@ -7,7 +7,7 @@ type Wire = components['schemas']; const placeOf = (name: string, target: string) => `${name} · ${target}`; -/** Live problems as active banner events: a stopped scheduler, each broken location, and late schedules per location. */ +/** Live problems as active banner events: a stopped scheduler, each broken location, and late schedules and low calendars per location. */ export function toPlatformEvents(problems: Wire['PlatformProblems'], now: number): PlatformEvent[] { const events: PlatformEvent[] = []; if (!problems.scheduler_leader) { @@ -53,6 +53,30 @@ export function toPlatformEvents(problems: Wire['PlatformProblems'], now: number message: ticks.length === 1 ? `${oldest.name} in ${place} is late.` : `${ticks.length} schedules in ${place} are late, starting with ${oldest.name}.` }); } + const low = new Map(); + for (const calendar of problems.low_calendars ?? []) { + const key = keyOf(calendar.location_id, calendar.target); + low.set(key, [...(low.get(key) ?? []), calendar]); + } + for (const [key, calendars] of low) { + const first = calendars[0]; + const place = placeOf(first.location_name, first.target); + const ends = (c: Wire['LowCalendar']) => c.horizon_end ?? c.refresh_at; + const soonest = calendars.reduce((a, b) => (ends(a) <= ends(b) ? a : b)); + events.push({ + id: `calendar:${key}`, + kind: 'calendar_horizon_low', + severity: 'warning', + occurredAt: ends(soonest), + active: true, + locationId: key, + subject: place, + message: + calendars.length === 1 + ? `${soonest.name} in ${place} ${soonest.horizon_end ? 'is running out of dates' : 'has no dates left'}; the location works out more at its next refresh.` + : `${calendars.length} calendars in ${place} are running out of dates, starting with ${soonest.name}.` + }); + } return events; } From 9918c5ea0140d8b6766bb1034cd732d3dd88ea29 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:17:15 +0000 Subject: [PATCH 44/49] feat(triggers): list each running schedule's next due times The person trigger listing now carries up to five due times from next_due_at on, worked out by the scheduler's own cron and calendar rules, so the UI no longer guesses later cron times. The trigger page also states the catch-up policy: one tick for the latest missed time. --- clients/typescript/src/schema.d.ts | 2 + .../declarations/_helpers/trigger_state.rs | 21 +++- .../src/declarations/constants.rs | 3 + .../declarations/main/list_person_triggers.rs | 14 ++- .../src/declarations/models.rs | 2 + .../tests/api/test_contracts.rs | 2 + .../tests/api/test_trigger_upcoming.rs | 118 ++++++++++++++++++ .../tests/api/test_types.rs | 7 ++ ui/src/lib/api/http-jobs.test.ts | 36 ++---- ui/src/lib/api/http-jobs.ts | 22 +--- 10 files changed, 180 insertions(+), 47 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index fc28b737d9..ab37ec64f6 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2166,6 +2166,8 @@ export interface components { /** @description A trigger as people list it, with its newest tick so a list needs no read per trigger. */ PersonTriggerRecord: components["schemas"]["TriggerRecord"] & { latest_tick?: null | components["schemas"]["TickRecord"]; + /** @description A running schedule's next due times from `next_due_at` on, at most five; empty otherwise. */ + upcoming: string[]; }; /** @description Observed physical evidence is not a live warehouse-existence guarantee. */ PhysicalEvidence: { diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs index b258882455..d000185371 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -2,8 +2,9 @@ use chrono::{DateTime, TimeDelta, Utc}; +use crate::declarations::constants::UPCOMING_TIMES; use crate::declarations::models::{ - PreviousTriggerState, Schedule, TriggerDefinition, TriggerStatus, + PreviousTriggerState, Schedule, TriggerDeclaration, TriggerDefinition, TriggerStatus, }; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_schedule_time::next_schedule_time; @@ -62,3 +63,21 @@ pub(crate) fn trigger_timing( }; Ok((status, next)) } + +/// The next due times of a running schedule from `next_due_at` on, as the scheduler works them out. +pub(crate) fn upcoming_times( + declaration: &TriggerDeclaration, + next_due_at: Option>, +) -> Result>, ControlError> { + let (TriggerDeclaration::Schedule(definition), Some(first)) = (declaration, next_due_at) else { + return Ok(Vec::new()); + }; + let mut times: Vec> = vec![first]; + while times.len() < UPCOMING_TIMES { + let Some(next) = next_schedule_time(&definition.schedule, times[times.len() - 1])? else { + break; + }; + times.push(next); + } + Ok(times) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/constants.rs b/crates/sqlbuild-orchestration/src/declarations/constants.rs index 00fc324482..125a59e986 100644 --- a/crates/sqlbuild-orchestration/src/declarations/constants.rs +++ b/crates/sqlbuild-orchestration/src/declarations/constants.rs @@ -6,3 +6,6 @@ pub(crate) const VARS_SETTING: &str = "vars"; pub(crate) const WITHHELD_SETTING: &str = "[REDACTED]"; /// Settings that name another target; a value that isn't a target name, such as a state path, is withheld. pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_"; + +/// How many due times a person's trigger listing gives for each running schedule. +pub(crate) const UPCOMING_TIMES: usize = 5; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs index 3e178777ac..94a4f0c976 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs @@ -7,6 +7,7 @@ use uuid::Uuid; use crate::authorization::main::authorize_listing::authorize_listing; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::declarations::_helpers::reads::{latest_ticks_in, triggers_in}; +use crate::declarations::_helpers::trigger_state::upcoming_times; use crate::declarations::models::PersonTriggerRecord; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; @@ -32,11 +33,14 @@ pub async fn list_person_triggers( .into_iter() .map(|tick| (tick.trigger_id, tick)) .collect(); - Ok(triggers + triggers .into_iter() - .map(|trigger| PersonTriggerRecord { - latest_tick: latest.remove(&trigger.trigger_id), - trigger, + .map(|trigger| { + Ok(PersonTriggerRecord { + latest_tick: latest.remove(&trigger.trigger_id), + upcoming: upcoming_times(&trigger.definition_json, trigger.next_due_at)?, + trigger, + }) }) - .collect()) + .collect() } diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 59d24c4e99..068f46283b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -179,6 +179,8 @@ pub struct PersonTriggerRecord { #[serde(flatten)] pub trigger: TriggerRecord, pub latest_tick: Option, + /// A running schedule's next due times from `next_due_at` on, at most five; empty otherwise. + pub upcoming: Vec>, } /// One operator change of a trigger, as a person reads it: what changed, who changed it, when and why. diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 39db9b74da..025a9bb80b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -84,6 +84,8 @@ mod test_reads; mod test_server_errors; #[path = "test_statistics_recovery.rs"] mod test_statistics_recovery; +#[path = "test_trigger_upcoming.rs"] +mod test_trigger_upcoming; #[path = "test_types.rs"] mod test_types; #[path = "test_ui_assets.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs b/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs new file mode 100644 index 0000000000..5897709eb4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs @@ -0,0 +1,118 @@ +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::test_types::UpcomingTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_person_triggers::list_person_triggers; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_running_and_paused_schedules_when_people_list_triggers_then_running_ones_carry_their_next_due_times() + { + let hourly = [ + "2031-01-01T01:00:00Z", + "2031-01-01T02:00:00Z", + "2031-01-01T03:00:00Z", + "2031-01-01T04:00:00Z", + "2031-01-01T05:00:00Z", + ]; + let calendar = [ + "2031-02-01T00:00:00Z", + "2031-03-01T00:00:00Z", + "2031-04-01T00:00:00Z", + ]; + let test_cases = [ + UpcomingTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_hourly: hourly, + expected_calendar: calendar, + }, + UpcomingTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_hourly: hourly, + expected_calendar: calendar, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({ + "hourly_orders": {"description": "Hourly orders", "job_key": "orders", + "starts_at": "2031-01-01T00:30:00Z", + "schedule": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"}}, + "paused_orders": {"description": "Paused orders", "job_key": "orders", + "default_status": "paused", "starts_at": "2031-01-01T00:30:00Z", + "schedule": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"}}, + "month_start_orders": {"description": "Month start orders", "job_key": "orders", + "schedule": {"kind": "calendar", "times": test_case.expected_calendar, + "refresh_at": "2031-01-15T00:00:00Z"}} + }), + ); + publish_snapshot(store, &input).await.expect("publication"); + let audit = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let administrator = bootstrap_user(store, &audit, "orders-admin") + .await + .expect("administrator"); + let listed: BTreeMap>> = list_person_triggers( + store, + (&audit, administrator), + (location, "development"), + DefaultRole::None, + ) + .await + .expect("person triggers") + .into_iter() + .map(|record| (record.trigger.name, record.upcoming)) + .collect(); + let times = |texts: &[&str]| -> Vec> { + texts + .iter() + .map(|text| text.parse().expect("time")) + .collect() + }; + assert_eq!( + listed, + BTreeMap::from([ + ( + "hourly_orders".to_owned(), + times(&test_case.expected_hourly) + ), + ( + "month_start_orders".to_owned(), + times(&test_case.expected_calendar) + ), + ("paused_orders".to_owned(), Vec::new()), + ]), + "{}: five cron times from the start, the calendar's remaining dates, none while paused", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index b3f09f5fd2..f5e8e9df1a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -220,3 +220,10 @@ pub(crate) struct PlanTestCase { pub(crate) expected_selection: &'static [&'static str], pub(crate) expected_warning: &'static str, } + +pub(crate) struct UpcomingTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_hourly: [&'static str; 5], + pub(crate) expected_calendar: [&'static str; 3], +} diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index 84fe805853..9a4ac9ab0c 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -7,8 +7,9 @@ type W = components['schemas']; const SNAPSHOT = '0191f0dd-0000-7000-8000-000000000000'; -function trigger(id: string, name: string, jobKey: string, definition: W['TriggerDeclaration'], status: W['TriggerStatus'] = 'running'): W['TriggerRecord'] { - return { trigger_id: id, name, job_key: jobKey, definition_json: definition, status, last_tick_for: null, next_due_at: '2026-11-07T06:00:00Z' }; +function trigger(id: string, name: string, jobKey: string, definition: W['TriggerDeclaration'], status: W['TriggerStatus'] = 'running'): W['PersonTriggerRecord'] { + const upcoming = 'schedule' in definition ? ['2026-11-07T06:00:00Z', '2026-11-08T06:00:00Z'] : []; + return { trigger_id: id, name, job_key: jobKey, definition_json: definition, status, last_tick_for: null, next_due_at: '2026-11-07T06:00:00Z', upcoming }; } const daily = trigger('t-daily', 'orders_daily', 'orders', { description: 'Daily orders', job_key: 'orders', schedule: { kind: 'cron', expression: '0 6 * * *', timezone: 'UTC' }, default_status: 'paused' }); @@ -51,8 +52,9 @@ describe('trigger mapping', () => { expect(tick.lateBySeconds).toBe(150); const detail = triggerDetailOf(toTrigger(onInventory, [], tick), onInventory, [tick], []); expect(detail.sensor).toMatchObject({ minimumIntervalSeconds: 60, cursorKnown: false }); - expect([detail.declaration, detail.upcoming, detail.evaluations, detail.stateHistory]).toEqual([null, null, null, []]); - expect(triggerDetailOf(toTrigger(daily, [], null), daily, [], []).sensor).toBeNull(); + expect([detail.declaration, detail.upcoming, detail.catchUp, detail.evaluations, detail.stateHistory]).toEqual([null, null, null, null, []]); + const schedule = triggerDetailOf(toTrigger(daily, [], null), daily, [], []); + expect([schedule.sensor, schedule.upcoming, schedule.catchUp]).toEqual([null, ['2026-11-07T06:00:00Z', '2026-11-08T06:00:00Z'], { mode: 'latest', max: 1 }]); }); it("shows a sensor's checks newest first, its cursor and when it moved, and flags code changed since the last check", () => { @@ -169,30 +171,18 @@ describe('runWindowOf', () => { }); describe('schedule due times', () => { - // Stored out of order on purpose; one time is before the server's next due time. + // Stored out of order on purpose. const times = ['2026-11-10T09:00:00Z', '2026-11-06T09:00:00Z', '2026-11-07T09:00:00Z', '2026-11-08T09:00:00Z']; - const calendar = (nextDueAt: string | null) => ({ - ...trigger('t-cal', 'business_days', 'orders', { description: 'Business days', job_key: 'orders', schedule: { kind: 'calendar', times, refresh_at: '2026-11-08T00:00:00Z' } }), - next_due_at: nextDueAt - }); - const detailOf = (wire: W['TriggerRecord']) => triggerDetailOf(toTrigger(wire, [], null), wire, [], []); - - it('lists a calendar from the next due time on, in time order, with the last stored time and next refresh', () => { - const detail = detailOf(calendar('2026-11-07T09:00:00Z')); - expect(detail.upcoming).toEqual(['2026-11-07T09:00:00Z', '2026-11-08T09:00:00Z', '2026-11-10T09:00:00Z']); - expect(detail.calendar).toMatchObject({ storedUntil: '2026-11-10T09:00:00.000Z', nextRefreshAt: '2026-11-08T00:00:00Z', horizon: null, low: false }); - }); - - it('has no upcoming calendar times once the server has no next due time', () => { - expect(detailOf(calendar(null)).upcoming).toEqual([]); - }); + const calendar = trigger('t-cal', 'business_days', 'orders', { description: 'Business days', job_key: 'orders', schedule: { kind: 'calendar', times, refresh_at: '2026-11-08T00:00:00Z' } }); + const detailOf = (wire: W['PersonTriggerRecord']) => triggerDetailOf(toTrigger(wire, [], null), wire, [], []); - it('shows only the next due time for cron, without working out later ones', () => { - expect(detailOf(daily).upcoming).toEqual(['2026-11-07T06:00:00Z']); + it('reports the last stored calendar time and the next refresh', () => { + expect(detailOf(calendar).calendar).toMatchObject({ storedUntil: '2026-11-10T09:00:00.000Z', nextRefreshAt: '2026-11-08T00:00:00Z', horizon: null, low: false }); expect(detailOf(daily).calendar).toBeNull(); }); - it('has no due times for a sensor', () => { + it('takes due times from the server, and has none for a sensor', () => { + expect(detailOf({ ...daily, upcoming: [] }).upcoming).toEqual([]); expect(detailOf(onInventory).upcoming).toBeNull(); }); }); diff --git a/ui/src/lib/api/http-jobs.ts b/ui/src/lib/api/http-jobs.ts index acc552c183..3b989c2ee9 100644 --- a/ui/src/lib/api/http-jobs.ts +++ b/ui/src/lib/api/http-jobs.ts @@ -80,21 +80,6 @@ export function toTrigger(wire: Wire['TriggerRecord'], runs: Run[], lastTick: Ti }; } -/** How many upcoming due times a calendar schedule lists. */ -const UPCOMING_LIMIT = 5; - -/** - * The server's next due time, and for calendars the stored times from it on, which the scheduler - * takes in order. Later cron times would need cron evaluation, which stays on the server. - */ -function upcomingOf(d: Wire['TriggerDeclaration'], nextDueAt: string | null): string[] | null { - if (!isSchedule(d)) return null; - if (d.schedule.kind === 'cron') return nextDueAt ? [nextDueAt] : []; - if (!nextDueAt) return []; - const next = Date.parse(nextDueAt); - return d.schedule.times.filter((t) => Date.parse(t) >= next).sort((a, b) => Date.parse(a) - Date.parse(b)).slice(0, UPCOMING_LIMIT); -} - /** A trigger's detail; parts with no server source are null and their widgets stay hidden. */ const CHANGE_KINDS: Record = { enable: 'enabled', @@ -146,7 +131,7 @@ function cursorMovedAt(state: Wire['PersonSensorState'], history: TriggerStateEv export function triggerDetailOf( trigger: Trigger, - wire: Wire['TriggerRecord'], + wire: Wire['PersonTriggerRecord'], ticks: Tick[], history: TriggerStateEvent[], state: Wire['PersonSensorState'] | null = null @@ -161,8 +146,9 @@ export function triggerDetailOf( definitionChanged: !!state && lastEvaluatedFingerprint !== null && lastEvaluatedFingerprint !== state.definition_fingerprint, declaration: null, targets: null, - catchUp: null, - upcoming: upcomingOf(d, wire.next_due_at ?? null), + // The scheduler ticks only the latest missed time after downtime, and a run still waiting covers it. + catchUp: isSchedule(d) ? { mode: 'latest', max: 1 } : null, + upcoming: isSchedule(d) ? wire.upcoming : null, calendar: calendar && { horizon: null, storedUntil: stored.length ? new Date(Math.max(...stored)).toISOString() : null, From 9083a29c5c91e0a363d1fe0528b20f9c595db797 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:22:09 +0000 Subject: [PATCH 45/49] fix(ui): describe the schedule catch-up as at most one run --- ui/src/routes/triggers/[id]/trigger-view.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/routes/triggers/[id]/trigger-view.svelte b/ui/src/routes/triggers/[id]/trigger-view.svelte index a6ab294819..08dc3fa3c4 100644 --- a/ui/src/routes/triggers/[id]/trigger-view.svelte +++ b/ui/src/routes/triggers/[id]/trigger-view.svelte @@ -100,7 +100,7 @@
Runs
{#if t.jobKey}{t.jobKey}{:else}its own code only{/if}
{#if t.cron}
Cron
{t.cron}
{/if} {#if t.timezone}
Time zone
{t.timezone}
{/if} - {#if t.catchUp}
After downtime
{t.catchUp.mode === 'latest' ? 'runs once for the latest missed time' : `runs for each missed time, up to ${t.catchUp.max}`}
{/if} + {#if t.catchUp}
After downtime
{t.catchUp.mode === 'latest' ? 'at most one run, for the latest missed time' : `runs for each missed time, up to ${t.catchUp.max}`}
{/if} {#if t.declaration}
Targets
{t.targets?.join(', ') ?? 'all targets'}
{/if}
In the code
{t.defaultState === 'running' ? 'on' : 'paused'}
{#if t.sourceFile}
Defined in
{t.sourceFile}
{/if} From aa7d2bc616b2212d0a0cc0fd597021e9b6eedd11 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:53:33 +0000 Subject: [PATCH 46/49] feat(triggers): show where each trigger is declared and its targets Publication records the project-relative file and line of each schedule, calendar and sensor, and the targets it is limited to; a job's schedule takes its job's. A sensor's fingerprint leaves both out, so moving its code does not flag the definition as changed. The trigger page shows them. --- clients/typescript/src/schema.d.ts | 6 ++ .../src/declarations/models.rs | 5 ++ .../src/scheduler/tests/helpers.rs | 4 ++ .../src/sensors/_helpers/facts.rs | 11 +++- .../src/sensors/models.rs | 7 ++- .../tests/sensors/test_person_state.rs | 34 ++++++++++++ .../compiled_project/snapshot_declarations.rs | 55 +++++++------------ .../native_host/orchestration_declarations.rs | 35 +++++++++++- .../native_host/orchestration_triggers.rs | 13 ++++- .../models/authoring.py | 8 +++ .../compiler/compiled_project/_test_types.py | 1 + .../publication/test_calendar.py | 4 ++ .../test_orchestration_snapshot.py | 21 +++++++ ui/src/lib/api/http-jobs.test.ts | 12 ++++ ui/src/lib/api/http-jobs.ts | 4 +- .../routes/triggers/[id]/trigger-view.svelte | 2 +- 16 files changed, 179 insertions(+), 43 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ab37ec64f6..37ba35a5fa 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2767,11 +2767,14 @@ export interface components { kind: "custom"; }; SensorDefinition: { + declared_at?: null | components["schemas"]["DeclaredAt"]; default_status?: components["schemas"]["TriggerStatus"]; description: string; job_key: string; /** Format: int32 */ minimum_interval_seconds: number; + /** @description The targets the declaration is limited to; empty for every target. */ + targets?: string[]; when: components["schemas"]["SensorCondition"]; }; /** @description How long the server keeps each kind of record. */ @@ -2963,12 +2966,15 @@ export interface components { /** @description Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. */ TriggerDeclaration: components["schemas"]["TriggerDefinition"] | components["schemas"]["SensorDefinition"]; TriggerDefinition: { + declared_at?: null | components["schemas"]["DeclaredAt"]; default_status?: components["schemas"]["TriggerStatus"]; description: string; job_key: string; schedule: components["schemas"]["Schedule"]; /** Format: date-time */ starts_at?: string | null; + /** @description The targets the declaration is limited to; empty for every target. */ + targets?: string[]; }; TriggerRecord: { definition_json: components["schemas"]["TriggerDeclaration"]; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 068f46283b..b67dadf618 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -86,6 +86,11 @@ pub struct TriggerDefinition { pub starts_at: Option>, #[serde(default)] pub default_status: TriggerStatus, + /// The targets the declaration is limited to; empty for every target. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub targets: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub declared_at: Option, } #[derive(Clone, Debug, Default, PartialEq)] diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs index f1a299164c..af58285254 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -35,6 +35,8 @@ fn calendar(seconds: impl Iterator) -> TriggerDefinition { job_key: "orders".into(), starts_at: Some(time(0)), default_status: TriggerStatus::Running, + targets: BTreeSet::new(), + declared_at: None, schedule: Schedule::Calendar { times: seconds.map(time).collect(), refresh_at: time(2000), @@ -203,6 +205,8 @@ pub(super) fn simulate_cron(expression: &str, wall_times: &[(u32, u32)]) -> Resu }, starts_at: Some(start), default_status: TriggerStatus::default(), + targets: BTreeSet::new(), + declared_at: None, }; let strategy = proptest::collection::vec((-1_440_i64..5_760, any::()), 0..256); let mut property_runner = TestRunner::new(Config { diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs index 4617403213..0e3f093e53 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs @@ -1,5 +1,7 @@ //! Sensor evaluation phases are recorded once; the latest cursor lives on the trigger's state. +use std::collections::BTreeSet; + use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; @@ -12,11 +14,16 @@ use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{Change, ControlTransaction, Parameter}; -/// Evaluations record the fingerprint of the definition they ran, so a later definition shows as changed. +/// Evaluations record the fingerprint of the definition they ran, so a later definition shows as changed; where it is written is not part of it. pub(crate) fn definition_fingerprint( definition: &SensorDefinition, ) -> Result { - Ok(sha256_hex(&serde_json::to_vec(definition).map_err( + let behaviour: SensorDefinition = SensorDefinition { + targets: BTreeSet::new(), + declared_at: None, + ..definition.clone() + }; + Ok(sha256_hex(&serde_json::to_vec(&behaviour).map_err( |error| ControlError::Invalid(error.to_string()), )?)) } diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 004d0a7527..33e8c7c098 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::declarations::models::TriggerStatus; +use crate::declarations::models::{DeclaredAt, TriggerStatus}; use crate::platform::models::PlatformEventKind; use crate::runs::models::{NodeOutcome, RunOutcome}; @@ -104,6 +104,11 @@ pub struct SensorDefinition { pub minimum_interval_seconds: NonZeroU32, #[serde(default)] pub default_status: TriggerStatus, + /// The targets the declaration is limited to; empty for every target. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub targets: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub declared_at: Option, } /// Storage order is authoritative even when a fact's occurrence time predates earlier pages. diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs b/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs index 94582ece70..511f861b17 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs @@ -130,6 +130,40 @@ fn given_a_checked_sensor_when_people_read_its_state_then_listing_view_gates_it_ "{}: both checks finished on the published definition", test_case.description ); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["health_orders"] + .as_object_mut() + .expect("sensor") + .extend([ + ("targets".into(), json!(["development"])), + ( + "declared_at".into(), + json!({"path": "orchestration/health.py", "line": 12}), + ), + ]); + publish_snapshot(store, &input) + .await + .expect("republish moved sensor"); + let moved = read(administrator, trigger.trigger_id, 1) + .await + .expect("administrator") + .expect("published sensor"); + let listed = list_triggers(store, roles.location_id, "development") + .await + .expect("triggers") + .remove(0); + assert_eq!( + ( + moved.definition_fingerprint, + json!(listed.definition_json)["declared_at"].clone(), + ), + ( + state.definition_fingerprint.clone(), + json!({"path": "orchestration/health.py", "line": 12}), + ), + "{}: where a sensor is written is published but does not change its definition", + test_case.description + ); let newest = read(administrator, trigger.trigger_id, 1) .await .expect("administrator") diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 96f339cb9f..c926c807f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -1,7 +1,7 @@ //! Compile authoring records against retained graph data before publication. use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Component, Path}; +use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; @@ -21,7 +21,7 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, module_jobs, + AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, module_jobs, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ compile_calendars, compile_cron, compile_sensors, @@ -56,33 +56,6 @@ pub(crate) fn discover_declarations<'py>( Ok(declarations) } -/// A declaration's file relative to the project; code outside the project keeps its location private. -fn declared_at(project_dir: &Path, written: Option<(String, u32)>) -> PyResult> { - let Some((file, line)) = written else { - return Ok(None); - }; - let file = Path::new(&file); - let roots = [ - project_dir.canonicalize()?, - std::path::absolute(project_dir)?, - ]; - let Some(root) = roots.iter().find(|root| file.starts_with(root)) else { - return Ok(None); - }; - let parts: Option> = file - .components() - .skip(root.components().count()) - .map(|component| match component { - Component::Normal(part) => part.to_str(), - _ => None, - }) - .collect(); - Ok(parts.map(|parts| DeclaredAt { - path: parts.join("/"), - line, - })) -} - pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, @@ -145,10 +118,16 @@ pub(crate) fn compile_declarations( .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) .collect(); + let declared_at: Option = declared_at(project_dir, job.declared_at)?; + let targets: BTreeSet = job.targets.into_iter().collect(); if let Some(schedule) = job.schedule { triggers.insert( job.name.clone(), - compile_cron(schedule.bind(py), &job.name, &job.description)?, + TriggerDefinition { + targets: targets.clone(), + declared_at: declared_at.clone(), + ..compile_cron(schedule.bind(py), &job.name, &job.description)? + }, ); } jobs.insert( @@ -160,8 +139,8 @@ pub(crate) fn compile_declarations( concurrency_groups: job.concurrency_groups.into_iter().collect(), select: Some(job.select), exclude: Some(job.exclude).filter(|exclude| !exclude.trim().is_empty()), - targets: job.targets.into_iter().collect(), - declared_at: declared_at(project_dir, job.declared_at)?, + targets, + declared_at, }, ); } @@ -190,10 +169,16 @@ pub(crate) fn compile_declarations( .iter() .flat_map(|name| jobs[name].selection.iter().cloned()) .collect(); + let declared_at: Option = declared_at(project_dir, group.declared_at)?; + let targets: BTreeSet = group.targets.into_iter().collect(); if let Some(schedule) = group.schedule { triggers.insert( group.name.clone(), - compile_cron(schedule.bind(py), &group.name, &group.description)?, + TriggerDefinition { + targets: targets.clone(), + declared_at: declared_at.clone(), + ..compile_cron(schedule.bind(py), &group.name, &group.description)? + }, ); } groups.insert( @@ -206,8 +191,8 @@ pub(crate) fn compile_declarations( concurrency_groups: group.concurrency_groups.into_iter().collect(), select: None, exclude: None, - targets: group.targets.into_iter().collect(), - declared_at: declared_at(project_dir, group.declared_at)?, + targets, + declared_at, }, jobs: dependencies, }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index de357c4f0a..1c4dda6f85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,7 +1,7 @@ //! The public authoring records enter native compilation here, never scheduler execution. use std::collections::{BTreeMap, HashSet}; -use std::path::Path; +use std::path::{Component, Path}; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -11,6 +11,7 @@ use serde_json::Value; use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_discovery::project_validation::models::OrchestrationNameFact; use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::DeclaredAt; use crate::bindings::_helpers::native_host::constants::{ AUTHORING_MODULE, CALENDAR_LABEL, CONSTRUCTING_MODULES, GROUP_LABEL, JOB_LABEL, @@ -79,6 +80,8 @@ pub(crate) struct AuthoredSensor { pub(crate) minimum_interval_seconds: Value, pub(crate) default_status: String, pub(crate) targets: Vec, + #[pyo3(attribute("_declared_at"))] + pub(crate) declared_at: Option<(String, u32)>, } pub(crate) enum AuthoredDeclaration { @@ -109,6 +112,36 @@ pub(crate) fn declaration_site(py: Python<'_>) -> PyResult Ok(None) } +/// A declaration's file relative to the project; code outside the project keeps its location private. +pub(crate) fn declared_at( + project_dir: &Path, + written: Option<(String, u32)>, +) -> PyResult> { + let Some((file, line)) = written else { + return Ok(None); + }; + let file = Path::new(&file); + let roots = [ + project_dir.canonicalize()?, + std::path::absolute(project_dir)?, + ]; + let Some(root) = roots.iter().find(|root| file.starts_with(root)) else { + return Ok(None); + }; + let parts: Option> = file + .components() + .skip(root.components().count()) + .map(|component| match component { + Component::Normal(part) => part.to_str(), + _ => None, + }) + .collect(); + Ok(parts.map(|parts| DeclaredAt { + path: parts.join("/"), + line, + })) +} + /// JSON authoring values cross once; the interpreter rejects circular user containers safely. pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { let text: String = value diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 583ae015e2..8779ef8608 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::native_host::constants::{ }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredSensor, authored_setting, + AuthoredSensor, authored_setting, declared_at, }; use crate::bindings::_helpers::native_host::orchestration_environment::{ callback_code_hash, orchestration_code_hash, @@ -56,6 +56,8 @@ fn compile_trigger( .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, default_status: serde_json::from_value(Value::String(controls.default_status)) .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + targets: BTreeSet::new(), + declared_at: None, }) } @@ -236,6 +238,13 @@ pub(crate) fn compile_calendars( let schedule = calendar_times(calendar, &session, &by_name, start)?; let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; trigger.starts_at.get_or_insert(start); + trigger.targets = calendar + .getattr("targets")? + .extract::>()? + .into_iter() + .collect(); + trigger.declared_at = + declared_at(project_dir, calendar.getattr("_declared_at")?.extract()?)?; definitions.triggers.insert(name, trigger); } Ok(definitions) @@ -295,6 +304,8 @@ pub(crate) fn compile_sensors( "when": condition, "minimum_interval_seconds": sensor.minimum_interval_seconds, "default_status": sensor.default_status, + "targets": sensor.targets, + "declared_at": declared_at(project_dir, sensor.declared_at)?, })) .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; definitions.sensors.insert(sensor.name, definition); diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index 233c6cc425..a904d71243 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -77,6 +77,10 @@ class CalendarSchedule: default_status: Literal["running", "paused"] = "running" targets: tuple[str, ...] = () function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: return replace(self, function=function) @@ -147,6 +151,10 @@ class Sensor: function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( default=None, repr=False, kw_only=True ) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: return replace(self, function=function) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 70dc4648db..eb871a9c8a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -277,6 +277,7 @@ class SnapshotDiscoveryTestCase: expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) expected_schedules: tuple[str, ...] = () expected_declared_at: dict[str, dict[str, object] | None] = field(default_factory=dict) + expected_trigger_declared_at: dict[str, dict[str, object] | None] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index f7d26b84fa..c5325d6f57 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -192,6 +192,10 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "fulfillment_calendar", "overdue_orders", } + assert first["documents"]["triggers"]["fulfillment_calendar"]["declared_at"] == { + "path": "orchestration/orders.py", + "line": 8, + } with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: stored, next_due = control.execute( "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 4788383218..c6e520bb1f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -532,6 +532,7 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select } trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] assert trigger["job_key"] == "nightly_orders" + assert trigger["targets"] == ["development"] assert trigger["schedule"] == { "kind": "cron", "expression": "15 2 * * *", @@ -749,6 +750,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 7, "default_status": "paused", + "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 61}, }, "edge_job_outcomes": { "description": "Keep warning and partial outcomes distinct from success.", @@ -760,6 +762,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "paused", + "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 77}, }, "edge_group_outcomes": { "description": "Observe group completion including cancellation and presumed failure.", @@ -772,9 +775,16 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "paused", + "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 84}, }, }, expected_schedules=("edge_snapshot_orders",), + expected_trigger_declared_at={ + "edge_snapshot_orders": { + "path": "orchestration/edge_cases/definitions.py", + "line": 14, + }, + }, expected_declared_at={ "edge_snapshot_orders": { "path": "orchestration/edge_cases/definitions.py", @@ -842,6 +852,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -866,6 +877,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -887,6 +899,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, "minimum_interval_seconds": 1, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 10}, } }, ), @@ -912,6 +925,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 1, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -935,6 +949,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -953,6 +968,8 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, "minimum_interval_seconds": 7, "default_status": "paused", + "targets": ["development"], + "declared_at": {"path": "orchestration/health.py", "line": 9}, } }, ), @@ -1003,6 +1020,10 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects assert { name: definitions[name].get("declared_at") for name in test_case.expected_declared_at } == test_case.expected_declared_at + assert { + name: publication["documents"]["triggers"][name].get("declared_at") + for name in test_case.expected_trigger_declared_at + } == test_case.expected_trigger_declared_at @pytest.mark.parametrize( diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index 9a4ac9ab0c..987a7ec3dc 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -41,6 +41,18 @@ describe('trigger mapping', () => { expect(toTrigger(onInventory, runs, null).lastRun).toBeNull(); }); + it('shows where a trigger is declared and the targets it is limited to, and nothing for an older snapshot', () => { + const declared = trigger('t-cal', 'fulfillment_calendar', 'orders', { + ...onInventory.definition_json, + targets: ['production'], + declared_at: { path: 'orchestration/orders.py', line: 8 } + } as W['SensorDefinition']); + const detail = triggerDetailOf(toTrigger(declared, [], null), declared, [], []); + expect([detail.sourceFile, detail.targets]).toEqual(['orchestration/orders.py:8', ['production']]); + const older = triggerDetailOf(toTrigger(daily, [], null), daily, [], []); + expect([older.sourceFile, older.targets]).toEqual([null, null]); + }); + it('links a group trigger to its group run, not a child listed before it', () => { const child = { ...run('r-child', 'orders_member', 't-daily'), parentRunId: 'r-group' } as Run; const group = { ...run('r-group', 'orders', 't-daily'), status: 'failed' } as Run; diff --git a/ui/src/lib/api/http-jobs.ts b/ui/src/lib/api/http-jobs.ts index 3b989c2ee9..518072be7f 100644 --- a/ui/src/lib/api/http-jobs.ts +++ b/ui/src/lib/api/http-jobs.ts @@ -74,7 +74,7 @@ export function toTrigger(wire: Wire['TriggerRecord'], runs: Run[], lastTick: Ti lastTick, // Group children copy the group's trigger, so only top-level runs are the trigger's own. lastRun: newest(runs, (r) => r.triggerId === wire.trigger_id && !r.parentRunId), - sourceFile: null, + sourceFile: d.declared_at ? `${d.declared_at.path}:${d.declared_at.line}` : null, definitionChanged: false, warning: null }; @@ -145,7 +145,7 @@ export function triggerDetailOf( ...trigger, definitionChanged: !!state && lastEvaluatedFingerprint !== null && lastEvaluatedFingerprint !== state.definition_fingerprint, declaration: null, - targets: null, + targets: d.targets?.length ? d.targets : null, // The scheduler ticks only the latest missed time after downtime, and a run still waiting covers it. catchUp: isSchedule(d) ? { mode: 'latest', max: 1 } : null, upcoming: isSchedule(d) ? wire.upcoming : null, diff --git a/ui/src/routes/triggers/[id]/trigger-view.svelte b/ui/src/routes/triggers/[id]/trigger-view.svelte index 08dc3fa3c4..154d6a5d59 100644 --- a/ui/src/routes/triggers/[id]/trigger-view.svelte +++ b/ui/src/routes/triggers/[id]/trigger-view.svelte @@ -101,7 +101,7 @@ {#if t.cron}
Cron
{t.cron}
{/if} {#if t.timezone}
Time zone
{t.timezone}
{/if} {#if t.catchUp}
After downtime
{t.catchUp.mode === 'latest' ? 'at most one run, for the latest missed time' : `runs for each missed time, up to ${t.catchUp.max}`}
{/if} - {#if t.declaration}
Targets
{t.targets?.join(', ') ?? 'all targets'}
{/if} + {#if t.targets}
Targets
{t.targets.join(', ')}
{/if}
In the code
{t.defaultState === 'running' ? 'on' : 'paused'}
{#if t.sourceFile}
Defined in
{t.sourceFile}
{/if} From a3d07207c46611be376b8056092ae29607db6b09 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:07:39 +0000 Subject: [PATCH 47/49] docs(server): say what happens when a location is newer than its server --- .../skills/sqlbuild/references/docs/cli/server.md | 12 ++++++++---- website/src/content/docs/docs/cli/server.mdx | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index a6f49ad7fe..a37f7ae52b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -26,15 +26,19 @@ Use `sqb server --help` for service, authentication and launcher options. ## Upgrade -Upgrade the server first, then the locations and workers that connect to it. A worker newer than -its server can send a run report the server cannot read. The worker then stops with: +Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the +server after a location or worker has connected from a newer release; the server can't read what +they wrote. + +| Newer than the server | What happens | Fix | +| --- | --- | --- | +| Worker | The worker refuses the run and stops with the message below. | Upgrade the server, then rerun the run. | +| Location | The server refuses the location's publication with `API rejected the request with HTTP 400 Bad Request`. The previous snapshot stays current. | Upgrade the server, then restart the location so it publishes again. | ```text the control server is older than this worker and refused its run report; upgrade the server before its workers ``` -Upgrade the server to the worker's release or newer, then rerun the run. - ## Set retention Set how much history the server keeps: diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 9c0e7bdbe5..0518129d53 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -15,15 +15,19 @@ Use `sqb server --help` for service, authentication and launcher options. ## Upgrade -Upgrade the server first, then the locations and workers that connect to it. A worker newer than -its server can send a run report the server cannot read. The worker then stops with: +Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the +server after a location or worker has connected from a newer release; the server can't read what +they wrote. + +| Newer than the server | What happens | Fix | +| --- | --- | --- | +| Worker | The worker refuses the run and stops with the message below. | Upgrade the server, then rerun the run. | +| Location | The server refuses the location's publication with `API rejected the request with HTTP 400 Bad Request`. The previous snapshot stays current. | Upgrade the server, then restart the location so it publishes again. | ```text the control server is older than this worker and refused its run report; upgrade the server before its workers ``` -Upgrade the server to the worker's release or newer, then rerun the run. - ## Set retention Set how much history the server keeps: From 47ad651df5dffd966a7f8eb9f85ef17c1aa649f2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:19:45 +0000 Subject: [PATCH 48/49] refactor(ui): link through one helper so the UI can be served under a path Links and navigations now go through $lib/paths, which adds the base path the UI is built for. sqb server serves it at the root as before; a build with SQB_UI_BASE set, such as a static demo at /demo, keeps every link under that path. --- .../components/admin/access-checker.svelte | 5 ++-- .../components/admin/assignment-list.svelte | 5 ++-- ui/src/lib/components/admin/audit-mini.svelte | 5 ++-- ui/src/lib/components/admin/role-cell.svelte | 3 ++- ui/src/lib/components/app-sidebar.svelte | 10 ++++--- .../components/build-area/build-source.svelte | 3 ++- .../components/platform/problem-list.svelte | 5 ++-- .../shell/platform-event-banner.svelte | 3 ++- .../components/timeline/run-timeline.svelte | 7 ++--- ui/src/lib/lineage/job-box-node.svelte | 5 ++-- ui/src/lib/paths.ts | 10 +++++++ ui/src/lib/state/app.svelte.ts | 8 +++--- ui/src/routes/+layout.svelte | 5 ++-- ui/src/routes/+layout.ts | 3 ++- ui/src/routes/+page.svelte | 21 ++++++++------- ui/src/routes/admin/audit/+page.svelte | 5 ++-- ui/src/routes/admin/check-access/+page.svelte | 3 ++- ui/src/routes/admin/people/+page.svelte | 7 ++--- .../routes/admin/people/[name]/+page.svelte | 3 ++- ui/src/routes/admin/roles/[name]/+page.svelte | 9 ++++--- ui/src/routes/admin/roles/roles-tab.svelte | 5 ++-- ui/src/routes/admin/roles/sso-tab.svelte | 3 ++- ui/src/routes/catalog/+page.svelte | 7 ++--- .../routes/catalog/[name]/checks-tab.svelte | 5 ++-- .../routes/catalog/[name]/history-tab.svelte | 5 ++-- ui/src/routes/catalog/[name]/node-view.svelte | 13 ++++----- .../routes/catalog/[name]/overview-tab.svelte | 9 ++++--- ui/src/routes/catalog/[name]/runs-tab.svelte | 5 ++-- .../catalog/[name]/warehouse-tab.svelte | 3 ++- ui/src/routes/changes/+page.svelte | 7 ++--- ui/src/routes/checks/+page.svelte | 9 ++++--- ui/src/routes/checks/[id]/check-view.svelte | 9 ++++--- ui/src/routes/connections/+page.svelte | 3 ++- ui/src/routes/jobs/+page.svelte | 9 ++++--- ui/src/routes/jobs/[name]/job-view.svelte | 27 ++++++++++--------- ui/src/routes/jobs/[name]/launch-form.svelte | 3 ++- ui/src/routes/lineage/+page.svelte | 17 ++++++------ ui/src/routes/locations/+page.svelte | 7 ++--- ui/src/routes/locations/[id]/+page.svelte | 5 ++-- ui/src/routes/login/+page.svelte | 3 ++- ui/src/routes/maintenance/+page.svelte | 5 ++-- ui/src/routes/plan/+page.svelte | 11 ++++---- ui/src/routes/providers/+page.svelte | 5 ++-- ui/src/routes/providers/[name]/+page.svelte | 9 ++++--- ui/src/routes/runs/+page.svelte | 3 ++- ui/src/routes/runs/[id]/failures-panel.svelte | 3 ++- ui/src/routes/runs/[id]/log-tab.svelte | 3 ++- ui/src/routes/runs/[id]/node-panel.svelte | 3 ++- ui/src/routes/runs/[id]/related-runs.svelte | 5 ++-- ui/src/routes/runs/[id]/run-banners.svelte | 3 ++- ui/src/routes/runs/[id]/run-gantt.svelte | 3 ++- ui/src/routes/runs/[id]/run-header.svelte | 13 ++++----- ui/src/routes/runs/[id]/state.svelte.ts | 3 ++- ui/src/routes/runs/run-list.svelte | 15 ++++++----- ui/src/routes/runs/run-queue.svelte | 11 ++++---- ui/src/routes/triggers/+page.svelte | 11 ++++---- .../routes/triggers/[id]/sensor-panel.svelte | 5 ++-- ui/src/routes/triggers/[id]/tick-log.svelte | 5 ++-- .../routes/triggers/[id]/trigger-view.svelte | 7 ++--- ui/svelte.config.js | 3 +++ 60 files changed, 239 insertions(+), 166 deletions(-) create mode 100644 ui/src/lib/paths.ts diff --git a/ui/src/lib/components/admin/access-checker.svelte b/ui/src/lib/components/admin/access-checker.svelte index 788c70ed01..1233d68b55 100644 --- a/ui/src/lib/components/admin/access-checker.svelte +++ b/ui/src/lib/components/admin/access-checker.svelte @@ -1,4 +1,5 @@ @@ -89,7 +90,7 @@
- +

{shortName(n?.key ?? name)}

@@ -104,7 +105,7 @@ {/if}
- View in lineage + View in lineage
{#if n} @@ -143,13 +144,13 @@
{:else if tab === 'sql'} {#if n.sql} - goto(`/catalog/${m}?tab=sql`)} /> + goto(to(`/catalog/${m}?tab=sql`))} /> {:else if n.code}
{/if} {:else if tab === 'drift' && n.sql} - goto(`/catalog/${m}?tab=drift`)} /> + goto(to(`/catalog/${m}?tab=drift`))} /> {:else if tab === 'lineage'} {#if graph.nodes.length} diff --git a/ui/src/routes/catalog/[name]/overview-tab.svelte b/ui/src/routes/catalog/[name]/overview-tab.svelte index 63e874ca49..2a63d18439 100644 --- a/ui/src/routes/catalog/[name]/overview-tab.svelte +++ b/ui/src/routes/catalog/[name]/overview-tab.svelte @@ -1,4 +1,5 @@