diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index d5b686bb6c..1e79419463 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -174,9 +174,10 @@ jobs: ;; esac case "${path}" in - ui/*|clients/typescript/*|.github/workflows/verify.yml) + ui/*|clients/typescript/*|.github/workflows/verify.yml|website/src/assets/logo-light.png|website/src/assets/logo-dark.png|website/public/favicon.png) # The web UI's own checks and unit tests; the wheels embed its build. It compiles - # against the generated control client, so client changes run them too. + # against the generated control client, so client changes run them too, and it + # imports the website's logos and favicon, so moving them fails its build. ui=true ;; esac @@ -357,11 +358,6 @@ jobs: node-version: "22" cache: npm cache-dependency-path: clients/typescript/package-lock.json - - name: Check generated control client against the built native API - run: | - npm --prefix clients/typescript ci - uv run --no-sync npm --prefix clients/typescript run check - uv run --no-sync npm --prefix clients/typescript test - name: Fetch the base for the new-duplicate check run: git fetch --no-tags --depth=1 origin "$BASE_REF" env: diff --git a/Makefile b/Makefile index a2c28cbad3..9e646384b7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill \ +.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \ check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \ test-e2e-duckdb-build-incremental \ test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \ @@ -706,7 +706,7 @@ verify-quick: exit $$status -check-ci: check-dupes-new check-e2e-shards check-native-layering check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill rust-lint +check-ci: check-dupes-new check-e2e-shards check-native-layering check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint uv run ruff format --check . uv run ruff check . uv run ty check src tests @@ -714,6 +714,13 @@ check-ci: check-dupes-new check-e2e-shards check-native-layering check-native-st uv run fensu check +# The generated TypeScript control client must match the OpenAPI document of the installed sqb. +check-control-client: + npm --prefix clients/typescript ci + uv run --no-sync npm --prefix clients/typescript run check + uv run --no-sync npm --prefix clients/typescript test + + check-native-layering: uv run python -m scripts.check_native_layering diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 6d8111d0c9..3efe664d23 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -338,6 +338,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/column-lineage": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonColumnLineage"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/connection-tests": { parameters: { query?: never; @@ -1334,6 +1350,21 @@ export interface components { CliLoginStartRequest: { client_hash: string; }; + /** @description One published column lineage edge; `transform` is the compiler's transform kind, such as `direct` or `cast`. */ + ColumnLineageEdge: { + source_column: string; + source_key: string; + target_column: string; + target_key: string; + transform: string; + }; + /** @description The column lineage reaching or leaving one node, from the current snapshot; empty before one publishes lineage. */ + ColumnLineageRead: { + edges: components["schemas"]["ColumnLineageEdge"][]; + nodes: components["schemas"]["LineageNodeColumns"][]; + /** Format: uuid */ + snapshot_id?: string | null; + }; /** @description A row whose key was already stored with different content; the first row is kept. */ ConflictEvidence: { existing_sha256: string; @@ -1472,6 +1503,16 @@ export interface components { target: string; window?: null | components["schemas"]["RunWindow"]; }; + LineageColumn: { + data_type?: string | null; + name: string; + }; + /** @description A node's columns as its compiled definition infers them; `data_type` is absent when inference could not tell. */ + LineageNodeColumns: { + columns: components["schemas"]["LineageColumn"][]; + kind: string; + node_key: string; + }; /** @description One location target the person can view, with the actions allowed on at least one of its nodes. */ LocationAccess: { actions: components["schemas"]["Action"][]; @@ -1989,6 +2030,11 @@ export interface components { claim_id: string; request: components["schemas"]["RunRequest"]; }; + /** + * @description The `sqb` command a run executes; job and group runs always build. + * @enum {string} + */ + RunCommand: "build" | "test" | "seed" | "audit" | "freshness"; RunErrorText: { error_type?: string | null; invocation_id: string; @@ -2121,6 +2167,7 @@ export interface components { }; /** @description The authorized selection and pinned execution inputs are immutable once requested. */ RunRequest: { + command?: components["schemas"]["RunCommand"]; concurrency_groups: string[]; image_id?: string | null; job_key?: string | null; @@ -3379,6 +3426,46 @@ export interface operations { }; }; }; + readPersonColumnLineage: { + parameters: { + query: { + /** @description The node whose upstream and downstream column lineage to return */ + node_key: string; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ColumnLineageRead"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; requestConnectionTest: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index d999dd7501..bd4293afd5 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -11,9 +11,10 @@ use sqlbuild_orchestration::runs::models::{CommandPlan, PlannerResource}; use sqlbuild_orchestration::runs::types::CommandPlanner; use crate::adhoc::constants::{ - ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, COMMAND_DEST, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, - POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SELECTOR_DESTS, + ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, COMMAND_DEST, FRESHNESS_VERB, INTEGER_CONVERTERS, + LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, }; +use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; use crate::argv::models::{ArgvSettings, Namespace, TypeFailure, Value}; @@ -52,13 +53,29 @@ impl CommandPlanner for CliPlanner { } if verb == BUILD_VERB { job_settings_arguments(&settings)?; + } else { + command_settings_arguments(verb, &settings)?; } let mut select: Vec = selectors.remove("select").unwrap_or_default(); select.extend(selectors.remove("positional_select").unwrap_or_default()); let exclude: Vec = selectors.remove("exclude").unwrap_or_default(); + let selection: BTreeSet = match verb.as_str() { + SEED_VERB => { + let seeds: BTreeSet = resolve(resources, &select, &exclude)? + .into_iter() + .filter(|key| key.starts_with("seed:")) + .collect(); + if seeds.is_empty() { + return Err("the selectors match no seeds in the current publication".into()); + } + seeds + } + FRESHNESS_VERB => freshness_sources(resources, &select, &exclude)?, + _ => resolve(resources, &select, &exclude)?, + }; Ok(CommandPlan { verb: verb.clone(), - selection: resolve(resources, &select, &exclude)?, + selection, settings, }) } @@ -163,18 +180,69 @@ fn resolve( resources: &[PlannerResource], select: &[String], exclude: &[String], +) -> Result, String> { + let selected: BTreeSet = resolved(resources, select, exclude)?; + if selected.is_empty() { + return Err("the selectors match no nodes in the current publication".into()); + } + Ok(selected) +} + +fn resolved( + resources: &[PlannerResource], + select: &[String], + exclude: &[String], ) -> Result, String> { let graph_resources: Vec = resources.iter().filter_map(graph_resource).collect(); let graph = build_project_graph(&graph_resources); - let selected: BTreeSet = resolve_selectors(&graph, select, exclude) + Ok(resolve_selectors(&graph, select, exclude) .map_err(|error| format!("{}: {}", error.code, error.message))? .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) + .collect()) +} + +/// Sources `sqb freshness` observes: selected or upstream sources, less those the exclusions reach. +fn freshness_sources( + resources: &[PlannerResource], + select: &[String], + exclude: &[String], +) -> Result, String> { + let selected: BTreeSet = + upstream_sources(resources, &resolved(resources, select, &[])?); + let excluded: BTreeSet = if exclude.is_empty() { + BTreeSet::new() + } else { + upstream_sources(resources, &resolved(resources, exclude, &[])?) + }; + let sources: BTreeSet = selected.difference(&excluded).cloned().collect(); + if sources.is_empty() { + return Err("no sources are selected or upstream of the selection".into()); + } + Ok(sources) +} + +fn upstream_sources(resources: &[PlannerResource], keys: &BTreeSet) -> BTreeSet { + let deps: BTreeMap<&str, &[String]> = resources + .iter() + .map(|resource| (resource.key.as_str(), resource.deps.as_slice())) .collect(); - if selected.is_empty() { - return Err("the selectors match no nodes in the current publication".into()); + let mut reached: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = keys.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if reached.insert(key) { + pending.extend( + deps.get(key) + .into_iter() + .flat_map(|deps| deps.iter().map(String::as_str)), + ); + } } - Ok(selected) + reached + .into_iter() + .filter(|key| key.starts_with("source:")) + .map(str::to_owned) + .collect() } /// A published node as the selector graph sees it; keys without a kind are not selectable. diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index 81f0d9ff24..9490e619b9 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -6,6 +6,12 @@ pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "fr /// The verb whose settings must also form valid job settings. pub(crate) const BUILD_VERB: &str = "build"; +/// The verb that runs only the seeds of its selection. +pub(crate) const SEED_VERB: &str = "seed"; + +/// The verb that observes sources, including those upstream of selected models. +pub(crate) const FRESHNESS_VERB: &str = "freshness"; + /// The parser destination holding the verb itself, which is not a setting. pub(crate) const COMMAND_DEST: &str = "command"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 03f1a5aeea..c4d2fd6386 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -180,6 +180,34 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select command: &["test", "-s", "orders", "--case", "happy_path"], expected_plan: Ok((&["model:orders"], json!({"case": "happy_path"}))), }, + AdhocPlanTestCase { + description: "audit settings come from the audit parser", + command: &["audit", "-s", "orders", "--concurrency", "2"], + expected_plan: Ok((&["model:orders"], json!({"concurrency": 2}))), + }, + AdhocPlanTestCase { + description: "seed runs only the seeds its selectors reach", + command: &["seed", "+orders"], + expected_plan: Ok((&["seed:currencies"], json!({}))), + }, + AdhocPlanTestCase { + description: "seed refuses a selection without seeds, which would run nothing", + command: &["seed", "orders"], + expected_plan: Err("the selectors match no seeds"), + }, + AdhocPlanTestCase { + description: "freshness observes every source upstream of a selected model", + command: &["freshness", "order_totals", "--state", "--fail-on-stale"], + expected_plan: Ok(( + &["source:shop.raw_orders"], + json!({"fail_on_stale": true, "state": true}), + )), + }, + AdhocPlanTestCase { + description: "freshness excludes the sources an excluded model reads", + command: &["freshness", "--exclude", "orders"], + expected_plan: Err("no sources are selected or upstream of the selection"), + }, ]; let planner = adhoc_planner(); for test_case in test_cases { diff --git a/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs new file mode 100644 index 0000000000..5087b6ff53 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs @@ -0,0 +1,30 @@ +//! Turn frozen settings back into one command's flags, against the same table as the command-line parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; +use crate::argv::_helpers::table::root_parser; + +/// Produce argument tokens for `command`, never shell text; a setting the command lacks is refused. +pub fn command_settings_arguments( + command: &str, + settings: &BTreeMap, +) -> Result, String> { + let parser = root_parser() + .actions + .iter() + .flat_map(|action| &action.parsers) + .find(|(name, _)| name == command) + .map(|(_, parser)| parser) + .ok_or_else(|| format!("{command} command is unavailable"))?; + let mut arguments: Vec = Vec::new(); + for (name, value) in settings { + if let Some(argument) = setting_argument(parser, name, value)? { + arguments.push(argument); + } + } + validate_mutex(parser, &arguments)?; + Ok(arguments) +} diff --git a/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs index 46d7b8ba46..4e6defc53a 100644 --- a/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs @@ -4,27 +4,13 @@ use std::collections::BTreeMap; use serde_json::Value as Json; -use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; use crate::argv::_helpers::normalization::cursor_error; -use crate::argv::_helpers::table::root_parser; use crate::argv::constants::BUILD_COMMAND; +use crate::argv::main::command_settings_arguments::command_settings_arguments; /// Produce argument tokens, never shell text; execution identity is owned by the run request. pub fn job_settings_arguments(settings: &BTreeMap) -> Result, String> { - let parser = root_parser() - .actions - .iter() - .flat_map(|action| &action.parsers) - .find(|(name, _)| name == BUILD_COMMAND) - .map(|(_, parser)| parser) - .ok_or_else(|| "build command is unavailable".to_owned())?; - let mut arguments: Vec = Vec::new(); - for (name, value) in settings { - if let Some(argument) = setting_argument(parser, name, value)? { - arguments.push(argument); - } - } - validate_mutex(parser, &arguments)?; + let arguments = command_settings_arguments(BUILD_COMMAND, settings)?; if let Some(message) = cursor_error(|name| settings.contains_key(name)) { return Err(format!("job settings: {message}")); } diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index d456903240..10bbe8475b 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,4 +1,5 @@ pub mod cli_namespace_defaults; +pub mod command_settings_arguments; pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 1ab8657bc6..4533664c83 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -146,6 +146,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/platform/problems", get(crate::api::_helpers::resources::person_reads::problems), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/column-lineage", + get(crate::api::_helpers::resources::person_operations::column_lineage), + ) .route( "/api/control/locations/{location_id}/targets/{target}/catalog", get(crate::api::_helpers::resources::person_operations::catalog), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 0bc0faabba..d510ca2834 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -23,12 +23,13 @@ use crate::runs::models::{ use crate::runs::types::CommandPlanner; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; +use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status; use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; use crate::warehouse_metadata::models::{ - CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus, - NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, + CatalogPage, CatalogQuery, ColumnLineageRead, FingerprintObservation, FingerprintQuery, + NodeStatus, NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, }; #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/statistics", operation_id = "readPersonStatistics", @@ -240,6 +241,28 @@ pub(crate) async fn fingerprint( )) } +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/column-lineage", operation_id = "readPersonColumnLineage", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("node_key" = String, Query, description = "The node whose upstream and downstream column lineage to return")), + responses((status = 200, body = ColumnLineageRead), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn column_lineage( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + Ok(Json( + read_person_column_lineage( + &store, + (&context, principal), + (location_id, &target, &options.node_key), + default_role, + ) + .await?, + )) +} + #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/node-status", operation_id = "readPersonNodeStatus", params(("location_id" = Uuid, Path), ("target" = String, Path), ("node_key" = String, Query, description = "One exact published node key")), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 6b302ac964..ca9d679cfd 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -42,6 +42,7 @@ use crate::api::_helpers::resources::{ person_operations::statistics, person_operations::fingerprint, person_operations::node_status, + person_operations::column_lineage, person_checks::list, person_checks::history, person_checks::sql, diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs index 00a01ffbb9..3e62a01a4f 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs @@ -128,26 +128,6 @@ pub(crate) fn decide_grant( GrantDecision::Denied(missing) } -/// The node keys of one snapshot. -pub(crate) async fn snapshot_nodes( - transaction: ControlTransaction, - snapshot_id: Uuid, -) -> Result<(ControlTransaction, BTreeSet), ControlError> { - let (transaction, nodes) = query( - transaction, - "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1", - vec![Parameter::Id(snapshot_id)], - ) - .await?; - Ok(( - transaction, - nodes - .iter() - .map(|row| row["node_key"].as_str().unwrap_or_default().to_owned()) - .collect(), - )) -} - /// Actions granted on every node, or on at least one node of a non-empty set. pub(crate) fn allowed( rows: &[Row], diff --git a/crates/sqlbuild-orchestration/src/authorization/main/person_lists_target.rs b/crates/sqlbuild-orchestration/src/authorization/main/person_lists_target.rs index 638066aea2..c4d8d5644f 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/person_lists_target.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/person_lists_target.rs @@ -3,9 +3,10 @@ use uuid::Uuid; use crate::authorization::_helpers::person_grants::{ - GrantDecision, decide_grant, person_role_rows, snapshot_nodes, + GrantDecision, decide_grant, person_role_rows, }; use crate::credentials::models::{Action, DefaultRole, PermissionScope}; +use crate::snapshots::main::read_snapshot_keys::read_snapshot_keys; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; @@ -29,7 +30,7 @@ pub(crate) async fn person_lists_target( commit(transaction).await?; return Ok(false); }; - let (transaction, nodes) = snapshot_nodes( + let (transaction, nodes) = read_snapshot_keys( transaction, serde_json::from_value(snapshot_id["snapshot_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?, diff --git a/crates/sqlbuild-orchestration/src/authorization/main/read_person_access.rs b/crates/sqlbuild-orchestration/src/authorization/main/read_person_access.rs index 7e6139fe30..33bf9e9ebf 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/read_person_access.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/read_person_access.rs @@ -4,9 +4,10 @@ use std::collections::BTreeSet; use uuid::Uuid; -use crate::authorization::_helpers::person_grants::{allowed, person_role_rows, snapshot_nodes}; +use crate::authorization::_helpers::person_grants::{allowed, person_role_rows}; use crate::authorization::models::{LocationAccess, PersonAccess}; use crate::credentials::models::{Action, DefaultRole}; +use crate::snapshots::main::read_snapshot_keys::read_snapshot_keys; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; @@ -51,7 +52,7 @@ pub(crate) async fn read_person_access( let location_id: Uuid = serde_json::from_value(current["location_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; let target = current["target"].as_str().unwrap_or_default().to_owned(); - let (next, nodes) = snapshot_nodes( + let (next, nodes) = read_snapshot_keys( transaction, serde_json::from_value(current["snapshot_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs index b54f550a6c..560a5c8395 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs @@ -1,4 +1,4 @@ -//! Attached checks are recorded beside nodes, but never join the frozen execution selection. +//! Attached checks, audits and SQL tests are recorded beside nodes, but never join the frozen execution selection. use crate::runs::constants::NODE_RUNNING; use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; @@ -16,10 +16,18 @@ pub(crate) async fn check_execution_history( _ => return Err(ControlError::Invalid("check fact required".into())), }; let (transaction, nodes) = query(transaction, - "SELECT n.node_key FROM sqb_runs r JOIN sqb_snapshot_nodes n ON n.snapshot_id=r.snapshot_id \ - WHERE r.run_id=$1 AND n.node_key=$2 AND n.node_kind='check' LIMIT 1", + "SELECT n.node_kind FROM sqb_runs r JOIN sqb_snapshot_nodes n ON n.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND n.node_key=$2 AND n.node_kind IN ('check','audit','sql_test') LIMIT 1", vec![Parameter::Id(request.run_id), Parameter::Text(node_key.clone())]).await?; - if !node_key.starts_with("check:") || nodes.is_empty() { + let attached = nodes + .first() + .and_then(|row| row["node_kind"].as_str()) + .is_some_and(|kind| { + node_key + .strip_prefix(kind) + .is_some_and(|rest| rest.starts_with(':')) + }); + if !attached { return Err(ControlError::Invalid( "check is outside the frozen snapshot".into(), )); diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index a6792c1b07..3fb6c0739b 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -13,10 +13,10 @@ use crate::runs::_helpers::requests::permissions::{authorize_execution, executio use crate::runs::_helpers::requests::publication::{ publish_request, record_authored, request_retry, }; -use crate::runs::constants::{ADHOC_AUDIT_TAG, LAUNCHABLE_VERBS}; +use crate::runs::constants::ADHOC_AUDIT_TAG; use crate::runs::models::{ - AdhocCheckRequest, AdhocPermission, AdhocRunRequest, PlannerResource, RunKind, RunOrigin, - RunRequest, + AdhocCheckRequest, AdhocPermission, AdhocRunRequest, PlannerResource, RunCommand, RunKind, + RunOrigin, RunRequest, }; use crate::runs::types::CommandPlanner; use crate::snapshots::main::job_write_set::job_write_set; @@ -49,6 +49,10 @@ pub(crate) async fn frozen_adhoc_request( Ok(plan) => plan, Err(refusal) => return Ok((transaction, Err(refusal))), }; + let Some(command) = RunCommand::of_verb(&plan.verb) else { + let refusal = format!("workers do not run sqb {}", plan.verb); + return Ok((transaction, Err(refusal))); + }; let (transaction, writers) = job_write_set(transaction, snapshot.snapshot_id, &plan.selection).await?; let (transaction, images) = query( @@ -80,6 +84,7 @@ pub(crate) async fn frozen_adhoc_request( snapshot_id: snapshot.snapshot_id, image_id, kind: RunKind::Adhoc, + command, origin: RunOrigin::Manual, job_key: None, parent_run_id: None, @@ -231,11 +236,6 @@ pub(in crate::runs) async fn request_adhoc_run_in( let (transaction, frozen) = frozen_adhoc_request(transaction, (&context, planner), (&check, request.run_id)).await?; let (verb, frozen) = frozen.map_err(ControlError::Invalid)?; - if !LAUNCHABLE_VERBS.contains(&verb.as_str()) { - return Err(ControlError::Invalid(format!( - "workers do not run sqb {verb} yet; only build can launch" - ))); - } let (transaction, context) = authorize_execution( transaction, (&context, principal_id), diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index f9c69bf0c7..3b563f265e 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use uuid::Uuid; use crate::declarations::main::frozen_definitions::frozen_definitions; -use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::runs::models::{RunCommand, RunKind, RunOrigin, RunRequest}; use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; @@ -46,6 +46,7 @@ pub(crate) async fn frozen_group_children( children.push(RunRequest { run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), kind: RunKind::Job, + command: RunCommand::Build, job_key: Some(name.clone()), parent_run_id: Some(request.run_id), rerun_of_run_id: request diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 036eccdf10..d77bf9c763 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -14,8 +14,6 @@ pub(crate) const RECALL_INVOCATION_ID_BYTES: usize = 256; /// Manual requests derive their audit event ID from the run ID so retries find the original. pub(crate) const MANUAL_JOB_AUDIT_TAG: &[u8] = b"manual-job-request"; pub(crate) const RERUN_AUDIT_TAG: &[u8] = b"rerun-request"; -/// Verbs a worker runs today; other verbs parse, resolve and authorise but are refused at launch. -pub(crate) const LAUNCHABLE_VERBS: [&str; 1] = ["build"]; /// Ad-hoc requests derive their audit event ID from the run ID so retries find the original. pub(crate) const ADHOC_AUDIT_TAG: &[u8] = b"adhoc-run-request"; diff --git a/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs b/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs index 25ed102225..4874d10070 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs @@ -6,7 +6,6 @@ use crate::credentials::models::AuditContext; use crate::runs::_helpers::requests::adhoc::{ frozen_adhoc_request, may_view_target, permission_decisions, }; -use crate::runs::constants::LAUNCHABLE_VERBS; use crate::runs::models::{AdhocCheck, AdhocCheckRequest}; use crate::runs::types::CommandPlanner; use crate::store::errors::ControlError; @@ -49,16 +48,14 @@ pub async fn check_adhoc_run( let (transaction, permissions) = permission_decisions(transaction, principal_id, &frozen).await?; commit(transaction).await?; - let error = (!LAUNCHABLE_VERBS.contains(&verb.as_str())) - .then(|| format!("workers do not run sqb {verb} yet; only build can launch")); Ok(AdhocCheck { snapshot_id: Some(frozen.snapshot_id), - launchable: error.is_none() && permissions.iter().all(|permission| permission.allowed), + launchable: permissions.iter().all(|permission| permission.allowed), verb: Some(verb), selection: frozen.selection.into_iter().collect(), write_set: frozen.write_set.into_iter().collect(), settings: frozen.settings, permissions, - error, + error: None, }) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index c84da1f294..2a362d89a0 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -2,7 +2,7 @@ use crate::credentials::models::AuditContext; use crate::declarations::models::JobDefinition; -use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::runs::models::{JobRunRequest, RunCommand, RunKind, RunOrigin, RunRequest}; use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -42,6 +42,7 @@ pub(crate) async fn frozen_job_request( snapshot_id, image_id, kind, + command: RunCommand::Build, origin: if request.window.is_some() { RunOrigin::Backfill } else { diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 5394fd2776..a1b9f0310e 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -28,6 +28,51 @@ impl RunKind { } } +/// The `sqb` command a run executes; job and group runs always build. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunCommand { + #[default] + Build, + Test, + Seed, + Audit, + Freshness, +} + +impl RunCommand { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Build => "build", + Self::Test => "test", + Self::Seed => "seed", + Self::Audit => "audit", + Self::Freshness => "freshness", + } + } + + /// The command a planned verb runs, when workers run it. + #[must_use] + pub fn of_verb(verb: &str) -> Option { + [ + Self::Build, + Self::Test, + Self::Seed, + Self::Audit, + Self::Freshness, + ] + .into_iter() + .find(|command| command.as_str() == verb) + } + + /// Build is the default, so job requests serialize as they always have. + #[must_use] + pub fn is_build(&self) -> bool { + *self == Self::Build + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] pub enum RunOrigin { @@ -98,6 +143,8 @@ pub struct RunRequest { pub snapshot_id: Uuid, pub image_id: Option, pub kind: RunKind, + #[serde(default, skip_serializing_if = "RunCommand::is_build")] + pub command: RunCommand, pub origin: RunOrigin, pub job_key: Option, pub parent_run_id: Option, diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index a5d40027ba..9576f5d4a0 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -1,7 +1,7 @@ //! Deterministic tick insertion gates all run facts; retries cannot queue a second run. use crate::runs::main::queue_run::queue_run; -use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::runs::models::{RunCommand, RunKind, RunOrigin, RunRequest}; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; @@ -48,6 +48,7 @@ pub(crate) async fn persist_ticks( snapshot_id: trigger.snapshot_id, image_id: trigger.image_id.clone(), kind: trigger.run_kind, + command: RunCommand::Build, origin: RunOrigin::Schedule, job_key: Some(trigger.job_key.clone()), parent_run_id: None, diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index 8c05cf753a..cfb2475cc5 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -11,5 +11,6 @@ pub mod publish_snapshot; pub(crate) mod read_current_record; pub mod read_person_node_sql; pub(crate) mod read_snapshot_documents; +pub(crate) mod read_snapshot_keys; pub(crate) mod read_snapshot_nodes; pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_keys.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_keys.rs new file mode 100644 index 0000000000..8bd38561a8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_keys.rs @@ -0,0 +1,26 @@ +//! Read a snapshot's node keys without loading any definition. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_keys( + transaction: ControlTransaction, + snapshot_id: Uuid, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1", + vec![Parameter::Id(snapshot_id)], + ) + .await?; + let keys: BTreeSet = rows + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect(); + Ok((transaction, keys)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs index a4b23af9ff..eee2ecc3e2 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs @@ -1,11 +1,20 @@ -//! Catalog listing rows from snapshot node definitions and each node's latest build. +//! Catalog listing rows and column lineage from snapshot node definitions and published edges. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; use serde_json::Value; use uuid::Uuid; use crate::sensors::constants::SOURCE_NODE_KIND; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::SnapshotNode; -use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, CatalogRelation}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; +use crate::warehouse_metadata::constants::LINEAGE_DEFINITION_BATCH; +use crate::warehouse_metadata::models::{ + CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn, + LineageNodeColumns, +}; /// A declared value from the definition itself, its schema entry or its config, in that order. fn declared<'a>(definition: &'a Value, key: &str) -> Option<&'a Value> { @@ -78,3 +87,138 @@ pub(crate) fn catalog_node( snapshot_id, } } + +/// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node. +pub(crate) fn published_edges( + document: Option<&Value>, + nodes: &BTreeSet, +) -> Vec { + let rows = document + .and_then(|document| document.get("edges")) + .and_then(Value::as_array); + rows.into_iter() + .flatten() + .filter_map(edge_row) + .filter(|edge| nodes.contains(&edge.source_key) && nodes.contains(&edge.target_key)) + .collect() +} + +fn edge_row(row: &Value) -> Option { + let [ + source_key, + source_column, + target_key, + target_column, + transform, + ] = row.as_array()?.as_slice() + else { + return None; + }; + let text = |value: &Value| value.as_str().map(str::to_owned); + Some(ColumnLineageEdge { + source_key: text(source_key)?, + source_column: text(source_column)?, + target_key: text(target_key)?, + target_column: text(target_column)?, + transform: text(transform)?, + }) +} + +/// Edges reached upstream and downstream from any column of `node_key`, each once, in published order. +pub(crate) fn neighbourhood(edges: &[ColumnLineageEdge], node_key: &str) -> Vec { + let mut reached: BTreeSet = walk(edges, node_key, true); + reached.extend(walk(edges, node_key, false)); + reached + .into_iter() + .map(|index| edges[index].clone()) + .collect() +} + +/// The edges followed from the node's columns towards sources (`upstream`) or consumers. +fn walk(edges: &[ColumnLineageEdge], node_key: &str, upstream: bool) -> BTreeSet { + let mut reached: BTreeSet = BTreeSet::new(); + let mut by_near: HashMap<(&str, &str), Vec> = HashMap::new(); + for (index, edge) in edges.iter().enumerate() { + by_near.entry(end(edge, !upstream)).or_default().push(index); + } + let mut pending: Vec<(&str, &str)> = by_near + .keys() + .filter(|(key, _)| *key == node_key) + .copied() + .collect(); + let mut seen: BTreeSet<(&str, &str)> = pending.iter().copied().collect(); + while let Some(column) = pending.pop() { + for &index in by_near.get(&column).into_iter().flatten() { + reached.insert(index); + let far = end(&edges[index], upstream); + if seen.insert(far) { + pending.push(far); + } + } + } + reached +} + +/// An edge's source column, or its target column when `source` is false. +fn end(edge: &ColumnLineageEdge, source: bool) -> (&str, &str) { + if source { + (&edge.source_key, &edge.source_column) + } else { + (&edge.target_key, &edge.target_column) + } +} + +/// The focal node and every node an edge touches, whose definitions the read loads. +pub(crate) fn touched_nodes(node_key: &str, edges: &[ColumnLineageEdge]) -> BTreeSet { + let mut keys: BTreeSet = BTreeSet::from([node_key.to_owned()]); + for edge in edges { + keys.extend([edge.source_key.clone(), edge.target_key.clone()]); + } + keys +} + +/// The inferred columns of each loaded node, in node key order. +pub(crate) fn node_columns(nodes: &BTreeMap) -> Vec { + nodes + .iter() + .map(|(key, node)| LineageNodeColumns { + node_key: key.clone(), + kind: node.kind.clone(), + columns: definition_columns(&node.definition), + }) + .collect() +} + +fn definition_columns(definition: &Value) -> Vec { + let columns = definition.get("columns").and_then(Value::as_array); + columns + .into_iter() + .flatten() + .filter_map(|column| { + Some(LineageColumn { + name: column.get("name")?.as_str()?.to_owned(), + data_type: column + .get("type") + .and_then(Value::as_str) + .map(str::to_owned), + }) + }) + .collect() +} + +/// The definitions of `keys`, read in bounded batches because a large snapshot's definitions dominate the read. +pub(crate) async fn touched_definitions( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + keys: &BTreeSet, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + let keys: Vec<&String> = keys.iter().collect(); + let mut nodes: BTreeMap = BTreeMap::new(); + for batch in keys.chunks(LINEAGE_DEFINITION_BATCH) { + let batch: BTreeSet = batch.iter().map(|key| (*key).clone()).collect(); + let (next, loaded) = read_snapshot_nodes(transaction, snapshot_id, Some(&batch)).await?; + transaction = next; + nodes.extend(loaded); + } + Ok((transaction, nodes)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index c52cb938ed..5ff4f0a6f6 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -3,3 +3,5 @@ pub const MAX_HISTORY_RECORDS: usize = 100; pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; pub(crate) const DEFAULT_STATISTICS_RETENTION_DAYS: u16 = 90; +/// Column-lineage node definitions read per query, well under every backend's bound parameter limit. +pub(crate) const LINEAGE_DEFINITION_BATCH: usize = 1000; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index f93dd8430f..f3f9f53dc5 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -10,6 +10,7 @@ pub mod ingest_statistics; pub mod list_person_catalog_nodes; pub(crate) mod prune_statistics; pub mod read_person_catalog; +pub mod read_person_column_lineage; pub mod read_person_fingerprint; pub mod read_person_node_status; pub mod read_person_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs new file mode 100644 index 0000000000..d119d5d222 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs @@ -0,0 +1,60 @@ +//! A person reads one node's column lineage with view on every current node, as the catalog listing requires. + +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::read_snapshot_keys::read_snapshot_keys; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_metadata::_helpers::catalog_nodes::{ + neighbourhood, node_columns, published_edges, touched_definitions, touched_nodes, +}; +use crate::warehouse_metadata::models::ColumnLineageRead; + +/// Columns and edges only, never SQL or expressions, which need view.sql. +pub async fn read_person_column_lineage( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location_id, target, node_key): (Uuid, &str, &str), + default_role: DefaultRole, +) -> Result { + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + credentials, + (location_id, target), + default_role, + ) + .await?; + let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?; + let Some(snapshot) = snapshot else { + commit(transaction).await?; + return Ok(ColumnLineageRead { + snapshot_id: None, + nodes: Vec::new(), + edges: Vec::new(), + }); + }; + let (transaction, current) = read_snapshot_keys(transaction, snapshot.snapshot_id).await?; + let (transaction, documents) = + read_snapshot_documents(transaction, snapshot.snapshot_id, Some(&["lineage"])).await?; + let edges = neighbourhood( + &published_edges(documents.get("lineage"), ¤t), + node_key, + ); + let (transaction, nodes) = touched_definitions( + transaction, + snapshot.snapshot_id, + &touched_nodes(node_key, &edges), + ) + .await?; + commit(transaction).await?; + Ok(ColumnLineageRead { + snapshot_id: Some(snapshot.snapshot_id), + nodes: node_columns(&nodes), + edges, + }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index cf6e2589c2..e71dcc67ad 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -645,3 +645,35 @@ pub struct CatalogNodeBuild { pub run_id: String, pub version_hash: String, } + +/// A node's columns as its compiled definition infers them; `data_type` is absent when inference could not tell. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LineageNodeColumns { + pub node_key: String, + pub kind: String, + pub columns: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LineageColumn { + pub name: String, + pub data_type: Option, +} + +/// One published column lineage edge; `transform` is the compiler's transform kind, such as `direct` or `cast`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ColumnLineageEdge { + pub source_key: String, + pub source_column: String, + pub target_key: String, + pub target_column: String, + pub transform: String, +} + +/// The column lineage reaching or leaving one node, from the current snapshot; empty before one publishes lineage. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ColumnLineageRead { + pub snapshot_id: Option, + pub nodes: Vec, + pub edges: Vec, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs new file mode 100644 index 0000000000..68593a8c42 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs @@ -0,0 +1,232 @@ +//! Published column lineage reaches people as one node's upstream and downstream columns, on both control stores. + +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::test_types::ColumnLineageTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; + +#[test] +fn given_published_column_lineage_when_people_read_a_node_then_only_its_reachable_columns_return() { + let test_cases = [ + ColumnLineageTestCase { + description: "SQLite column lineage", + backend: Backend::Sqlite, + expected_orders_edges: vec![ + ("model:raw_orders", "model:orders"), + ("model:orders", "model:order_totals"), + ("source:shop.raw_orders", "model:raw_orders"), + ("seed:currencies", "model:orders"), + ], + expected_leaf_edges: 4, + }, + ColumnLineageTestCase { + description: "PostgreSQL column lineage", + backend: Backend::Postgres, + expected_orders_edges: vec![ + ("model:raw_orders", "model:orders"), + ("model:orders", "model:order_totals"), + ("source:shop.raw_orders", "model:raw_orders"), + ("seed:currencies", "model:orders"), + ], + expected_leaf_edges: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let node = |kind: &str, columns: serde_json::Value| SnapshotNode { + kind: kind.into(), + definition: json!({"materialization": "table", "columns": columns}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }; + let model = |columns: serde_json::Value| node("model", columns); + let mut input = snapshot(location_id); + input.nodes = [ + ( + "source:shop.raw_orders", + node("source", json!([{"name": "amount", "type": "VARCHAR"}])), + ), + ( + "seed:currencies", + node("seed", json!([{"name": "code", "type": "VARCHAR"}])), + ), + ( + "model:raw_orders", + model(json!([{"name": "amount", "type": "VARCHAR"}])), + ), + ( + "model:orders", + model(json!([{"name": "amount", "type": "DECIMAL(10,2)"}])), + ), + ( + "model:order_totals", + model(json!([{"name": "total", "type": null}])), + ), + ( + "model:inventory", + model(json!([{"name": "quantity", "type": "INTEGER"}])), + ), + ( + "model:inventory_report", + model(json!([{"name": "quantity", "type": "INTEGER"}])), + ), + ] + .into_iter() + .map(|(key, node)| (key.to_owned(), node)) + .collect(); + // A chain longer than one definition batch, so the read loads definitions in several queries. + let chain: Vec = (0..1200).map(|i| format!("model:chain_{i:04}")).collect(); + for key in &chain { + input.nodes.insert( + key.clone(), + model(json!([{"name": "id", "type": "INTEGER"}])), + ); + } + let mut edges: Vec = vec![ + json!([ + "model:raw_orders", + "amount", + "model:orders", + "amount", + "cast" + ]), + json!([ + "model:orders", + "amount", + "model:order_totals", + "total", + "aggregate" + ]), + json!([ + "model:inventory", + "quantity", + "model:inventory_report", + "quantity", + "direct" + ]), + json!([ + "model:retired_orders", + "amount", + "model:orders", + "amount", + "direct" + ]), + json!([ + "source:shop.raw_orders", + "amount", + "model:raw_orders", + "amount", + "direct" + ]), + json!(["seed:currencies", "code", "model:orders", "amount", "join"]), + ]; + edges.extend( + chain + .windows(2) + .map(|pair| json!([pair[0], "id", pair[1], "id", "direct"])), + ); + input + .documents + .insert("lineage".into(), json!({ "edges": edges })); + publish_snapshot(store, &input).await.expect("publish"); + let admin = bootstrap_user(store, &audit, "orders-admin") + .await + .expect("administrator"); + let read = |node_key: &'static str, person: Uuid| { + read_person_column_lineage( + store, + (&audit, person), + (location_id, "development", node_key), + DefaultRole::None, + ) + }; + let orders = read("model:orders", admin).await.expect("orders lineage"); + assert_eq!( + orders + .edges + .iter() + .map(|e| (e.source_key.as_str(), e.target_key.as_str())) + .collect::>(), + test_case.expected_orders_edges, + "{}: both directions, never another node's chain or a node outside the snapshot", + test_case.description + ); + assert_eq!( + json!(orders.nodes), + json!([ + {"node_key": "model:order_totals", "kind": "model", "columns": [{"name": "total", "data_type": null}]}, + {"node_key": "model:orders", "kind": "model", "columns": [{"name": "amount", "data_type": "DECIMAL(10,2)"}]}, + {"node_key": "model:raw_orders", "kind": "model", "columns": [{"name": "amount", "data_type": "VARCHAR"}]}, + {"node_key": "seed:currencies", "kind": "seed", "columns": [{"name": "code", "data_type": "VARCHAR"}]}, + {"node_key": "source:shop.raw_orders", "kind": "source", "columns": [{"name": "amount", "data_type": "VARCHAR"}]}, + ]), + "{}", + test_case.description + ); + let leaf = read("model:order_totals", admin) + .await + .expect("leaf lineage"); + assert_eq!( + leaf.edges.len(), + test_case.expected_leaf_edges, + "{}: upstream is followed transitively", + test_case.description + ); + let middle = read("model:chain_0600", admin) + .await + .expect("chain lineage"); + assert_eq!( + ( + middle.edges.len(), + middle + .nodes + .iter() + .filter(|node| node.columns.len() == 1) + .count() + ), + (1199, 1200), + "{}: every touched node's columns load across definition batches", + test_case.description + ); + let stranger = create_principal(store, &audit, "orders-stranger") + .await + .expect("person"); + assert!( + matches!( + read("model:orders", stranger).await, + Err(ControlError::Denied { .. }) + ), + "{}: lineage needs view on every node", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 9265f0c7f9..59ff175e30 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -38,6 +38,8 @@ mod test_changes; mod test_checks; #[path = "test_cli_login.rs"] mod test_cli_login; +#[path = "test_column_lineage.rs"] +mod test_column_lineage; #[path = "test_contract.rs"] mod test_contract; #[path = "test_custom_roles.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 7f73e0b6c4..3fa0ad9d3f 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -90,6 +90,13 @@ pub(crate) struct LocationWorkTestCase { pub(crate) expected_generation: i64, } +pub(crate) struct ColumnLineageTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_orders_edges: Vec<(&'static str, &'static str)>, + pub(crate) expected_leaf_edges: usize, +} + pub(crate) struct WarehouseMetadataTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 4594bf388d..a19bd5068d 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -10,7 +10,7 @@ use crate::credentials::models::{AuditContext, BuiltinRole, RoleAssignmentReques use crate::credentials::tests::location_helpers::create_location; use crate::runs::_helpers::requests::publication::publish_request; use crate::runs::models::{ - AdhocCheck, AdhocCheckRequest, AdhocRunRequest, RunKind, RunOrigin, RunRequest, + AdhocCheck, AdhocCheckRequest, AdhocRunRequest, RunCommand, RunKind, RunOrigin, RunRequest, }; use crate::runs::tests::test_types::{AdhocCommandTestCase, AdhocPeople}; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -78,6 +78,7 @@ pub(crate) fn request( snapshot_id, image_id: Some("sha256:synthetic-orders-image".into()), kind: RunKind::Adhoc, + command: RunCommand::Build, origin: RunOrigin::Manual, job_key: None, parent_run_id: None, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index 7cf277f638..24e87728a1 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -117,14 +117,14 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ expected_launch: Err(403), }, AdhocCommandTestCase { - description: "verbs workers cannot run yet are refused at launch", + description: "other commands launch with their own verb and write nothing", person: AdhocPerson::Developer, command: &["test", "model:orders"], priority: 0, reason: "Check orders", expected_check: json!({"selection": ["model:orders"], "write_set": [], "permissions": [["run.adhoc", true]], - "error": "workers do not run sqb test yet; only build can launch", "launchable": false}), - expected_launch: Err(400), + "error": null, "launchable": true}), + expected_launch: Ok(true), }, AdhocCommandTestCase { description: "a refused flag is refused before any permission", @@ -236,6 +236,24 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ ], "{backend:?}: the worker receives the exact resolved selection of the pinned publication" ); + let tested = test_cases + .iter() + .position(|test_case| test_case.command[0] == "test") + .expect("a test launch"); + let commands = rows( + store, + "SELECT request_json FROM sqb_runs WHERE run_id=$1 OR run_id=$2 ORDER BY run_id", + vec![Parameter::Id(run_ids[0]), Parameter::Id(run_ids[tested])], + ) + .await + .iter() + .map(|row| json_column(&row["request_json"]).get("command").cloned()) + .collect::>>(); + assert_eq!( + commands, + vec![None, Some(json!("test"))], + "{backend:?}: the worker receives the command, and a build request is serialized as before" + ); let audit = rows( store, "SELECT after_json FROM sqb_security_audit WHERE action='run.adhoc_requested'", @@ -255,6 +273,7 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ vec![ json!(["build", {}, "Refresh orders"]), json!(["build", {"full_refresh": true, "warehouse": "BUILD_XS"}, "Rebuild orders now"]), + json!(["test", {}, "Check orders"]), ], "{backend:?}: each launch is audited once with its command, reason and warehouse" ); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs index 0fe7151327..00f6ded985 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs @@ -1,4 +1,4 @@ -//! Check observations are claim-bound facts, never implicit runnable selection members. +//! Check, audit and SQL test observations are claim-bound facts, never implicit runnable selection members. use std::collections::BTreeMap; @@ -68,6 +68,22 @@ fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membershi }, ); } + for (key, kind) in [ + ("audit:not_null:model:orders:id", "audit"), + ("sql_test:orders_keeps_amount", "sql_test"), + ("sql_test:mislabelled", "audit"), + ] { + input.nodes.insert( + key.into(), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": key}), + raw_sql: None, + compiled_sql: Some("SELECT 1".into()), + python_source: None, + }, + ); + } input .nodes .insert("check:products".into(), input.nodes["products"].clone()); @@ -112,6 +128,9 @@ fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membershi RunChange::CheckStarted { node_key: "check:products".into(), }, + RunChange::CheckStarted { + node_key: "sql_test:mislabelled".into(), + }, RunChange::CheckStarted { node_key: "orders".into(), }, @@ -124,15 +143,17 @@ fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membershi .expect("rejected facts rolled back"), before ); - for (name, outcome) in [ - ("failed", NodeOutcome::Failed), - ("warned", NodeOutcome::Warn), - ("skipped", NodeOutcome::Skipped), - ("cancelled", NodeOutcome::Cancelled), - ("passed", NodeOutcome::Success), + for (key, outcome) in [ + ("check:failed", NodeOutcome::Failed), + ("check:warned", NodeOutcome::Warn), + ("check:skipped", NodeOutcome::Skipped), + ("check:cancelled", NodeOutcome::Cancelled), + ("check:passed", NodeOutcome::Success), + ("audit:not_null:model:orders:id", NodeOutcome::Failed), + ("sql_test:orders_keeps_amount", NodeOutcome::Success), ] { let started = report(RunChange::CheckStarted { - node_key: format!("check:{name}"), + node_key: key.into(), }); assert!( change_run(store, RunAuthority::Scheduler(&lease), &started) @@ -162,7 +183,7 @@ fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membershi .expect("idempotent start") ); let finished = report(RunChange::CheckFinished { - node_key: format!("check:{name}"), + node_key: key.into(), outcome, }); assert!( @@ -210,14 +231,14 @@ fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membershi .iter() .filter(|row| row["event_type"] == "run_check_started") .count(), - 5 + 7 ); assert_eq!( facts .iter() .filter(|row| row["event_type"] == "run_check_finished") .count(), - 6 + 8 ); for change in [ RunChange::NodeStarted { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index aaa617d9ad..f1deefee12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -1,6 +1,7 @@ //! `sqb audit`: compile the selected audits, run them with nested progress, and report. use std::collections::BTreeSet; +use std::sync::Arc; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyString}; @@ -13,6 +14,7 @@ use sqlbuild_core::audit_command::models::AuditCounts; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::{AuditExecutionResult, AuditOutcome}; +use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::boundary::python_calls::{call_with, keywords, within}; use crate::bindings::_helpers::build_progress::audit_progress::AuditProgressReporter; @@ -23,6 +25,7 @@ use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; +use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::host; @@ -35,6 +38,8 @@ use crate::bindings::_helpers::result_output::documents::{ audit_execution_json, audit_projection, write_execution_json_output, }; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; +use crate::bindings::_helpers::result_output::projection::audit_check_id; +use crate::bindings::_helpers::result_output::resource_ids::audit_identity; const EXECUTION_COMMAND: &str = "sqb audit"; @@ -54,6 +59,7 @@ fn literal_location<'py>(value: &Bound<'py, PyAny>) -> PyResult>, event_writer: Py, plan: Py, connection: Py, @@ -62,6 +68,15 @@ struct AuditCallbackRelay { #[pymethods] impl AuditCallbackRelay { fn on_result(&self, py: Python<'_>, result: Bound<'_, PyAny>) -> PyResult<()> { + if let Some(worker) = &self.worker { + let outcome = match AuditExecutionResult::read(&result)?.outcome { + AuditOutcome::Pass => NodeOutcome::Success, + AuditOutcome::Warn | AuditOutcome::Insufficient => NodeOutcome::Warn, + AuditOutcome::Skipped => NodeOutcome::Skipped, + AuditOutcome::Error => NodeOutcome::Failed, + }; + worker.finish_attached(py, &audit_check_id(&result)?, outcome)?; + } call_with( &self.event_writer.bind(py).getattr("write_build_result")?, &[ @@ -289,10 +304,15 @@ fn write_headers( fn execute<'py>( invocation: &Bound<'py, PyAny>, pipeline: &Bound<'py, PyAny>, - run: &AuditRun<'py>, + (run, worker): (&AuditRun<'py>, Option>), ) -> PyResult>>> { let py: Python<'py> = invocation.py(); let plan: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; + if let Some(worker) = &worker { + for entry in native_field(&plan, "audit_entries")?.try_iter()? { + worker.start_attached(py, &audit_identity(&entry?, "name")?)?; + } + } let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; let event_writer: Bound<'py, PyAny> = ExecutionEventWriter::new(py, None) .and_then(|value| pyo3::Bound::new(py, value)) @@ -300,6 +320,7 @@ fn execute<'py>( let relay: Bound<'py, AuditCallbackRelay> = Bound::new( py, AuditCallbackRelay { + worker, event_writer: event_writer.clone().unbind(), plan: plan.clone().unbind(), connection: run.connection.clone().unbind(), @@ -509,15 +530,25 @@ fn complete( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_audit_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + run_audit(py, request, None) +} + +/// Run one `sqb audit` request; a claimed run's audits come from its verified inputs and report to its worker. +pub(crate) fn run_audit<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, + verified: Option>, +) -> PyResult { + let (worker, discovered) = verified.unzip(); let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, &record_class(py, RecordId::AuditInvocation)?, - true, + (true, discovered), )?; let pipeline: Bound<'_, PyAny> = compile_audits(request, &invocation)?; let run: AuditRun<'_> = prepare_run(request, &invocation, &pipeline)?; write_headers(&invocation, &pipeline, run.concurrency)?; - let results = execute(&invocation, &pipeline, &run)?; + let results = execute(&invocation, &pipeline, (&run, worker))?; complete(request, &invocation, &results, &run) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 5352acf42c..f9b8a4d5c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -649,7 +649,7 @@ pub(crate) fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, &record_class(py, RecordId::CheckInvocation)?, - true, + (true, None), )?; let stream: Bound<'_, PyAny> = native_field(&invocation, "progress_stream")?; stream.call_method1("write", ("\n",))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 7c0ebc870f..4b600d1dae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -404,7 +404,7 @@ fn run_implementation<'py>( "run_debug_command" => run_debug_command(py, options())?, "run_diff_command" => run_diff_command(py, &request()?)?, "run_format_command" => run_format_command(py, options())?, - "run_freshness_command" => run_freshness_command(py, &request()?)?, + "run_freshness_command" => run_freshness_command(py, &request()?, None)?, "run_init_command" => run_init_command( py, argument(&arguments, "project_dir")?.filter(|value| !value.is_none()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 64e2663f1b..45f161496f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -68,7 +68,7 @@ pub(crate) fn resolve_adapter_connection_context<'py>( fn invocation_context<'py>( request: &Bound<'py, PyAny>, - require_connection: bool, + (require_connection, verified): (bool, Option<&Bound<'py, PyAny>>), ) -> PyResult> { let py: Python<'py> = request.py(); let requested: Bound<'py, PyAny> = native_field(request, "project_dir")?; @@ -77,10 +77,13 @@ fn invocation_context<'py>( } else { requested }; - let discovered: Bound<'py, PyAny> = discover_project_inputs( - py, - Some(keywords(py, &[("project_dir", project_dir.clone())])?), - )?; + let discovered: Bound<'py, PyAny> = match verified { + Some(discovered) => discovered.clone(), + None => discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), + )?, + }; let adapter_context: Bound<'py, PyAny> = resolve_adapter_connection_context( &discovered, &project_dir, @@ -134,12 +137,13 @@ fn context_fields<'py>( ]) } -/// `invocation_type(...)` with discovery, adapter, connection and output context. +/// `invocation_type(...)` with discovery (or a worker's `verified` inputs), adapter, connection and output context. pub(crate) fn resolve_command_invocation<'py>( request: &Bound<'py, PyAny>, invocation_type: &Bound<'py, PyAny>, + verified: Option<&Bound<'py, PyAny>>, ) -> PyResult> { - let context: InvocationContext<'py> = invocation_context(request, true)?; + let context: InvocationContext<'py> = invocation_context(request, (true, verified))?; call_with(invocation_type, &context_fields(&context)?) } @@ -147,10 +151,11 @@ pub(crate) fn resolve_command_invocation<'py>( pub(crate) fn resolve_reported_command_invocation<'py>( request: &Bound<'py, PyAny>, invocation_type: &Bound<'py, PyAny>, - require_connection: bool, + (require_connection, verified): (bool, Option<&Bound<'py, PyAny>>), ) -> PyResult> { let py: Python<'py> = request.py(); - let context: InvocationContext<'py> = invocation_context(request, require_connection)?; + let context: InvocationContext<'py> = + invocation_context(request, (require_connection, verified))?; let InvocationContext { adapter_context, use_color, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index edd96b77f1..e6f1dfb8cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -1,13 +1,16 @@ //! Frozen selection, executor callbacks and completion belong to one verified worker build. use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; use sqlbuild_core::models::ResourceKind; -use sqlbuild_orchestration::runs::models::{NodeOutcome, RunChange, RunExecution, RunOutcome}; +use sqlbuild_orchestration::runs::models::{ + NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome, +}; use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION}; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; @@ -20,17 +23,29 @@ pub(crate) struct WorkerBuild { execution: RunExecution, completed: Mutex>, checks: Mutex>>, + /// The snapshot's audits and SQL tests, each with the node keys it checks. + attached: BTreeMap>, + /// The worst attached outcome of each node, for commands that only run tests or audits. + checked: Mutex>, + /// Whether the command returned, having run every check it planned, rather than being refused or raising. + returned: AtomicBool, /// Why the worker refused the run before executing anything, reported as the run's status reason. refusal: Mutex>, } impl WorkerBuild { - pub(crate) fn new(worker: &Bound<'_, NativeOrchestrationWorker>) -> PyResult> { + pub(crate) fn new( + worker: &Bound<'_, NativeOrchestrationWorker>, + attached: BTreeMap>, + ) -> PyResult> { let state = Arc::new(Self { execution: worker.get().execution(worker.py())?, worker: worker.clone().unbind(), completed: Mutex::default(), checks: Mutex::default(), + attached, + checked: Mutex::default(), + returned: AtomicBool::new(false), refusal: Mutex::default(), }); for key in state.execution.skipped_nodes.keys() { @@ -39,6 +54,20 @@ impl WorkerBuild { Ok(state) } + pub(crate) fn command(&self) -> RunCommand { + self.execution.claim.request.command + } + + /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. + fn checks_only(&self) -> bool { + matches!(self.command(), RunCommand::Test | RunCommand::Audit) + } + + /// Record that the command returned instead of being refused or raising. + pub(crate) fn command_returned(&self) { + self.returned.store(true, Ordering::SeqCst); + } + pub(crate) fn has_work(&self) -> bool { self.execution .claim @@ -61,6 +90,24 @@ impl WorkerBuild { Ok(()) } + /// Reject writers outside the frozen write set before any executor connection. + pub(crate) fn enforce_writers( + &self, + py: Python<'_>, + keys: &[String], + ) -> Result<(), pyo3::PyErr> { + self.check_cancelled(py)?; + match keys + .iter() + .find(|key| !self.execution.claim.request.write_set.contains(*key)) + { + Some(key) => Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))), + None => Ok(()), + } + } + /// Reject an implicitly expanded write set before any executor connection or schema change. pub(crate) fn enforce_plan_writers( &self, @@ -157,17 +204,105 @@ impl WorkerBuild { "table" | "view" | "snapshot" | "custom" => "model", kind => kind, }; - let key = format!("{kind}:{name}"); - if self.execution.claim.request.selection.contains(&key) { + self.start_key(py, &format!("{kind}:{name}")) + } + + /// Report a selected node as started; the deterministic event ID makes a repeat a no-op. + pub(crate) fn start_key(&self, py: Python<'_>, key: &str) -> PyResult<()> { + self.check_cancelled(py)?; + if self.execution.claim.request.selection.contains(key) { self.worker.bind(py).get().change( py, &format!("node-started/{key}"), - RunChange::NodeStarted { node_key: key }, + RunChange::NodeStarted { + node_key: key.into(), + }, )?; } Ok(()) } + /// Report a selected node's outcome when the command reports no executor result for it. + pub(crate) fn finish_key( + &self, + py: Python<'_>, + key: &str, + outcome: NodeOutcome, + ) -> PyResult<()> { + self.complete(py, key, (outcome, None, None)) + } + + /// A test or audit the snapshot records has started; the node it attaches to starts with it. + pub(crate) fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { + self.check_cancelled(py)?; + let Some(nodes) = self.attached.get(key) else { + return Ok(()); + }; + if self + .checks + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + self.worker.bind(py).get().change( + py, + &format!("check-started/{key}"), + RunChange::CheckStarted { + node_key: key.into(), + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), None); + if self.checks_only() { + for node in nodes { + self.start_key(py, node)?; + } + } + Ok(()) + } + + /// A test or audit finished; a check the snapshot does not record is not reported. + pub(crate) fn finish_attached( + &self, + py: Python<'_>, + key: &str, + outcome: NodeOutcome, + ) -> PyResult<()> { + let Some(nodes) = self.attached.get(key) else { + return Ok(()); + }; + if outcome != NodeOutcome::Skipped { + self.start_attached(py, key)?; + } + let outcome = if outcome == NodeOutcome::Failed + && self.worker.bind(py).get().refresh_cancellation(py)? + { + NodeOutcome::Cancelled + } else { + outcome + }; + self.finish_check(py, key, outcome)?; + if outcome == NodeOutcome::Skipped { + return Ok(()); + } + let mut checked = self.checked.lock().map_err(|_| poisoned())?; + for node in nodes { + let worst = checked.get(node).map_or(outcome, |previous| { + if severity(outcome) > severity(*previous) { + outcome + } else { + *previous + } + }); + checked.insert(node.clone(), worst); + } + Ok(()) + } + /// Existing native progress callbacks publish selected-node evidence without Python relays. pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { if self.execution.claim.request.selection.contains(key) { @@ -275,9 +410,27 @@ impl WorkerBuild { Ok(()) } - /// Never call missing selected nodes successful merely because the executor returned zero. + /// Never call unseen nodes successful unless the executor returned; a refused command proves only its failures. pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; + let checks_only = self.checks_only(); + if checks_only && !cancelled { + let checked = self.checked.lock().map_err(|_| poisoned())?.clone(); + let returned = self.returned.load(Ordering::SeqCst); + for key in &self.execution.claim.request.selection { + let seen = checked.get(key).copied(); + let outcome = if returned { + Some(seen.unwrap_or(NodeOutcome::Success)) + } else { + seen.filter(|outcome| { + matches!(outcome, NodeOutcome::Failed | NodeOutcome::Cancelled) + }) + }; + if let Some(outcome) = outcome { + self.complete(py, key, (outcome, None, None))?; + } + } + } for key in &self.execution.claim.request.selection { self.complete( py, @@ -295,10 +448,11 @@ impl WorkerBuild { } let outcome = { let completed = self.completed.lock().map_err(|_| poisoned())?; - let good = completed - .values() + let outcomes: Vec = completed.values().copied().collect(); + let good = outcomes + .iter() .any(|value| matches!(value, NodeOutcome::Success | NodeOutcome::Warn)); - let bad = completed.values().any(|value| { + let bad = outcomes.iter().any(|value| { matches!( value, NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled @@ -310,7 +464,7 @@ impl WorkerBuild { RunOutcome::Partial } else if bad || exit_code != 0 { RunOutcome::Failed - } else if completed.values().any(|value| *value == NodeOutcome::Warn) { + } else if outcomes.contains(&NodeOutcome::Warn) { RunOutcome::Warn } else { RunOutcome::Success @@ -360,6 +514,20 @@ impl WorkerBuild { } } +/// A claimed run's worker build and the project inputs it verified against the frozen snapshot. +pub(crate) type VerifiedRun<'a, 'py> = (Arc, &'a Bound<'py, PyAny>); + +/// Failure outranks cancellation, then warnings, then success. +fn severity(outcome: NodeOutcome) -> u8 { + match outcome { + NodeOutcome::Skipped => 0, + NodeOutcome::Success => 1, + NodeOutcome::Warn => 2, + NodeOutcome::Cancelled => 3, + NodeOutcome::Failed => 4, + } +} + fn poisoned() -> pyo3::PyErr { PyRuntimeError::new_err("worker build outcome lock is unavailable") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index f39fbfaa59..bdb83a33d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -9,14 +9,16 @@ use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::freshness_text::main::freshness_exit_code::freshness_exit_code; use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness_text; -use sqlbuild_core::freshness_text::models::FreshnessSourceResult; +use sqlbuild_core::freshness_text::models::{FreshnessSourceResult, FreshnessStatus}; use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; +use crate::bindings::_helpers::compiled_project::snapshot_execution::VerifiedRun; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; @@ -255,6 +257,15 @@ fn observe( Ok(results) } +/// An unreadable source or an age policy error fails; an unknown or warning age warns. +fn source_outcome(result: &FreshnessSourceResult) -> NodeOutcome { + match (result.status, result.age_status.as_deref()) { + (FreshnessStatus::Error, _) | (_, Some("error")) => NodeOutcome::Failed, + (FreshnessStatus::Unknown, _) | (_, Some("warn" | "unknown")) => NodeOutcome::Warn, + _ => NodeOutcome::Success, + } +} + fn write_json_output( py: Python<'_>, payload: &str, @@ -268,8 +279,13 @@ fn write_json_output( ) } -/// Run one `sqb freshness` request. -pub(crate) fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { +/// Run one `sqb freshness` request; a claimed run observes its verified sources and reports to its worker. +pub(crate) fn run_freshness_command<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, + verified: Option>, +) -> PyResult { + let (worker, verified_inputs) = verified.unzip(); let fail_on_stale: bool = native_field(request, "fail_on_stale")?.extract()?; let fail_on_error: bool = native_field(request, "fail_on_error")?.extract()?; if fail_on_stale && !native_field(request, "compare_state")?.extract::()? { @@ -282,8 +298,10 @@ pub(crate) fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) directory if !directory.is_none() => directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'_, PyAny> = - discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?; + let discovered: Bound<'_, PyAny> = match verified_inputs { + Some(discovered) => discovered.clone(), + None => discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?, + }; let select: Bound<'_, PyAny> = native_field(request, "select")?; let exclude: Bound<'_, PyAny> = native_field(request, "exclude")?; reject_unit_test_selectors(&select, &exclude, &discovered)?; @@ -312,6 +330,11 @@ pub(crate) fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) )?), )?; let names: Vec = selected_source_names(&graph, (&select, &exclude))?; + if let Some(worker) = &worker { + for name in &names { + worker.start_key(py, &format!("source:{name}"))?; + } + } let results: Vec = if names.is_empty() { Vec::new() } else { @@ -322,6 +345,15 @@ pub(crate) fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) &names, )? }; + if let Some(worker) = &worker { + for result in &results { + worker.finish_key( + py, + &format!("source:{}", result.name), + source_outcome(result), + )?; + } + } let payload: String = freshness_json(&results).map_err(|error| PyValueError::new_err(format!("{error:?}")))?; let json_output_path: Bound<'_, PyAny> = native_field(request, "json_output_path")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index c47a4df49e..3a87a55767 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -343,6 +343,16 @@ impl ProjectColumnLineage { Ok(()) } + /// Every edge's fields in model order, from the compact records, without building Python edges. + pub(crate) fn edge_keys(&mut self, py: Python<'_>) -> PyResult> { + self.ensure_records(&LineageClasses::new(py)?)?; + Ok(self + .records + .as_ref() + .map(|records| records.keys.clone()) + .unwrap_or_default()) + } + fn positions(&self, select: impl Fn(&EdgeIndex) -> Option<&Vec>) -> Vec { self.records .as_ref() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index ffee78c649..c372c93cc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -1,5 +1,16 @@ +use sqlbuild_orchestration::runs::models::RunCommand; + /// Location actions accepted by the interpreter-side one-shot publisher. pub(crate) const PUBLISH_ACTION: &str = "publish"; /// Workers verify the pinned nodes and environment without reevaluating declarations. pub(crate) const EXECUTE_ACTION: &str = "execute"; + +/// The native implementation each frozen run command dispatches to. +pub(crate) const COMMAND_IMPLEMENTATIONS: [(RunCommand, &str); 5] = [ + (RunCommand::Build, "run_build_command"), + (RunCommand::Test, "run_test_command"), + (RunCommand::Seed, "run_seed_command"), + (RunCommand::Audit, "run_audit_command"), + (RunCommand::Freshness, "run_freshness_command"), +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 79131a3bb0..5c4301a8d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod command; mod constants; mod runtime; mod snapshot; +mod snapshot_lineage; mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index e5e7028962..6e2353b734 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; @@ -79,7 +80,11 @@ pub(crate) fn build_orchestration_snapshot( )?; } input = compile_project_roles(py, input, (&project_dir, graph.bind(py).get().graph()))?; - input.nodes = identity_nodes(project.into_pyobject(py)?.as_any(), input.nodes)?; + let project = project.into_pyobject(py)?; + if let Some(lineage) = column_lineage_document(project.as_any())? { + input.documents.insert("lineage".into(), lineage); + } + input.nodes = identity_nodes(project.as_any(), input.nodes)?; Ok(NativeOrchestrationSnapshot { input }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs new file mode 100644 index 0000000000..8152d20ee1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs @@ -0,0 +1,36 @@ +//! The project's column lineage edges, published so people can trace columns without the project. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use serde_json::{Value, json}; + +use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; +use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; +use crate::bindings::_helpers::records::construct::enum_member; +use crate::bindings::_helpers::records::ids::RecordId; + +/// FAST column lineage of every model as `[source key, column, target key, column, transform]` rows; `None` without SQL analysis. +pub(crate) fn column_lineage_document(project: &Bound<'_, PyAny>) -> PyResult> { + let py = project.py(); + let mode = enum_member(py, RecordId::ColumnLineageMode, "FAST")?; + let Some(lineage) = build_project_column_lineage(project, None, &mode, None)? else { + return Ok(None); + }; + let keys = lineage + .bind(py) + .cast::()? + .borrow_mut() + .edge_keys(py)?; + let rows: Vec = keys + .into_iter() + .map(|key| { + json!([ + format!("{}:{}", key.source_type, key.source_name), + key.source_column, + format!("model:{}", key.target_model), + key.target_column, + key.transform, + ]) + }) + .collect(); + Ok(Some(json!({ "edges": rows }))) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index b8000d071a..934cb34d5d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -1,17 +1,23 @@ //! Execute only the API's frozen request after compiling and verifying the unmodified baseline. +use std::collections::BTreeMap; use std::sync::{Arc, mpsc}; use std::time::Duration; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; +use serde_json::Value; +use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_orchestration::runs::models::{RunKind, RunOutcome, RunRequest, RunWindow}; +use sqlbuild_orchestration::runs::models::{ + RunCommand, RunKind, RunOutcome, RunRequest, RunWindow, +}; +use crate::bindings::_helpers::audit_command::command::run_audit; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; use crate::bindings::_helpers::build_command::command::run_build; use crate::bindings::_helpers::cli::argv::parse_cli_invocation; @@ -23,14 +29,51 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; +use crate::bindings::_helpers::location_command::constants::COMMAND_IMPLEMENTATIONS; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::fields::{field, replaced}; +use crate::bindings::_helpers::seed_command::command::run_seed; +use crate::bindings::_helpers::sql_test_command::test_command::run_test; + +/// Each audit and SQL test the snapshot publishes, with the node keys it checks. +fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap> { + let mut attached: BTreeMap> = BTreeMap::new(); + for (key, node) in &snapshot.input.nodes { + let definition: &Value = &node.definition; + let nodes: Vec = match node.kind.as_str() { + "audit" => definition + .get("target") + .and_then(|target| { + Some(format!( + "{}:{}", + target.get("kind")?.as_str()?, + target.get("name")?.as_str()? + )) + }) + .into_iter() + .collect(), + "sql_test" => definition + .pointer("/test/payload/expected_model_names") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(|name| format!("model:{name}")) + .collect(), + _ => continue, + }; + attached.insert(key.clone(), nodes); + } + attached +} -fn build_request<'py>( +/// The frozen command's request and warehouse scope; build carries the verified inputs in its request. +fn command_request<'py>( py: Python<'py>, - discovered: Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, arguments: &KeywordArguments<'py>, worker: &Bound<'py, NativeOrchestrationWorker>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -55,8 +98,15 @@ fn build_request<'py>( settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); } - let mut argv: Vec> = std::iter::once("build".to_owned()) - .chain(job_settings_arguments(&settings).map_err(PyValueError::new_err)?) + let command: RunCommand = frozen.command; + let flags: Vec = if command == RunCommand::Build { + job_settings_arguments(&settings) + } else { + command_settings_arguments(command.as_str(), &settings) + } + .map_err(PyValueError::new_err)?; + let mut argv: Vec> = std::iter::once(command.as_str().to_owned()) + .chain(flags) .map(|argument| PyString::new(py, &argument)) .collect(); argv.extend([ @@ -79,18 +129,41 @@ fn build_request<'py>( let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; if args.is_none() { - return Err(PyValueError::new_err("invalid frozen build settings")); + return Err(PyValueError::new_err("invalid frozen command settings")); } let Dispatch::Run(route, None) = command_dispatch(&command_facts(&args)?) else { return Err(PyValueError::new_err( - "frozen settings must dispatch a build", + "frozen settings must dispatch the frozen command", )); }; - let request = Invocation::new(&args)?.value(route.arguments[0].source)?; - Ok(( - replaced(&request, &[("discovered_inputs", discovered)])?, - warehouse_scope(&args)?, - )) + if !COMMAND_IMPLEMENTATIONS.contains(&(command, route.implementation)) { + return Err(PyValueError::new_err( + "frozen settings must dispatch the frozen command", + )); + } + let mut request = Invocation::new(&args)?.value(route.arguments[0].source)?; + if command == RunCommand::Build { + request = replaced(&request, &[("discovered_inputs", discovered.clone())])?; + } + Ok((request, warehouse_scope(&args)?)) +} + +/// Run the frozen command from the verified inputs, reporting its progress and outcomes to `worker`. +fn run_command<'py>( + py: Python<'py>, + worker: &Arc, + (request, discovered): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult { + let verified = Some((Arc::clone(worker), discovered)); + let code = match worker.command() { + RunCommand::Build => run_build(py, request, Some(Arc::clone(worker))), + RunCommand::Test => run_test(py, request, verified), + RunCommand::Seed => run_seed(py, request, verified), + RunCommand::Audit => run_audit(py, request, verified), + RunCommand::Freshness => run_freshness_command(py, request, verified), + }?; + worker.command_returned(); + Ok(code) } /// Why an ad-hoc run's `--warehouse` is refused; published jobs keep their authored warehouse. @@ -125,10 +198,10 @@ fn unconfigured_warehouse( } /// The heartbeat is native; only driver cancellation crosses the executor host boundary. -fn run_cancellable( - py: Python<'_>, +fn run_cancellable<'py>( + py: Python<'py>, worker: &Arc, - request: &Bound<'_, PyAny>, + command: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult { let state = Arc::clone(worker); let (stop, stopping) = mpsc::channel::<()>(); @@ -148,7 +221,7 @@ fn run_cancellable( } }) .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let result = run_build(py, request, Some(Arc::clone(worker))); + let result = run_command(py, worker, command); drop(stop); py.detach(|| thread.join()) .map_err(|_| PyRuntimeError::new_err("worker cancellation thread failed"))?; @@ -172,7 +245,7 @@ pub(super) fn execute_worker<'py>( ) .map_err(|error| PyValueError::new_err(error.to_string()))?, )?; - let worker = WorkerBuild::new(&owner)?; + let worker = WorkerBuild::new(&owner, attached_nodes(snapshot))?; if let Some(recall) = recall { *recall .worker @@ -194,9 +267,11 @@ pub(super) fn execute_worker<'py>( write_stderr(py, &format!("Run refused: {reason}\n"))?; return Err(PyValueError::new_err(reason)); } - let (request, scope) = build_request(py, discovered, arguments, &owner)?; + let (request, scope) = command_request(py, &discovered, arguments, &owner)?; write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?; - within(&scope, |_| run_cancellable(py, &worker, &request)) + within(&scope, |_| { + run_cancellable(py, &worker, (&request, &discovered)) + }) })(); let code = result.as_ref().copied().unwrap_or(1); let outcome = if recall.is_some() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index 86bd0533a6..933ba326cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -67,7 +67,7 @@ pub(crate) fn sql_test_check_id(result: &Bound<'_, PyAny>) -> PyResult { /// `audit_resource_id` of one audit result. pub(crate) fn audit_check_id(result: &Bound<'_, PyAny>) -> PyResult { - audit_identity(result) + audit_identity(result, "audit_name") } /// The canonical resource name and identifier of an executor result. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs index 82b98a8bc7..8f6ae011e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs @@ -80,8 +80,11 @@ pub(crate) fn sql_test_identity( ) } -/// `audit_resource_id` of one audit result. -pub(crate) fn audit_identity(result: &Bound<'_, PyAny>) -> PyResult { +/// `audit_resource_id` of one audit result or plan entry, whose name is in `audit_name_field`. +pub(crate) fn audit_identity( + result: &Bound<'_, PyAny>, + audit_name_field: &str, +) -> PyResult { let target_kind: Bound<'_, PyAny> = native_field(result, "attached_target_kind")?; let kind: Bound<'_, PyAny> = if target_kind.is_none() { native_field(result, "attachment_kind")? @@ -89,7 +92,7 @@ pub(crate) fn audit_identity(result: &Bound<'_, PyAny>) -> PyResult { target_kind }; audit_identity_of( - &native_field(result, "audit_name")?, + &native_field(result, audit_name_field)?, &kind, ( &native_field(result, "attached_target_name")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index c5938f72c1..771ad4ab1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -1,6 +1,7 @@ //! `sqb seed`: compile the selected seeds, load them, and report. use std::collections::HashMap; +use std::sync::Arc; use std::time::Instant; use pyo3::exceptions::PyKeyError; @@ -25,6 +26,7 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invo use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; +use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_entries::command_plans::build_seed_command_plan; @@ -115,6 +117,7 @@ fn connection_relay<'py>( /// Write each finished seed's row and its execution event. #[pyclass(module = "sqlbuild._native", frozen)] struct SeedCompletionRelay { + worker: Option>, stream: Py, use_color: bool, ordinals: HashMap, @@ -128,6 +131,9 @@ struct SeedCompletionRelay { impl SeedCompletionRelay { fn on_seed_complete(&self, py: Python<'_>, result: Bound<'_, PyAny>) -> PyResult<()> { let name: String = native_field(&result, "seed_name")?.extract()?; + if let Some(worker) = &self.worker { + worker.completed(&result, &format!("seed:{name}"))?; + } let mut result: Bound<'_, PyAny> = result; if let Some(projector) = &self.projector { let duration: Bound<'_, PyAny> = call_with( @@ -321,7 +327,7 @@ fn write_header( fn load_seeds<'py>( invocation: &Bound<'py, PyAny>, pipeline: &Bound<'py, PyAny>, - concurrency: i64, + (concurrency, worker): (i64, Option>), ) -> PyResult>> { let py: Python<'py> = invocation.py(); let stream: Bound<'py, PyAny> = native_field(invocation, "progress_stream")?; @@ -356,6 +362,7 @@ fn load_seeds<'py>( let completion: Bound<'py, SeedCompletionRelay> = Bound::new( py, SeedCompletionRelay { + worker, stream: stream.clone().unbind(), use_color: use_color.is_truthy()?, ordinals, @@ -435,9 +442,32 @@ fn load_seeds<'py>( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let invocation: Bound<'_, PyAny> = - resolve_command_invocation(request, &record_class(py, RecordId::SeedInvocation)?)?; + run_seed(py, request, None) +} + +/// Run one `sqb seed` request; a claimed run's seeds come from its verified inputs and stay in its write set. +pub(crate) fn run_seed<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, + verified: Option>, +) -> PyResult { + let (worker, discovered) = verified.unzip(); + let invocation: Bound<'_, PyAny> = resolve_command_invocation( + request, + &record_class(py, RecordId::SeedInvocation)?, + discovered, + )?; let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; + if let Some(worker) = &worker { + let seeds: Vec = seed_names(&pipeline)? + .into_iter() + .map(|name| format!("seed:{name}")) + .collect(); + worker.enforce_writers(py, &seeds)?; + for seed in &seeds { + worker.start_key(py, seed)?; + } + } let concurrency: i64 = resolve_concurrency( &native_field(request, "concurrency")?, &native_field(&invocation, "discovered_inputs")?, @@ -447,7 +477,7 @@ pub(crate) fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py .max(1); write_header(&invocation, &pipeline, concurrency)?; let started: Instant = Instant::now(); - let results: Vec> = load_seeds(&invocation, &pipeline, concurrency)?; + let results: Vec> = load_seeds(&invocation, &pipeline, (concurrency, worker))?; let elapsed: f64 = started.elapsed().as_secs_f64(); let mut passed: usize = 0; let mut failed: usize = 0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 777750ca28..35accda3ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; +use std::sync::Arc; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -20,6 +21,8 @@ use sqlbuild_core::test_command::main::test_outcome_status::test_outcome_status; use sqlbuild_core::test_command::models::{TestFailure, TestInspectionEntry, TestPlanWarning}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::{SqlTestExecutionResult, SqlTestOutcome}; +use sqlbuild_host::types::ResultEnum; +use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; @@ -42,6 +45,7 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_com use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; +use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; @@ -104,6 +108,13 @@ fn name_width(entries: &Bound<'_, PyAny>) -> PyResult { fn outcome_status(outcome: &Bound<'_, PyAny>) -> PyResult<&'static str> { Ok(test_outcome_status( + test_outcome(outcome)?.map_or("", ResultEnum::value), + )) +} + +/// The typed outcome of a test or step outcome member; `None` for a value this build does not know. +fn test_outcome(outcome: &Bound<'_, PyAny>) -> PyResult> { + Ok(SqlTestOutcome::from_value( &outcome.getattr("value")?.extract::()?, )) } @@ -163,6 +174,7 @@ fn entry_group(entry: &Bound<'_, PyAny>) -> PyResult { /// Forward test progress, completions and connection timing to their reporters. #[pyclass(module = "sqlbuild._native", frozen)] struct TestCallbackRelay { + worker: Option>, progress: Py, event_writer: Py, plan_output: Py, @@ -172,6 +184,9 @@ struct TestCallbackRelay { #[pymethods] impl TestCallbackRelay { fn on_test_start(&self, py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + if let Some(worker) = &self.worker { + worker.start_attached(py, &sql_test_identity(entry, "name")?)?; + } call_with( &self.progress.bind(py).getattr("on_item_start")?, &[ @@ -191,16 +206,24 @@ impl TestCallbackRelay { if group.is_empty() { group = UNKNOWN_GROUP.to_owned(); } - let resource_id: Bound<'_, PyAny> = text(py, &sql_test_identity(result, "test_name")?); + let identity: String = sql_test_identity(result, "test_name")?; + let outcome: Bound<'_, PyAny> = result.getattr("outcome")?; + let status: &str = outcome_status(&outcome)?; + if let Some(worker) = &self.worker { + let outcome = if test_outcome(&outcome)? == Some(SqlTestOutcome::Pass) { + NodeOutcome::Success + } else { + NodeOutcome::Failed + }; + worker.finish_attached(py, &identity, outcome)?; + } + let resource_id: Bound<'_, PyAny> = text(py, &identity); call_with( &self.progress.bind(py).getattr("on_item_complete")?, &[ ("group_name", text(py, &group)), ("item_name", text(py, &test_label(result, "test_name")?)), - ( - "status_text", - text(py, outcome_status(&result.getattr("outcome")?)?), - ), + ("status_text", text(py, status)), ("child_rows", expectation_rows(result)?.into_any()), ("detail", text(py, &real_model_detail(result)?)), ("error_code", result.getattr("error_code")?), @@ -481,7 +504,7 @@ fn select_changed<'py>( fn run_tests<'py>( invocation: &Bound<'py, PyAny>, pipeline: &Bound<'py, PyAny>, - (concurrency, timeout): (i64, &RunTimeout), + (concurrency, timeout, worker): (i64, &RunTimeout, Option>), ) -> PyResult>>> { let py: Python<'py> = invocation.py(); let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; @@ -555,6 +578,7 @@ fn run_tests<'py>( let relay: Bound<'py, TestCallbackRelay> = Bound::new( py, TestCallbackRelay { + worker, progress: progress.unbind(), event_writer: event_writer.clone().unbind(), plan_output: plan_output.clone().unbind(), @@ -666,11 +690,21 @@ fn complete( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + run_test(py, request, None) +} + +/// Run one `sqb test` request; a claimed run's tests come from its verified inputs and report to its worker. +pub(crate) fn run_test<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, + verified: Option>, +) -> PyResult { + let (worker, discovered) = verified.unzip(); let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, &record_class(py, RecordId::TestInvocation)?, - !inspect_only, + (!inspect_only, discovered), )?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; let project_dir: Bound<'_, PyAny> = native_field(&invocation, "effective_project_dir")?; @@ -729,7 +763,7 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py TimeoutKind::Test, )?; let results: Vec>> = - run_tests(&invocation, &pipeline, (concurrency, &timeout))?; + run_tests(&invocation, &pipeline, (concurrency, &timeout, worker))?; write_test_runtime_target( ( &native_field(&invocation, "effective_project_dir")? diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 05cafdfc59..625cdf7873 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -165,15 +165,22 @@ would be accepted. A dry run queues nothing and records no audit entry. To launch, `POST /api/control/runs/adhoc` with the same fields plus a new `run_id` and a `reason`. The response is `{"applied": true}`; sending the same `run_id` again returns `{"applied": false}` and starts nothing. The server resolves selectors against the current -publication, and the worker runs exactly the resolved nodes. The target is always the one in the -request. +publication, and the worker runs exactly the resolved nodes, from the project it verified against +that publication; files that change on the location after that check are not used. The target is +always the one in the request. -| Verb | Ad-hoc runs | +| Verb | What the worker runs | | --- | --- | -| `build` | Dry run and launch | -| `test`, `seed`, `audit`, `freshness` | Dry run only; workers can't run them yet | +| `build` | The selected nodes, with their tests and audits | +| `test` | The SQL tests of the selected models | +| `audit` | The audits of the selected nodes, against what is already built | +| `seed` | The selected seeds; other selected nodes are left out | +| `freshness` | The selected sources and every source upstream of the selection | | Any other verb | Refused | +Only `build` and `seed` write, so `test`, `audit` and `freshness` runs never wait for other runs. +A `test` or `audit` run reports each selected node with the worst result of its tests or audits. + Selectors use the CLI grammar: names, `seed:`, `source:`, `tag:`, `path:` and `test:`, with `+` for upstream or downstream nodes. Positional selectors work as on the command line. diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 9fd65cab7a..8291e2a98c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -233,6 +233,13 @@ class SnapshotEnvironmentTestCase: expected_environment_count: int +@dataclass(frozen=True) +class SnapshotLineageTestCase: + description: str + files: dict[str, str] + expected_edges: list[list[str]] + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 9c8f2db146..0692f9df09 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -342,6 +342,20 @@ class ChangedJobTestCase(WorkerExecutionTestCase): expected_runs: tuple[tuple[str, frozenset[str], dict[str, int]], ...] +@dataclass(frozen=True) +class AdhocVerbTestCase: + """An ad-hoc verb a worker runs after its prerequisites, and the run's node and check rows.""" + + description: str + prerequisites: list[list[str]] + command: list[str] + expected_selection: list[str] + expected_run_nodes: list[tuple[str, str, str]] + # Files the worker's checkout gains after it verified the final run, which must not run. + rewrites: dict[str, str] = field(default_factory=dict) + expected_status: str = "success" + + @dataclass(frozen=True) class AdhocRunTestCase: """An ad-hoc command a person checks and launches through `sqb server`, and what follows.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index f8054f779d..cd3ce745b0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -1,20 +1,24 @@ """People check and launch ad-hoc commands through `sqb server`; workers run exactly what was resolved.""" +import json from pathlib import Path from typing import Any from urllib.error import HTTPError from uuid import uuid4 +import duckdb import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( AdhocRunTestCase, + AdhocVerbTestCase, WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( duckdb_rows, scheduled_worker, security_audit_rows, + sqlite_rows, wait_for_run, worker_statements, ) @@ -185,3 +189,164 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ worker.project / "warehouse.duckdb", "SELECT count(*) FROM information_schema.tables WHERE table_name='customers'", ) == [(test_case.expected_customers_tables,)] + + +SEED: list[str] = ["seed", "--select", "regions"] +BUILD: list[str] = ["build", "--select", "customers"] +CUSTOMERS_SQL: str = ( + "MODEL (description 'Customers.', materialized table, " + "columns (id (type INTEGER, audits [not_null])));\n" + 'SELECT count(*)::INTEGER AS id FROM __seed("regions")\n' +) +SEEDS_YML: str = ( + "seeds:\n - name: regions\n description: Regions.\n columns:\n" + " - name: region\n type: {region_type}\n" + " - name: label\n type: VARCHAR\n" +) +SOURCES_YML: str = ( + "sources:\n - name: raw_orders\n description: Raw orders\n" + " schema: main\n table: {table}\n freshness:\n" + " column: loaded_at\n columns:\n" + " - name: loaded_at\n type: TIMESTAMP\n" +) +CUSTOMERS_TEST: str = ( + 'TEST (name "customers_have_ids");\n' + "WITH __seed__regions AS (SELECT 'EU' AS region, 'Europe' AS label),\n" + "__expected__customers AS (SELECT {expected} AS id)\n" +) +# Project Python is imported during discovery after the SQL and YAML were read, so once the test +# writes the flag, the final run's worker rewrites its checkout after verification, as a deploy +# landing mid-run would. +REWRITE_CHECKOUT: str = ( + "import json, pathlib\n" + "root = pathlib.Path(__file__).resolve().parent.parent\n" + "flag = root / 'rewrite.json'\n" + "if flag.exists():\n" + " for name, text in json.loads(flag.read_text()).items():\n" + " (root / name).write_text(text)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + AdhocVerbTestCase( + description="seed loads only the selected seeds, as verified", + prerequisites=[], + command=SEED, + expected_selection=["seed:regions"], + expected_run_nodes=[("seed:regions", "node", "success")], + rewrites={"seeds/seeds.yml": SEEDS_YML.format(region_type="INTEGER")}, + ), + AdhocVerbTestCase( + description="audit reports each verified audit and its node's worst result", + prerequisites=[SEED, BUILD], + command=["audit", "--select", "customers"], + expected_selection=["model:customers"], + expected_run_nodes=[ + ("audit:not_null:model:customers:id", "check", "success"), + ("model:customers", "node", "success"), + ], + rewrites={"models/customers.sql": CUSTOMERS_SQL.replace("not_null", "unique_x")}, + ), + AdhocVerbTestCase( + description="test reports each verified SQL test and the model it tests", + prerequisites=[], + command=["test", "--select", "customers"], + expected_selection=["model:customers"], + expected_run_nodes=[ + ("model:customers", "node", "success"), + ("sql_test:customers_have_ids", "check", "success"), + ], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), + AdhocVerbTestCase( + description="freshness observes the verified sources upstream of the selection", + prerequisites=[], + command=["freshness", "--select", "raw_orders"], + expected_selection=["source:raw_orders"], + expected_run_nodes=[("source:raw_orders", "node", "success")], + rewrites={"sources/raw.yml": SOURCES_YML.format(table="missing_orders")}, + ), + AdhocVerbTestCase( + description="a refused test run leaves its unchecked node skipped", + prerequisites=[], + command=["test", "--select", "customers", "--warehouse", "ANALYST_XS"], + expected_selection=["model:customers"], + expected_run_nodes=[("model:customers", "node", "skipped")], + expected_status="failed", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_ad_hoc_verb_when_the_checkout_changes_after_verification_then_the_worker_runs_the_verified_project( + test_case: AdhocVerbTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + worker_case: WorkerExecutionTestCase = WorkerExecutionTestCase( + description=test_case.description, + model_sql="SELECT 5 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(5,)], + files={ + "models/customers.sql": CUSTOMERS_SQL, + "tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=1), + "seeds/regions.csv": "region,label\nEU,Europe\n", + "seeds/seeds.yml": SEEDS_YML.format(region_type="VARCHAR"), + "sources/raw.yml": SOURCES_YML.format(table="raw_orders"), + "python/rewrite_checkout.py": REWRITE_CHECKOUT, + }, + ) + with scheduled_worker( + tmp_path, + monkeypatch, + worker_case, + server_arguments=("--password-origin", "https://control.example.com"), + ) as worker: + _ = wait_for_run(worker.api, worker.process, worker.log, job_key="annual_orders") + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + session: tuple[str, str] = person_login(worker.api, admin["name"]) + scope: dict[str, Any] = {"location_id": worker.api.location_id, "target": "development"} + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + _ = warehouse.execute("CREATE TABLE raw_orders AS SELECT now() AS loaded_at") + + def launch(command: list[str]) -> tuple[dict[str, Any], str, dict[str, Any]]: + check: dict[str, Any] = person_request( + worker.api, + "/api/control/runs/adhoc/check", + body=scope | {"command": command, "priority": 0}, + session=session, + ) + assert (check["error"], check["launchable"]) == (None, True), command + run_id: str = str(uuid4()) + assert person_request( + worker.api, + "/api/control/runs/adhoc", + body=scope + | {"run_id": run_id, "command": command, "reason": "Check verbs", "priority": 0}, + session=session, + ) == {"applied": True} + return ( + check, + run_id, + wait_for_run(worker.api, worker.process, worker.log, run_id=run_id), + ) + + for command in test_case.prerequisites: + assert launch(command)[2]["status"] == "success", (command, worker.log.read_text()) + _ = (worker.project / "rewrite.json").write_text(json.dumps(test_case.rewrites)) + check, run_id, completed = launch(test_case.command) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + assert check["selection"] == test_case.expected_selection + assert ( + sqlite_rows( + worker.control_store, + "SELECT node_key, kind, status FROM sqb_run_nodes " + f"WHERE run_id = '{run_id}' ORDER BY node_key", + ) + == test_case.expected_run_nodes + ) + for name, text in test_case.rewrites.items(): + assert (worker.project / name).read_text() == text, "the checkout changed under the run" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 7437f8ca6a..9824426203 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -35,6 +35,7 @@ SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, SnapshotIdentityTestCase, + SnapshotLineageTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, ) @@ -326,6 +327,59 @@ def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snaps ) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotLineageTestCase( + description="published column lineage follows passthroughs, casts and aggregations", + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.');\n\n" + "SELECT 1 AS order_id, 'eu' AS region, '5.00' AS amount_text\n" + ), + "models/orders.sql": ( + "MODEL (description 'Typed orders.');\n\n" + "SELECT order_id, region, CAST(amount_text AS DECIMAL(10, 2)) AS amount\n" + 'FROM __ref("stg_orders")\n' + ), + "models/region_revenue.sql": ( + "MODEL (description 'Revenue by region.');\n\n" + "SELECT region, SUM(amount) AS revenue\n" + 'FROM __ref("orders")\nGROUP BY region\n' + ), + }, + expected_edges=[ + ["model:orders", "amount", "model:region_revenue", "revenue", "aggregation"], + ["model:orders", "region", "model:region_revenue", "region", "direct"], + ["model:stg_orders", "amount_text", "model:orders", "amount", "cast"], + ["model:stg_orders", "order_id", "model:orders", "order_id", "direct"], + ["model:stg_orders", "region", "model:orders", "region", "direct"], + ], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_snapshot_document( + test_case: SnapshotLineageTestCase, + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + publication: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + edges: list[list[str]] = publication["documents"]["lineage"]["edges"] + assert sorted(edges) == test_case.expected_edges + sources: set[str] = {edge[0] for edge in edges} + targets: set[str] = {edge[2] for edge in edges} + assert sources | targets <= set(publication["nodes"]) + + @pytest.mark.parametrize( "test_case", [ diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index e0fb906e31..0e66db6d41 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -56,8 +56,11 @@ export interface ApiClient extends PlatformApi { navCounts(scope: Scope): Promise; // ---- BUILD: overview, catalog and lineage, checks, changes (types/build.ts) ---- - /** Health, attention items, metrics, build status by group and recent changes. */ - overview(scope: Scope): Promise; + /** + * Health, attention items, metrics, build status by group and recent changes; null when the + * person may view only part of the target, because its listings need view on every node. + */ + overview(scope: Scope): Promise; /** Every node in the current snapshot with its build state; also the lineage graph. */ listCatalogNodes(scope: Scope): Promise; getCatalogNode(scope: Scope, key: string): Promise; diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index d8c3d60150..43a203dd0a 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { findNode, nodeDetailOf, nodeJobsOf, toCatalogNodes, toNodeRuns } from './http-catalog'; +import { findNode, nodeDetailOf, nodeJobsOf, toCatalogNodes, toColumnLineage, toNodeRuns } from './http-catalog'; import { toCheck } from './http-checks'; type W = components['schemas']; @@ -132,3 +132,38 @@ describe('catalog node runs and jobs', () => { expect([d.recentOutcomes, d.lastDurationMs, d.jobs]).toEqual([['success', 'failed', 'skipped'], 1500, []]); }); }); + +describe('column lineage mapping', () => { + const nodes = toCatalogNodes([node('model:raw_orders', { materialization: 'view' }), node('model:orders', { materialization: 'table' })]); + const edge = (source: string, target: string, transform: string): W['ColumnLineageEdge'] => ({ source_key: 'model:raw_orders', source_column: source, target_key: 'model:orders', target_column: target, transform }); + + it('keeps a name through direct or star as a passthrough, a new name as a rename, and names any other transform', () => { + const lineage = toColumnLineage( + { + snapshot_id: 's', + nodes: [ + { node_key: 'model:raw_orders', kind: 'model', columns: [{ name: 'amount', data_type: null }] }, + { node_key: 'model:orders', kind: 'model', columns: [{ name: 'amount', data_type: 'DECIMAL(10,2)' }] }, + { node_key: 'model:retired', kind: 'model', columns: [] } + ], + edges: [edge('id', 'id', 'direct'), edge('note', 'note', 'star'), edge('cust', 'customer_id', 'direct'), edge('amount', 'amount', 'cast')] + }, + nodes + ); + expect(lineage?.edges.map(([, target, lens, label]) => [target, lens, label])).toEqual([ + ['model:orders.id', 'passthrough', undefined], + ['model:orders.note', 'passthrough', undefined], + ['model:orders.customer_id', 'rename', undefined], + ['model:orders.amount', 'transform', 'Cast'] + ]); + expect(lineage?.models.map((m) => [m.id, m.name, m.kind, m.columns])).toEqual([ + ['model:raw_orders', 'raw_orders', 'view', [{ name: 'amount', type: '' }]], + ['model:orders', 'orders', 'table', [{ name: 'amount', type: 'DECIMAL(10,2)' }]] + ]); + expect([lineage?.sql, lineage?.expressions]).toEqual([{}, {}]); + }); + + it('has nothing to show when no lineage reaches the node', () => { + expect(toColumnLineage({ snapshot_id: 's', nodes: [], edges: [] }, nodes)).toBeNull(); + }); +}); diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index d7afd7683b..15c85d72c9 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -1,6 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { shortName } from '$lib/build-area/labels'; -import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, NodeJob, NodeOutcome, NodeRunEntry, StaleReason } from './types/build'; +import { cardStatus } from '$lib/lineage/catalog-graph'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, ColumnLens, ColumnLineage, NodeJob, NodeOutcome, NodeRunEntry, StaleReason } from './types/build'; import type { NodeKind } from './types/operate'; type Wire = components['schemas']; @@ -141,3 +142,34 @@ export function nodeDetailOf( history: null }; } + +/** A kept name through `direct` or `star` is a passthrough, a new name a rename; every other transform changes the value. */ +function lensOf(edge: Wire['ColumnLineageEdge']): ColumnLens { + if (edge.transform !== 'direct' && edge.transform !== 'star') return 'transform'; + return edge.source_column === edge.target_column ? 'passthrough' : 'rename'; +} + +/** The compiler's transform kind as a label, such as `Cast`; passthroughs and renames keep the lens's own name. */ +function transformLabel(edge: Wire['ColumnLineageEdge']): string | undefined { + if (lensOf(edge) !== 'transform') return undefined; + return edge.transform === 'unknown' ? 'Transformed' : edge.transform.charAt(0).toUpperCase() + edge.transform.slice(1); +} + +/** + * Column lineage around one node, named and coloured from the catalog listing. The read carries no + * SQL or expressions, so those stay empty; null before a snapshot publishes lineage for the node. + */ +export function toColumnLineage(read: Wire['ColumnLineageRead'], nodes: CatalogNode[]): ColumnLineage | null { + if (!read.edges.length) return null; + const listed = new Map(nodes.map((n) => [n.key, n])); + return { + models: read.nodes.flatMap((n) => { + const node = listed.get(n.node_key); + const columns = n.columns.map((c) => ({ name: c.name, type: c.data_type ?? '' })); + return node ? [{ id: n.node_key, name: shortName(n.node_key), kind: node.kind, status: cardStatus(node), columns }] : []; + }), + edges: read.edges.map((e) => [`${e.source_key}.${e.source_column}`, `${e.target_key}.${e.target_column}`, lensOf(e), transformLabel(e)]), + sql: {}, + expressions: {} + }; +} diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index 4e0ca36b56..1145cff64a 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -326,6 +326,145 @@ function queuedAdhocRun(id: string) { }; } +describe('overview', () => { + function server(nodesStatus: number) { + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/nodes') && nodesStatus !== 200) return Response.json({ error: 'refused' }, { status: nodesStatus }); + if (path.endsWith('/platform/problems')) return Response.json({ scheduler_leader: 'server-1', scheduler_lease_expires_at: null, broken_locations: [], overdue_ticks: [] }); + return Response.json([]); + }) + ); + return createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + } + + it('is null for a person who may view only part of the target, with no partial counts', async () => { + expect(await server(403).overview(ORDERS)).toBeNull(); + }); + + it('still fails on any other error', async () => { + await expect(server(500).overview(ORDERS)).rejects.toMatchObject({ status: 500 }); + }); + + it('summarises the target when every listing is allowed', async () => { + expect((await server(200).overview(ORDERS))?.metrics.runs24h.total).toBe(0); + }); +}); + +describe('administration writes', () => { + const ada = { principal_id: 'p-ada', name: 'ada', created_at: '2026-11-01T00:00:00Z', disabled_at: null, has_password: true, groups: [], active_sessions: 2, last_signed_in_at: null }; + + function server(refuse = false) { + const posts: [string, unknown][] = []; + const state = { disabled: false, assignments: [{ assignment_id: 'a-1', principal_id: 'p-ada', name: 'ada', role_key: 'operator', location_id: 'loc-orders', target: 'development' }] }; + vi.stubGlobal('document', { cookie: 'sqb-csrf=token-1' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = new URL(request.url).pathname; + if (request.method === 'POST') { + const body = await request.json(); + posts.push([path, body]); + if (refuse) return Response.json({ error: 'cannot remove or disable the final installation administrator' }, { status: 403 }); + if (body === 'disable') state.disabled = true; + if (body?.role && !body.revoke) state.assignments.push({ assignment_id: 'a-2', principal_id: body.principal_id, name: 'ada', role_key: body.role, location_id: body.location_id, target: body.target }); + return Response.json({ applied: true }); + } + if (path === '/api/admin/people') return Response.json([{ ...ada, disabled_at: state.disabled ? '2026-11-09T00:00:00Z' : null }]); + if (path === '/api/admin/roles') return Response.json({ assignments: state.assignments, group_roles: [] }); + return Response.json([]); + }) + ); + return { posts, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('gives a role in one location target by person ID and returns the new assignment', async () => { + const { posts, client } = server(); + const given = await client.createRoleAssignment({ principal: 'ada', role: 'developer', scope: { locationId: 'loc-orders:development' } }); + expect(posts).toEqual([['/api/admin/roles', { principal_id: 'p-ada', role: 'developer', location_id: 'loc-orders', target: 'development', revoke: false }]]); + expect([given.id, given.scope.locationId]).toEqual(['a-2', 'loc-orders:development']); + }); + + it('gives a role everywhere with no location or target', async () => { + const { posts, client } = server(); + await client.createRoleAssignment({ principal: 'ada', role: 'viewer', scope: { locationId: null } }); + expect(posts[0][1]).toMatchObject({ location_id: null, target: null }); + }); + + it('removes an assignment by revoking exactly its person, role and scope', async () => { + const { posts, client } = server(); + await client.deleteRoleAssignment('a-1'); + expect(posts).toEqual([['/api/admin/roles', { principal_id: 'p-ada', role: 'operator', location_id: 'loc-orders', target: 'development', revoke: true }]]); + }); + + it('disables a person by ID and returns them disabled', async () => { + const { posts, client } = server(); + const account = await client.disableAccount('ada'); + expect(posts).toEqual([['/api/admin/people/p-ada', 'disable']]); + expect(account.disabled).toBe(true); + }); + + it('reports a refusal from the server', async () => { + const { client } = server(true); + await expect(client.disableAccount('ada')).rejects.toMatchObject({ status: 403 }); + }); +}); + +describe('testConnection', () => { + const record = (completed: string | null, outcome: unknown = null) => ({ + generation: 1, + requested_at: '2026-11-09T10:00:00Z', + requested_by: 'ada', + expires_at: '2026-11-09T10:05:00Z', + completed_at: completed, + request_json: { location_id: 'loc', target: 'development', request_id: 'r', reason: 'x', work: { kind: 'check_connection' } }, + result_json: outcome ? { claim_id: 'c', session_id: 's', outcome } : null + }); + + function server(answers: unknown[]) { + const calls: string[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + calls.push(request.method); + if (request.method === 'POST') return Response.json({ applied: true }); + return Response.json(answers.length > 1 ? answers.shift() : answers[0]); + }) + ); + return { calls, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('asks the location once and waits until it answers', async () => { + vi.setSystemTime(Date.parse('2026-11-09T10:00:00Z')); + const { calls, client } = server([null, record(null), record('2026-11-09T10:00:03Z', { kind: 'connection', healthy: false })]); + const result = client.testConnection(ORDERS, 'warehouse'); + await vi.advanceTimersByTimeAsync(3000); + expect(await result).toMatchObject({ outcome: 'failed', durationMs: 3000, trigger: 'manual' }); + expect(calls).toEqual(['POST', 'GET', 'GET', 'GET']); + }); + + it('reports a test replaced by a newer request instead of a result', async () => { + const { client } = server([record('2026-11-09T10:00:01Z', { kind: 'superseded' })]); + const result = client.testConnection(ORDERS, 'warehouse'); + const settled = expect(result).rejects.toThrow('A newer request replaced this connection test'); + await vi.advanceTimersByTimeAsync(1000); + await settled; + }); + + it('gives up when the location never answers', async () => { + const { calls, client } = server([null]); + const result = client.testConnection(ORDERS, 'warehouse'); + const settled = expect(result).rejects.toMatchObject({ status: 504 }); + await vi.advanceTimersByTimeAsync(61_000); + await settled; + expect(calls.length).toBe(61); + }); +}); + describe('getCatalogNode', () => { /** Answers the reads a node page makes; anything else, such as node status, is refused. */ function serve(actions: string[], sqlStatus = 200) { diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index d23ef1f429..4d6d125f21 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -2,19 +2,21 @@ import createFetch from 'openapi-fetch'; import type { components, paths } from '$control/schema.d.ts'; import { ApiError, type ApiClient } from './client'; import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks'; -import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toNodeRuns } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toNodeRuns, toColumnLineage } from './http-catalog'; import { auditMatches, toAuditEntry } from './http-audit'; import { toSnapshotChanges } from './http-changes'; -import { toLocationSummary, warehouseSyncOf } from './http-locations'; +import { manualConnectionTest, toLocationSummary, warehouseSyncOf } from './http-locations'; import { toAccount, toRoleAssignments } from './http-people'; +import { toMaintenanceHistory } from './http-maintenance'; +import { overviewOf } from './http-overview'; import { toPlatformEvents } from './http-problems'; import { roleDetailOf, toRoleDefinition } from './http-roles'; import { jobDetailOf, toJob, toTick, toTrigger, triggerDetailOf } from './http-jobs'; import { RUN_WINDOW, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, PlatformEvent, Scope, WarehouseSync } from './types'; -import type { Account, AuditEntry, AuditQuery, LocationSummary, RoleAssignment, RoleDefinition, RoleDetail } from './types/platform'; -import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build'; +import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationSummary, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; import type { AdhocRunCheck, AdhocRunRequest, Job, JobDetail, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunOutputLine, RunStatement, StatementSql } from './types/operate'; type Wire = components['schemas']; @@ -23,6 +25,7 @@ type Wire = components['schemas']; const SUPPORTED: ReadonlySet = new Set([ 'supports', 'bootstrap', + 'overview', 'changes', 'listRunRows', 'runTabCounts', @@ -49,20 +52,31 @@ const SUPPORTED: ReadonlySet = new Set([ 'getCheckEvidence', 'listCatalogNodes', 'getCatalogNode', + 'getColumnLineage', 'listSnapshotChanges', 'listAuditEntries', 'listAccounts', 'listRoleAssignments', + 'createRoleAssignment', + 'deleteRoleAssignment', + 'disableAccount', + 'enableAccount', 'listLocations', 'listPlatformEvents', 'listRoles', 'getRole', 'getWarehouseSync', + 'maintenanceHistory', + 'testConnection', 'listSnapshotNodes', 'checkAdhocRun', 'launchAdhocRun' ]); +/** How often a connection test asks for the location's answer, and the longest it waits. */ +const CONNECTION_TEST_POLL_MS = 1000; +const CONNECTION_TEST_WAIT_MS = 60_000; + /** The server's largest audit page, and how many the audit log reads before it stops looking. */ const AUDIT_PAGE = 100; const AUDIT_PAGES = 10; @@ -374,6 +388,38 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!data) throw new ApiError(errorText(error, response), response.status); return warehouseSyncOf(data); }, + /** Asks the location to connect, then waits for its answer until the request expires. */ + async testConnection(scope: Scope): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const path = { location_id: locationId, target }; + const requestId = crypto.randomUUID(); + const asked = await http.POST('/api/control/locations/{location_id}/targets/{target}/connection-tests', { + body: { request_id: requestId, reason: 'Tested from the web UI' }, + params: { path, header: { 'x-sqb-csrf': csrfToken(document.cookie) } } + }); + if (asked.response.status === 401) onUnauthorized(); + if (!asked.response.ok) throw new ApiError(errorText(asked.error, asked.response), asked.response.status); + let expiresAt = Date.now() + CONNECTION_TEST_WAIT_MS; + while (Date.now() < expiresAt) { + await new Promise((resolve) => setTimeout(resolve, CONNECTION_TEST_POLL_MS)); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}', { + params: { path: { ...path, request_id: requestId } } + }); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + const finished = manualConnectionTest(data ?? null); + if (finished) return finished; + if (data) expiresAt = Math.min(expiresAt, Date.parse(data.expires_at)); + } + throw new ApiError('The location did not answer the connection test in time; check that it is running', 504); + }, + async maintenanceHistory(scope: Scope): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/maintenance', { params: { path: { location_id: locationId, target } } }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toMaintenanceHistory(data); + }, async listPlatformEvents(): Promise { const { data, error, response } = await http.GET('/api/control/platform/problems'); if (response.status === 401) onUnauthorized(); @@ -406,11 +452,38 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return role ? roleDetailOf(role, assignments, locationId) : null; }, async listRoleAssignments(): Promise { - const { data, error, response } = await http.GET('/api/admin/roles'); + // Group members come from the people read; without it, SSO-derived roles stay unlisted. + return toRoleAssignments(await roleHoldings(), await people().catch(() => [])); + }, + async createRoleAssignment(request: RoleAssignmentRequest): Promise { + const person = await personNamed(request.principal); + const { locationId, target } = request.scope.locationId ? parseLocationKey(request.scope.locationId) : { locationId: null, target: null }; + await changeRole({ principal_id: person.principal_id, role: request.role, location_id: locationId, target, revoke: false }); + // The server answers only whether it changed anything; an assignment that already existed is found too. + const given = (await implemented.listRoleAssignments()).find( + (a) => a.source === 'manual' && a.principal.name === request.principal && a.role === request.role && a.scope.locationId === request.scope.locationId + ); + if (!given) throw new ApiError(`${request.role} was not given to ${request.principal}`, 409); + return given; + }, + async deleteRoleAssignment(assignmentId: string): Promise { + const held = (await roleHoldings()).assignments.find((a) => a.assignment_id === assignmentId); + if (!held) throw new ApiError('That role assignment no longer exists', 404); + await changeRole({ principal_id: held.principal_id, role: held.role_key, location_id: held.location_id ?? null, target: held.target ?? null, revoke: true }); + }, + disableAccount: (name: string): Promise => changePerson(name, 'disable'), + enableAccount: (name: string): Promise => changePerson(name, 'enable'), + async getColumnLineage(scope: Scope, ref: string): Promise { + const nodes = await implemented.listCatalogNodes(scope); + const node = findNode(nodes, ref); + if (!node) return null; + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/column-lineage', { + params: { path: { location_id: locationId, target }, query: { node_key: node.key } } + }); if (response.status === 401) onUnauthorized(); if (!data) throw new ApiError(errorText(error, response), response.status); - // Group members come from the people read; without it, SSO-derived roles stay unlisted. - return toRoleAssignments(data, await people().catch(() => [])); + return toColumnLineage(data, nodes); }, async getCatalogNode(scope: Scope, ref: string): Promise { const nodes = await implemented.listCatalogNodes(scope); @@ -434,6 +507,24 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt // Status and SQL have their own permissions; a refusal leaves that part out rather than failing the page. return nodeDetailOf(nodes, node, status.data ?? null, sql?.data ?? null, checks.map(toCheck), { runs: runs?.data ? toNodeRuns(runs.data) : null, jobs }); }, + async overview(scope: Scope): Promise { + // A week of publications for the models metric; the server returns at most 100. + const reads = Promise.all([ + recentRuns(scope), + implemented.listChecks(scope), + implemented.listCatalogNodes(scope), + implemented.listSnapshotChanges(scope, { limit: 100 }), + implemented.listPlatformEvents() + ]); + // Every listing needs view on all of the target, so a partial viewer gets no counts at all. + const read = await reads.catch((error: unknown) => { + if (error instanceof ApiError && error.status === 403) return null; + throw error; + }); + if (!read) return null; + const [runs, checks, nodes, changes, events] = read; + return overviewOf({ runs, checks, nodes, changes, events, now: Date.now() }); + }, listChecks: async (scope: Scope): Promise => (await checkResults(scope)).map(toCheck), getCheck: async (scope: Scope, id: string): Promise => checkDetailOf(await checkResults(scope, id)), async getCheckSql(scope: Scope, checkId: string): Promise { @@ -544,6 +635,37 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return data; } + async function roleHoldings() { + const { data, error, response } = await http.GET('/api/admin/roles'); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return data; + } + + async function changeRole(body: Wire['RoleAssignmentRequest']) { + const { error, response } = await http.POST('/api/admin/roles', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + } + + async function personNamed(name: string) { + const person = (await people()).find((p) => p.name === name); + if (!person) throw new ApiError(`Unknown person ${name}`, 404); + return person; + } + + /** The server refuses to disable the last enabled installation administrator. */ + async function changePerson(name: string, change: 'disable' | 'enable'): Promise { + const person = await personNamed(name); + const { error, response } = await http.POST('/api/admin/people/{principal_id}', { + body: change, + params: { path: { principal_id: person.principal_id }, header: { 'x-sqb-csrf': csrfToken(document.cookie) } } + }); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + return toAccount(await personNamed(name)); + } + async function people() { const { data, error, response } = await http.GET('/api/admin/people'); if (response.status === 401) onUnauthorized(); diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index 4136768b3e..cc674f3c05 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -35,6 +35,24 @@ export function connectionTestOf(observations: Wire['LocationHealthObservation'] return { outcome: ok ? 'ok' : 'failed', at: latest.received_at, durationMs: 0, trigger: 'publish', by: null, message: ok ? 'Connected after publishing' : 'Could not connect after publishing' }; } +/** + * A finished manual connection test, or null while the location hasn't answered. The location + * reports only whether it connected; `sqb debug` in the location shows why it couldn't. + */ +export function manualConnectionTest(record: Wire['LocationWorkRecord'] | null): ConnectionTest | null { + const outcome = record?.result_json?.outcome; + if (!record?.completed_at || !outcome) return null; + if (outcome.kind !== 'connection') throw new Error('A newer request replaced this connection test; test again'); + return { + outcome: outcome.healthy ? 'ok' : 'failed', + at: record.completed_at, + durationMs: Date.parse(record.completed_at) - Date.parse(record.requested_at), + trigger: 'manual', + by: null, + message: outcome.healthy ? 'Connected to the warehouse' : 'Could not connect to the warehouse; run sqb debug in the location to see why' + }; +} + /** `observations` is null when the person may not list the whole target. */ export function toLocationSummary(location: Location, observations: Wire['LocationHealthObservation'][] | null, published: Wire['PublishedSnapshot'] | null, now: number): LocationSummary { const known = observations ? healthOf(observations, now) : { health: null, liveInstances: 0, lastSeenAt: null }; diff --git a/ui/src/lib/api/http-maintenance.test.ts b/ui/src/lib/api/http-maintenance.test.ts new file mode 100644 index 0000000000..c94783fd59 --- /dev/null +++ b/ui/src/lib/api/http-maintenance.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest'; +import type { components } from '$control/schema.d.ts'; +import { formatRelation } from '$lib/platform/labels'; +import { oldNameStatus } from '$lib/platform/maintenance'; +import { toMaintenanceHistory } from './http-maintenance'; + +type W = components['schemas']; + +function migration(eventId: string, originSchema: string | null, createdAt: string): W['WarehouseMigrationRecord'] { + return { + state_schema: 'sqlbuild_state', + event_id: eventId, + target_name: 'development', + origin_model: null, + origin_schema: originSchema, + origin_name: `${eventId}_old`, + destination_model: `model:${eventId}`, + destination_schema: 'analytics', + destination_name: eventId, + origin_version_hash: 'v1', + discovery: 'manual', + decision: 'renamed', + run_id: 'run-1', + created_at: createdAt + }; +} + +function step(migrationEventId: string, eventType: string, createdAt: string, facts: Partial = {}): W['WarehouseOldNameRecord'] { + return { + state_schema: 'sqlbuild_state', + event_id: `${migrationEventId}-${eventType}`, + event_type: eventType, + migration_event_id: migrationEventId, + destination_model: `model:${migrationEventId}`, + old_schema: 'analytics', + old_name: `${migrationEventId}_old`, + new_schema: 'analytics', + new_name: migrationEventId, + run_id: `run-${eventType}`, + created_at: createdAt, + ...facts + }; +} + +describe('maintenance history mapping', () => { + // Moves arrive newest first; old-name steps arrive oldest first, interleaved across moves. + const history = toMaintenanceHistory({ + migrations: [migration('customers', 'analytics', '2026-11-08T10:00:00Z'), migration('orders', null, '2026-11-07T10:00:00Z')], + old_name_views: [ + step('orders', 'required', '2026-11-07T10:00:00Z', { view_retention: '30d' }), + step('customers', 'required', '2026-11-08T10:00:00Z', { view_retention: 'false' }), + step('orders', 'view_created', '2026-11-07T10:01:00Z', { expires_at: '2026-12-07T10:01:00Z', column_aliases: '{"amount":"total","id":"order_id"}', grants_copied: '["GRANT SELECT"]' }), + step('orders', 'view_dropped', '2026-11-08T11:00:00Z', { drop_reason: 'Removed by hand' }) + ], + janitor_events: [] + }); + + it('keeps one view per move, in the order of the moves, with its steps oldest first', () => { + expect(history.oldNameViews.map((v) => [v.migrationEventId, v.steps.map((s) => s.eventType)])).toEqual([ + ['customers', ['required']], + ['orders', ['required', 'view_created', 'view_dropped']] + ]); + }); + + it('takes each fact from the step that recorded it', () => { + const orders = history.oldNameViews[1]; + expect([orders.retention, orders.expiresAt, orders.dropReason, orders.grantsCopied]).toEqual(['30d', '2026-12-07T10:01:00Z', 'Removed by hand', 1]); + expect(orders.columnAliases).toEqual([ + { old: 'amount', new: 'total' }, + { old: 'id', new: 'order_id' } + ]); + expect(oldNameStatus(orders, Date.parse('2026-11-09T00:00:00Z'))).toBe('dropped'); + expect(oldNameStatus(history.oldNameViews[0], Date.parse('2026-11-09T00:00:00Z'))).toBe('not_kept'); + }); + + it('shows a relation recorded without its schema by its bare name', () => { + expect(history.migrations.map((m) => formatRelation(m.origin))).toEqual(['analytics.customers_old', 'orders_old']); + }); +}); diff --git a/ui/src/lib/api/http-maintenance.ts b/ui/src/lib/api/http-maintenance.ts new file mode 100644 index 0000000000..8a82971247 --- /dev/null +++ b/ui/src/lib/api/http-maintenance.ts @@ -0,0 +1,79 @@ +import type { components } from '$control/schema.d.ts'; +import type { JanitorEvent, MaintenanceHistory, MigrationEvent, OldNameEventType, OldNameView, RelationName } from './types'; + +type Wire = components['schemas']; + +const relation = (database: string | null | undefined, schema: string | null | undefined, name: string): RelationName => ({ database: database ?? null, schema: schema ?? '', name }); + +function toMigration(w: Wire['WarehouseMigrationRecord']): MigrationEvent { + return { + eventId: w.event_id, + targetName: w.target_name ?? '', + originModel: w.origin_model ?? null, + origin: relation(w.origin_database, w.origin_schema, w.origin_name), + destinationModel: w.destination_model, + destination: relation(w.destination_database, w.destination_schema, w.destination_name), + discovery: w.discovery as MigrationEvent['discovery'], + decision: w.decision as MigrationEvent['decision'], + runId: w.run_id, + createdAt: w.created_at + }; +} + +/** Stored as compact JSON of `{old: new}`. */ +function columnAliases(text: string | null | undefined): OldNameView['columnAliases'] { + return text ? Object.entries(JSON.parse(text) as Record).map(([old, renamed]) => ({ old, new: renamed })) : []; +} + +/** Stored as a JSON list of the grant statements copied to the view. */ +const grantCount = (text: string | null | undefined): number | null => (text ? (JSON.parse(text) as string[]).length : null); + +/** One view per move, from its append-only steps (oldest first); each fact comes from the newest step that records it. */ +function toOldNameViews(rows: Wire['WarehouseOldNameRecord'][], migrationOrder: string[]): OldNameView[] { + const steps = new Map(); + for (const row of rows) steps.set(row.migration_event_id, [...(steps.get(row.migration_event_id) ?? []), row]); + const rank = (id: string) => (migrationOrder.includes(id) ? migrationOrder.indexOf(id) : migrationOrder.length); + return [...steps.entries()] + .sort(([a], [b]) => rank(a) - rank(b)) + .map(([migrationEventId, recorded]) => { + const first = recorded[0]; + const newest = (key: K) => [...recorded].reverse().find((r) => r[key] != null)?.[key] ?? null; + return { + migrationEventId, + destinationModel: first.destination_model, + old: relation(first.old_database, first.old_schema, first.old_name), + new: relation(first.new_database, first.new_schema, first.new_name), + retention: (newest('view_retention') as string | null) ?? '', + archiveName: newest('archive_name') as string | null, + columnAliases: columnAliases(newest('column_aliases') as string | null), + grantsCopied: grantCount(newest('grants_copied') as string | null), + expiresAt: newest('expires_at') as string | null, + dropReason: newest('drop_reason') as string | null, + steps: recorded.map((r) => ({ eventType: r.event_type as OldNameEventType, at: r.created_at, runId: r.run_id })) + }; + }); +} + +function toJanitorEvent(w: Wire['WarehouseJanitorRecord']): JanitorEvent { + return { + eventId: w.event_id, + eventType: w.event_type as JanitorEvent['eventType'], + occurredAt: w.occurred_at, + runId: w.run_id, + relationSchema: w.relation_schema, + relationType: (w.relation_type ?? null) as JanitorEvent['relationType'], + originalQualifiedName: w.original_qualified_name ?? null, + archiveQualifiedName: w.archive_qualified_name, + archivedAt: w.archived_at + }; +} + +/** Synced maintenance history; the server lists the newest 100 moves and clean-ups. */ +export function toMaintenanceHistory(wire: Wire['MaintenanceHistory']): MaintenanceHistory { + const migrations = wire.migrations.map(toMigration); + return { + migrations, + oldNameViews: toOldNameViews(wire.old_name_views, migrations.map((m) => m.eventId)), + janitorEvents: wire.janitor_events.map(toJanitorEvent) + }; +} diff --git a/ui/src/lib/api/http-overview.test.ts b/ui/src/lib/api/http-overview.test.ts new file mode 100644 index 0000000000..ec1b730f72 --- /dev/null +++ b/ui/src/lib/api/http-overview.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; +import { overviewOf } from './http-overview'; +import type { CatalogNode, Check, NodeChange, PlatformEvent, Run, SnapshotChange } from './types'; + +const now = Date.parse('2026-11-09T12:00:00Z'); +const hoursAgo = (h: number) => new Date(now - h * 3_600_000).toISOString(); + +function run(id: string, status: Run['status'], queuedHoursAgo: number, endedHoursAgo: number | null, parentRunId: string | null = null): Run { + return { id, status, parentRunId, jobKey: 'orders', kind: 'job', statusReason: null, queuedAt: hoursAgo(queuedHoursAgo), startedAt: hoursAgo(queuedHoursAgo), endedAt: endedHoursAgo === null ? null : hoursAgo(endedHoursAgo) } as Run; +} + +const node = (key: string, kind: CatalogNode['kind']) => ({ key, kind }) as CatalogNode; +const check = (id: string, outcome: string | null) => ({ id, latest: outcome ? { outcome } : null }) as unknown as Check; +const change = (publishedHoursAgo: number, nodes: [NodeChange['op'], string][]) => + ({ snapshotId: `s-${publishedHoursAgo}`, publishedAt: hoursAgo(publishedHoursAgo), nodes: nodes.map(([op, key]) => ({ key, op })) }) as SnapshotChange; +const event = (id: string, active: boolean) => ({ id, active }) as PlatformEvent; + +describe('overview from live reads', () => { + // Newest queued first, as the runs listing returns them. + const runs = [ + run('active', 'running', 1, null), + run('child-failed', 'failed', 2, 2, 'group'), + run('failed', 'failed', 3, 2.5), + run('presumed', 'presumed_failed', 4, 3.5), + run('partial', 'partial', 5, 4.5), + run('failed-4th', 'failed', 6, 5.5), + run('ok', 'success', 7, 0.5), + run('stuck', 'unresponsive', 8, null), + run('warn', 'warn', 23.5, 23), + run('yesterday-failed', 'failed', 25, 24.5) + ]; + const overview = overviewOf({ + runs, + checks: [check('a', 'pass'), check('b', 'fail'), check('c', 'error'), check('d', 'warn'), check('e', null)], + nodes: [node('model:orders', 'table'), node('model:totals', 'view'), node('model:events', 'incremental'), node('source:raw', 'source'), node('seed:regions', 'seed')], + changes: [change(1, [['added', 'model:orders'], ['added', 'seed:regions'], ['removed', 'model:old']]), change(24 * 8, [['added', 'model:totals']])], + events: [event('scheduler', true), event('cleared', false)], + now + }); + + it('counts top-level runs queued in the last 24 hours', () => { + expect(overview.metrics.runs24h).toEqual({ total: 8, success: 1, warn: 1, partial: 1, failed: 3, active: 2 }); + }); + + it('lists the three newest failed runs that ended today, then unresponsive runs and active platform events', () => { + expect(overview.attention.map((i) => (i.kind === 'platform_event' ? `event:${i.event.id}` : 'run' in i ? `${i.kind}:${i.run.id}` : i.kind))).toEqual([ + 'run_problem:failed', + 'run_problem:presumed', + 'run_problem:partial', + 'run_unresponsive:stuck', + 'event:scheduler' + ]); + }); + + it('counts models and this week’s model changes only', () => { + expect(overview.metrics.models).toEqual({ total: 3, addedThisWeek: 1, removedThisWeek: 1 }); + }); + + it('counts check outcomes, with errors as failing', () => { + expect(overview.metrics.checks).toEqual({ total: 5, passing: 1, warning: 1, failing: 2 }); + }); + + it('names the run that ended last, not the newest queued', () => { + expect(overview.latestRun?.id).toBe('ok'); + }); + + it('leaves parts without a server read empty instead of zero', () => { + expect([overview.metrics.stale, overview.metrics.cost, overview.groups]).toEqual([null, null, null]); + }); +}); diff --git a/ui/src/lib/api/http-overview.ts b/ui/src/lib/api/http-overview.ts new file mode 100644 index 0000000000..9f2c47ce48 --- /dev/null +++ b/ui/src/lib/api/http-overview.ts @@ -0,0 +1,67 @@ +import { checkCountsOf } from './http-catalog'; +import type { AttentionItem, AttentionRun, CatalogNode, Check, Overview, PlatformEvent, Run, SnapshotChange } from './types'; + +const DAY = 24 * 60 * 60 * 1000; +const MODEL_KINDS = new Set(['view', 'table', 'incremental']); +const PROBLEM_STATUSES = new Set(['failed', 'partial', 'presumed_failed']); +const ACTIVE_STATUSES = new Set(['queued', 'launched', 'running', 'unresponsive']); +/** The failed runs the health banner lists; the runs page has the rest. */ +const PROBLEM_RUNS = 3; +const RECENT_CHANGES = 4; + +export type OverviewInputs = { + /** Listed runs, newest first; the listing holds the newest 1,000, so a busier day undercounts. */ + runs: Run[]; + checks: Check[]; + nodes: CatalogNode[]; + /** Publications, newest first. */ + changes: SnapshotChange[]; + events: PlatformEvent[]; + now: number; +}; + +const attentionRun = ({ id, jobKey, kind, status, statusReason, startedAt, endedAt }: Run): AttentionRun => ({ id, jobKey, kind, status, statusReason, startedAt, endedAt }); + +/** + * The overview from reads the person already has. The server has no project-wide build state, cost + * or group read yet, so stale counts and group status are null and cost is left out; the page hides them. + */ +export function overviewOf({ runs, checks, nodes, changes, events, now }: OverviewInputs): Overview { + const topLevel = runs.filter((r) => !r.parentRunId); + const recent = topLevel.filter((r) => Date.parse(r.queuedAt) >= now - DAY); + const count = (statuses: string[]) => recent.filter((r) => statuses.includes(r.status)).length; + const attention: AttentionItem[] = [ + ...topLevel + .filter((r) => r.endedAt && Date.parse(r.endedAt) >= now - DAY && PROBLEM_STATUSES.has(r.status)) + .slice(0, PROBLEM_RUNS) + .map((r): AttentionItem => ({ kind: 'run_problem', run: attentionRun(r) })), + ...topLevel.filter((r) => r.status === 'unresponsive').map((r): AttentionItem => ({ kind: 'run_unresponsive', run: attentionRun(r), lastHeartbeatAt: null })), + ...events.filter((e) => e.active).map((event): AttentionItem => ({ kind: 'platform_event', event })) + ]; + const thisWeek = changes.filter((c) => Date.parse(c.publishedAt) >= now - 7 * DAY).flatMap((c) => c.nodes); + const latest = topLevel.filter((r) => r.endedAt).sort((a, b) => Date.parse(b.endedAt!) - Date.parse(a.endedAt!))[0]; + return { + metrics: { + models: { + total: nodes.filter((n) => MODEL_KINDS.has(n.kind)).length, + addedThisWeek: thisWeek.filter((c) => c.op === 'added' && c.key.startsWith('model:')).length, + removedThisWeek: thisWeek.filter((c) => c.op === 'removed' && c.key.startsWith('model:')).length + }, + runs24h: { + total: recent.length, + success: count(['success']), + warn: count(['warn']), + partial: count(['partial']), + failed: count(['failed', 'presumed_failed']), + active: recent.filter((r) => ACTIVE_STATUSES.has(r.status)).length + }, + checks: checkCountsOf(checks), + stale: null, + cost: null + }, + attention, + groups: null, + recentChanges: changes.slice(0, RECENT_CHANGES), + latestRun: latest ? { id: latest.id, status: latest.status, endedAt: latest.endedAt } : null + }; +} diff --git a/ui/src/lib/api/mock/build.ts b/ui/src/lib/api/mock/build.ts index ba93c80a50..907039815c 100644 --- a/ui/src/lib/api/mock/build.ts +++ b/ui/src/lib/api/mock/build.ts @@ -218,7 +218,7 @@ export class MockBuild { // ---- overview ---- - async overview(scope: Scope): Promise { + async overview(scope: Scope): Promise { const states = this.#states(scope); const checks = this.#checks(scope, states); const nodes = SNAPSHOT_NODES.map((sn) => this.#catalogNode(states.get(sn.key)!, checks)); diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 7f814a49db..2d73379a8d 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -942,7 +942,15 @@ export class MockOperate { const resolution = resolveSelection(parts.select, parts.exclude); if (resolution.unknown.length) return refused(`no nodes match selector '${resolution.unknown[0]}'`); if (!resolution.nodes.length) return refused('the selection resolves to no nodes'); - const writable = resolution.nodes.filter((n) => snapshotNode(n)?.kind !== 'source'); + const isSource = (n: string) => snapshotNode(n)?.kind === 'source'; + const selection = + parts.verb === 'seed' + ? resolution.nodes.filter((n) => snapshotNode(n)?.kind === 'seed') + : parts.verb === 'freshness' + ? [...new Set(resolution.nodes.flatMap((n) => [n, ...ancestors(n)]).filter(isSource))] + : resolution.nodes; + if (!selection.length) return refused(parts.verb === 'seed' ? 'the selectors match no seeds in the current publication' : 'no sources are selected or upstream of the selection'); + const writable = selection.filter((n) => !isSource(n)); const writeSet = parts.verb === 'build' ? writable.filter((n) => !(parts.flags['--no-load'] && snapshotNode(n)?.kind === 'loader')) @@ -951,7 +959,7 @@ export class MockOperate { : []; const scopeTerms = ADHOC_SCOPE[this.#ctx.role()]; const granted = scopeTerms === null ? null : new Set(writeSetOf(scopeTerms.length ? scopeTerms : ['__none__'], [])); - const covered = this.#ctx.can('run.adhoc') && (granted === null || resolution.nodes.every((n) => granted.has(n))); + const covered = this.#ctx.can('run.adhoc') && (granted === null || selection.every((n) => granted.has(n))); const decide = (action: Capability, allowed: boolean): AdhocPermission => ({ action, allowed, reason: allowed ? null : `missing ${action} permission for this target` }); const gates = [...new Set(Object.keys(parts.flags).flatMap((f) => flagSpec(parts.verb, f)?.requires ?? []))]; const permissions = [ @@ -959,15 +967,14 @@ export class MockOperate { ...gates.map((action) => decide(action, this.#ctx.can(action))), ...(request.priority > 0 ? [decide('run.prioritise', this.#ctx.can('run.prioritise'))] : []) ]; - const error = parts.verb === 'build' ? null : `workers do not run sqb ${parts.verb} yet; only build can launch`; return { verb: parts.verb, - selection: resolution.nodes, + selection, writeSet, settings: Object.fromEntries(Object.entries(parts.flags).map(([f, v]) => [f.replace(/^--/, '').replaceAll('-', '_'), v])), permissions, - error, - launchable: error === null && permissions.every((p) => p.allowed) + error: null, + launchable: permissions.every((p) => p.allowed) }; } diff --git a/ui/src/lib/api/platform-client.ts b/ui/src/lib/api/platform-client.ts index e1f7771434..ccbacb8dd7 100644 --- a/ui/src/lib/api/platform-client.ts +++ b/ui/src/lib/api/platform-client.ts @@ -80,7 +80,7 @@ export interface PlatformApi { getRole(locationId: string, name: string): Promise; /** `role.assign` */ listRoleAssignments(): Promise; - /** You can only give a role you hold in that scope. `role.assign` */ + /** Built-in roles everywhere or in one location, custom roles in their location, admin everywhere only. `role.assign` */ createRoleAssignment(request: RoleAssignmentRequest): Promise; /** Manual assignments only; SSO, policy and built-in ones follow their source. `role.assign` */ deleteRoleAssignment(assignmentId: string): Promise; diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 906f9e77df..0e2b541e0b 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -193,9 +193,9 @@ export type CatalogNodeDetail = Omit & { export type ColumnLens = 'raw' | 'passthrough' | 'rename' | 'transform'; export type ColumnLineage = { - models: { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src'; columns: { name: string; type: string }[] }[]; - /** `model.column` pairs from upstream to downstream. */ - edges: [string, string, ColumnLens][]; + models: { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; columns: { name: string; type: string }[] }[]; + /** `model.column` pairs from upstream to downstream, with an optional label naming the transform, such as `Cast`. */ + edges: [string, string, ColumnLens, string?][]; sql: Record; /** The defining expression of each column, per model. */ expressions: Record>; @@ -352,8 +352,8 @@ export type OverviewMetrics = { models: { total: number; addedThisWeek: number; removedThisWeek: number }; runs24h: { total: number; success: number; warn: number; partial: number; failed: number; active: number }; checks: CheckCounts; - /** Distinct stale nodes, and how many are changed or behind (a node can be both). */ - stale: { nodes: number; changed: number; behind: number; sources: number }; + /** Distinct stale nodes, and how many are changed or behind (a node can be both); null when the server reports no project-wide build state. */ + stale: { nodes: number; changed: number; behind: number; sources: number } | null; /** Null when the adapter has no cost data. */ cost: { last24h: number; baseline: number; baselineDays: number; unit: CostUnit } | null; }; @@ -364,7 +364,8 @@ export type GroupStatus = { group: string; total: number; current: number; chang export type Overview = { metrics: OverviewMetrics; attention: AttentionItem[]; - groups: GroupStatus[]; + /** Null when the server reports no per-group build state. */ + groups: GroupStatus[] | null; recentChanges: SnapshotChange[]; latestRun: (Pick & { status: RunStatus }) | null; }; diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 27fe6998d1..195b9fbd38 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -470,7 +470,7 @@ export type AdhocRunCheck = { /** Accepted flags as the worker receives them, by setting name. */ settings: Record; permissions: AdhocPermission[]; - /** Why the command cannot run regardless of permissions, such as a refused flag or a verb workers can't run yet. */ + /** Why the command cannot run regardless of permissions, such as a refused flag or a selection that matches nothing. */ error: string | null; launchable: boolean; }; diff --git a/ui/src/lib/assets/favicon.svg b/ui/src/lib/assets/favicon.svg deleted file mode 100644 index cc5dc66a3a..0000000000 --- a/ui/src/lib/assets/favicon.svg +++ /dev/null @@ -1 +0,0 @@ -svelte-logo \ No newline at end of file diff --git a/ui/src/lib/branding.test.ts b/ui/src/lib/branding.test.ts new file mode 100644 index 0000000000..d58d57f11f --- /dev/null +++ b/ui/src/lib/branding.test.ts @@ -0,0 +1,26 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +// The UI imports its logos and favicon from the website's approved files through `$website`, so a +// branding change reaches both. A copy under ui/ would drift again, so none may exist. +const ui = join(import.meta.dirname, '..', '..'); + +const files = (dir: string): string[] => + readdirSync(dir, { withFileTypes: true }).flatMap((entry) => + entry.isDirectory() ? files(join(dir, entry.name)) : [join(dir, entry.name)] + ); + +describe('branding', () => { + it('keeps no image copies in the UI', () => { + const images = [...files(join(ui, 'static')), ...files(join(ui, 'src'))].filter((path) => /\.(png|svg|ico|jpe?g|webp)$/i.test(path)); + expect(images).toEqual([]); + }); + + it('loads every logo and favicon from the website', () => { + const references = files(join(ui, 'src')) + .filter((path) => path.endsWith('.svelte') || path.endsWith('.html')) + .flatMap((path) => [...readFileSync(path, 'utf8').matchAll(/(?:src|href)="([^"{]*(?:logo|favicon|icon)[^"]*)"/g)].map((match) => `${path}: ${match[1]}`)); + expect(references).toEqual([]); + }); +}); diff --git a/ui/src/lib/build-area/attention.ts b/ui/src/lib/build-area/attention.ts index 06a7444a9e..c9f35164d9 100644 --- a/ui/src/lib/build-area/attention.ts +++ b/ui/src/lib/build-area/attention.ts @@ -38,7 +38,7 @@ export function attentionView(item: AttentionItem, now: number): AttentionView { key: `unresponsive:${item.run.id}`, tone: 'warning', title: `${item.run.jobKey ?? item.run.id} is unresponsive`, - detail: item.lastHeartbeatAt ? `Not heard from since ${formatAgo(item.lastHeartbeatAt, now)}` : 'Not heard from yet', + detail: item.lastHeartbeatAt ? `Not heard from since ${formatAgo(item.lastHeartbeatAt, now)}` : (item.run.statusReason ?? 'Not heard from yet'), since: formatAgo(item.run.startedAt, now), href: `/runs/${item.run.id}`, action: 'View run' diff --git a/ui/src/lib/components/admin/assignment-list.svelte b/ui/src/lib/components/admin/assignment-list.svelte index e790d4766d..3d81e09145 100644 --- a/ui/src/lib/components/admin/assignment-list.svelte +++ b/ui/src/lib/components/admin/assignment-list.svelte @@ -30,14 +30,21 @@ let newLocation = $state(''); let newRole = $state(''); - /** Roles defined in every location when giving one everywhere, else the location's roles. */ + /** + * Everywhere, only built-in roles, since a custom role belongs to the location that publishes it. + * In one location, its roles except admin, which is only given for every location. + */ const choices = $derived.by(() => { - if (newLocation) return rolesByLocation.get(newLocation) ?? []; + if (newLocation) return (rolesByLocation.get(newLocation) ?? []).filter((r) => r.name !== 'admin'); const lists = [...rolesByLocation.values()]; if (!lists.length) return []; - return lists[0].filter((r) => lists.every((l) => l.some((x) => x.name === r.name))); + return lists[0].filter((r) => r.source === 'built_in' && lists.every((l) => l.some((x) => x.name === r.name))); }); const roleOf = (name: string) => allRoles.find((r) => r.name === name); + // A role picked for another place may not be on offer here. + $effect(() => { + if (newRole && !choices.some((r) => r.name === newRole)) newRole = ''; + }); async function act(key: string, action: () => Promise) { busy = key; @@ -124,7 +131,7 @@ {#each choices as r (r.name)}{/each} Give role - You can only give roles you hold yourself in that location. + Admin is given for every location; custom roles for the location that defines them. {/if} {#if error}

{error}

{/if} diff --git a/ui/src/lib/components/admin/audit-mini.svelte b/ui/src/lib/components/admin/audit-mini.svelte index 3b488f6243..82462576a9 100644 --- a/ui/src/lib/components/admin/audit-mini.svelte +++ b/ui/src/lib/components/admin/audit-mini.svelte @@ -22,7 +22,7 @@ {e.outcome === 'denied' ? 'Denied: ' : ''}{auditActionLabel(e.action)}{#if e.scope.subject} · {e.scope.subject}{/if} - {#if e.decisionId} + {#if e.decisionId && app.client.supports('lookupDecision')} Check this {/if} @@ -69,7 +69,9 @@ font-size: 12.5px; } code { + min-width: 0; font-family: var(--font-mono); font-size: 11.5px; + overflow-wrap: anywhere; } diff --git a/ui/src/lib/components/app-sidebar.svelte b/ui/src/lib/components/app-sidebar.svelte index 933625c20c..93234dbdf6 100644 --- a/ui/src/lib/components/app-sidebar.svelte +++ b/ui/src/lib/components/app-sidebar.svelte @@ -1,4 +1,6 @@ - + {#if signingIn} diff --git a/ui/src/routes/+page.svelte b/ui/src/routes/+page.svelte index 0be8b21009..c747d7bcd1 100644 --- a/ui/src/routes/+page.svelte +++ b/ui/src/routes/+page.svelte @@ -16,9 +16,14 @@ import { attentionRank, attentionView, type AttentionTone } from '$lib/build-area/attention'; import { changeOpLabel, formatCredits } from '$lib/build-area/labels'; import { summarizeChange } from '$lib/build-area/changes'; + import { pagesBeyondTarget } from '$lib/nav'; const overview = createResource((client, scope) => client.overview(scope), ['runs', 'checks', 'node_status', 'snapshot', 'platform_events', 'triggers']); const data = $derived(overview.data ?? null); + // Null data means the person may view only part of the target, so the page offers others instead. + // A reload keeps the previous data, so only a settled, error-free null counts. + const partial = $derived(overview.data === null && !overview.loading && !overview.error); + const otherPages = $derived(pagesBeyondTarget(app.client, app.capabilities)); const attentionIcon: Record = { error: AlertCircleIcon, @@ -41,6 +46,9 @@ const successRate = $derived(m && finished ? Math.round(((m.runs24h.success + m.runs24h.warn) / finished) * 100) : null); const costChange = $derived(m?.cost && m.cost.baseline ? Math.round(((m.cost.last24h - m.cost.baseline) / m.cost.baseline) * 100) : null); const groups = $derived(data?.groups ?? []); + // Parts without a server read stay hidden rather than showing zeros. + const showGroups = $derived(data?.groups !== null); + const showTimeline = $derived(app.client.supports('timeline')); let rebuildOpen = $state(false); @@ -52,135 +60,153 @@
-
- {#if overview.error}
{overview.error}
{/if} -
-
-
- {#if issues.length}{:else}{/if} -
-
- {issues.length ? `${app.location?.name} needs attention` : `${app.location?.name} is healthy`} - {issues.length ? `${issues.length} ${issues.length === 1 ? 'item' : 'items'} · ${latestText}` : latestText} -
- {#if m} -
- {#if m.stale.changed}{m.stale.changed} changed{/if} - {#if m.stale.behind}{m.stale.behind} behind{/if} - {#if m.checks.failing}{m.checks.failing} failing {m.checks.failing === 1 ? 'check' : 'checks'}{/if} + {#if overview.error}
{overview.error}
{/if} + {#if partial} +
+

You can see only part of {app.location?.name} · {app.target?.name}

+

The Overview sums up the whole target, so it needs view on every node in it. An administrator can give you view on the rest.

+ {#if otherPages.length} +

Pages you can use:

+ + {/if} + {#if (app.bootstrap?.locations.length ?? 0) > 1}

Or choose another location from the menu at the top of the sidebar.

{/if} +
+ {:else} +
+
+
+
+ {#if issues.length}{:else}{/if}
- {/if} - {#if latest}View latest run →{/if} -
- {#if issues.length} -
- {#each issues as { item, view } (view.key)} - {@const Glyph = attentionIcon[view.tone]} -
- - {view.title} - {view.detail} - {view.since} - - {#if item.kind === 'stale_models'} - (rebuildOpen = true)}>Rebuild {staleSelection.length} - {/if} - {view.action} → - +
+ {issues.length ? `${app.location?.name} needs attention` : `${app.location?.name} is healthy`} + {issues.length ? `${issues.length} ${issues.length === 1 ? 'item' : 'items'} · ${latestText}` : latestText} +
+ {#if m} +
+ {#if m.stale?.changed}{m.stale.changed} changed{/if} + {#if m.stale?.behind}{m.stale.behind} behind{/if} + {#if m.checks.failing}{m.checks.failing} failing {m.checks.failing === 1 ? 'check' : 'checks'}{/if}
- {/each} + {/if} + {#if latest}View latest run →{/if}
- {/if} -
+ {#if issues.length} +
+ {#each issues as { item, view } (view.key)} + {@const Glyph = attentionIcon[view.tone]} +
+ + {view.title} + {view.detail} + {view.since} + + {#if item.kind === 'stale_models'} + (rebuildOpen = true)}>Rebuild {staleSelection.length} + {/if} + {view.action} → + +
+ {/each} +
+ {/if} +
-
- {#if m} -
- Models -
{m.models.total}deployed
- 0}>{m.models.addedThisWeek || m.models.removedThisWeek ? `+${m.models.addedThisWeek} / −${m.models.removedThisWeek} this week` : 'No changes this week'} -
-
- Runs · 24h -
{m.runs24h.total}{successRate === null ? 'none finished' : `${successRate}% succeeded`}
- 0}>{m.runs24h.partial} partial · {m.runs24h.failed} failed · {m.runs24h.active} active -
-
- Checks passing -
{m.checks.passing}of {m.checks.total}
- 0}>{m.checks.failing} failing · {m.checks.warning} warning -
-
- Stale -
{m.stale.nodes}nodes
- {m.stale.changed} changed · {m.stale.behind} behind · {m.stale.sources} {m.stale.sources === 1 ? 'source' : 'sources'} late -
- {#if m.cost} +
+ {#if m} +
+ Models +
{m.models.total}deployed
+ 0}>{m.models.addedThisWeek || m.models.removedThisWeek ? `+${m.models.addedThisWeek} / −${m.models.removedThisWeek} this week` : 'No changes this week'} +
- Cost · 24h -
{formatCredits(m.cost.last24h)}
- = 20}>{costChange === null ? '—' : `${costChange >= 0 ? '+' : ''}${costChange}%`} vs {m.cost.baselineDays}-day average + Runs · 24h +
{m.runs24h.total}{successRate === null ? 'none finished' : `${successRate}% succeeded`}
+ 0}>{m.runs24h.partial} partial · {m.runs24h.failed} failed · {m.runs24h.active} active
+
+ Checks passing +
{m.checks.passing}of {m.checks.total}
+ 0}>{m.checks.failing} failing · {m.checks.warning} warning +
+ {#if m.stale} +
+ Stale +
{m.stale.nodes}nodes
+ {m.stale.changed} changed · {m.stale.behind} behind · {m.stale.sources} {m.stale.sources === 1 ? 'source' : 'sources'} late +
+ {/if} + {#if m.cost} +
+ Cost · 24h +
{formatCredits(m.cost.last24h)}
+ = 20}>{costChange === null ? '—' : `${costChange >= 0 ? '+' : ''}${costChange}%`} vs {m.cost.baselineDays}-day average +
+ {/if} {/if} +
+ + {#if showTimeline} +
+
+

Activity

Past runs, and when schedules will run next
+ Open full timeline → +
+ +
{/if} -
-
-
-

Activity

Past runs, and when schedules will run next
- Open full timeline → +
+ {#if showGroups}
+
+

Build status by group

Current, stale and failed nodes · {app.location?.name}
+ Open catalog → +
+
+
GroupCurrentChangedBehindFailed
+ {#each groups as g (g.group)} + + {g.group} + + + + + + {g.current}/{g.total} + {g.changed || '—'} + {g.behind || '—'} + {g.failed || '—'} + + {:else} +
{overview.loading ? 'Loading…' : 'Nothing deployed yet. Deploy the project to see its models here.'}
+ {/each} +
+
{/if} +
+
+

Recent changes

Recent deploys
+ All changes → +
+ +
- -
- -
-
-
-

Build status by group

Current, stale and failed nodes · {app.location?.name}
- Open catalog → -
-
-
GroupCurrentChangedBehindFailed
- {#each groups as g (g.group)} - - {g.group} - - - - - - {g.current}/{g.total} - {g.changed || '—'} - {g.behind || '—'} - {g.failed || '—'} - - {:else} -
{overview.loading ? 'Loading…' : 'Nothing deployed yet. Deploy the project to see its models here.'}
- {/each} -
-
-
-
-

Recent changes

Recent deploys
- All changes → -
- -
-
+ {/if}
diff --git a/ui/src/routes/login/+page.svelte b/ui/src/routes/login/+page.svelte index 298181bdd4..4727d67cc5 100644 --- a/ui/src/routes/login/+page.svelte +++ b/ui/src/routes/login/+page.svelte @@ -1,4 +1,6 @@