diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 32140c3e3f..f985d67ffd 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -434,6 +434,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/jobs/{job_key}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonJob"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/maintenance": { parameters: { query?: never; @@ -578,6 +594,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/nodes/{node_key}/error": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonRunNodeError"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/output": { parameters: { query?: never; @@ -1479,6 +1511,31 @@ export interface components { failed: components["schemas"]["Table"][]; failure?: string | null; }; + /** @description One job or group as declared, for a person who may list the target; free-form setting values are withheld. */ + JobDetailRecord: { + concurrency_groups: string[]; + /** @description Selected nodes whose failure stops a group. */ + critical_nodes: string[]; + description: string; + job_key: string; + kind: components["schemas"]["RunKind"]; + /** @description The groups this job belongs to. */ + member_of: string[]; + /** @description A group's member jobs in run order; empty for a job. */ + members: components["schemas"]["JobMember"][]; + selection: string[]; + /** @description Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. */ + settings: { + [key: string]: unknown; + }; + /** Format: uuid */ + snapshot_id: string; + }; + JobMember: { + job_key: string; + /** @description Members this one waits for, from lineage and from the group's `after`. */ + waits_for: string[]; + }; /** @description A job or group of the current published snapshot. Settings are left out of person reads. */ JobRecord: { concurrency_groups: string[]; @@ -1492,6 +1549,8 @@ export interface components { }; /** @description A trusted operator names a published job; execution inputs are never supplied by the caller. */ JobRunRequest: { + /** @description Rebuild every selected model from scratch; it rewrites history, as a window does. */ + full_refresh?: boolean; job_key: string; /** Format: uuid */ location_id: string; @@ -2011,6 +2070,8 @@ export interface components { node_key: string; } | { error_code?: string | null; + /** @description Redacted and bounded on the way in, whoever sends it; absent from older workers. */ + error_message?: string | null; /** @enum {string} */ kind: "node_finished"; node_key: string; @@ -2114,6 +2175,15 @@ export interface components { }; /** @enum {string} */ RunKind: "job" | "group" | "adhoc"; + /** @description A node's outcome error in one run: redacted, bounded text and the project file of the node, never its SQL. */ + RunNodeFailure: { + error_code?: string | null; + /** @description Null when the worker reported none or the run's history has been pruned. */ + message?: string | null; + node_key: string; + /** @description Relative to the project root, with `/` separators; null when the run's snapshot is gone. */ + path?: string | null; + }; RunNodeRecord: { /** Format: date-time */ ended_at?: string | null; @@ -3687,6 +3757,52 @@ export interface operations { }; }; }; + readPersonJob: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + job_key: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["JobDetailRecord"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Needs view on every current node of the target */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description No published job by that name */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; readPersonMaintenance: { parameters: { query?: { @@ -4071,6 +4187,53 @@ export interface operations { }; }; }; + readPersonRunNodeError: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + node_key: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunNodeFailure"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Needs view.evidence on the node */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Node not found in this run */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; readPersonRunOutput: { parameters: { query?: never; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index cbce05d41e..35d9b69dfd 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -183,6 +183,7 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result> = @@ -15,8 +15,12 @@ pub(crate) static BEARER: LazyLock> = LazyLock::new(|| compiled(BEARER_PATTERN)); pub(crate) static SECRET_VALUE: LazyLock> = LazyLock::new(|| compiled(SECRET_VALUE_PATTERN)); +pub(crate) static DDL_PASSWORD: LazyLock> = + LazyLock::new(|| compiled(DDL_PASSWORD_PATTERN)); pub(crate) static URI_CREDENTIAL: LazyLock> = LazyLock::new(|| compiled(URI_CREDENTIAL_PATTERN)); +static BARE_USERINFO: LazyLock> = + LazyLock::new(|| compiled(BARE_USERINFO_PATTERN)); fn compiled(pattern: &str) -> Result { RegexBuilder::new(pattern) @@ -35,3 +39,19 @@ pub(crate) fn substituted( Err(_) => text.to_owned(), } } + +/// A port is all digits and the authority ends after it (`/`, `?`, `#`); anything else may be a password. +pub(crate) fn bare_userinfo_redacted(text: &str) -> String { + match BARE_USERINFO.as_ref() { + Ok(regex) => regex + .replace_all(text, |found: &Captures<'_>| { + if !found[3].is_empty() && found[2].bytes().all(|byte| byte.is_ascii_digit()) { + found[0].to_owned() + } else { + format!("{}[REDACTED]{}", &found[1], &found[3]) + } + }) + .into_owned(), + Err(_) => text.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/redaction/constants.rs b/crates/sqlbuild-core/src/redaction/constants.rs index 6b823c8bb1..6ff44b2168 100644 --- a/crates/sqlbuild-core/src/redaction/constants.rs +++ b/crates/sqlbuild-core/src/redaction/constants.rs @@ -13,10 +13,11 @@ pub(crate) const SECRET_FIELD_PARTS: [&str; 11] = [ "secret", "token", ]; +/// A key may be glued to a prefix, as in `db_password` or `PGPASSWORD`; `pwd` stands alone so `OLDPWD` and `cwd` keep their paths. pub(crate) const SECRET_VALUE_PATTERN: &str = concat!( - r"(?i)\b(api[-_]?key|access[-_]?key|private[-_]?key|authorization|", + r"(?i)\b([a-z0-9_.-]*?(?:api[-_]?key|access[-_]?key|private[-_]?key|authorization|", r"client[-_]?secret|refresh[-_]?token|", - r"password|passwd|pwd|secret|token|dsn|connection_string)", + r"password|passwd|secret|token|dsn|connection_string)|pwd)", r#"(\s*[=:]\s*)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})"#, ); pub(crate) const AUTHORIZATION_PATTERN: &str = concat!( @@ -24,10 +25,24 @@ pub(crate) const AUTHORIZATION_PATTERN: &str = concat!( r#"("[^"\r\n]{1,512}"|'[^'\r\n]{1,512}'|[^\s,;]{1,512})"#, ); pub(crate) const BEARER_PATTERN: &str = r"(?i)\bbearer\s+[a-z0-9._~+/=-]{1,512}"; -pub(crate) const URI_CREDENTIAL_PATTERN: &str = - r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s@]+)(@)"; +/// The password runs to the last `@` before the host, so an unencoded `@` inside it is covered. +pub(crate) const URI_CREDENTIAL_PATTERN: &str = concat!( + r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]+:)([^\s]+?)", + r#"(@[^\s@/?#,;'"]*)([\s/?#,;'"]|$)"#, +); +/// SQL that sets a password: `PASSWORD '…'` (quoted only, so prose such as "password authentication" stays) and `IDENTIFIED [WITH plugin] BY …`. +pub(crate) const DDL_PASSWORD_PATTERN: &str = concat!( + r#"(?i)\b(password)(\s+)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}')|"#, + r#"\b(identified\s+(?:with\s+\w+\s+)?by)(\s+)("[^"\r\n]{0,512}"|'[^'\r\n]{0,512}'|[^\s,;]{1,512})"#, +); +pub(crate) const DDL_REPLACEMENT: &str = "${1}${2}${4}${5}[REDACTED]"; pub(crate) const PAIR_REPLACEMENT: &str = "${1}${2}[REDACTED]"; pub(crate) const BEARER_REPLACEMENT: &str = "Bearer [REDACTED]"; -pub(crate) const URI_REPLACEMENT: &str = "${1}[REDACTED]${3}"; +pub(crate) const URI_REPLACEMENT: &str = "${1}[REDACTED]${3}${4}"; /// Compiled size allowance for the bounded Unicode repetitions above. pub(crate) const PATTERN_SIZE_LIMIT: usize = 1 << 28; +/// `scheme://name:value` with no `@` after it, as in a cut connection string; group 3 ends the authority. +pub(crate) const BARE_USERINFO_PATTERN: &str = + r#"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@'"]+:)([^\s/?#@'"]+)([/?#]?)"#; +/// The most characters of error text kept after redaction. +pub(crate) const ERROR_TEXT_CHARS: usize = 2000; diff --git a/crates/sqlbuild-core/src/redaction/main/mod.rs b/crates/sqlbuild-core/src/redaction/main/mod.rs index 47aff8ca43..a02ca5d2de 100644 --- a/crates/sqlbuild-core/src/redaction/main/mod.rs +++ b/crates/sqlbuild-core/src/redaction/main/mod.rs @@ -1,2 +1,3 @@ +pub mod redact_error_text; pub mod redact_text; pub mod secret_field_name; diff --git a/crates/sqlbuild-core/src/redaction/main/redact_error_text.rs b/crates/sqlbuild-core/src/redaction/main/redact_error_text.rs new file mode 100644 index 0000000000..5aa7856753 --- /dev/null +++ b/crates/sqlbuild-core/src/redaction/main/redact_error_text.rs @@ -0,0 +1,17 @@ +//! `redact_error_text`: error text redacted before it is cut, so a cut can never expose part of a secret. + +use crate::redaction::_helpers::patterns::bare_userinfo_redacted; +use crate::redaction::constants::ERROR_TEXT_CHARS; +use crate::redaction::main::redact_text::redact_text; + +/// Named secrets and URI passwords redacted, even where a warehouse cut the URI before its host, then at most 2,000 characters; a second pass changes nothing. +#[must_use] +pub fn redact_error_text(text: &str) -> String { + let redacted = bare_userinfo_redacted(&redact_text(text)); + if redacted.chars().count() <= ERROR_TEXT_CHARS { + return redacted; + } + let mut kept: String = redacted.chars().take(ERROR_TEXT_CHARS - 1).collect(); + kept.push('…'); + kept +} diff --git a/crates/sqlbuild-core/src/redaction/main/redact_text.rs b/crates/sqlbuild-core/src/redaction/main/redact_text.rs index 9764c88a0e..e80044a4f2 100644 --- a/crates/sqlbuild-core/src/redaction/main/redact_text.rs +++ b/crates/sqlbuild-core/src/redaction/main/redact_text.rs @@ -1,15 +1,18 @@ //! `redact_text`: obvious named secrets and URI passwords removed from diagnostic text. use crate::redaction::_helpers::patterns::{ - AUTHORIZATION, BEARER, SECRET_VALUE, URI_CREDENTIAL, substituted, + AUTHORIZATION, BEARER, DDL_PASSWORD, SECRET_VALUE, URI_CREDENTIAL, substituted, +}; +use crate::redaction::constants::{ + BEARER_REPLACEMENT, DDL_REPLACEMENT, PAIR_REPLACEMENT, URI_REPLACEMENT, }; -use crate::redaction::constants::{BEARER_REPLACEMENT, PAIR_REPLACEMENT, URI_REPLACEMENT}; -/// Authorization headers, bearer tokens, `name=value` secrets, then URI passwords redacted. +/// Authorization headers, bearer tokens, `name=value` secrets, SQL passwords, then URI passwords redacted. #[must_use] pub fn redact_text(value: &str) -> String { let authorization: String = substituted(&AUTHORIZATION, value, PAIR_REPLACEMENT); let bearer: String = substituted(&BEARER, &authorization, BEARER_REPLACEMENT); let named: String = substituted(&SECRET_VALUE, &bearer, PAIR_REPLACEMENT); - substituted(&URI_CREDENTIAL, &named, URI_REPLACEMENT) + let statement: String = substituted(&DDL_PASSWORD, &named, DDL_REPLACEMENT); + substituted(&URI_CREDENTIAL, &statement, URI_REPLACEMENT) } diff --git a/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs b/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs index a998e3bc7c..6b7305b63e 100644 --- a/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs +++ b/crates/sqlbuild-core/src/redaction/tests/test_redact_text.rs @@ -1,3 +1,4 @@ +use crate::redaction::main::redact_error_text::redact_error_text; use crate::redaction::main::redact_text::redact_text; use crate::redaction::tests::test_types::RedactTestCase; @@ -19,6 +20,61 @@ fn given_diagnostic_text_when_redacting_then_matches_python_substitutions() { text: "nothing to see", expected_text: "nothing to see", }, + RedactTestCase { + description: "a key glued to a prefix with a separator", + text: "db_password=orders-pass host=warehouse", + expected_text: "db_password=[REDACTED] host=warehouse", + }, + RedactTestCase { + description: "a key glued to a prefix without a separator", + text: "PGPASSWORD=orders-pass psql", + expected_text: "PGPASSWORD=[REDACTED] psql", + }, + RedactTestCase { + description: "near miss: a key that only starts with a secret word keeps its value", + text: "password_policy=strict max_tokens=5 tokenizer: words passwords: 3 tries", + expected_text: "password_policy=strict max_tokens=5 tokenizer: words passwords: 3 tries", + }, + RedactTestCase { + description: "near miss: pwd stays a whole word, so working directories are kept", + text: "OLDPWD=/srv/orders cwd=/srv/orders pwd=orders-pass", + expected_text: "OLDPWD=/srv/orders cwd=/srv/orders pwd=[REDACTED]", + }, + RedactTestCase { + description: "a quoted PASSWORD in DDL", + text: "CREATE ROLE orders_app WITH LOGIN PASSWORD 'orders-pass' VALID UNTIL 'infinity'", + expected_text: "CREATE ROLE orders_app WITH LOGIN PASSWORD [REDACTED] VALID UNTIL 'infinity'", + }, + RedactTestCase { + description: "IDENTIFIED BY with a quoted and a bare password", + text: "CREATE USER orders IDENTIFIED BY 'orders-pass'; ALTER USER stock IDENTIFIED BY stockpass", + expected_text: "CREATE USER orders IDENTIFIED BY [REDACTED]; ALTER USER stock IDENTIFIED BY [REDACTED]", + }, + RedactTestCase { + description: "IDENTIFIED WITH a plugin BY a password, as ClickHouse writes it", + text: "CREATE USER orders IDENTIFIED WITH sha256_password BY 'orders-pass' HOST ANY", + expected_text: "CREATE USER orders IDENTIFIED WITH sha256_password BY [REDACTED] HOST ANY", + }, + RedactTestCase { + description: "near miss: prose about passwords and IDENTIFIED WITH keep their words", + text: "password authentication failed for user \"orders\"; IDENTIFIED WITH caching_sha2_password", + expected_text: "password authentication failed for user \"orders\"; IDENTIFIED WITH caching_sha2_password", + }, + RedactTestCase { + description: "an unencoded @ inside a URI password", + text: "url=postgres://orders:p@ss@warehouse.example.test/orders", + expected_text: "url=postgres://orders:[REDACTED]@warehouse.example.test/orders", + }, + RedactTestCase { + description: "near miss: an @ after the host belongs to the path or query, not the password", + text: "postgres://orders:pw@warehouse.example.test/orders?contact=support@example.com, https://api.example.test/v1", + expected_text: "postgres://orders:[REDACTED]@warehouse.example.test/orders?contact=support@example.com, https://api.example.test/v1", + }, + RedactTestCase { + description: "two URIs in a list each lose their password", + text: "postgres://orders:one@h1.example.test,postgres://stock:two@h2.example.test", + expected_text: "postgres://orders:[REDACTED]@h1.example.test,postgres://stock:[REDACTED]@h2.example.test", + }, ]; for test_case in test_cases { @@ -30,3 +86,43 @@ fn given_diagnostic_text_when_redacting_then_matches_python_substitutions() { ); } } + +#[test] +fn given_error_text_with_cut_uris_when_redacting_then_position_tells_a_port_from_a_password() { + let test_cases = [ + RedactTestCase { + description: "an all-digit password in a URI cut before its host", + text: "could not parse 'postgres://orders:1234", + expected_text: "could not parse 'postgres://orders:[REDACTED]", + }, + RedactTestCase { + description: "a password cut at a closing quote", + text: "dsn 'postgres://orders:orders-pass' rejected", + expected_text: "dsn 'postgres://orders:[REDACTED]' rejected", + }, + RedactTestCase { + description: "a cut password containing / is not a port", + text: "cut 'postgres://orders:pa/ss", + expected_text: "cut 'postgres://orders:[REDACTED]/ss", + }, + RedactTestCase { + description: "near miss: a port the authority ends after stays", + text: "GET http://warehouse.example.test:8080/health failed; retry http://warehouse.example.test:8443?attempt=2", + expected_text: "GET http://warehouse.example.test:8080/health failed; retry http://warehouse.example.test:8443?attempt=2", + }, + RedactTestCase { + description: "a complete URI is redacted once and a second pass changes nothing", + text: "postgres://orders:[REDACTED]@warehouse.example.test:5432/orders", + expected_text: "postgres://orders:[REDACTED]@warehouse.example.test:5432/orders", + }, + ]; + + for test_case in test_cases { + assert_eq!( + redact_error_text(test_case.text), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 4533664c83..aa338502b5 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -126,6 +126,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/jobs", get(crate::api::_helpers::resources::person_reads::jobs), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/jobs/{job_key}", + get(crate::api::_helpers::resources::person_reads::job), + ) .route( "/api/control/locations/{location_id}/targets/{target}/triggers", get(crate::api::_helpers::resources::person_reads::triggers), @@ -182,6 +186,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/output", get(crate::api::_helpers::resources::person_reads::run_output), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/nodes/{node_key}/error", + get(crate::api::_helpers::resources::person_reads::run_node_error), + ) .route( "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/runs", get(crate::api::_helpers::resources::person_reads::node_runs), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index 5b8f666c47..0eddfb9c89 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -16,6 +16,7 @@ pub(crate) mod request_json; pub(crate) mod router; pub(crate) mod run_reads; pub(crate) mod runs; +pub(crate) mod server_errors; pub(crate) mod snapshots; pub(crate) mod trigger_reads; pub(crate) mod ui_assets; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index b8854f4d1f..aa9846a7a2 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -1,4 +1,4 @@ -//! Person reads of run nodes, node run history, catalog nodes, jobs, triggers, ticks and publications; refusal precedes every lookup, so unknown objects look forbidden. +//! Person reads of run nodes and their errors, node run history, catalog nodes, jobs and one job's declaration, triggers, ticks and publications; refusal precedes every lookup, so unknown objects look forbidden. use axum::extract::{Path, Query, State}; use axum::http::StatusCode; @@ -6,17 +6,19 @@ use axum::{Extension, Json}; use uuid::Uuid; use crate::api::errors::ApiError; -use crate::api::models::{ReadQuery, RunEventsQuery, StatementSqlQuery}; +use crate::api::models::{ReadQuery, RunEventsQuery, RunNodePath, StatementSqlQuery}; use crate::credentials::models::{AuditContext, AuthenticatedPerson, DefaultRole}; use crate::declarations::main::list_person_jobs::list_person_jobs; use crate::declarations::main::list_person_triggers::list_person_triggers; -use crate::declarations::models::{JobRecord, PersonTriggerRecord}; +use crate::declarations::main::read_person_job::read_person_job; +use crate::declarations::models::{JobDetailRecord, JobRecord, PersonTriggerRecord}; use crate::platform::main::read_person_location_health::read_person_location_health; use crate::platform::main::read_person_problems::read_person_problems; use crate::platform::models::{LocationHealthObservation, PlatformProblems}; use crate::runs::main::list_person_run_nodes::list_person_run_nodes; use crate::runs::models::{ - NodeRunRecord, RunEventRecord, RunNodeRecord, RunOutput, RunStatementRecord, RunStatementSql, + NodeRunRecord, RunEventRecord, RunNodeFailure, RunNodeRecord, RunOutput, RunStatementRecord, + RunStatementSql, }; use crate::scheduler::main::list_person_ticks::list_person_ticks; use crate::scheduler::models::TickRecord; @@ -25,6 +27,7 @@ use crate::snapshots::models::PublishedSnapshot; use crate::store::models::ControlStore; use crate::warehouse_history::main::read_person_node_runs::read_person_node_runs; use crate::warehouse_history::main::read_person_run_events::read_person_run_events; +use crate::warehouse_history::main::read_person_run_node_error::read_person_run_node_error; use crate::warehouse_history::main::read_person_run_output::read_person_run_output; use crate::warehouse_history::main::read_person_run_statement_sql::read_person_run_statement_sql; use crate::warehouse_history::main::read_person_run_statements::read_person_run_statements; @@ -190,6 +193,28 @@ pub(crate) async fn run_statement_sql( .ok_or_else(|| not_found("statement not found")) } +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/nodes/{node_key}/error", operation_id = "readPersonRunNodeError", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), ("node_key" = String, Path)), + responses((status = 200, body = RunNodeFailure), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody, description = "Needs view.evidence on the node"), + (status = 404, description = "Node not found in this run")), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn run_node_error( + (State(store), Extension(context)): (State, Extension), + Path(path): Path, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + read_person_run_node_error( + &store, + (&context, principal), + (path.location_id, &path.target), + (path.run_id, &path.node_key), + ) + .await? + .map(Json) + .ok_or_else(|| not_found("node not found in this run")) +} + #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/output", operation_id = "readPersonRunOutput", params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path)), responses((status = 200, body = RunOutput), (status = 401, description = "Session rejected"), @@ -213,6 +238,29 @@ pub(crate) async fn run_output( .ok_or_else(|| not_found("run not found")) } +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/jobs/{job_key}", operation_id = "readPersonJob", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("job_key" = String, Path)), + responses((status = 200, body = JobDetailRecord), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody, description = "Needs view on every current node of the target"), + (status = 404, description = "No published job by that name")), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn job( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + Path((location_id, target, job_key)): Path<(Uuid, String, String)>, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + read_person_job( + &store, + (&context, principal), + (location_id, &target, &job_key), + default_role, + ) + .await? + .map(Json) + .ok_or_else(|| not_found("job not found")) +} + #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/jobs", operation_id = "listPersonJobs", params(("location_id" = Uuid, Path), ("target" = String, Path)), responses((status = 200, body = Vec), (status = 401, description = "Session rejected"), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/server_errors.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/server_errors.rs new file mode 100644 index 0000000000..a8933a2d52 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/server_errors.rs @@ -0,0 +1,22 @@ +//! Axum's own 5xx rejections are plain text naming Rust types; clients get the generic JSON error instead. + +use axum::http::header; +use axum::response::{IntoResponse, Response}; + +use crate::api::errors::ApiError; + +pub(crate) async fn generic_server_errors(response: Response) -> Response { + let plain = response + .headers() + .get(header::CONTENT_TYPE) + .is_some_and(|value| value.as_bytes().starts_with(b"text/plain")); + if !(plain && response.status().is_server_error()) { + return response; + } + ApiError { + status: response.status(), + message: "internal server error".into(), + decision_id: None, + } + .into_response() +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 29eb042961..1397e69ed2 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -4,6 +4,7 @@ use axum::routing::get; use axum::{Extension, Json, Router, middleware}; use crate::api::_helpers::authentication::{bearer::local_request, configure::configure}; +use crate::api::_helpers::resources::server_errors::generic_server_errors; use crate::api::_helpers::resources::ui_assets::{UI_ASSETS, serve_ui}; use crate::api::_helpers::resources::{changes, router::router}; use crate::api::main::openapi_document::openapi_document; @@ -79,10 +80,11 @@ fn development_application( address: std::net::SocketAddr, ) -> Router { let application = serve_ui(application, UI_ASSETS); - match scope { + let application = match scope { Some(scope) => application .layer(Extension(scope)) .layer(middleware::from_fn_with_state(address, local_request)), None => application, - } + }; + application.layer(middleware::map_response(generic_server_errors)) } diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index ca9d679cfd..c57fbc085b 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -32,7 +32,9 @@ use crate::api::_helpers::resources::{ person_reads::run_statements, person_reads::run_statement_sql, person_reads::run_output, + person_reads::run_node_error, person_reads::jobs, + person_reads::job, person_reads::triggers, person_reads::ticks, person_reads::snapshots, @@ -132,6 +134,14 @@ pub(crate) struct CheckResultPath { pub(crate) event_id: uuid::Uuid, } +#[derive(Debug, Deserialize)] +pub(crate) struct RunNodePath { + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) run_id: uuid::Uuid, + pub(crate) node_key: String, +} + #[derive(Debug, Deserialize)] pub(crate) struct RunRecallPath { pub(crate) location_id: uuid::Uuid, diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs new file mode 100644 index 0000000000..7eaeb7eda8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs @@ -0,0 +1,93 @@ +//! Display projections of a declaration: run order of a group's members and settings safe to show. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; +use sqlbuild_core::redaction::main::redact_text::redact_text; + +use crate::declarations::constants::{DEFER_SETTING_PREFIX, VARS_SETTING, WITHHELD_SETTING}; +use crate::declarations::models::{Definitions, GroupDefinition, JobMember}; +use crate::snapshots::models::SnapshotNode; + +/// Members whose waits are all earlier come next, by name among equals; validation rejected cycles. +pub(crate) fn members_in_order(group: &GroupDefinition) -> Vec { + let mut placed: BTreeSet<&str> = BTreeSet::new(); + let mut members: Vec = Vec::new(); + while members.len() < group.jobs.len() { + let ready = ready_members(group, &placed); + if ready.is_empty() { + break; + } + for (name, waits) in ready { + placed.insert(name); + members.push(JobMember { + job_key: name.clone(), + waits_for: waits.clone(), + }); + } + } + members +} + +/// Unplaced members whose every wait is already placed. +fn ready_members<'a>( + group: &'a GroupDefinition, + placed: &BTreeSet<&str>, +) -> Vec<(&'a String, &'a BTreeSet)> { + group + .jobs + .iter() + .filter(|(name, waits)| !placed.contains(name.as_str()) && all_placed(waits, placed)) + .collect() +} + +fn all_placed(waits: &BTreeSet, placed: &BTreeSet<&str>) -> bool { + waits.iter().all(|wait| placed.contains(wait.as_str())) +} + +/// `vars` can carry credentials and a deferral can name a storage path, so only their presence shows. +pub(crate) fn shown_settings(settings: &BTreeMap) -> BTreeMap { + settings + .iter() + .map(|(name, value)| { + let shown = match value { + _ if withheld(name, value) => Value::String(WITHHELD_SETTING.into()), + Value::String(text) => Value::String(redact_text(text)), + other => other.clone(), + }; + (name.clone(), shown) + }) + .collect() +} + +fn withheld(name: &str, value: &Value) -> bool { + name == VARS_SETTING + || (name.starts_with(DEFER_SETTING_PREFIX) && !value.as_str().is_some_and(target_name)) +} + +/// Target names are TOML bare keys in the project file: letters, digits, `_` and `-`. +fn target_name(text: &str) -> bool { + !text.is_empty() + && text + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') +} + +/// The groups of the snapshot that run `job_key`. +pub(crate) fn groups_running(definitions: &Definitions, job_key: &str) -> BTreeSet { + definitions + .groups + .iter() + .filter(|(_, group)| group.jobs.contains_key(job_key)) + .map(|(name, _)| name.clone()) + .collect() +} + +/// Selected nodes marked critical, whose failure stops a group. +pub(crate) fn critical_nodes(nodes: &BTreeMap) -> BTreeSet { + nodes + .iter() + .filter(|(_, node)| node.definition["config"]["critical"] == Value::Bool(true)) + .map(|(key, _)| key.clone()) + .collect() +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index dd82f3f543..c0e012e8e8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod development; pub(crate) mod documents; +pub(crate) mod job_detail; pub(crate) mod operator_change; pub(crate) mod operator_state; pub(crate) mod publication; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs index 84e2af675a..f75bdfa563 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs @@ -45,6 +45,7 @@ pub(crate) async fn trigger_scope( reason: request.reason.clone(), priority: 0, window: None, + full_refresh: false, }; let (transaction, frozen) = frozen_job_request(transaction, context, &job).await?; let (transaction, nodes) = diff --git a/crates/sqlbuild-orchestration/src/declarations/constants.rs b/crates/sqlbuild-orchestration/src/declarations/constants.rs new file mode 100644 index 0000000000..00fc324482 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/constants.rs @@ -0,0 +1,8 @@ +//! Names the declaration reads compare against. + +/// The free-form job setting that can carry credentials, so person reads show only that it is set. +pub(crate) const VARS_SETTING: &str = "vars"; +/// What a withheld setting shows instead of its value. +pub(crate) const WITHHELD_SETTING: &str = "[REDACTED]"; +/// Settings that name another target; a value that isn't a target name, such as a state path, is withheld. +pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_"; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index 0729a456c1..ec6d61efd4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -8,4 +8,5 @@ pub mod list_trigger_events; pub mod list_triggers; pub mod pause_development_triggers; pub(crate) mod publish_definitions; +pub mod read_person_job; pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs b/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs new file mode 100644 index 0000000000..abc034ffe5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs @@ -0,0 +1,85 @@ +//! One job's declaration needs the same view as the job list: every current node of the target. + +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::declarations::_helpers::job_detail::{ + critical_nodes, groups_running, members_in_order, shown_settings, +}; +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::declarations::models::{JobDetailRecord, StoredJob}; +use crate::runs::models::RunKind; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query_records::query_records}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; + +/// `None` when the target publishes no job by that name; refusal comes first either way. +pub async fn read_person_job( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (location_id, target, job_key): (Uuid, &str, &str), + default_role: DefaultRole, +) -> Result, ControlError> { + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + (context, credentials), + (location_id, target), + default_role, + ) + .await?; + let (transaction, published) = + published_snapshot(transaction, (location_id, target, job_key)).await?; + let Some(snapshot_id) = published else { + commit(transaction).await?; + return Ok(None); + }; + let (transaction, definitions) = frozen_definitions(transaction, snapshot_id).await?; + let (kind, job, members) = match ( + definitions.jobs.get(job_key), + definitions.groups.get(job_key), + ) { + (Some(job), _) => (RunKind::Job, job.clone(), Vec::new()), + (None, Some(group)) => (RunKind::Group, group.job.clone(), members_in_order(group)), + (None, None) => { + return Err(ControlError::Invalid( + "published job is absent from its snapshot".into(), + )); + } + }; + let (transaction, nodes) = + read_snapshot_nodes(transaction, snapshot_id, Some(&job.selection)).await?; + commit(transaction).await?; + Ok(Some(JobDetailRecord { + job_key: job_key.into(), + kind, + description: job.description, + snapshot_id, + settings: shown_settings(&job.settings), + members, + member_of: groups_running(&definitions, job_key), + critical_nodes: critical_nodes(&nodes), + selection: job.selection, + concurrency_groups: job.concurrency_groups, + })) +} + +/// The snapshot whose job list holds `job_key`, if the target publishes it. +async fn published_snapshot( + transaction: ControlTransaction, + (location_id, target, job_key): (Uuid, &str, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, stored): (_, Vec) = query_records( + transaction, + "SELECT job_key, kind, snapshot_id, definition_json FROM sqb_jobs \ + WHERE location_id=$1 AND target=$2 AND job_key=$3", + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(job_key.into()), + ], + ) + .await?; + Ok((transaction, stored.first().map(|job| job.snapshot_id))) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs index 5359000d16..f147865646 100644 --- a/crates/sqlbuild-orchestration/src/declarations/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 8a981a57b4..a3113d39cc 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -108,6 +108,32 @@ pub(crate) struct StoredJob { pub definition_json: JobDefinition, } +/// One job or group as declared, for a person who may list the target; free-form setting values are withheld. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct JobDetailRecord { + pub job_key: String, + pub kind: crate::runs::models::RunKind, + pub description: String, + pub snapshot_id: uuid::Uuid, + pub selection: BTreeSet, + pub concurrency_groups: BTreeSet, + /// Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. + pub settings: BTreeMap, + /// A group's member jobs in run order; empty for a job. + pub members: Vec, + /// The groups this job belongs to. + pub member_of: BTreeSet, + /// Selected nodes whose failure stops a group. + pub critical_nodes: BTreeSet, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct JobMember { + pub job_key: String, + /// Members this one waits for, from lineage and from the group's `after`. + pub waits_for: BTreeSet, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct TriggerRecord { pub trigger_id: uuid::Uuid, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs index e43c5c3acd..44894e2600 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs @@ -165,6 +165,7 @@ async fn finish_group( rows_affected: node.rows_affected, error_code: node.error_code.clone(), skipped_because_of: node.skipped_because_of.clone(), + error_message: None, }, ) .await?; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs new file mode 100644 index 0000000000..c5914d86e2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs @@ -0,0 +1,11 @@ +//! Node error text is normalized while a run change is read, so no caller stores or returns it raw. + +use serde::{Deserialize, Deserializer}; + +use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; + +pub(crate) fn bounded_error_text<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Ok(Option::::deserialize(deserializer)?.map(|text| redact_error_text(&text))) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 21b643ed02..1c4f2e737c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -2,6 +2,7 @@ pub(in crate::runs) mod change_run; pub(in crate::runs) mod changes; pub(in crate::runs) mod checks; pub(in crate::runs) mod claims; +pub(crate) mod error_text; pub(in crate::runs) mod events; pub(in crate::runs) mod nodes; pub(in crate::runs) mod output; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs index f753703706..a41eacfc2d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs @@ -23,6 +23,7 @@ pub(crate) async fn project_node( rows_affected, error_code, skipped_because_of, + .. } => ( node_key, outcome.as_str(), diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs index 9c62c054b8..bef39000b3 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs @@ -1,8 +1,9 @@ -//! Initial run and node rows written when a run is queued. +//! Initial run and node rows written when a run is queued; `rewrite_history` records what the request's own frozen inputs rewrite. use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use crate::runs::_helpers::requests::permissions::rewrites_history; use crate::runs::models::RunRequest; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -22,20 +23,21 @@ pub(crate) async fn project_queued_run( ); (transaction, _) = query(transaction, "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \ - requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21) \ + requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json, rewrite_history) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21,$22) \ ON CONFLICT (run_id) DO UPDATE SET location_id=excluded.location_id,target=excluded.target,snapshot_id=excluded.snapshot_id, \ image_id=excluded.image_id,kind=excluded.kind,origin=excluded.origin,job_key=excluded.job_key,parent_run_id=excluded.parent_run_id, \ root_run_id=excluded.root_run_id,trigger_id=excluded.trigger_id,tick_id=excluded.tick_id,rerun_of_run_id=excluded.rerun_of_run_id, \ requested_by=excluded.requested_by,priority=excluded.priority,selection_json=excluded.selection_json,write_set_digest=excluded.write_set_digest, \ status=excluded.status,queued_at=excluded.queued_at,node_counts_json=excluded.node_counts_json,last_event_storage_id=excluded.last_event_storage_id, \ - request_json=excluded.request_json,rewrite_history=FALSE,status_reason=NULL,launched_at=NULL,started_at=NULL,ended_at=NULL,cancel_requested_at=NULL", + request_json=excluded.request_json,rewrite_history=excluded.rewrite_history,status_reason=NULL,launched_at=NULL,started_at=NULL,ended_at=NULL,cancel_requested_at=NULL", vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id), Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()), Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id), Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id), Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest), - Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?; + Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request)), + Parameter::Boolean(rewrites_history(request, std::slice::from_ref(request)))]).await?; for node in &request.selection { (transaction, _) = query(transaction, "INSERT INTO sqb_run_nodes (run_id, node_key, status, writes) VALUES ($1,$2,'pending',$3)", diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index 3b563f265e..939a2c7c4f 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -5,6 +5,7 @@ use std::collections::BTreeSet; use uuid::Uuid; use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::runs::constants::FULL_REFRESH_SETTING; use crate::runs::models::{RunCommand, RunKind, RunOrigin, RunRequest}; use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; @@ -43,6 +44,10 @@ pub(crate) async fn frozen_group_children( if selection.is_empty() { continue; } + let mut settings = job.settings.clone(); + if let Some(full_refresh) = request.settings.get(FULL_REFRESH_SETTING) { + settings.insert(FULL_REFRESH_SETTING.into(), full_refresh.clone()); + } children.push(RunRequest { run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), kind: RunKind::Job, @@ -54,7 +59,7 @@ pub(crate) async fn frozen_group_children( .map(|id| Uuid::new_v5(&id, name.as_bytes())), write_set: selection.intersection(&writers).cloned().collect(), selection, - settings: job.settings.clone(), + settings, concurrency_groups: job.concurrency_groups.clone(), ..request.clone() }); diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs index 531d0b8286..6e83f5519c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -57,6 +57,11 @@ pub(crate) async fn request_job_run_in( "backfill window start must not follow end".into(), )); } + if request.full_refresh && request.window.is_some() { + return Err(ControlError::Invalid( + "a backfill window and a full refresh cannot be combined".into(), + )); + } let event_id = Uuid::new_v5(&request.run_id, MANUAL_JOB_AUDIT_TAG); let (transaction, retry) = request_retry(transaction, &context, (request.run_id, &json!(request))).await?; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs index 9ee5c9fd27..4a9acad803 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs @@ -91,7 +91,8 @@ pub(crate) fn execution_scopes( .collect() } -fn rewrites_history(request: &RunRequest, executions: &[RunRequest]) -> bool { +/// A window, a full refresh, a cursor setting or an ad-hoc destructive flag on any of `executions`. +pub(in crate::runs) fn rewrites_history(request: &RunRequest, executions: &[RunRequest]) -> bool { if request.window.is_some() { return true; } diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index d77bf9c763..9a7bea5f08 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -17,3 +17,6 @@ pub(crate) const RERUN_AUDIT_TAG: &[u8] = b"rerun-request"; /// Ad-hoc requests derive their audit event ID from the run ID so retries find the original. pub(crate) const ADHOC_AUDIT_TAG: &[u8] = b"adhoc-run-request"; + +/// The build setting a full-refresh launch adds to the frozen request and each group child. +pub(crate) const FULL_REFRESH_SETTING: &str = "full_refresh"; diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index 2a362d89a0..85e8a26843 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -2,6 +2,7 @@ use crate::credentials::models::AuditContext; use crate::declarations::models::JobDefinition; +use crate::runs::constants::FULL_REFRESH_SETTING; use crate::runs::models::{JobRunRequest, RunCommand, RunKind, RunOrigin, RunRequest}; use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; @@ -25,12 +26,17 @@ pub(crate) async fn frozen_job_request( })?; let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; - let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + let mut definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; let image_id = serde_json::from_value(row["image_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; let kind: RunKind = serde_json::from_value(row["kind"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.full_refresh { + definition + .settings + .insert(FULL_REFRESH_SETTING.into(), serde_json::Value::Bool(true)); + } let (transaction, write_set) = job_write_set(transaction, snapshot_id, &definition.selection).await?; Ok(( diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index a1b9f0310e..50649157a9 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -174,6 +174,9 @@ pub struct JobRunRequest { pub priority: i64, #[serde(default, skip_serializing_if = "Option::is_none")] pub window: Option, + /// Rebuild every selected model from scratch; it rewrites history, as a window does. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub full_refresh: bool, } /// A retry derives its execution scope from a terminal run, never from caller selectors. @@ -343,6 +346,17 @@ pub struct RunErrorText { pub message: String, } +/// A node's outcome error in one run: redacted, bounded text and the project file of the node, never its SQL. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunNodeFailure { + pub node_key: String, + pub error_code: Option, + /// Null when the worker reported none or the run's history has been pruned. + pub message: Option, + /// Relative to the project root, with `/` separators; null when the run's snapshot is gone. + pub path: Option, +} + /// One build attempt of a node by a run in this target; the same metadata as the run's node list. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct NodeRunRecord { @@ -428,6 +442,13 @@ pub enum RunChange { rows_affected: Option, error_code: Option, skipped_because_of: Option, + /// Redacted and bounded on the way in, whoever sends it; absent from older workers. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::runs::_helpers::lifecycle::error_text::bounded_error_text" + )] + error_message: Option, }, /// Observational attached-check facts never expand the frozen execution selection. CheckStarted { diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs index 237959031b..69a24fad30 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -47,6 +47,7 @@ pub(crate) async fn queue_sensor_run( reason: "Sensor condition matched".into(), priority: 0, window: None, + full_refresh: false, }; let (transaction, mut frozen) = frozen_job_request(transaction, &context, &authored).await?; if frozen.snapshot_id != evaluation.snapshot_id { diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs index b87929e1b4..0f0a8efe59 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs @@ -6,6 +6,7 @@ pub mod read_person_maintenance; pub mod read_person_microbatch_coverage; pub mod read_person_node_runs; pub mod read_person_run_events; +pub mod read_person_run_node_error; pub mod read_person_run_output; pub mod read_person_run_statement_sql; pub mod read_person_run_statements; diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_node_error.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_node_error.rs new file mode 100644 index 0000000000..c976559688 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_node_error.rs @@ -0,0 +1,85 @@ +//! Error text can quote warehouse data, so it needs view.evidence on that node; the read is audited like evidence. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::begin_authorized::begin_authorized; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::models::{RunChange, RunChangeRequest, RunNodeFailure}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter}; + +/// `None` when the node is not in that run of this location and target; refusal comes first either way. +pub async fn read_person_run_node_error( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location_id, target): (Uuid, &str), + (run_id, node_key): (Uuid, &str), +) -> Result, ControlError> { + let scope = PermissionScope { + action: Action::ViewEvidence, + location_id: Some(location_id), + target: Some(target.into()), + nodes: [node_key.into()].into(), + }; + let (transaction, context) = + begin_authorized(store, credentials, &scope, DefaultRole::None).await?; + let (mut transaction, rows) = query( + transaction, + "SELECT r.snapshot_id, n.error_code, e.payload FROM sqb_runs r \ + JOIN sqb_run_nodes n ON n.run_id=r.run_id AND n.kind='node' AND n.node_key=$4 \ + LEFT JOIN recall_events e ON e.storage_id=n.finished_storage_id \ + WHERE r.run_id=$1 AND r.location_id=$2 AND r.target=$3", + vec![ + Parameter::Id(run_id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(node_key.into()), + ], + ) + .await?; + let mut result = None; + if let Some(row) = rows.into_iter().next() { + let message = match &row["payload"] { + Value::Null => None, + payload => match serde_json::from_value::(payload["change"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .change + { + RunChange::NodeFinished { error_message, .. } => error_message, + _ => None, + }, + }; + let snapshot_id: Uuid = serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let nodes; + (transaction, nodes) = + read_snapshot_nodes(transaction, snapshot_id, Some(&scope.nodes)).await?; + result = Some(RunNodeFailure { + node_key: node_key.into(), + error_code: row["error_code"].as_str().map(str::to_owned), + message, + path: nodes + .get(node_key) + .and_then(|node| node.definition.get("path")) + .and_then(Value::as_str) + .map(|path| path.replace('\\', "/")), + }); + } + let transaction = append_audit( + transaction, + &context, + (Uuid::now_v7(), "error.read"), + ( + json!(scope), + json!(null), + json!({"run_id": run_id, "found": result.is_some()}), + ), + ) + .await?; + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs index 813c99c3ac..3fc18ce7a0 100644 --- a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -3,6 +3,7 @@ use sqlbuild_recall::records::models::OutputChunk; use uuid::Uuid; +use crate::api::errors::ClientError; use crate::api::main::client_response::client_response; use crate::api::models::{HeartbeatResponse, MutationResponse}; use crate::runs::models::{OutputPublication, RunChange, RunChangeRequest, RunExecution}; @@ -42,7 +43,13 @@ impl WorkerClient { .post(format!("{}events", self.run_url)) .json(&request), ) - .await?; + .await + .map_err(|error| match error { + ClientError::Rejected(reqwest::StatusCode::UNPROCESSABLE_ENTITY) => { + WorkerError::OlderServer + } + other => other.into(), + })?; Ok(result.applied) } diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs index 49240df632..e6c79b88b2 100644 --- a/crates/sqlbuild-orchestration/src/worker/errors.rs +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -8,6 +8,11 @@ pub enum WorkerError { Transport(#[from] reqwest::Error), #[error("worker API rejected the request with HTTP {0}")] Rejected(reqwest::StatusCode), + /// A server only refuses the shape of a worker's own report when it is older than the worker. + #[error( + "the control server is older than this worker and refused its run report; upgrade the server before its workers" + )] + OlderServer, #[error("worker heartbeat could not start: {0}")] Heartbeat(#[from] std::io::Error), #[error(transparent)] diff --git a/crates/sqlbuild-orchestration/tests/api/test_changes.rs b/crates/sqlbuild-orchestration/tests/api/test_changes.rs index c61c5fa997..4db85856e1 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_changes.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_changes.rs @@ -176,6 +176,7 @@ fn given_independent_http_observers_when_committed_state_changes_then_scoped_sse reason: "manual verification".into(), priority: 0, window: None, + full_refresh: false, }; assert!( request_job_run(store, &audit, &request) diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 59ff175e30..a1b7ab1c0d 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -76,6 +76,8 @@ mod test_proxy; mod test_publish_encoding; #[path = "test_reads.rs"] mod test_reads; +#[path = "test_server_errors.rs"] +mod test_server_errors; #[path = "test_statistics_recovery.rs"] mod test_statistics_recovery; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_denials.rs b/crates/sqlbuild-orchestration/tests/api/test_denials.rs index c3fe067e9e..3983b08d29 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_denials.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_denials.rs @@ -91,6 +91,7 @@ fn given_refused_requests_when_people_send_them_then_no_node_is_named_and_unknow reason: "Refresh orders".into(), priority: 0, window: None, + full_refresh: false, }; assert!( request_job_run(store, &roles.context, &queued) diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_live.rs b/crates/sqlbuild-orchestration/tests/api/test_person_live.rs index 600de855ad..31b6c4250a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_live.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_live.rs @@ -169,6 +169,7 @@ fn given_people_with_different_roles_when_they_bootstrap_and_subscribe_then_only reason: "manual verification".into(), priority: 0, window: None, + full_refresh: false, }; assert!( request_job_run(store, &roles.context, &request) diff --git a/crates/sqlbuild-orchestration/tests/api/test_server_errors.rs b/crates/sqlbuild-orchestration/tests/api/test_server_errors.rs new file mode 100644 index 0000000000..4de2859582 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_server_errors.rs @@ -0,0 +1,67 @@ +//! Axum's 5xx rejections name Rust types, so they reach clients as the generic JSON error; 4xx text describes the request. + +use axum::extract::Path; +use axum::routing::get; +use axum::{Extension, Router, middleware}; +use uuid::Uuid; + +use crate::api::_helpers::resources::server_errors::generic_server_errors; +use crate::api::errors::ApiError; +use crate::api::tests::test_types::ServerErrorTestCase; +use crate::store::errors::ControlError; +use crate::store::tests::store_helpers::run; + +#[test] +fn given_framework_rejections_when_served_then_server_errors_hide_their_text_and_request_errors_keep_it() + { + let test_cases = [ + ServerErrorTestCase { + description: "a missing extension names a Rust type", + path: "/missing", + expected_status: 500, + expected_body: r#"{"error":"internal server error"}"#, + }, + ServerErrorTestCase { + description: "a malformed path value describes the caller's own request", + path: "/runs/not-a-run", + expected_status: 400, + expected_body: "Invalid URL: Cannot parse `run_id` with value `not-a-run`", + }, + ServerErrorTestCase { + description: "a handler's JSON error is already generic", + path: "/failed", + expected_status: 500, + expected_body: r#"{"error":"control-store operation failed"}"#, + }, + ]; + run(async { + let application = Router::new() + .route("/missing", get(|Extension(_): Extension| async {})) + .route("/runs/{run_id}", get(|Path(_): Path| async {})) + .route( + "/failed", + get(|| async { + Err::<(), ApiError>(ControlError::Schema("synthetic".into()).into()) + }), + ) + .layer(middleware::map_response(generic_server_errors)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("socket"); + let url = format!("http://{}", listener.local_addr().expect("address")); + let server = tokio::spawn(async move { axum::serve(listener, application).await }); + for test_case in test_cases { + let response = reqwest::get(format!("{url}{}", test_case.path)) + .await + .expect("request"); + let status = response.status().as_u16(); + let body = response.text().await.expect("body"); + assert!( + status == test_case.expected_status && body.starts_with(test_case.expected_body), + "{}: {status} {body}", + test_case.description + ); + } + server.abort(); + }); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 3fa0ad9d3f..0b878cb812 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -28,6 +28,12 @@ pub(crate) struct DevelopmentTestCase { pub(crate) expected_status: crate::declarations::models::TriggerStatus, } +pub(crate) struct OlderServerReportTestCase { + pub(crate) description: &'static str, + pub(crate) error_message: Option<&'static str>, + pub(crate) expected_upgrade_refusal: bool, +} + pub(crate) struct WorkerRedirectTestCase { pub(crate) description: &'static str, pub(crate) expected_status: reqwest::StatusCode, @@ -177,3 +183,33 @@ pub(crate) struct UiPathTestCase { pub(crate) expected_status: u16, pub(crate) expected_body: &'static str, } + +pub(crate) struct ServerErrorTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + pub(crate) expected_status: u16, + pub(crate) expected_body: &'static str, +} + +/// The node report shape a server had before failed nodes carried error text. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +#[allow(dead_code)] +pub(crate) struct OlderNodeReport { + event_id: uuid::Uuid, + run_id: uuid::Uuid, + claim_id: Option, + change: OlderNodeFinished, +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +#[allow(dead_code)] +pub(crate) struct OlderNodeFinished { + kind: String, + node_key: String, + outcome: crate::runs::models::NodeOutcome, + rows_affected: Option, + error_code: Option, + skipped_because_of: Option, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs index ffc06aa6ce..4c4f7f78a2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_worker.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -25,7 +25,8 @@ use crate::api::tests::helpers::{start_api, stop_api}; use crate::api::tests::output_helpers::{chunk, output_contract}; use crate::api::tests::recall_helpers::recall_contract; use crate::api::tests::test_types::{ - ApiContractTestCase, TestApi, WorkerDeadlineTestCase, WorkerRedirectTestCase, + ApiContractTestCase, OlderNodeReport, OlderServerReportTestCase, TestApi, + WorkerDeadlineTestCase, WorkerRedirectTestCase, }; use crate::credentials::tests::location_helpers::create_location; use crate::runs::tests::helpers::{queue, request, snapshot}; @@ -198,6 +199,7 @@ fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_a rows_affected: Some(7), error_code: None, skipped_because_of: None, + error_message: None, }, ) .await @@ -387,6 +389,87 @@ fn given_a_credentialed_worker_when_the_server_redirects_then_the_redirect_is_no } } +#[test] +fn given_an_older_server_when_a_worker_reports_error_text_then_the_refusal_asks_for_a_server_upgrade() + { + let test_cases = [ + OlderServerReportTestCase { + description: "a report without error text keeps working", + error_message: None, + expected_upgrade_refusal: false, + }, + OlderServerReportTestCase { + description: "error text the older server cannot read names the upgrade order", + error_message: Some("Conversion Error: could not convert 'n/a' to INT32"), + expected_upgrade_refusal: true, + }, + ]; + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated older server"); + let url = format!("http://{}", listener.local_addr().expect("address")); + let application = Router::new().route( + "/api/runs/{run_id}/events", + axum::routing::post(|axum::Json(_): axum::Json| async { + axum::Json(serde_json::json!({"applied": true})) + }), + ); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, application) + .with_graceful_shutdown(async { + let _ = signal.await; + }) + .await + .map_err(ControlError::from) + }); + let api = TestApi { + url, + shutdown, + task, + }; + let client = connect_worker( + &api.url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("worker client"); + for test_case in test_cases { + let reported = client + .report( + Uuid::now_v7(), + RunChange::NodeFinished { + node_key: "model:orders".into(), + outcome: NodeOutcome::Failed, + rows_affected: None, + error_code: Some("R002".into()), + skipped_because_of: None, + error_message: test_case.error_message.map(str::to_owned), + }, + ) + .await; + let message = reported.as_ref().err().map(ToString::to_string); + assert_eq!( + ( + reported.is_ok(), + matches!(reported, Err(WorkerError::OlderServer)), + message + .is_some_and(|text| text.contains("upgrade the server before its workers")), + ), + ( + !test_case.expected_upgrade_refusal, + test_case.expected_upgrade_refusal, + test_case.expected_upgrade_refusal + ), + "{}", + test_case.description + ); + } + stop_api(api).await; + }); +} + #[test] fn given_a_slow_recall_server_when_the_delivery_deadline_expires_then_the_native_store_stops_waiting() { diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs index 186b36662b..502b771942 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs @@ -79,6 +79,7 @@ fn given_expired_audit_rows_when_pruned_then_undelivered_rows_survive_and_retrie reason: "Refresh orders".into(), priority: 1, window: None, + full_refresh: false, }; assert!( request_job_run(store, &context, &request) diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs index 6b2c0e2b1d..f761083db0 100644 --- a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -45,6 +45,7 @@ pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { rows_affected: Some(1), error_code: None, skipped_because_of: None, + error_message: None, }), ); changes.push(RunChange::Finished { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs index 00f6ded985..08b78b9dc2 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs @@ -250,6 +250,7 @@ fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membershi rows_affected: Some(1), error_code: None, skipped_because_of: None, + error_message: None, }, RunChange::Finished { outcome: RunOutcome::Failed, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index 0c5abb3f2d..d4ef1c1b64 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -20,10 +20,14 @@ mod test_faults; mod test_group_admission; #[path = "test_groups.rs"] mod test_groups; +#[path = "test_job_detail.rs"] +mod test_job_detail; #[path = "test_lifecycle.rs"] mod test_lifecycle; #[path = "test_monitor.rs"] mod test_monitor; +#[path = "test_node_errors.rs"] +mod test_node_errors; #[path = "test_permissions.rs"] mod test_permissions; #[path = "test_requests.rs"] diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 8e0e670966..84e674cfa8 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -92,6 +92,7 @@ fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queu reason: "Exercise mixed queue ordering".into(), priority: test_case.parent_priority, window: None, + full_refresh: false, }, ) .await @@ -191,6 +192,7 @@ fn given_group_children_sharing_a_limit_of_one_when_claiming_then_they_run_one_a reason: "Load the warehouse once".into(), priority: 1, window: None, + full_refresh: false, }, ) .await @@ -292,6 +294,7 @@ fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknow reason: "Exercise launcher eligibility".into(), priority: 1, window: None, + full_refresh: false, }, ) .await @@ -529,6 +532,7 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th reason: "Exercise capacity-aware admission".into(), priority: index as i64 + 1, window: None, + full_refresh: false, }, ) .await @@ -654,6 +658,7 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t reason: "Older group".into(), priority: 1, window: None, + full_refresh: false, }; request_job_run(store, &actor, &older) .await @@ -829,6 +834,7 @@ fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelate reason: "Exercise blocked group queue".into(), priority: 1, window: None, + full_refresh: false, }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index 10f1b53bf8..4911455b32 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -79,6 +79,7 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a reason: "Build the group".into(), priority: 1, window: None, + full_refresh: false, }; assert!( request_job_run(store, &context, &request) @@ -125,6 +126,7 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a rows_affected: Some(11), error_code: None, skipped_because_of: None, + error_message: None, }, RunChange::Finished { outcome: RunOutcome::Success, @@ -353,6 +355,7 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups reason: "Build independent groups".into(), priority: 1, window: None, + full_refresh: false, }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs new file mode 100644 index 0000000000..eac63a861d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -0,0 +1,284 @@ +//! One job's declaration reads like the job list; a full-refresh launch needs run.rewrite_history down to group children and records it. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::{BuiltinRole, DefaultRole, RoleAssignmentRequest}; +use crate::credentials::tests::role_helpers::{ + SUPPORT_ROLE, assign, credentials, orders_grant, role_fixture, +}; +use crate::declarations::main::read_person_job::read_person_job; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::main::{ + advance_groups::advance_groups, list_runs::list_runs, request_person_run::request_person_run, +}; +use crate::runs::models::{JobRunRequest, PersonRunRequest, RunRecord, RunRequest, RunWindow}; +use crate::runs::tests::helpers::{json_column, rows, snapshot}; +use crate::runs::tests::test_types::JobDetailTestCase; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_rewrite_history_gate_them() + { + let test_cases = [ + JobDetailTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_order: ["daily_orders", "customers"], + }, + JobDetailTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_order: ["daily_orders", "customers"], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let mut input = snapshot(roles.location_id); + input.generation = 2; + input.documents.insert( + "roles".into(), + json!({SUPPORT_ROLE: {"description": "Answer order questions", + "grants": [orders_grant(json!(["view"]), &["orders"])]}}), + ); + input.documents.insert("jobs".into(), json!({ + "daily_orders": {"description": "Build orders", "selection": ["orders"], "settings": { + "vars": "{\"api_key\": \"orders-key-123\"}", + "defer_to": "/srv/orders/state", + "defer_sources_to": "production", + "warehouse": "postgres://orders:orders-db-pass@warehouse.example.test/orders", + "concurrency": 4}}, + "customers": {"description": "Build customers", "selection": ["customers"]}, + })); + input.documents.insert( + "groups".into(), + json!({"orders_group": { + "description": "Orders, then customers", "selection": ["customers", "orders"], + "jobs": {"customers": ["daily_orders"], "daily_orders": []}}}), + ); + input + .nodes + .get_mut("orders") + .expect("orders node") + .definition = json!({"materialization": "table", "config": {"critical": true}}); + publish_snapshot(store, &input).await.expect("publish jobs"); + let administrator = (&roles.context, credentials(&roles.administrator)); + let location = roles.location_id; + let read = |who, job: &'static str| { + read_person_job( + store, + who, + (location, "development", job), + DefaultRole::None, + ) + }; + + let job = read(administrator, "daily_orders") + .await + .expect("administrator") + .expect("published job"); + assert_eq!( + ( + json!(job.settings), + json!(job.critical_nodes), + json!(job.member_of), + job.members.len() + ), + ( + json!({"concurrency": 4, "vars": "[REDACTED]", "defer_to": "[REDACTED]", + "defer_sources_to": "production", + "warehouse": "postgres://orders:[REDACTED]@warehouse.example.test/orders"}), + json!(["orders"]), + json!(["orders_group"]), + 0 + ), + "{}: vars and a deferral path are withheld, a target name shows, URI passwords are redacted", + test_case.description + ); + let group = read(administrator, "orders_group") + .await + .expect("administrator") + .expect("published group"); + assert_eq!( + group + .members + .iter() + .map(|member| (member.job_key.as_str(), json!(member.waits_for))) + .collect::>(), + [ + (test_case.expected_order[0], json!([])), + (test_case.expected_order[1], json!(["daily_orders"])) + ], + "{}: members in run order, not name order, with their waits", + test_case.description + ); + assert!( + read(administrator, "nightly") + .await + .expect("administrator") + .is_none(), + "{}: an unknown job is not found", + test_case.description + ); + let member = (&roles.context, roles.member_id); + for job in ["daily_orders", "nightly"] { + assert!( + matches!(read(member, job).await, Err(ControlError::Denied { .. })), + "{}: view on one node does not list the target, and refusal hides whether {job} exists", + test_case.description + ); + } + + let operator = create_principal(store, &roles.context, "orders-operator") + .await + .expect("operator"); + assign( + store, + (&roles.context, &roles.administrator), + RoleAssignmentRequest { + principal_id: operator, + role: BuiltinRole::Operator.into(), + location_id: None, + target: None, + revoke: false, + }, + ) + .await; + let launch = |job_key: &str| JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: job_key.into(), + reason: "Rebuild orders from scratch".into(), + priority: 0, + window: None, + full_refresh: true, + }; + let refused = launch("daily_orders"); + assert!( + matches!( + request_person_run( + store, + (&roles.context, operator), + PersonRunRequest::Job(&refused) + ) + .await, + Err(ControlError::Denied { .. }) + ), + "{}: an operator may run the job but not rewrite its history", + test_case.description + ); + let both = JobRunRequest { + window: Some(RunWindow::Integer { start: 1, end: 2 }), + ..launch("daily_orders") + }; + assert!( + matches!( + request_person_run(store, administrator, PersonRunRequest::Job(&both)).await, + Err(ControlError::Invalid(_)) + ), + "{}: a window and a full refresh are refused together", + test_case.description + ); + let group_launch = launch("orders_group"); + let full_job = launch("daily_orders"); + for request in [full_job.clone(), group_launch.clone()] { + assert!( + request_person_run(store, administrator, PersonRunRequest::Job(&request)) + .await + .expect("administrator launch"), + "{}: an administrator may rebuild from scratch", + test_case.description + ); + let stored = rows( + store, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await; + assert_eq!( + json_column(&stored[0]["request_json"])["settings"]["full_refresh"], + json!(true), + "{}: the frozen request carries the full refresh", + test_case.description + ); + } + let plain = JobRunRequest { + full_refresh: false, + ..launch("customers") + }; + request_person_run(store, administrator, PersonRunRequest::Job(&plain)) + .await + .expect("ordinary launch"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let scope = (location, "development"); + advance_groups(store, &lease, (&BTreeMap::new(), scope)) + .await + .expect("start the group"); + let runs = list_runs(store, scope, 50).await.expect("runs"); + let rewrites = |matches: &dyn Fn(&RunRecord) -> bool| { + runs.iter() + .filter(|run| matches(run)) + .map(|run| run.rewrite_history) + .collect::>() + }; + assert_eq!( + ( + rewrites(&|run| run.run_id == full_job.run_id), + rewrites(&|run| run.run_id == group_launch.run_id), + rewrites(&|run| run.run_id == plain.run_id), + ), + (vec![true], vec![true], vec![false]), + "{}: full refreshes record that they rewrite history, an ordinary run does not", + test_case.description + ); + let children = rewrites(&|run| run.parent_run_id == Some(group_launch.run_id)); + assert!( + !children.is_empty() && children.iter().all(|flag| *flag), + "{}: started group children record the full refresh: {children:?}", + test_case.description + ); + let stored = rows( + store, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(group_launch.run_id)], + ) + .await; + let parent: RunRequest = + serde_json::from_value(json_column(&stored[0]["request_json"])).expect("request"); + let transaction = begin(store, TransactionMode::Read).await.expect("read"); + let (transaction, children) = frozen_group_children(transaction, &parent) + .await + .expect("children"); + commit(transaction).await.expect("commit"); + assert!( + children.len() == 2 + && children + .iter() + .all(|child| child.settings.get("full_refresh") == Some(&json!(true))), + "{}: every group child rebuilds from scratch", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index f032ef356a..25e79224a6 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -246,6 +246,7 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ rows_affected: Some(17), error_code: (outcome == NodeOutcome::Failed).then(|| "E100".into()), skipped_because_of: None, + error_message: None, }, ..started.clone() }; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_node_errors.rs b/crates/sqlbuild-orchestration/tests/runs/test_node_errors.rs new file mode 100644 index 0000000000..e8572b9594 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_node_errors.rs @@ -0,0 +1,195 @@ +//! A failed node's error text is redacted and bounded once a worker reports it, and only evidence on that node reads it. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::role_helpers::{credentials, orders_grant, role_fixture}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChangeRequest}; +use crate::runs::tests::helpers::{queue, request, rows, snapshot}; +use crate::runs::tests::test_types::RunNodeErrorTestCase; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::Parameter; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_history::main::read_person_run_node_error::read_person_run_node_error; + +const EXPECTED_MESSAGE: &str = "Conversion Error: could not convert 'n/a' to INT32 \ + (password=[REDACTED], url=postgres://orders:[REDACTED]@warehouse.example.test/orders, \ + cut 'postgres://orders:[REDACTED] by the warehouse, port http://warehouse.example.test:8080/health) "; + +#[test] +fn given_a_failed_node_when_people_read_its_error_then_only_evidence_on_that_node_sees_redacted_bounded_text() + { + let test_cases = [ + RunNodeErrorTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_message: EXPECTED_MESSAGE, + }, + RunNodeErrorTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_message: EXPECTED_MESSAGE, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let mut input = snapshot(roles.location_id); + input.generation = 2; + input.documents.insert( + "roles".into(), + json!({"orders_support": {"description": "Answer order questions", + "grants": [orders_grant(json!(["view", "view.evidence"]), &["orders"])]}}), + ); + input + .nodes + .get_mut("orders") + .expect("orders node") + .definition = json!({"materialization": "table", "path": "models\\orders.sql"}); + let published = publish_snapshot(store, &input).await.expect("publish"); + queue( + store, + &request( + (roles.location_id, published.snapshot_id), + 1, + 1, + &["customers", "orders"], + ), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("queued run"); + let run_id = claim.request.run_id; + let raw = format!( + "Conversion Error: could not convert 'n/a' to INT32 (password=orders-db-pass, \ + url=postgres://orders:orders-db-pass@warehouse.example.test/orders, \ + cut 'postgres://orders:orders-d... by the warehouse, port http://warehouse.example.test:8080/health) {}", + "x".repeat(3000) + ); + for change in [ + json!({"kind": "started"}), + json!({"kind": "node_finished", "node_key": "orders", "outcome": "failed", + "rows_affected": null, "error_code": "T002", "skipped_because_of": null, + "error_message": raw}), + json!({"kind": "node_finished", "node_key": "customers", "outcome": "skipped", + "rows_affected": null, "error_code": null, "skipped_because_of": "orders"}), + ] { + let reported: RunChangeRequest = serde_json::from_value(json!({ + "event_id": Uuid::now_v7(), "run_id": run_id, + "claim_id": claim.claim_id, "change": change, + })) + .expect("a worker's report body"); + change_run( + store, + RunAuthority::Worker { + run_id, + claim_id: claim.claim_id, + }, + &reported, + ) + .await + .expect("worker report"); + } + let member = (&roles.context, roles.member_id); + let location = roles.location_id; + let read = |who, run_id, node: &'static str| { + read_person_run_node_error(store, who, (location, "development"), (run_id, node)) + }; + let error = read(member, run_id, "orders") + .await + .expect("evidence on orders") + .expect("orders ran"); + let message = error.message.expect("reported message"); + assert_eq!( + ( + error.error_code.as_deref(), + error.path.as_deref(), + message.chars().count() + ), + (Some("T002"), Some("models/orders.sql"), 2000), + "{}: code, a portable path, and the bound", + test_case.description + ); + assert!( + message.starts_with(test_case.expected_message) && message.ends_with('…'), + "{}: secrets are redacted before the cut: {message}", + test_case.description + ); + assert!( + matches!( + read(member, run_id, "customers").await, + Err(ControlError::Denied { .. }) + ), + "{}: evidence on another node does not reach this one", + test_case.description + ); + let administrator = (&roles.context, credentials(&roles.administrator)); + let skipped = read(administrator, run_id, "customers") + .await + .expect("administrator") + .expect("customers was selected"); + assert_eq!( + (skipped.message, skipped.path), + (None, None), + "{}: no reported text and no published path", + test_case.description + ); + for (run_id, node) in [(run_id, "products"), (Uuid::from_u128(9), "orders")] { + assert!( + read(administrator, run_id, node) + .await + .expect("administrator") + .is_none(), + "{}: a node outside the run and an unknown run are not found", + test_case.description + ); + } + let stored = rows( + store, + "SELECT payload FROM recall_events WHERE invocation_id=$1 AND event_type='run_node_finished'", + vec![Parameter::Text(format!("scheduler/{run_id}"))], + ) + .await; + assert!( + stored.len() == 2 + && stored + .iter() + .all(|row| !row.to_string().contains("orders-d")), + "{}: the stored fact never held the secret", + test_case.description + ); + let audited = rows( + store, + "SELECT count(*) AS reads FROM sqb_security_audit WHERE action='error.read'", + vec![], + ) + .await; + assert_eq!( + audited[0]["reads"].as_i64(), + Some(4), + "{}: every allowed read is audited", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index f6d4c726fb..c47dc489cd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -140,6 +140,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi reason: "Refresh order totals".into(), priority: 0, window: None, + full_refresh: false, }; assert!( matches!( diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index a66096e888..e5301db2af 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -72,6 +72,7 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an start: "2026-07-03T01:17:23+05:30".parse().expect("start"), end: "2026-07-04T01:17:23+05:30".parse().expect("end"), }), + full_refresh: false, }; for invalid in [ JobRunRequest { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index cc08da0422..6d3856bd86 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -81,6 +81,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ reason: "Build orders".into(), priority: 1, window: Some(RunWindow::Integer { start: -3, end: 11 }), + full_refresh: false, }, ) .await @@ -142,6 +143,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ error_code: None, skipped_because_of: (outcome == NodeOutcome::Skipped) .then(|| "orders".into()), + error_message: None, }); } changes.push(RunChange::Finished { @@ -267,6 +269,16 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ execution.description ); let runs = list_runs(store, scope, 20).await.expect("list reruns"); + assert!( + runs.iter() + .filter(|run| { + run.run_id == request.run_id + || run.parent_run_id == Some(request.run_id) + }) + .all(|run| run.rewrite_history), + "{}: a backfill's rerun and its children record that they rewrite history", + execution.description + ); assert_eq!( runs.iter() .find(|run| run.run_id == request.run_id) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 0f4cc521f6..d937ebc730 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -191,3 +191,15 @@ impl crate::runs::types::CommandPlanner for ExactKeyPlanner { Ok(plan) } } + +pub(crate) struct RunNodeErrorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_message: &'static str, +} + +pub(crate) struct JobDetailTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_order: [&'static str; 2], +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 4960f0ab8f..32366c394a 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -169,6 +169,7 @@ pub(crate) async fn finish_run( reason: "Exercise terminal sensor scope".into(), priority: 10, window: None, + full_refresh: false, }, ) .await @@ -215,6 +216,7 @@ pub(crate) async fn finish_claimed_run( rows_affected: None, error_code: None, skipped_because_of: None, + error_message: None, }); } for change in changes { diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs index 472d2b5e20..9d195c33c3 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -105,6 +105,7 @@ fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_mu reason: "Exercise wide execution evidence".into(), priority: 10, window: None, + full_refresh: false, }, ) .await @@ -132,6 +133,7 @@ fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_mu rows_affected: None, error_code: None, skipped_because_of: None, + error_message: None, })) .chain(std::iter::once(RunChange::Finished { outcome: RunOutcome::Success, diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs index fe02a84690..560cdbcef7 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs @@ -177,6 +177,7 @@ fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queu reason: "Deliver before whole-run completion".into(), priority: 10, window: None, + full_refresh: false, }, ) .await @@ -206,6 +207,7 @@ fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queu rows_affected: None, error_code: None, skipped_because_of: None, + error_message: None, }, ]; for change in changes { @@ -304,6 +306,7 @@ fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queu rows_affected: None, error_code: None, skipped_because_of: None, + error_message: None, }, ] { change_run( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index e6f1dfb8cd..a5fe394f0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -8,6 +8,7 @@ use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; use sqlbuild_core::models::ResourceKind; +use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; use sqlbuild_orchestration::runs::models::{ NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome, }; @@ -49,7 +50,7 @@ impl WorkerBuild { refusal: Mutex::default(), }); for key in state.execution.skipped_nodes.keys() { - state.complete(worker.py(), key, (NodeOutcome::Skipped, None, None))?; + state.complete(worker.py(), key, (NodeOutcome::Skipped, None, (None, None)))?; } Ok(state) } @@ -142,7 +143,7 @@ impl WorkerBuild { } else { NodeOutcome::Skipped }; - self.complete(pipeline.py(), &key, (outcome, None, None))?; + self.complete(pipeline.py(), &key, (outcome, None, (None, None)))?; } } } @@ -152,7 +153,7 @@ impl WorkerBuild { /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { for key in keys { - self.complete(py, key, (NodeOutcome::Success, None, None))?; + self.complete(py, key, (NodeOutcome::Success, None, (None, None)))?; } Ok(()) } @@ -161,7 +162,11 @@ impl WorkerBuild { &self, py: Python<'_>, key: &str, - (outcome, rows_affected, error_code): (NodeOutcome, Option, Option), + (outcome, rows_affected, (error_code, error_message)): ( + NodeOutcome, + Option, + (Option, Option), + ), ) -> PyResult<()> { if !self.execution.claim.request.selection.contains(key) { return Ok(()); @@ -183,6 +188,7 @@ impl WorkerBuild { rows_affected, error_code, skipped_because_of: self.execution.skipped_nodes.get(key).cloned(), + error_message: error_message.as_deref().map(redact_error_text), }, )?; self.completed @@ -229,7 +235,7 @@ impl WorkerBuild { key: &str, outcome: NodeOutcome, ) -> PyResult<()> { - self.complete(py, key, (outcome, None, None)) + self.complete(py, key, (outcome, None, (None, None))) } /// A test or audit the snapshot records has started; the node it attaches to starts with it. @@ -344,7 +350,11 @@ impl WorkerBuild { Some(value) => value.extract()?, None => None, }; - self.complete(result.py(), key, (outcome, rows, error))?; + let message = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + self.complete(result.py(), key, (outcome, rows, (error, message)))?; } Ok(()) } @@ -427,7 +437,7 @@ impl WorkerBuild { }) }; if let Some(outcome) = outcome { - self.complete(py, key, (outcome, None, None))?; + self.complete(py, key, (outcome, None, (None, None)))?; } } } @@ -442,7 +452,7 @@ impl WorkerBuild { NodeOutcome::Skipped }, None, - None, + (None, None), ), )?; } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 625cdf7873..ce21db6214 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -8,6 +8,7 @@ Online: https://sqlbuild.com/docs/cli/server/ ## Contents +- Upgrade - Set retention - Grant roles to OIDC groups (beta) - Inspect published SQL @@ -22,6 +23,17 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Upgrade + +Upgrade the server first, then the locations and workers that connect to it. A worker newer than +its server can send a run report the server cannot read. The worker then stops with: + +```text +the control server is older than this worker and refused its run report; upgrade the server before its workers +``` + +Upgrade the server to the worker's release or newer, then rerun the run. + ## Set retention Set how much history the server keeps: diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index 0e66db6d41..f4349b7da0 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -22,6 +22,7 @@ import type { RunDetail, RunEventPage, RunListQuery, + RunNodeError, RunOutputLine, RunRowPage, RunRowQuery, @@ -94,6 +95,8 @@ export interface ApiClient extends PlatformApi { /** One statement's SQL, fetched when its row is expanded. `view.sql` */ getStatementSql(scope: Scope, runId: string, statementId: string): Promise; listRunOutput(scope: Scope, runId: string): Promise; + /** One failed node's error text, fetched when its row is expanded. `view.evidence` on the node */ + getRunNodeError(scope: Scope, runId: string, nodeKey: string): Promise; /** Queued runs in claim order, with why each one waits. */ listQueue(scope: Scope): Promise; timeline(scope: Scope, window: { since: string; until: string }): Promise; diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index 1368d6c0c3..38e35f1815 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -436,10 +436,18 @@ describe('launchJob', () => { expect(posts[0]).toMatchObject({ job_key: 'daily', target: 'development', window: { kind: 'integer', start: 1200, end: 1500 } }); }); - it('leaves the window out of an ordinary run', async () => { + it('leaves the window and full refresh out of an ordinary run', async () => { const { posts, client } = server(Response.json({ error: 'stop here' }, { status: 409 })); await expect(client.launchJob(ORDERS, 'daily')).rejects.toThrow(); expect(posts[0]).not.toHaveProperty('window'); + expect(posts[0]).not.toHaveProperty('full_refresh'); + }); + + it("asks for a full refresh and shows the server's refusal as it comes", async () => { + const { posts, client } = server(Response.json({ error: 'run.rewrite_history is required; decision 7' }, { status: 403 })); + await expect(client.launchJobFromScratch(ORDERS, 'daily')).rejects.toMatchObject({ status: 403, message: expect.stringContaining('run.rewrite_history') }); + expect(posts[0]).toMatchObject({ job_key: 'daily', full_refresh: true, reason: 'Full refresh started from the web UI' }); + expect(posts[0]).not.toHaveProperty('window'); }); it("shows a plain-text refusal of a body the server couldn't read", async () => { @@ -468,6 +476,22 @@ describe('launchJob', () => { }); }); +describe('getJob', () => { + function server(job: Response) { + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => (new URL(request.url).pathname.endsWith('/jobs/nightly') ? job.clone() : Response.json([]))) + ); + return createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + } + + it('treats an unknown job as not found, and keeps a refusal as an error', async () => { + await expect(server(Response.json({ error: 'job not found' }, { status: 404 })).getJob(ORDERS, 'nightly')).resolves.toBeNull(); + await expect(server(Response.json({ error: 'view is required; decision 3' }, { status: 403 })).getJob(ORDERS, 'nightly')).rejects.toMatchObject({ status: 403 }); + }); +}); + describe('administration writes', () => { const ada = { principal_id: 'p-ada', name: 'ada', created_at: '2026-11-01T00:00:00Z', disabled_at: null, has_password: true, groups: [], active_sessions: 2, last_signed_in_at: null }; @@ -623,3 +647,57 @@ describe('getCatalogNode', () => { expect(paths.filter((p) => p.endsWith('/sql'))).toHaveLength(1); }); }); + +describe('node error text', () => { + const RUN = '0191f0ee-0000-7000-8000-000000000001'; + + /** Evidence holds on orders but not on customers, as for a person whose grant covers one node. */ + function serve(actions: string[]) { + const paths: string[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = decodeURIComponent(new URL(request.url).pathname); + paths.push(path); + if (path.endsWith('/bootstrap')) { + return Response.json({ login_mode: 'password', access: { name: 'cara', principal_id: 'p', actions: [], locations: [{ location_id: '0191f0aa-orders', name: 'Orders', target: 'development', actions }] } }); + } + if (path.endsWith(`/runs/${RUN}/nodes/model:orders/error`)) { + return Response.json({ node_key: 'model:orders', error_code: 'T002', message: 'Conversion Error: password=[REDACTED]\nLINE 3', path: 'models/orders.sql' }); + } + return Response.json({ error: 'missing view.evidence permission' }, { status: 403 }); + }) + ); + return paths; + } + + it('reads one node by its own key and maps the file', async () => { + const paths = serve(['view', 'view.evidence']); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + await client.bootstrap(); + expect(await client.getRunNodeError(ORDERS, RUN, 'model:orders')).toEqual({ + nodeKey: 'model:orders', + errorCode: 'T002', + message: 'Conversion Error: password=[REDACTED]\nLINE 3', + file: 'models/orders.sql' + }); + expect(paths.at(-1)).toBe(`/api/control/locations/0191f0aa-orders/targets/development/runs/${RUN}/nodes/model:orders/error`); + }); + + it('asks for a refused node once, so each audited refusal happens only once', async () => { + const paths = serve(['view', 'view.evidence']); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + await client.bootstrap(); + for (let i = 0; i < 2; i++) await expect(client.getRunNodeError(ORDERS, RUN, 'model:customers')).rejects.toMatchObject({ status: 403 }); + expect(paths.filter((p) => p.endsWith('/error'))).toHaveLength(1); + }); + + it('sends no request where the location grants no evidence', async () => { + const paths = serve(['view']); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + await client.bootstrap(); + await expect(client.getRunNodeError(ORDERS, RUN, 'model:orders')).rejects.toMatchObject({ status: 403 }); + expect(paths.filter((p) => p.endsWith('/error'))).toHaveLength(0); + }); +}); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index f7aec54ce7..5ae4b62c85 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -17,7 +17,7 @@ import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState import type { AuthMode, Bootstrap, Capability, Location, PlatformEvent, Scope, WarehouseSync } from './types'; import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, LocationSummary, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; -import type { AdhocRunCheck, AdhocRunRequest, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunOutputLine, RunStatement, StatementSql } from './types/operate'; +import type { AdhocRunCheck, AdhocRunRequest, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; type Wire = components['schemas']; @@ -36,11 +36,13 @@ const SUPPORTED: ReadonlySet = new Set([ 'listRunStatements', 'getStatementSql', 'listRunOutput', + 'getRunNodeError', 'cancelRun', 'rerunRun', 'listJobs', 'getJob', 'launchJob', + 'launchJobFromScratch', 'listTriggers', 'getTrigger', 'setTriggerStates', @@ -235,6 +237,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt // The same for run output, which needs view.evidence on every node the run selected. let evidenceLocations = new Set(); const outputRefused = new Set(); + // A node's error text needs view.evidence on that node alone. + const errorRefused = new Set(); const implemented = { supports: (method: keyof ApiClient) => SUPPORTED.has(method), changes: new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)), @@ -294,6 +298,17 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!data) throw new ApiError(errorText(error, response), response.status); return toRunOutput(data); }, + async getRunNodeError(scope: Scope, runId: string, nodeKey: string): Promise { + const refusal = `${scope.locationId}|${runId}|${nodeKey}`; + if (!evidenceLocations.has(scope.locationId) || errorRefused.has(refusal)) throw new ApiError("You may not view this node's error text", 403); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/nodes/{node_key}/error', { + params: { path: { ...runPath(scope, runId), node_key: nodeKey } } + }); + if (response.status === 401) onUnauthorized(); + if (response.status === 403) errorRefused.add(refusal); + if (!data) throw new ApiError(errorText(error, response), response.status); + return { nodeKey: data.node_key, errorCode: data.error_code ?? null, message: data.message ?? null, file: data.path ?? null }; + }, async cancelRun(scope: Scope, runId: string): Promise { const body = { run_id: runId, event_id: crypto.randomUUID(), reason: 'Cancelled from the web UI' }; const { error, response } = await http.POST('/api/control/runs/cancel', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); @@ -317,21 +332,22 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return jobs.map((job) => toJob(job, triggers, runs)); }, async getJob(scope: Scope, key: string): Promise { - const [jobs, triggers, runs] = await Promise.all([listedJobs(scope), triggersOf(scope), recentRuns(scope)]); - const wire = jobs.find((job) => job.job_key === key); - return wire ? jobDetailOf(toJob(wire, triggers, runs), wire) : null; - }, - /** A window makes it a backfill, which the server also checks against `run.rewrite_history`. */ - async launchJob(scope: Scope, key: string, options?: JobLaunchOptions): Promise { const { locationId, target } = parseLocationKey(scope.locationId); - const window = options?.window ? runWindowOf(options.window) : undefined; - const body = { run_id: crypto.randomUUID(), location_id: locationId, target, job_key: key, priority: 0, reason: window ? 'Backfill started from the web UI' : 'Started from the web UI', window }; - const { error, response } = await http.POST('/api/control/runs', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); - inFlight.clear(); + const [read, triggers, runs] = await Promise.all([ + http.GET('/api/control/locations/{location_id}/targets/{target}/jobs/{job_key}', { params: { path: { location_id: locationId, target, job_key: key } } }), + triggersOf(scope), + recentRuns(scope) + ]); + const { data, error, response } = read; if (response.status === 401) onUnauthorized(); - if (!response.ok) throw new ApiError(errorText(error, response), response.status); - return requireRun(scope, body.run_id); + if (response.status === 404) return null; + if (!data) throw new ApiError(errorText(error, response), response.status); + return jobDetailOf(toJob(data, triggers, runs), data); }, + /** A window makes it a backfill, which the server also checks against `run.rewrite_history`. */ + launchJob: (scope: Scope, key: string, options?: JobLaunchOptions): Promise => startJob(scope, key, options, false), + /** The server checks a full refresh against `run.rewrite_history` and refuses it with a window. */ + launchJobFromScratch: (scope: Scope, key: string, options?: JobLaunchOptions): Promise => startJob(scope, key, options, true), async listCatalogNodes(scope: Scope): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/nodes', { params: { path: { location_id: locationId, target } } }); @@ -756,6 +772,19 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return index < 0 ? null : { run: listed[index], wire: wires[index], listed }; } + /** A window makes it a backfill; the server checks either against `run.rewrite_history` and refuses both together. */ + async function startJob(scope: Scope, key: string, options: JobLaunchOptions | undefined, fullRefresh: boolean): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const window = options?.window ? runWindowOf(options.window) : undefined; + const reason = fullRefresh ? 'Full refresh started from the web UI' : window ? 'Backfill started from the web UI' : 'Started from the web UI'; + const body = { run_id: crypto.randomUUID(), location_id: locationId, target, job_key: key, priority: 0, reason, window, full_refresh: fullRefresh || undefined }; + const { error, response } = await http.POST('/api/control/runs', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); + inFlight.clear(); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + return requireRun(scope, body.run_id); + } + async function requireRun(scope: Scope, runId: string): Promise { const found = await findRun(scope, runId); if (!found) throw new ApiError('The run is not in the recent runs of this location', 404); diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index 246be981fc..31b1881d0e 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -63,11 +63,42 @@ describe('job mapping', () => { const job = toJob(wire, triggers, [run('r-inv', 'inventory', null), run('r-ord', 'orders', null)]); expect(job).toMatchObject({ key: 'orders', kind: 'job', nodeCount: 2, triggerIds: ['t-daily'], concurrencyGroups: ['warehouse_heavy'], select: null, overlap: null, priority: null }); expect(job.lastRun?.id).toBe('r-ord'); - const detail = jobDetailOf(job, wire); + const detail = jobDetailOf(job, { ...wire, settings: { vars: '[REDACTED]', concurrency: 4, empty_ok: true, models: ['a', 'b'] }, members: [], member_of: ['nightly'], critical_nodes: ['model:orders'] }); expect(detail.resolvedNodes).toEqual(['model:customers', 'model:orders']); - expect([detail.settings, detail.concurrency, detail.memberOf, detail.cursorType]).toEqual([null, null, null, null]); + expect(detail.settings).toEqual({ vars: '[REDACTED]', concurrency: '4', empty_ok: true, models: '["a","b"]' }); + expect([detail.groupMembers, detail.members, detail.memberOf, detail.criticalNodes]).toEqual([null, null, ['nightly'], ['model:orders']]); + expect([detail.concurrency, detail.cursorType, detail.declaration]).toEqual([null, null, null]); expect(toJob({ ...wire, job_key: 'nightly', kind: 'group' }, triggers, []).kind).toBe('group'); }); + + it("steps a group's members by what they wait for, in the server's run order rather than by name", () => { + const wire: W['JobDetailRecord'] = { + job_key: 'nightly', + kind: 'group', + description: 'Everything nightly', + selection: [], + concurrency_groups: [], + snapshot_id: SNAPSHOT, + settings: {}, + member_of: [], + critical_nodes: [], + members: [ + { job_key: 'stock', waits_for: [] }, + { job_key: 'orders', waits_for: [] }, + { job_key: 'customers', waits_for: ['orders'] }, + { job_key: 'audit', waits_for: ['customers', 'stock'] } + ] + }; + const detail = jobDetailOf(toJob(wire, [], []), wire); + expect(detail.groupMembers?.map((m) => [m.jobKey, m.stage, m.upstream.map((u) => u.jobKey)])).toEqual([ + ['stock', 0, []], + ['orders', 0, []], + ['customers', 1, ['orders']], + ['audit', 2, ['customers', 'stock']] + ]); + expect(detail.groupMembers?.flatMap((m) => m.upstream.map((u) => u.source))).toEqual([null, null, null]); + expect(detail.members).toEqual(['stock', 'orders', 'customers', 'audit']); + }); }); describe('runWindowOf', () => { diff --git a/ui/src/lib/api/http-jobs.ts b/ui/src/lib/api/http-jobs.ts index 97d95e4f61..63e332e7d3 100644 --- a/ui/src/lib/api/http-jobs.ts +++ b/ui/src/lib/api/http-jobs.ts @@ -1,6 +1,6 @@ import type { components } from '$control/schema.d.ts'; import { ApiError } from './client'; -import type { CursorWindow, Job, JobDetail, Run, RunRef, Tick, Trigger, TriggerDetail, TriggerKind } from './types/operate'; +import type { CursorWindow, Job, JobDetail, JobGroupMember, Run, RunRef, Tick, Trigger, TriggerDetail, TriggerKind } from './types/operate'; type Wire = components['schemas']; @@ -153,18 +153,33 @@ export function toJob(wire: Wire['JobRecord'], triggers: Trigger[], runs: Run[]) }; } -export function jobDetailOf(job: Job, wire: Wire['JobRecord']): JobDetail { +/** A setting as shown: text and flags as stored, anything else as JSON. */ +const shownSetting = (value: unknown): string | boolean => (typeof value === 'string' || typeof value === 'boolean' ? value : JSON.stringify(value)); + +/** Members in the server's run order; a member's step follows the latest step it waits for. */ +function groupMembersOf(members: Wire['JobMember'][]): JobGroupMember[] { + const stageOf = new Map(); + return members.map((m) => { + const stage = Math.max(-1, ...m.waits_for.map((w) => stageOf.get(w) ?? -1)) + 1; + stageOf.set(m.job_key, stage); + // The server reports the waits, not whether lineage or `after` caused each one. + return { jobKey: m.job_key, upstream: m.waits_for.map((jobKey) => ({ jobKey, source: null })), stage }; + }); +} + +export function jobDetailOf(job: Job, wire: Wire['JobDetailRecord']): JobDetail { return { ...job, + members: wire.kind === 'group' ? wire.members.map((m) => m.job_key) : null, declaration: null, exclude: [], - settings: null, + settings: Object.fromEntries(Object.entries(wire.settings).map(([name, value]) => [name, shownSetting(value)])), resolvedNodes: wire.selection, concurrency: null, targets: null, - groupMembers: null, - criticalNodes: [], - memberOf: null, + groupMembers: wire.kind === 'group' ? groupMembersOf(wire.members) : null, + criticalNodes: wire.critical_nodes, + memberOf: wire.member_of, cursorType: null }; } diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index 49e78f99f0..f84884dfcc 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -19,6 +19,7 @@ import type { RunDetail, RunEventPage, RunListQuery, + RunNodeError, RunOutputLine, RunStatement, Scope, @@ -214,6 +215,12 @@ export class MockApiClient implements ApiClient { return this.#operate.listRunOutput(scope, runId); } + async getRunNodeError(scope: Scope, runId: string, nodeKey: string): Promise { + const node = (await this.getRunDetail(scope, runId))?.nodes.find((n) => n.runId === runId && n.nodeKey === nodeKey); + if (!node) throw new ApiError('node not found in this run', 404); + return { nodeKey, errorCode: node.errorCode, message: node.errorDetail ?? node.errorMessage, file: node.errorLocation?.file ?? null }; + } + async listQueue(scope: Scope): Promise { this.#location(scope); return this.#operate.listQueue(scope); diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 4f21e277e9..8c80ffa930 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -210,8 +210,8 @@ export type RunNode = { rowsAffected: number | null; errorCode: string | null; errorMessage: string | null; - /** Where in the project the failure points, e.g. the model file and line of the failing statement. */ - errorLocation: { file: string; line: number } | null; + /** Where in the project the failure points: the model file, and the line when it is known. */ + errorLocation: { file: string; line: number | null } | null; /** Full error text (warehouse message, statement, trace); `errorMessage` is its first line. */ errorDetail: string | null; skippedBecauseOf: SkippedBecauseOf | null; @@ -219,6 +219,16 @@ export type RunNode = { critical: boolean; }; +/** A failed node's error, read on request because it can quote warehouse data. `view.evidence` on the node */ +export type RunNodeError = { + nodeKey: string; + errorCode: string | null; + /** Redacted and cut to 2,000 characters by the server; null when none was recorded. */ + message: string | null; + /** The node's file, relative to the project root. */ + file: string | null; +}; + export type RunEdge = { from: string; to: string }; export const RECALL_EVENT_TYPES = [ @@ -526,14 +536,14 @@ export type ConcurrencyGroup = { export type JobGroupMember = { jobKey: string; - /** Jobs this member waits for: lineage-derived plus declared `after`. */ - upstream: { jobKey: string; source: 'lineage' | 'after' }[]; + /** Jobs this member waits for: lineage-derived plus declared `after`; null where the cause isn't reported. */ + upstream: { jobKey: string; source: 'lineage' | 'after' | null }[]; /** Position in the derived order; members in the same stage run side by side. */ stage: number; }; export type JobDetail = Job & { - /** Null where the server doesn't report it, as for `settings`, `concurrency`, `groupMembers` and `memberOf`. */ + /** Null where the server doesn't report it, as for `declaration`, `concurrency` and `targets`. */ declaration: { sourceFile: string; line: number; code: string } | null; exclude: string[]; /** Settings are CLI flags validated against the command table. */ diff --git a/ui/src/routes/jobs/[name]/group-order.svelte b/ui/src/routes/jobs/[name]/group-order.svelte index fe89275eba..a06b994e6a 100644 --- a/ui/src/routes/jobs/[name]/group-order.svelte +++ b/ui/src/routes/jobs/[name]/group-order.svelte @@ -20,8 +20,8 @@ {m.jobKey} {#if m.upstream.length} {#each m.upstream as u (u.jobKey)} - - after {u.jobKey} · {u.source === 'after' ? 'set in code' : 'uses its data'} + + after {u.jobKey}{u.source ? ` · ${u.source === 'after' ? 'set in code' : 'uses its data'}` : ''} {/each} {:else} diff --git a/ui/src/routes/runs/[id]/failures-panel.svelte b/ui/src/routes/runs/[id]/failures-panel.svelte index a96c91ce36..bbaa7f3ea1 100644 --- a/ui/src/routes/runs/[id]/failures-panel.svelte +++ b/ui/src/routes/runs/[id]/failures-panel.svelte @@ -1,23 +1,43 @@ @@ -32,22 +52,34 @@
    {#each failed as n (`${n.runId}/${n.nodeKey}`)} {@const key = `${n.runId}/${n.nodeKey}`} + {@const loaded = read(key)} + {@const where = place(n, key)} + {@const detail = n.errorDetail ?? loaded?.message ?? null}
  • {#if app.client.supports('listRunStatements')}{:else}{n.nodeKey}{/if} {#if n.runId !== view.run?.id}in {n.jobKey ?? shortRunId(n.runId)}{/if} - {n.errorMessage ?? 'Failed'} - {#if n.errorLocation} - {n.errorLocation.file}:{n.errorLocation.line} - + {n.errorMessage ?? loaded?.message?.split('\n')[0] ?? (n.errorCode ? `Failed with ${n.errorCode}` : 'Failed')} + {#if where} + {where} + {/if} - {#if n.errorDetail} + {#if n.errorDetail || readable} {/if}
    - {#if open.has(key) && n.errorDetail} -
    {n.errorDetail}
    - {#if n.errorCode}

    Error code {n.errorCode}

    {/if} + {#if open.has(key)} + {@const got = errors[key]} + {#if detail} +
    {detail}
    + {:else if got && 'failure' in got} +

    {got.failure}

    + {:else if got} +

    No error text was recorded for this node.

    + {:else if !n.errorDetail} +

    Loading the error…

    + {/if} + {#if n.errorCode && detail}

    Error code {n.errorCode}

    {/if} {/if}
  • {/each} diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index ec563d7817..eb9077cd87 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -13,6 +13,17 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Upgrade + +Upgrade the server first, then the locations and workers that connect to it. A worker newer than +its server can send a run report the server cannot read. The worker then stops with: + +```text +the control server is older than this worker and refused its run report; upgrade the server before its workers +``` + +Upgrade the server to the worker's release or newer, then rerun the run. + ## Set retention Set how much history the server keeps: