diff --git a/Cargo.lock b/Cargo.lock index e76a2dcf85..25a14905cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1175,6 +1175,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "siphasher" version = "1.0.4" @@ -1225,12 +1245,14 @@ dependencies = [ "arrayvec", "bstr", "either", + "errno", "memchr", "once_cell", "regex-automata", "syn 2.0.119", "syn 3.0.3", "thiserror 2.0.21", + "windows-sys", ] [[package]] @@ -1350,6 +1372,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", + "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", "sqlbuild-cache", diff --git a/Cargo.toml b/Cargo.toml index 129467fd7f..84be0319c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,8 +105,13 @@ ruff_text_size = "=0.0.4" # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`. # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output. rustc-hash = "2" +# The native SIGTERM handler: sets a flag the CLI watches, without unsafe code in the workspace. +signal-hook = { version = "0.3", default-features = false } +# Features signal-hook's registry enables, matched in the sqb build through sqlbuild-build-features. +errno = "0.3" +windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] } # Recall's local store: SQLite compiled in, so every build has the same engine version. -rusqlite = { version = "0.37", features = ["bundled"] } +rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } saphyr-parser = "0.1" regex = "1" diff --git a/crates/sqlbuild-build-features/Cargo.toml b/crates/sqlbuild-build-features/Cargo.toml index 84a5d1ee63..949dead6d8 100644 --- a/crates/sqlbuild-build-features/Cargo.toml +++ b/crates/sqlbuild-build-features/Cargo.toml @@ -22,6 +22,13 @@ once_cell.workspace = true regex-automata.workspace = true thiserror.workspace = true +# signal-hook's registry, in the extension only, enables these on the platforms that use them. +[target.'cfg(unix)'.dependencies] +errno.workspace = true + +[target.'cfg(windows)'.dependencies] +windows-sys.workspace = true + # Proc-macro dependencies resolve with build dependencies, so the syn features PyO3's macros use # are enabled here. The empty build script makes Cargo resolve them. [build-dependencies] diff --git a/crates/sqlbuild-cli/src/argv/_helpers/table.rs b/crates/sqlbuild-cli/src/argv/_helpers/table.rs index 94b61666df..12bfc5b45b 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/table.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/table.rs @@ -131,6 +131,15 @@ impl TableReader { help: help(choice.get("help").unwrap_or(&Json::Null)), }) .collect(), + removed_choices: items(json, "removed_choices") + .iter() + .map(|removed| { + ( + text(removed, "value").unwrap_or_default(), + text(removed, "instead").unwrap_or_default(), + ) + }) + .collect(), } } } diff --git a/crates/sqlbuild-cli/src/argv/_helpers/values.rs b/crates/sqlbuild-cli/src/argv/_helpers/values.rs index df7da8834b..79602bc483 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/values.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/values.rs @@ -114,6 +114,16 @@ impl Run<'_> { if choices.iter().any(|choice| choice == text) { return Ok(()); } + if let Some((_, instead)) = action + .removed_choices + .iter() + .find(|(removed, _)| removed == text) + { + return Err(argument_error( + action, + &format!("{} was removed; {instead}", self.host.repr(text)), + )); + } let listed: Vec = choices .iter() .map(|choice| python_str_repr(choice)) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 9947a4dcdf..d2b8b61be1 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -4397,13 +4397,21 @@ "--format" ], "dest": "query_format", - "default": "long", "choices": [ "long", - "table", - "json", - "csv" + "json" + ], + "removed_choices": [ + { + "value": "table", + "instead": "the default output prints a table when the result fits; use --format long for one record per row" + }, + { + "value": "csv", + "instead": "use --json for programs, or the warehouse's own export for bulk data" + } ], + "help": "by default a table when the result fits, else one record per row; long always prints records; json for programs", "mutex_group": 0, "group": 1 }, @@ -7183,14 +7191,21 @@ "--format" ], "dest": "recall_format", - "default": "long", "choices": [ "long", - "table", - "json", - "csv" - ], - "help": "long (default) for reading, json for programs, csv for export; table is deprecated", + "json" + ], + "removed_choices": [ + { + "value": "table", + "instead": "the default output prints a table when the result fits; use --format long for one record per row" + }, + { + "value": "csv", + "instead": "use --json for programs" + } + ], + "help": "by default a table when the result fits, else one record per row; long always prints records; json for programs", "mutex_group": 0, "group": 1 }, @@ -7374,14 +7389,21 @@ "--format" ], "dest": "recall_format", - "default": "long", "choices": [ "long", - "table", - "json", - "csv" - ], - "help": "long (default) for reading, json for programs, csv for export; table is deprecated", + "json" + ], + "removed_choices": [ + { + "value": "table", + "instead": "use the default long output" + }, + { + "value": "csv", + "instead": "use --json for programs" + } + ], + "help": "long (default) for reading, json for programs", "mutex_group": 0, "group": 1 }, diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 29d4726dc1..ef1b833fbb 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -159,7 +159,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ ("max_columns", NamespaceDefault::None), ("query_sql", NamespaceDefault::None), ("query_file", NamespaceDefault::None), - ("query_format", NamespaceDefault::Text("long")), + ("query_format", NamespaceDefault::None), ("query_limit", NamespaceDefault::Int(20)), ("query_no_limit", NamespaceDefault::Bool(false)), ("cost_selector", NamespaceDefault::Text("latest")), @@ -228,7 +228,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ ("recall_command", NamespaceDefault::None), ("recall_sql", NamespaceDefault::None), ("recall_backend", NamespaceDefault::Text("logs")), - ("recall_format", NamespaceDefault::Text("long")), + ("recall_format", NamespaceDefault::None), ("recall_limit", NamespaceDefault::Int(100)), ("recall_no_limit", NamespaceDefault::Bool(false)), ("recall_max_cell_bytes", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index 037208ddb0..44d88ac46a 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -98,6 +98,8 @@ pub struct ActionSpec { /// Sub-command names and their parsers, in order. pub parsers: Vec<(String, ParserSpec)>, pub choices_help: Vec, + /// Values no longer accepted, each with what to use instead; never listed in help. + pub removed_choices: Vec<(String, String)>, } /// One argument group's title and description. diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 71b7c4de91..550b31ebe3 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -12,6 +12,8 @@ pub(crate) const DBT_INIT_SUBCOMMAND: &str = "init"; pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; /// `sqb recall` reads history and is never recorded itself. pub(crate) const RECALL_COMMAND_NAME: &str = "recall"; +/// A `--select-file` line starting with this is a comment. +pub(crate) const SELECTOR_COMMENT_PREFIX: &str = "#"; const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir); const SELECT: Field = field("select", Source::Select); diff --git a/crates/sqlbuild-cli/src/dispatch/main/mod.rs b/crates/sqlbuild-cli/src/dispatch/main/mod.rs index 7cc59e2303..594f4fab79 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/mod.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/mod.rs @@ -3,4 +3,5 @@ pub mod command_dispatch; pub mod event_export_warning; pub mod failure_report; pub mod invocation_policy; +pub mod selector_file_selectors; pub mod unavailable_handler; diff --git a/crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs b/crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs new file mode 100644 index 0000000000..ea47a6e995 --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs @@ -0,0 +1,17 @@ +//! The selectors a `--select-file` lists. + +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::dispatch::constants::SELECTOR_COMMENT_PREFIX; + +/// One selector per stripped line of `text`, without blank lines and `#` comments. +#[must_use] +pub fn selector_file_selectors(text: &str) -> Vec { + python_splitlines(text) + .into_iter() + .map(python_strip) + .filter(|line| !line.is_empty() && !line.starts_with(SELECTOR_COMMENT_PREFIX)) + .map(str::to_owned) + .collect() +} diff --git a/crates/sqlbuild-config/src/recall/constants.rs b/crates/sqlbuild-config/src/recall/constants.rs index 74a066bec1..76f2a2fab1 100644 --- a/crates/sqlbuild-config/src/recall/constants.rs +++ b/crates/sqlbuild-config/src/recall/constants.rs @@ -2,7 +2,8 @@ pub const RECALL_SECTION: &str = "recall"; /// Keys `[recall]` and `targets..recall` accept, sorted. -pub const RECALL_KEYS: [&str; 7] = [ +pub const RECALL_KEYS: [&str; 8] = [ + "audit_evidence", "diagnostics_min_severity", "enabled", "max_output_bytes", @@ -15,6 +16,8 @@ pub const RECALL_KEYS: [&str; 7] = [ pub const REMOVED_CONNECTION_KEY: &str = "connection"; /// The diagnostic severities a Recall store can keep from, least severe first. pub const RECALL_SEVERITIES: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; +/// Which audit outcomes keep their evidence rows in Recall, the default first. +pub const AUDIT_EVIDENCE_MODES: [&str; 3] = ["warn-and-error", "error", "off"]; /// The longest shutdown flush budget, in milliseconds. pub const MAX_SHUTDOWN_TIMEOUT_MS: u64 = 600_000; /// The longest retention, in days. diff --git a/crates/sqlbuild-config/src/recall/main/recall_section.rs b/crates/sqlbuild-config/src/recall/main/recall_section.rs index fce31d1f59..981b50b83a 100644 --- a/crates/sqlbuild-config/src/recall/main/recall_section.rs +++ b/crates/sqlbuild-config/src/recall/main/recall_section.rs @@ -3,7 +3,8 @@ use crate::models::ConfigValue; use crate::recall::_helpers::values::{non_empty, output_bytes, retention_days, timeout_ms}; use crate::recall::constants::{ - MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES, REMOVED_CONNECTION_KEY, + AUDIT_EVIDENCE_MODES, MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES, + REMOVED_CONNECTION_KEY, }; use crate::recall::models::RecallSection; @@ -65,6 +66,16 @@ pub fn recall_section(value: &ConfigValue, label: &str) -> Result { + let text = non_empty(value, label, key)?; + if !AUDIT_EVIDENCE_MODES.contains(&text) { + return Err(format!( + "{label}.audit_evidence must be one of: {}", + AUDIT_EVIDENCE_MODES.join(", ") + )); + } + section.audit_evidence = Some(text.to_owned()); + } other => { return Err(format!( "{label} contains unknown key(s): {other}. Allowed keys: {}", diff --git a/crates/sqlbuild-config/src/recall/models.rs b/crates/sqlbuild-config/src/recall/models.rs index 6b8e21e9ec..219353c31b 100644 --- a/crates/sqlbuild-config/src/recall/models.rs +++ b/crates/sqlbuild-config/src/recall/models.rs @@ -13,4 +13,6 @@ pub struct RecallSection { pub diagnostics_min_severity: Option, /// Bytes of each output stream to record; `Some(0)` records everything. pub max_output_bytes: Option, + /// Which audit outcomes keep their evidence rows: one of [`AUDIT_EVIDENCE_MODES`](crate::recall::constants::AUDIT_EVIDENCE_MODES). + pub audit_evidence: Option, } diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs index 61ca10eb0e..96e9812512 100644 --- a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -17,6 +17,7 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { ("shutdown_timeout", text("1500ms")), ("diagnostics_min_severity", text("debug")), ("max_output_bytes", ConfigValue::Integer(4_194_304)), + ("audit_evidence", text("error")), ]), expected_section: RecallSection { enabled: Some(false), @@ -26,6 +27,7 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { shutdown_timeout_ms: Some(1500), diagnostics_min_severity: Some("debug".to_owned()), max_output_bytes: Some(4_194_304), + audit_evidence: Some("error".to_owned()), }, }, ValidSectionTestCase { @@ -64,6 +66,13 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { #[test] fn given_invalid_sections_when_validated_then_the_message_names_the_key() { let test_cases = [ + InvalidSectionTestCase { + description: "audit evidence names one of its modes", + value: table(&[("audit_evidence", text("always"))]), + label: "targets.prod.recall", + expected_message_prefix: "targets.prod.recall.audit_evidence must be one of: \ + warn-and-error, error, off", + }, InvalidSectionTestCase { description: "output bytes must not be negative", value: table(&[("max_output_bytes", ConfigValue::Integer(-1))]), diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index cd6572eb5b..a56ae32466 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -79,7 +79,7 @@ pub(crate) const FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: [&str; 8] = [ "sql", "statement_sql", ]; -pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [ +pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 56] = [ "adapter", "attached_column_name", "attached_target_kind", @@ -92,6 +92,7 @@ pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [ "binding_key", "command", "configured_batch_size", + "cost_status", "cursor_end_exclusive", "cursor_start", "definition_fingerprint", @@ -653,7 +654,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ event_type: "invocation_completed", required_correlations: &[], required_payload: &[], - allowed_payload: &["command", "duration_ms", "exit_code"], + allowed_payload: &["command", "cost_status", "duration_ms", "exit_code"], terminal: true, }, EventDefinition { @@ -662,11 +663,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ required_payload: &[], allowed_payload: &[ "command", + "cost_status", "duration_ms", "error_code", "error_message", "error_type", "exit_code", + "signal_number", ], terminal: true, }, @@ -1010,6 +1013,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ "intent", "job_id", "metadata", + "outcome", "purpose", "query_id", "row_count", diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 96c15345fc..36ee9d8e7e 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -113,7 +113,6 @@ pub(crate) const QUERY_COMMAND: &str = "query"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; -pub(crate) const CSV_FORMAT: &str = "csv"; /// What replaces the cut tail of a transient status message. pub(crate) const STATUS_ELLIPSIS: &str = "..."; /// The resource kind of audits and the prefix of their resource identifiers. diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs index e483f76b46..7a6c3bc531 100644 --- a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -1,11 +1,11 @@ //! Whether a command's stdout carries machine output, so human progress must use stderr. use crate::progress::constants::{ - CSV_FORMAT, DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, + DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, }; use crate::progress::models::OutputModeFacts; -/// `--json`, `sqb dbt ... --json`, `lineage --format json` or `query --format csv|json`. +/// `--json`, `sqb dbt ... --json`, `lineage --format json` or `query --format json`. #[must_use] pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); @@ -13,9 +13,7 @@ pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { _ if facts.json => true, Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, - Some(QUERY_COMMAND) => { - [CSV_FORMAT, JSON_FORMAT].contains(&format(&facts.query_format).as_str()) - } + Some(QUERY_COMMAND) => format(&facts.query_format) == JSON_FORMAT, _ => false, } } diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs index 4bee5218ae..b1e56752e3 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs @@ -269,9 +269,9 @@ fn given_parsed_options_when_choosing_output_mode_then_machine_output_is_detecte expected_machine_output: true, }, OutputModeTestCase { - description: "query csv", + description: "query json", facts: OutputModeFacts { - query_format: Some("csv".to_owned()), + query_format: Some("json".to_owned()), ..facts("query", &[]) }, expected_machine_output: true, diff --git a/crates/sqlbuild-core/src/query_command/_helpers/boxed.rs b/crates/sqlbuild-core/src/query_command/_helpers/boxed.rs new file mode 100644 index 0000000000..dfed793a8d --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/_helpers/boxed.rs @@ -0,0 +1,109 @@ +//! Query rows in a box: names and types centered over aligned columns. + +use crate::query_command::constants::{ + BOX_BOTTOM, BOX_HORIZONTAL, BOX_RULE, BOX_TOP, BOX_VERTICAL, +}; +use crate::query_command::models::QueryDisplay; +use crate::text::main::cell_len::cell_len; + +/// Whether `value` prints on one line, so a box can hold it. +fn single_line(value: &str) -> bool { + !value.chars().any(char::is_control) +} + +fn rule(widths: &[usize], [left, middle, right]: [char; 3]) -> String { + let segments: Vec = widths + .iter() + .map(|width| BOX_HORIZONTAL.to_string().repeat(width + 2)) + .collect(); + format!("{left}{}{right}", segments.join(&middle.to_string())) +} + +fn centered(text: &str, width: usize) -> String { + let spare: usize = width.saturating_sub(cell_len(text)); + let left: usize = spare / 2; + format!("{}{text}{}", " ".repeat(left), " ".repeat(spare - left)) +} + +fn aligned(text: &str, width: usize, right: bool) -> String { + let spare: String = " ".repeat(width.saturating_sub(cell_len(text))); + if right { + format!("{spare}{text}") + } else { + format!("{text}{spare}") + } +} + +fn line(cells: Vec) -> String { + let joined: String = cells.join(&format!(" {BOX_VERTICAL} ")); + format!("{BOX_VERTICAL} {joined} {BOX_VERTICAL}") +} + +/// The box's lines when it is at most `limit` cells wide and every value is one line; `None` otherwise, so nothing is cut to fit. +pub(crate) fn boxed_lines(display: &QueryDisplay, limit: usize) -> Option> { + let types: Vec<&str> = (0..display.columns.len()) + .map(|index| { + display + .types + .get(index) + .and_then(Option::as_deref) + .unwrap_or_default() + }) + .collect(); + let mut widths: Vec = display + .columns + .iter() + .zip(&types) + .map(|(column, kind)| cell_len(column).max(cell_len(kind))) + .collect(); + for row in &display.rows { + for (index, value) in row.iter().enumerate() { + if !single_line(value) { + return None; + } + if let Some(width) = widths.get_mut(index) { + *width = (*width).max(cell_len(value)); + } + } + } + let total: usize = widths.iter().map(|width| width + 3).sum::() + 1; + if total > limit || display.columns.iter().any(|column| !single_line(column)) { + return None; + } + let mut lines: Vec = vec![ + rule(&widths, BOX_TOP), + line( + display + .columns + .iter() + .zip(&widths) + .map(|(column, width)| centered(column, *width)) + .collect(), + ), + ]; + if types.iter().any(|kind| !kind.is_empty()) { + lines.push(line( + types + .iter() + .zip(&widths) + .map(|(kind, width)| centered(kind, *width)) + .collect(), + )); + } + lines.push(rule(&widths, BOX_RULE)); + for row in &display.rows { + lines.push(line( + widths + .iter() + .enumerate() + .map(|(index, width)| { + let value: &str = row.get(index).map_or("", String::as_str); + let right: bool = display.numeric.get(index).copied().unwrap_or(false); + aligned(value, *width, right) + }) + .collect(), + )); + } + lines.push(rule(&widths, BOX_BOTTOM)); + Some(lines) +} diff --git a/crates/sqlbuild-core/src/query_command/_helpers/footer.rs b/crates/sqlbuild-core/src/query_command/_helpers/footer.rs index e145c57f3f..d6697b1545 100644 --- a/crates/sqlbuild-core/src/query_command/_helpers/footer.rs +++ b/crates/sqlbuild-core/src/query_command/_helpers/footer.rs @@ -1,20 +1,18 @@ -//! The row count and truncation lines under long and table output. +//! The row count line under every text layout. +use crate::query_command::constants::MANY_ROWS; use crate::query_command::models::QueryDisplay; -/// A blank line, ` row(s)`, and the truncation hint when the limit cut rows off. +/// A blank line and the row count: when the limit cut rows off, the limit and how to change it; above [`MANY_ROWS`], a hint to aggregate or limit the query. pub(crate) fn footer(display: &QueryDisplay) -> Vec { let count: usize = display.rows.len(); - let mut lines: Vec = vec![ - String::new(), - format!("{count} {}", if count == 1 { "row" } else { "rows" }), - ]; - if display.truncated { - lines.push(display.limit.map_or_else(String::new, |limit| { - format!( - "Showing {limit} rows. Use --limit to show more or --no-limit to disable the limit." - ) - })); - } - lines + let noun: &str = if count == 1 { "row" } else { "rows" }; + let line: String = match display.limit.filter(|_| display.truncated) { + Some(limit) => format!("{count} {noun} shown (limit {limit}; use --limit or --no-limit)"), + None if count > MANY_ROWS => { + format!("{count} {noun}; to read fewer, aggregate with GROUP BY or add a LIMIT") + } + None => format!("{count} {noun}"), + }; + vec![String::new(), line] } diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_long.rs b/crates/sqlbuild-core/src/query_command/_helpers/long.rs similarity index 94% rename from crates/sqlbuild-core/src/query_command/main/render_query_long.rs rename to crates/sqlbuild-core/src/query_command/_helpers/long.rs index d56e24dab7..a751bbaad6 100644 --- a/crates/sqlbuild-core/src/query_command/main/render_query_long.rs +++ b/crates/sqlbuild-core/src/query_command/_helpers/long.rs @@ -6,7 +6,7 @@ use crate::query_command::models::QueryDisplay; /// `-[ RECORD n ]---+` then `column | value` lines per row, then the row count. #[must_use] -pub fn render_query_long(display: &QueryDisplay) -> String { +pub(crate) fn long_text(display: &QueryDisplay) -> String { let width: usize = display .columns .iter() diff --git a/crates/sqlbuild-core/src/query_command/_helpers/mod.rs b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs index c8f10e5269..32089b756c 100644 --- a/crates/sqlbuild-core/src/query_command/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs @@ -1 +1,3 @@ +pub(crate) mod boxed; pub(crate) mod footer; +pub(crate) mod long; diff --git a/crates/sqlbuild-core/src/query_command/constants.rs b/crates/sqlbuild-core/src/query_command/constants.rs index 7d9aac4965..a13e21d4b5 100644 --- a/crates/sqlbuild-core/src/query_command/constants.rs +++ b/crates/sqlbuild-core/src/query_command/constants.rs @@ -1,6 +1,11 @@ //! Query output layout. pub(crate) const RECORD_RULE_WIDTH: usize = 40; -pub(crate) const TABLE_SEPARATOR: &str = " | "; -pub(crate) const CSV_DELIMITER: char = ','; -pub(crate) const CSV_QUOTE: char = '"'; +/// Above this many rows the footer suggests aggregating or limiting the query. +pub(crate) const MANY_ROWS: usize = 50; +/// The box layout's borders: corners, tees, crosses and lines. +pub(crate) const BOX_TOP: [char; 3] = ['┌', '┬', '┐']; +pub(crate) const BOX_RULE: [char; 3] = ['├', '┼', '┤']; +pub(crate) const BOX_BOTTOM: [char; 3] = ['└', '┴', '┘']; +pub(crate) const BOX_HORIZONTAL: char = '─'; +pub(crate) const BOX_VERTICAL: char = '│'; diff --git a/crates/sqlbuild-core/src/query_command/main/mod.rs b/crates/sqlbuild-core/src/query_command/main/mod.rs index 7496e3a434..5d9e192526 100644 --- a/crates/sqlbuild-core/src/query_command/main/mod.rs +++ b/crates/sqlbuild-core/src/query_command/main/mod.rs @@ -1,5 +1,3 @@ pub mod render_dag_summary; -pub mod render_query_csv; pub mod render_query_json; -pub mod render_query_long; -pub mod render_query_table; +pub mod render_query_text; diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs b/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs deleted file mode 100644 index 1247dfa793..0000000000 --- a/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Query rows as CSV, as Python's default `csv.writer` writes them. - -use crate::query_command::constants::{CSV_DELIMITER, CSV_QUOTE}; - -fn field(value: &str) -> String { - if value - .chars() - .any(|character| matches!(character, CSV_DELIMITER | CSV_QUOTE | '\n' | '\r')) - { - format!("\"{}\"", value.replace('"', "\"\"")) - } else { - value.to_owned() - } -} - -fn record(values: &[String]) -> String { - if let [only] = values - && only.is_empty() - { - return "\"\"\n".to_owned(); - } - let fields: Vec = values.iter().map(|value| field(value)).collect(); - format!("{}\n", fields.join(",")) -} - -/// The header record then one record per row; fields quoted only when they must be. -#[must_use] -pub fn render_query_csv(columns: &[String], rows: &[Vec]) -> String { - let mut text: String = record(columns); - for row in rows { - text.push_str(&record(row)); - } - text -} diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_table.rs b/crates/sqlbuild-core/src/query_command/main/render_query_table.rs deleted file mode 100644 index 1ad9803744..0000000000 --- a/crates/sqlbuild-core/src/query_command/main/render_query_table.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Query rows as an aligned table. - -use crate::query_command::_helpers::footer::footer; -use crate::query_command::constants::TABLE_SEPARATOR; -use crate::query_command::models::QueryDisplay; -use crate::text::main::python_isspace::python_isspace; - -fn padded(value: &str, width: usize) -> String { - format!("{value: String { - let mut widths: Vec = display - .columns - .iter() - .map(|column| column.chars().count()) - .collect(); - for row in &display.rows { - for (index, value) in row.iter().enumerate() { - if let Some(width) = widths.get_mut(index) { - *width = (*width).max(value.chars().count()); - } - } - } - let width_at = |index: usize| widths.get(index).copied().unwrap_or(0); - let mut lines: Vec = vec![ - display - .columns - .iter() - .enumerate() - .map(|(index, column)| padded(column, width_at(index))) - .collect::>() - .join(TABLE_SEPARATOR), - widths - .iter() - .map(|width| "-".repeat(*width)) - .collect::>() - .join(TABLE_SEPARATOR), - ]; - for row in &display.rows { - let line: String = row - .iter() - .enumerate() - .map(|(index, value)| padded(value, width_at(index))) - .collect::>() - .join(TABLE_SEPARATOR); - lines.push(line.trim_end_matches(python_isspace).to_owned()); - } - lines.extend(footer(display)); - format!("{}\n", lines.join("\n")) -} diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_text.rs b/crates/sqlbuild-core/src/query_command/main/render_query_text.rs new file mode 100644 index 0000000000..8d2b4a2323 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/main/render_query_text.rs @@ -0,0 +1,19 @@ +//! Query rows as text: a box when they fit, one record per row otherwise. + +use crate::query_command::_helpers::boxed::boxed_lines; +use crate::query_command::_helpers::footer::footer; +use crate::query_command::_helpers::long::long_text; +use crate::query_command::models::{QueryDisplay, QueryLayout}; + +/// The rows in `layout`, then the row count. A fitted layout never drops columns or rows: a result wider than the width, or with a value spanning lines, prints one record per row. +#[must_use] +pub fn render_query_text(display: &QueryDisplay, layout: QueryLayout) -> String { + let QueryLayout::Fitted { width } = layout else { + return long_text(display); + }; + let Some(mut lines) = boxed_lines(display, width) else { + return long_text(display); + }; + lines.extend(footer(display)); + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-core/src/query_command/models.rs b/crates/sqlbuild-core/src/query_command/models.rs index 724b1349f8..324f3a9309 100644 --- a/crates/sqlbuild-core/src/query_command/models.rs +++ b/crates/sqlbuild-core/src/query_command/models.rs @@ -1,10 +1,23 @@ //! A query result as its text formats display it. -/// Column names, each row's cells as display text, and whether the limit cut rows off. +/// Column names and types, each row's cells as display text, and whether the limit cut rows off. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct QueryDisplay { pub columns: Vec, + /// Each column's type as the warehouse names it, where known; empty when no type is known. + pub types: Vec>, + /// Columns whose values are numbers, right-aligned in the box layout. + pub numeric: Vec, pub rows: Vec>, pub truncated: bool, pub limit: Option, } + +/// How a result is laid out as text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QueryLayout { + /// A box when every row fits `width` terminal cells, else one record per row. + Fitted { width: usize }, + /// One record per row. + Long, +} diff --git a/crates/sqlbuild-core/src/query_command/tests/helpers.rs b/crates/sqlbuild-core/src/query_command/tests/helpers.rs index e611510a34..795f72a478 100644 --- a/crates/sqlbuild-core/src/query_command/tests/helpers.rs +++ b/crates/sqlbuild-core/src/query_command/tests/helpers.rs @@ -1,4 +1,25 @@ +use crate::query_command::models::QueryDisplay; + /// Owned copies of string literals. pub(super) fn texts(values: &[&str]) -> Vec { values.iter().map(|value| (*value).to_owned()).collect() } + +/// A small invocation-count result with one NULL count. +pub(super) fn invocations() -> QueryDisplay { + QueryDisplay { + columns: texts(&["status", "origin", "invocations"]), + types: vec![ + Some("varchar".to_owned()), + Some("varchar".to_owned()), + Some("int32".to_owned()), + ], + numeric: vec![false, false, true], + rows: vec![ + texts(&["completed", "integration", "133"]), + texts(&["cancelled", "local", "NULL"]), + ], + truncated: false, + limit: None, + } +} diff --git a/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs index 0cdd792d93..3551975fe7 100644 --- a/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs +++ b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs @@ -1,52 +1,75 @@ use crate::json::models::JsonValue; use crate::query_command::main::render_dag_summary::render_dag_summary; -use crate::query_command::main::render_query_csv::render_query_csv; use crate::query_command::main::render_query_json::render_query_json; -use crate::query_command::main::render_query_long::render_query_long; -use crate::query_command::main::render_query_table::render_query_table; -use crate::query_command::models::QueryDisplay; -use crate::query_command::tests::helpers::texts; +use crate::query_command::main::render_query_text::render_query_text; +use crate::query_command::models::{QueryDisplay, QueryLayout}; +use crate::query_command::tests::helpers::{invocations, texts}; use crate::query_command::tests::test_types::{ - CsvTestCase, DagTestCase, DisplayTestCase, JsonTestCase, + DagTestCase, DisplayTestCase, FooterTestCase, JsonTestCase, }; #[test] -fn given_rows_when_rendering_text_then_aligns_and_counts() { +fn given_rows_when_rendering_text_then_a_fitting_result_is_boxed_and_others_are_records() { let test_cases = [ DisplayTestCase { - description: "two rows with unicode and a truncation hint", + description: "a narrow result is boxed with types and right-aligned numbers", + display: invocations(), + layout: QueryLayout::Fitted { width: 120 }, + expected_text: "┌───────────┬─────────────┬─────────────┐\n\ + │ status │ origin │ invocations │\n\ + │ varchar │ varchar │ int32 │\n\ + ├───────────┼─────────────┼─────────────┤\n\ + │ completed │ integration │ 133 │\n\ + │ cancelled │ local │ NULL │\n\ + └───────────┴─────────────┴─────────────┘\n\n2 rows\n", + }, + DisplayTestCase { + description: "a result wider than the limit prints every value as records", + display: invocations(), + layout: QueryLayout::Fitted { width: 40 }, + expected_text: "-[ RECORD 1 ]---------------------------+\nstatus | completed\n\ + origin | integration\ninvocations | 133\n\n\ + -[ RECORD 2 ]---------------------------+\nstatus | cancelled\n\ + origin | local\ninvocations | NULL\n\n2 rows\n", + }, + DisplayTestCase { + description: "a value spanning lines prints as records, and the limit is named", display: QueryDisplay { - columns: texts(&["id", "name"]), - rows: vec![texts(&["1", "héllo"]), texts(&["22", ""])], + columns: texts(&["sql"]), + rows: vec![texts(&["SELECT 1\nFROM orders"])], truncated: true, - limit: Some(2), + limit: Some(1), + ..QueryDisplay::default() }, - expected_long: "-[ RECORD 1 ]---------------------------+\nid | 1\nname | héllo\n\n-[ RECORD 2 ]---------------------------+\nid | 22\nname | \n\n2 rows\nShowing 2 rows. Use --limit to show more or --no-limit to disable the limit.\n", - expected_table: "id | name \n-- | -----\n1 | héllo\n22 |\n\n2 rows\nShowing 2 rows. Use --limit to show more or --no-limit to disable the limit.\n", + layout: QueryLayout::Fitted { width: 120 }, + expected_text: "-[ RECORD 1 ]---------------------------+\nsql | SELECT 1\nFROM orders\n\n\ + 1 row shown (limit 1; use --limit or --no-limit)\n", }, DisplayTestCase { - description: "no rows and an unlimited truncation leaves an empty line", + description: "no rows still show the columns, without a type line when none is known", display: QueryDisplay { - columns: texts(&["x"]), - rows: Vec::new(), - truncated: true, - limit: None, + columns: texts(&["order_id"]), + ..QueryDisplay::default() }, - expected_long: "\n0 rows\n\n", - expected_table: "x\n-\n\n0 rows\n\n", + layout: QueryLayout::Fitted { width: 120 }, + expected_text: "┌──────────┐\n│ order_id │\n├──────────┤\n└──────────┘\n\n0 rows\n", + }, + DisplayTestCase { + description: "the long layout is kept when forced", + display: QueryDisplay { + columns: texts(&["id"]), + rows: vec![texts(&["1"])], + ..QueryDisplay::default() + }, + layout: QueryLayout::Long, + expected_text: "-[ RECORD 1 ]---------------------------+\nid | 1\n\n1 row\n", }, ]; for test_case in test_cases { assert_eq!( - render_query_long(&test_case.display), - test_case.expected_long, - "{}", - test_case.description - ); - assert_eq!( - render_query_table(&test_case.display), - test_case.expected_table, + render_query_text(&test_case.display, test_case.layout), + test_case.expected_text, "{}", test_case.description ); @@ -54,23 +77,31 @@ fn given_rows_when_rendering_text_then_aligns_and_counts() { } #[test] -fn given_rows_when_rendering_csv_then_quotes_minimally() { - let test_cases = [CsvTestCase { - description: "delimiters, quotes, newlines and a lone empty field", - columns: texts(&["a", "b"]), - rows: vec![ - texts(&["x,y", "q\"z"]), - texts(&["l\nm", ""]), - texts(&[""]), - Vec::new(), - ], - expected_csv: "a,b\n\"x,y\",\"q\"\"z\"\n\"l\nm\",\n\"\"\n\n", - }]; +fn given_many_rows_when_rendering_text_then_only_more_than_fifty_suggest_aggregating() { + let test_cases = [ + FooterTestCase { + description: "fifty rows print only the count", + row_count: 50, + expected_footer: "\n50 rows\n", + }, + FooterTestCase { + description: "more than fifty rows suggest aggregating", + row_count: 51, + expected_footer: "\n51 rows; to read fewer, aggregate with GROUP BY or add a LIMIT\n", + }, + ]; for test_case in test_cases { - assert_eq!( - render_query_csv(&test_case.columns, &test_case.rows), - test_case.expected_csv, + let display = QueryDisplay { + columns: texts(&["n"]), + rows: (0..test_case.row_count).map(|_| texts(&["7"])).collect(), + ..QueryDisplay::default() + }; + + let rendered: String = render_query_text(&display, QueryLayout::Fitted { width: 120 }); + + assert!( + rendered.ends_with(test_case.expected_footer), "{}", test_case.description ); diff --git a/crates/sqlbuild-core/src/query_command/tests/test_types.rs b/crates/sqlbuild-core/src/query_command/tests/test_types.rs index 99d7d0b43a..eddd4ac5c4 100644 --- a/crates/sqlbuild-core/src/query_command/tests/test_types.rs +++ b/crates/sqlbuild-core/src/query_command/tests/test_types.rs @@ -1,18 +1,17 @@ use crate::json::models::JsonValue; -use crate::query_command::models::QueryDisplay; +use crate::query_command::models::{QueryDisplay, QueryLayout}; pub(super) struct DisplayTestCase { pub(super) description: &'static str, pub(super) display: QueryDisplay, - pub(super) expected_long: &'static str, - pub(super) expected_table: &'static str, + pub(super) layout: QueryLayout, + pub(super) expected_text: &'static str, } -pub(super) struct CsvTestCase { +pub(super) struct FooterTestCase { pub(super) description: &'static str, - pub(super) columns: Vec, - pub(super) rows: Vec>, - pub(super) expected_csv: &'static str, + pub(super) row_count: usize, + pub(super) expected_footer: &'static str, } pub(super) struct JsonTestCase { diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 637dbe5d5f..1087c31918 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -709,6 +709,16 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Executor, }, + HostEntry { + name: "audit_evidence_scope", + kind: HostKind::Function, + edge: HostEdge::Executor, + }, + HostEntry { + name: "cancel_running_statements", + kind: HostKind::Function, + edge: HostEdge::Executor, + }, HostEntry { name: "EventDispatcher", kind: HostKind::Class, @@ -719,6 +729,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, + HostEntry { + name: "StatementLifecycle", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, HostEntry { name: "ObservabilityValidationError", kind: HostKind::Exception, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 44cb7aa246..86621a8546 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -342,8 +342,11 @@ pub enum ExecutorHost { RunStatus, ExecutionDurationLimitError, ExecutionDeadlineScope, + AuditEvidenceScope, + CancelRunningStatements, EventDispatcher, OperationLifecycle, + StatementLifecycle, ObservabilityValidationError, CreateLifecycleEvent, CurrentExecutionIdentity, diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index a0d4b49fb4..2ffda3e998 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -23,6 +23,7 @@ rayon.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true +signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true sqlbuild-cache.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 620d94adcf..bfa9c04ff5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -5,6 +5,7 @@ use std::collections::BTreeSet; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_core::audit_command::main::audit_exit_code::audit_exit_code; use sqlbuild_core::audit_command::main::render_audit_header::render_audit_header; use sqlbuild_core::audit_command::main::render_audit_summary::render_audit_summary; @@ -43,6 +44,15 @@ fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } +/// A target database or schema as a storage location: `preserve` names no location, so audit history is stored beside the models, as a build stores it. +fn literal_location<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if !value.is_none() && value.extract::()? == PRESERVE_TARGET_VALUE { + return Ok(py.None().into_bound(py)); + } + Ok(value.clone()) +} + /// Forward audit results to the event writer and connection timing to its reporter. #[pyclass(module = "sqlbuild._native", frozen)] struct AuditCallbackRelay { @@ -388,11 +398,11 @@ fn execute<'py>( ("run_id", run_id.clone()), ( "storage_database", - native_field(&project, "effective_target_database")?, + literal_location(&native_field(&project, "effective_target_database")?)?, ), ( "storage_schema", - native_field(&project, "effective_target_schema")?, + literal_location(&native_field(&project, "effective_target_schema")?)?, ), ], )? @@ -452,6 +462,22 @@ fn audit_payload<'py>( Ok(PyString::new(py, &format!("{dumped}\n")).into_any()) } +/// The warning line for audit results that could not be stored in the warehouse history table; audit outcomes are unaffected. +fn projection_warning(py: Python<'_>, style: CliStyle) -> PyResult> { + let projection: Bound<'_, PyAny> = + host::entry(py, ExecutorHost::CurrentAuditResultProjection)?.call0()?; + if projection.is_none() || !native_field(&projection, "degraded")?.is_truthy()? { + return Ok(None); + } + let failed: i64 = native_field(&projection, "failed_count")?.extract()?; + let reason: Option = native_field(&projection, "failure_reason")?.extract()?; + let reason: String = reason.map_or_else(String::new, |reason| format!(": {reason}")); + Ok(Some(format!( + "{} {failed} audit result(s) were not written to the audit history table{reason}\n", + style.warning("warning:") + ))) +} + /// Write the summary footer and JSON output; returns the exit code. fn complete( request: &Bound<'_, PyAny>, @@ -481,7 +507,11 @@ fn complete( let style: CliStyle = CliStyle { use_color: native_field(invocation, "use_color")?.is_truthy()?, }; - stream.call_method1("write", (render_audit_summary(counts, style),))?; + let warning: String = projection_warning(py, style)?.unwrap_or_default(); + stream.call_method1( + "write", + (format!("{warning}{}", render_audit_summary(counts, style)),), + )?; stream.call_method0("flush")?; let payload: Bound<'_, PyAny> = audit_payload( &PyTuple::new(py, results)?.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index f255f63599..f92b9941c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -46,12 +46,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_cli::diff_report::main::parse_diff_name_range::parse_diff_name_range; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; +use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; use sqlbuild_cli::dispatch::main::unavailable_handler::unavailable_handler; use sqlbuild_cli::dispatch::models::{ ClassRef, CommandFacts, DiffSide, Dispatch, DispatchFailure, Field, Route, Source, }; use sqlbuild_core::format_command::main::format_path_selectors::format_path_selectors; -use sqlbuild_core::text::main::python_strip::python_strip; use crate::bindings::_helpers::boundary::python_calls::{imported, within}; use crate::bindings::_helpers::records::construct::error_class; @@ -60,7 +60,6 @@ use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const PATHLIB_MODULE: &str = "pathlib"; -const COMMENT_PREFIX: &str = "#"; const UTF8: &str = "utf-8"; /// The code of a `--select-file` that cannot be read. const SELECTOR_FILE_CODE: &str = "C114"; @@ -265,15 +264,10 @@ fn selector_inputs<'py>( Ok(text) => text, Err(error) => return Err(selector_file_error(py, &path, &error)?), }; - let lines: Vec = text.call_method0("splitlines")?.extract()?; - let mut count: usize = 0; - for line in &lines { - let stripped: &str = python_strip(line); - if stripped.is_empty() || stripped.starts_with(COMMENT_PREFIX) { - continue; - } - selectors.push(PyString::new(py, stripped).into_any()); - count += 1; + let listed: Vec = selector_file_selectors(&text.extract::()?); + let count: usize = listed.len(); + for selector in &listed { + selectors.push(PyString::new(py, selector).into_any()); } files.push(new_record_with( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index c2496e31fd..3962b79206 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -1,15 +1,15 @@ //! The `sqb` entry: invocation scopes, parsing, observed dispatch and the error projection. use std::sync::Arc; -use std::time::Instant; +use std::time::{Instant, SystemTime}; use pyo3::exceptions::PySystemExit; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{ - PyBool, PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, - PyTypeMethods, + PyBool, PyCFunction, PyDateTime, PyDict, PyDictMethods, PyInt, PyList, PyString, + PyStringMethods, PyTuple, PyTypeMethods, PyTzInfo, }; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::dispatch::main::failure_report::failure_report; @@ -28,7 +28,12 @@ use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; +use crate::bindings::_helpers::cli::termination::{ + CommandTerminated, TERMINATED_MESSAGE, TerminationHandler, handle_sigterm, interrupting_signal, + signal_exit_code, terminated, terminated_exit, +}; use crate::bindings::_helpers::compile_command::report::format_compile_error_json; +use crate::bindings::_helpers::cost_report::attribution::take_cost_status; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; @@ -36,9 +41,9 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo use crate::bindings::_helpers::recall::constants::{JSON_FORMAT, RECALL_SETTINGS_CODE}; use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::recall::session::{ - InvocationRecall, finish_invocation_recall, start_invocation_recall, + InvocationRecall, effective_selection, finish_invocation_recall, start_invocation_recall, }; -use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; +use crate::bindings::_helpers::recall::sinks::{RecallLifecycleSink, publish_lifecycle_event}; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::construct::{error_class, new_error}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -218,8 +223,9 @@ fn write_machine_error( let format: Option = invocation .args .getattr_opt("recall_format")? - .map(|format| format.extract()) - .transpose()?; + .map(|format| format.extract::>()) + .transpose()? + .flatten(); format.as_deref() == Some(JSON_FORMAT) } else if invocation.command_is("janitor") { invocation.flag("json")? && !take_machine_output_written() @@ -476,13 +482,15 @@ fn subscribe_recall( RecallLifecycleSink { session, replacements: Arc::clone(&recall.replacements), + running_sql: Arc::default(), + invocation_events: Arc::clone(&recall.invocation_events), }, )?; dispatcher.call_method1("subscribe_history", (sink,))?; Ok(()) } -/// `target`, `adapter` and `selection_count` of the started invocation, when known. +/// `target`, `adapter` and `selection_count` of the started invocation, when known; the count includes what selector files list. fn add_invocation_context( invocation: &Invocation<'_>, payload: &Bound<'_, PyDict>, @@ -495,7 +503,8 @@ fn add_invocation_context( } let select: Bound<'_, PyAny> = invocation.args.getattr("select")?; if !select.is_none() { - payload.set_item("selection_count", select.len()?)?; + let (selectors, _) = effective_selection(&invocation.args)?; + payload.set_item("selection_count", selectors.len())?; } Ok(()) } @@ -542,7 +551,9 @@ fn observed_dispatch<'py>( } let command: String = invocation.command_name(); drop(take_failure_cause()); + drop(take_cost_status()); let started: Instant = Instant::now(); + let started_at: SystemTime = SystemTime::now(); let elapsed = |started: Instant| -> PyResult> { u64::try_from(started.elapsed().as_millis()) .unwrap_or(u64::MAX) @@ -555,23 +566,57 @@ fn observed_dispatch<'py>( invocation.project_dir()?.unbind(), ), )?; + let started_payload = || -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("command", &command)?; + add_invocation_context(invocation, &payload)?; + Ok(payload) + }; + let failed_payload = |error: &PyErr| -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("command", &command)?; + payload.set_item("duration_ms", elapsed(started)?)?; + let (error_type, code, message): (String, String, String) = raised_cause(py, error); + payload.set_item("error_type", error_type)?; + if !code.is_empty() { + payload.set_item("error_code", code)?; + } + payload.set_item("error_message", message)?; + if let Some(signal) = interrupting_signal(py, error) { + payload.set_item("signal_number", signal)?; + payload.set_item("exit_code", signal_exit_code(signal))?; + } + if let Some(status) = take_cost_status() { + payload.set_item("cost_status", status)?; + } + Ok(payload) + }; let scope: Bound<'py, PyAny> = observability.clone().into_any(); - within(&scope, |dispatcher| { + let observed: PyResult> = within(&scope, |dispatcher| { let publish = |event: Bound<'py, PyAny>| -> PyResult<()> { dispatcher.call_method1("publish_lifecycle", (event,))?; Ok(()) }; subscribe_recall((&observability, dispatcher), &invocation.recall)?; - let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item("command", &command)?; - add_invocation_context(invocation, &payload)?; - publish(lifecycle_event(py, "invocation_started", payload)?)?; - match dispatch_command(&invocation.args, invocation.handlers.clone()) { + publish(lifecycle_event( + py, + "invocation_started", + started_payload()?, + )?)?; + let dispatched: PyResult> = + match dispatch_command(&invocation.args, invocation.handlers.clone()) { + Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), + dispatched => dispatched, + }; + match dispatched { Ok(exit_code) => { let payload: Bound<'py, PyDict> = PyDict::new(py); payload.set_item("command", &command)?; payload.set_item("duration_ms", elapsed(started)?)?; payload.set_item("exit_code", &exit_code)?; + if let Some(status) = take_cost_status() { + payload.set_item("cost_status", status)?; + } let terminal: &str = if exit_code.eq(0)? { "invocation_completed" } else { @@ -586,21 +631,65 @@ fn observed_dispatch<'py>( Ok(exit_code) } Err(error) => { - let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item("command", &command)?; - payload.set_item("duration_ms", elapsed(started)?)?; - let (error_type, code, message): (String, String, String) = - raised_cause(py, &error); - payload.set_item("error_type", error_type)?; - if !code.is_empty() { - payload.set_item("error_code", code)?; - } - payload.set_item("error_message", message)?; - publish(lifecycle_event(py, "invocation_failed", payload)?)?; + publish(lifecycle_event( + py, + "invocation_failed", + failed_payload(&error)?, + )?)?; Err(error) } } - }) + }); + if let (Err(error), Some(session)) = (&observed, &invocation.recall.session) + && interrupting_signal(py, error).is_some() + && !invocation.recall.invocation_events.ended() + { + let started: Option>> = + (!invocation.recall.invocation_events.started()).then(started_payload); + let recorded: PyResult<()> = started.transpose().and_then(|started| { + record_interrupted_end( + py, + session, + started.map(|payload| (started_at, payload)), + failed_payload(error)?, + ) + }); + drop(recorded); + } + observed +} + +/// Record the end of an invocation interrupted before its own lifecycle events were all recorded, such as during project discovery, so Recall shows it cancelled instead of running. `started` holds the start event's time and payload when that event was not recorded either. +fn record_interrupted_end( + py: Python<'_>, + session: &sqlbuild_recall::writer::models::RecallSession, + started: Option<(SystemTime, Bound<'_, PyDict>)>, + failed: Bound<'_, PyDict>, +) -> PyResult<()> { + if let Some((started_at, payload)) = started { + let seconds: f64 = started_at + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0.0, |elapsed| elapsed.as_secs_f64()); + let occurred_at: Bound<'_, PyDateTime> = + PyDateTime::from_timestamp(py, seconds, Some(&PyTzInfo::utc(py)?.to_owned()))?; + let started: Bound<'_, PyAny> = call_with( + &host::entry(py, ExecutorHost::CreateLifecycleEvent)?, + &[ + ( + "event_type", + PyString::new(py, "invocation_started").into_any(), + ), + ("payload", payload.into_any()), + ("occurred_at", occurred_at.into_any()), + ], + )?; + publish_lifecycle_event(py, session, &started)?; + } + publish_lifecycle_event( + py, + session, + &lifecycle_event(py, "invocation_failed", failed)?, + ) } /// Dispatch once and project every expected failure to its coded error and exit code. @@ -690,6 +779,8 @@ fn parsed_invocation<'py>( RecallLifecycleSink { session, replacements: Arc::clone(&recall.replacements), + running_sql: Arc::default(), + invocation_events: Arc::clone(&recall.invocation_events), }, )? .into_any(), @@ -719,23 +810,32 @@ fn parsed_invocation<'py>( Ok(handled_dispatch(&invocation)?.unbind()) }, )?; - let dispatched: PyResult> = if policy.creates_project { - call_with(&operation.into_any(), &[]) - } else { - call_with( - &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, - &[ - ("args", args), - ("identity", identity.clone()), - ("operation", operation.into_any()), - ("recall", recall_sink), - ( - "capture", - PyBool::new(py, policy.captured).to_owned().into_any(), - ), - ], - ) - }; + let evidence_scope: Bound<'py, PyAny> = call_with( + &host::entry(py, ExecutorHost::AuditEvidenceScope)?, + &[( + "mode", + PyString::new(py, recall.audit_evidence.setting()).into_any(), + )], + )?; + let dispatched: PyResult> = within(&evidence_scope, |_| { + if policy.creates_project { + call_with(&operation.into_any(), &[]) + } else { + call_with( + &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, + &[ + ("args", args), + ("identity", identity.clone()), + ("operation", operation.into_any()), + ("recall", recall_sink), + ( + "capture", + PyBool::new(py, policy.captured).to_owned().into_any(), + ), + ], + ) + } + }); let exit_status: Option = match &dispatched { Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), Ok(_) | Err(_) => None, @@ -782,6 +882,21 @@ fn run_cli<'py>( .get_item(pyo3::types::PySlice::new(py, 1, isize::MAX, 1))?, }; let argv: Bound<'py, PyAny> = offered_launch_record(&argv)?; + let mut termination: TerminationHandler = handle_sigterm(py)?; + let ran: PyResult> = invoke(py, &argv, handlers); + if termination.settle(py) { + drop(ran); + return terminated_exit(py).into_bound_py_any(py); + } + ran +} + +/// Run one parsed `argv` inside its invocation, external-context and diagnostics scopes. +fn invoke<'py>( + py: Python<'py>, + argv: &Bound<'py, PyAny>, + handlers: Option>, +) -> PyResult> { let external: Bound<'py, PyAny> = external_context(py)?; let invocation_scope: Bound<'py, PyAny> = host::entry(py, ExecutorHost::InvocationScope)?.call0()?; @@ -799,7 +914,7 @@ fn run_cli<'py>( )], )?; within(&diagnostics, |_| { - parsed_invocation(&argv, handlers.clone(), identity) + parsed_invocation(argv, handlers.clone(), identity) }) }) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 17fbdc632b..c26744d81a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -8,3 +8,4 @@ pub(crate) mod namespace; pub(crate) mod observability; pub(crate) mod registration; pub(crate) mod surrogates; +pub(crate) mod termination; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 90cea0cf56..7c15b912eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{progress, progress_reporters, progress_text}; -use crate::bindings::_helpers::cli::{argv, entry, launch_state, namespace, observability}; +use crate::bindings::_helpers::cli::{ + argv, entry, launch_state, namespace, observability, termination, +}; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, @@ -19,6 +21,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { launch_state::register(module)?; namespace::register(module)?; observability::register(module)?; + termination::register(module)?; progress::register(module)?; progress_reporters::register(module)?; progress_text::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs new file mode 100644 index 0000000000..3256513caa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs @@ -0,0 +1,220 @@ +//! SIGTERM as an interrupt: the command cancels its running statements, stops the way Ctrl+C stops it, records its end and exits with the conventional code. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::thread::JoinHandle; +use std::time::Duration; + +use pyo3::create_exception; +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_host::entries::models::ExecutorHost; + +use crate::bindings::_helpers::boundary::python_calls::imported; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::console::write_stderr; +use crate::bindings::_helpers::records::fields::field as native_field; + +create_exception!( + sqlbuild._native, + CommandTerminated, + PyKeyboardInterrupt, + "Raised when the command receives SIGTERM; handled like Ctrl+C." +); + +/// The number of SIGTERM, as POSIX defines it. +pub(crate) const SIGTERM_NUMBER: i64 = 15; +/// The number of SIGINT, which Python raises as `KeyboardInterrupt`. +pub(crate) const SIGINT_NUMBER: i64 = 2; +/// The exit code of a command stopped by SIGTERM: 128 plus the signal number. +pub(crate) const TERMINATED_EXIT_CODE: i64 = 128 + SIGTERM_NUMBER; +/// The exit code of a command stopped by Ctrl+C: 128 plus the signal number. +pub(crate) const INTERRUPTED_EXIT_CODE: i64 = 128 + SIGINT_NUMBER; +pub(crate) const TERMINATED_MESSAGE: &str = "terminated by SIGTERM"; +/// `signal.SIG_DFL`, as `signal.getsignal` reports a signal Python leaves to its default action. +const DEFAULT_ACTION: i64 = 0; +/// How often the watcher looks for a termination request. +const WATCH_INTERVAL: Duration = Duration::from_millis(20); + +/// The process's SIGTERM actions, registered once: take the default action while `default` is set, otherwise set `terminated`. +struct SigtermFlags { + default: Arc, + terminated: Arc, +} + +static SIGTERM_FLAGS: OnceLock> = OnceLock::new(); + +/// Register the SIGTERM actions; `None` when the platform refuses them. +fn registered_flags() -> Option<&'static SigtermFlags> { + SIGTERM_FLAGS + .get_or_init(|| { + let flags = SigtermFlags { + default: Arc::new(AtomicBool::new(true)), + terminated: Arc::new(AtomicBool::new(false)), + }; + let signal: i32 = signal_hook::consts::SIGTERM; + let registered = + signal_hook::flag::register_conditional_default(signal, Arc::clone(&flags.default)) + .and_then(|_| { + signal_hook::flag::register(signal, Arc::clone(&flags.terminated)) + }); + match registered { + Ok(_) => Some(flags), + Err(_unsupported_signal) => None, + } + }) + .as_ref() +} + +/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. Holds only Rust values: while it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again. +pub(crate) struct TerminationHandler { + flags: Option<&'static SigtermFlags>, + stop: Arc, + watcher: Option>, +} + +impl TerminationHandler { + /// Give SIGTERM its default action back and wait for the watcher to finish; whether SIGTERM arrived while the command ran. + pub(crate) fn settle(&mut self, py: Python<'_>) -> bool { + if let Some(flags) = self.flags { + flags.default.store(true, Ordering::SeqCst); + } + self.stop.store(true, Ordering::SeqCst); + if let Some(watcher) = self.watcher.take() { + py.detach(|| drop(watcher.join())); + } + terminated() + } +} + +impl Drop for TerminationHandler { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(watcher) = self.watcher.take() { + Python::attach(|py| py.detach(|| drop(watcher.join()))); + } + if let Some(flags) = self.flags { + flags.default.store(true, Ordering::SeqCst); + flags.terminated.store(false, Ordering::SeqCst); + } + } +} + +/// Where the watcher sends the interrupt: the main thread, and whether Python handles Ctrl+C there. +#[derive(Clone, Copy)] +struct MainThread { + ident: u64, + handles_interrupt: bool, +} + +/// Once SIGTERM arrives: let a second one end the process at once, cancel every running statement, and interrupt the main thread the way Ctrl+C does, where the platform can signal one thread. +fn watch(flags: &SigtermFlags, stop: &AtomicBool, main: MainThread) { + while !stop.load(Ordering::SeqCst) { + if flags.terminated.load(Ordering::SeqCst) { + flags.default.store(true, Ordering::SeqCst); + Python::attach(|py| { + let cancelled = host::entry(py, ExecutorHost::CancelRunningStatements) + .and_then(|cancel| cancel.call0()); + drop(cancelled); + if main.handles_interrupt { + let interrupted = imported(py, "signal", "pthread_kill") + .and_then(|kill| kill.call1((main.ident, SIGINT_NUMBER))); + drop(interrupted); + } + }); + return; + } + std::thread::sleep(WATCH_INTERVAL); + } +} + +/// Handle SIGTERM while the command runs: only on the main thread, and only when Python leaves SIGTERM to its default action, so an embedding application keeps its own handler. +pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult { + let threading: Bound<'_, PyAny> = py.import("threading")?.into_any(); + let main_thread: Bound<'_, PyAny> = threading.call_method0("main_thread")?; + let on_main: bool = threading.call_method0("current_thread")?.is(&main_thread); + let action = |signal: i64| -> PyResult> { + imported(py, "signal", "getsignal")?.call1((signal,)) + }; + let default: bool = action(SIGTERM_NUMBER)? + .extract::() + .is_ok_and(|action| action == DEFAULT_ACTION); + let stop: Arc = Arc::new(AtomicBool::new(false)); + let Some(flags) = registered_flags().filter(|_| on_main && default) else { + return Ok(TerminationHandler { + flags: None, + stop, + watcher: None, + }); + }; + let main = MainThread { + ident: native_field(&main_thread, "ident")?.extract()?, + handles_interrupt: action(SIGINT_NUMBER)?.is_callable(), + }; + flags.terminated.store(false, Ordering::SeqCst); + let watched: Arc = Arc::clone(&stop); + let watcher: JoinHandle<()> = std::thread::Builder::new() + .name("sqlbuild-termination".to_owned()) + .spawn(move || watch(flags, &watched, main))?; + flags.default.store(false, Ordering::SeqCst); + Ok(TerminationHandler { + flags: Some(flags), + stop, + watcher: Some(watcher), + }) +} + +/// Whether SIGTERM arrived while the command runs. +pub(crate) fn terminated() -> bool { + SIGTERM_FLAGS + .get() + .and_then(Option::as_ref) + .is_some_and(|flags| flags.terminated.load(Ordering::SeqCst)) +} + +/// Whether SIGTERM arrived while the command runs, so a statement error can be reported as the termination it is. +#[pyfunction] +fn termination_requested() -> bool { + terminated() +} + +/// The signal that interrupted the command: SIGTERM once one arrived, SIGINT for any other `KeyboardInterrupt`. +pub(crate) fn interrupting_signal(py: Python<'_>, error: &PyErr) -> Option { + if !error.is_instance_of::(py) { + return None; + } + Some( + if terminated() || error.is_instance_of::(py) { + SIGTERM_NUMBER + } else { + SIGINT_NUMBER + }, + ) +} + +/// The exit code a run stopped by `signal` ends with. +pub(crate) fn signal_exit_code(signal: i64) -> i64 { + if signal == SIGTERM_NUMBER { + TERMINATED_EXIT_CODE + } else { + INTERRUPTED_EXIT_CODE + } +} + +/// After SIGTERM: drop the interrupt it left pending for the main thread, write the termination notice to stderr and give the conventional exit code. A stream that cannot be written changes nothing. +pub(crate) fn terminated_exit(py: Python<'_>) -> i64 { + drop(py.check_signals()); + drop(write_stderr(py, &format!("error: {TERMINATED_MESSAGE}\n"))); + TERMINATED_EXIT_CODE +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "CommandTerminated", + module.py().get_type::(), + )?; + module.add_function(wrap_pyfunction!(termination_requested, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 1a10104441..11fe1c157d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -1,6 +1,6 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; @@ -39,10 +39,11 @@ use crate::bindings::_helpers::compile_outputs::runtime_target::{ }; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::construct::{enum_member, error_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; +use crate::bindings::_helpers::records::fields::path_posix_text; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::sql_test_glue::plan_driver::plan_compiled_sql_test_artifacts; use crate::bindings::_helpers::sql_test_glue::plan_driver::sql_test_model_chain_names_by_key; @@ -57,6 +58,9 @@ const TESTS_DIR: &str = "tests"; const MANIFEST_FILE: &str = "manifest.json"; /// The compile cache file stamping the compiled tree the last stale-file walk left. const COMPILED_TREE_STAMP_FILE: &str = "compiled-tree.bin"; +/// The model files compiles wrote under `target/compiled/`, one per line, relative to it. +const COMPILED_MODELS_FILE: &str = "compiled-models.txt"; + const SQL_FILE_SUFFIX: &str = ".sql"; const SQL_TEST_CACHE_LOGGER: &str = "sqlbuild.compile.reuse"; /// The miss reason of a SQL test the cache file holds no record for. @@ -68,6 +72,8 @@ const PUBLISHED_TREE: &str = "tree"; const TARGET_OPTIONS: (&[&str], &[&str]) = (&["target_dir", "adapter", "project"], &["manifest"]); const STAGING_CHANGED_HELP: &str = "Rerun the compile; avoid deleting target/ while a compile is running."; +/// The first line of a compiled model whose Rules failed. +const RULE_FAILED_HEADER: &str = "-- sqb: compile failed for this model; not buildable"; /// `base / part / ...`. fn joined<'py>(base: &Bound<'py, PyAny>, parts: &[&str]) -> PyResult> { @@ -318,8 +324,15 @@ fn write_planned_target<'py>( managed.push(path); } batch.flush(py)?; + let managed: Vec = extracted(&managed)?; + record_compiled_models( + &target_dir.extract::()?, + &compiled.extract::()?, + &managed, + record_mode(remove_stale_files), + )?; if remove_stale_files { - remove_stale(target_dir, extracted(&managed)?, None)?; + remove_stale(target_dir, managed, None)?; } write_manifest(target_dir, manifest, None)?; written_target( @@ -723,9 +736,15 @@ pub(crate) fn write_static_target<'py>( })(); let (test_paths, diagnostics) = batch.settle(py, queued)?; managed.extend(test_paths); + let mut paths: Vec = managed_models; + paths.extend(extracted(&managed)?); + record_compiled_models( + &target_dir.extract::()?, + &compiled_dir, + &paths, + record_mode(remove_stale_files), + )?; if remove_stale_files { - let mut paths: Vec = managed_models; - paths.extend(extracted(&managed)?); let cache_dir: Bound<'py, PyAny> = native_field(project, "compile_cache_dir")?; let cache_dir: Option = if cache_dir.is_none() { None @@ -748,6 +767,182 @@ pub(crate) fn write_static_target<'py>( ) } +/// `path` relative to `project_dir` with `/` separators, as model files name themselves. +fn project_relative(path: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>) -> PyResult { + let text: String = path_posix_text(path)?; + if !Path::new(&text).is_absolute() { + return Ok(text); + } + let root: PathBuf = std::fs::canonicalize(path_posix_text(project_dir)?)?; + Ok(Path::new(&text) + .strip_prefix(&root) + .map_or(text.clone(), |relative| { + relative + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") + })) +} + +/// Each source file's Rule errors as compiled-file header lines: code, location and message. +fn rule_error_lines( + diagnostics: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult>> { + let py: Python<'_> = diagnostics.py(); + let rule_phase: Bound<'_, PyAny> = enum_member(py, RecordId::DiagnosticPhase, "RULE")?; + let mut lines: HashMap> = HashMap::new(); + for diagnostic in diagnostics.try_iter()? { + let diagnostic: Bound<'_, PyAny> = diagnostic?; + let path: Bound<'_, PyAny> = native_field(&diagnostic, "path")?; + if path.is_none() + || !native_field(&diagnostic, "is_error")?.is_truthy()? + || !native_field(&diagnostic, "phase")?.is(&rule_phase) + { + continue; + } + let relative: String = project_relative(&path, project_dir)?; + let position = |field: &str| -> PyResult { + let value: Bound<'_, PyAny> = native_field(&diagnostic, field)?; + Ok(if value.is_none() { + String::new() + } else { + format!(":{}", value.str()?) + }) + }; + let message: String = native_field(&diagnostic, "message")?.str()?.to_string(); + let line: String = format!( + "-- {} at {relative}{}{}: {}", + native_field(&diagnostic, "code")?.str()?, + position("line")?, + position("column")?, + message.split_whitespace().collect::>().join(" "), + ); + lines.entry(relative).or_default().push(line); + } + Ok(lines) +} + +/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. An earlier manifest is removed, since it no longer describes `target/`, and so is a model file an earlier compile wrote for a model the project no longer has. Every other artifact stays as the last compile left it, for the next passing compile to publish or remove. +pub(crate) fn write_rule_failed_models<'py>( + target_dir: &Bound<'py, PyAny>, + (project, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + diagnostics: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = target_dir.py(); + let errors: HashMap> = rule_error_lines(diagnostics, project_dir)?; + let _ = prepared_target(target_dir)?; + let manifest: PathBuf = target_dir.extract::()?.join(MANIFEST_FILE); + match std::fs::remove_file(&manifest) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error.into()), + _ => {} + } + let compiled_dir: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; + let mut batch: ArtifactBatch = ArtifactBatch::new(true, None); + let mut written: Vec = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let (relative_path, query_sql): (String, String) = + if let Ok(native) = model.cast::() { + let (relative_path, query_sql) = native.get().output_texts(py)?; + (relative_path, query_sql.to_string()) + } else { + ( + path_posix_text(&native_field(&model, "relative_path")?)?, + native_field(&model, "query_sql")?.extract()?, + ) + }; + let sql: String = match errors.get(&relative_path) { + Some(lines) => format!("{RULE_FAILED_HEADER}\n{}\n{query_sql}", lines.join("\n")), + None => query_sql, + }; + let path: PathBuf = native_model_output_path(&compiled_dir, &relative_path); + batch.queue_sql_at(py, path.clone(), &sql)?; + written.push(path); + } + batch.flush(py)?; + record_compiled_models( + &target_dir.extract::()?, + &compiled_dir, + &written, + ModelRecord::RemoveRetired, + ) +} + +fn record_mode(removes_stale_files: bool) -> ModelRecord { + if removes_stale_files { + ModelRecord::Replace + } else { + ModelRecord::Extend + } +} + +/// How a compile updates the record of the model files SQLBuild wrote under `target/compiled/`. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ModelRecord { + /// A passing compile that removed stale files: the record is exactly what it wrote. + Replace, + /// A passing compile that kept the files already there: the record gains what it wrote. + Extend, + /// A compile whose Rules failed: other recorded models are retired, so their files are removed. + RemoveRetired, +} + +/// Update the record of written model files, one path per line relative to `compiled_dir`. +fn record_compiled_models( + target: &Path, + compiled_dir: &Path, + written: &[PathBuf], + mode: ModelRecord, +) -> PyResult<()> { + let record: PathBuf = target.join(COMPILED_MODELS_FILE); + let previous: String = match std::fs::read_to_string(&record) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(error.into()), + }; + let models_dir: PathBuf = compiled_dir.join(MODELS_DIR); + let mut lines: BTreeSet = BTreeSet::new(); + for path in written { + match path.strip_prefix(&models_dir) { + Ok(relative) => { + let _ = lines.insert(format!( + "{MODELS_DIR}/{}", + relative.to_string_lossy().replace('\\', "/") + )); + } + Err(_not_a_model) => continue, + } + } + for line in previous.lines().filter(|line| !line.is_empty()) { + if lines.contains(line) { + continue; + } + match mode { + ModelRecord::Replace => {} + ModelRecord::Extend => { + let _ = lines.insert(line.to_owned()); + } + ModelRecord::RemoveRetired => { + let path: PathBuf = compiled_dir.join(line); + if !path.starts_with(&models_dir) { + continue; + } + match std::fs::remove_file(&path) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(error.into()); + } + _ => {} + } + } + } + } + let contents: String = lines.iter().map(|line| format!("{line}\n")).collect(); + if contents != previous { + std::fs::write(&record, contents)?; + } + Ok(()) +} + /// Plan uncached SQL tests and report their errors without writing artifacts or cache. pub(crate) fn planning_diagnostics<'py>( target_dir: &Bound<'py, PyAny>, @@ -783,7 +978,7 @@ pub(crate) fn publish_staged_target<'py>( .extract()?; let (staged, compiled): (PathBuf, PathBuf) = (staged_dir.extract()?, compiled_dir.extract()?); let (kind, published) = record_compile_timing("physical_write_ms", || { - publish_staged_artifacts(py, staged, compiled, expected) + publish_staged_artifacts(py, staged, compiled.clone(), expected.clone()) })?; if kind == PUBLISHED_STAGING_CHANGED { return Err(staging_changed( @@ -807,6 +1002,16 @@ pub(crate) fn publish_staged_target<'py>( remove_stale(target_dir, extracted(&managed)?, None)?; } } + let written: Vec = expected + .iter() + .map(|relative| compiled.join(relative)) + .collect(); + record_compiled_models( + &target_dir.extract::()?, + &compiled, + &written, + ModelRecord::Replace, + )?; write_manifest(target_dir, manifest, recorder)?; written_target( target_dir, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 902d4fa14c..6b7a54e1b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -26,6 +26,7 @@ use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_ use crate::bindings::_helpers::compile_command::reuse_recording::{ CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, }; +use crate::bindings::_helpers::compile_command::target_writer::write_rule_failed_models; use crate::bindings::_helpers::compile_context::input_reads::record_compile_input_reads; use crate::bindings::_helpers::compile_driver::analysis::analyze; use crate::bindings::_helpers::compile_driver::phases::{ @@ -358,6 +359,16 @@ fn compiled_with_status<'py>( (prepared_artifacts, recorder), status, )?; + if rules_failed && !skip_write { + write_rule_failed_models( + &target_dir(&project_dir)?, + ( + &native_field(&native_field(&analysis, "graph")?, "project")?, + &project_dir, + ), + &native_field(&analysis, "diagnostics")?, + )?; + } let timings: Bound<'py, PyDict> = timings_ms( (&analysis, write_ms), (&recorders.detailed_timings, reuse_timings), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/attribution.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/attribution.rs new file mode 100644 index 0000000000..7feb2891a0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/attribution.rs @@ -0,0 +1,22 @@ +//! How far the running command attributed warehouse cost, for the invocation's terminal event. + +use std::cell::RefCell; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +thread_local! { + /// Per thread, since an invocation finalizes cost and reports on the thread that runs it. + static COST_STATUS: RefCell> = const { RefCell::new(None) }; +} + +/// Record the cost status of the latest finalization; a later one replaces it, so a run that stops before collecting keeps `pending`. +pub(crate) fn report_cost_status(status: &Bound<'_, PyAny>) -> PyResult<()> { + let status: String = status.str()?.to_string(); + COST_STATUS.with(|slot| *slot.borrow_mut() = Some(status)); + Ok(()) +} + +/// The cost status the command reported, cleared so the next invocation starts without one. +pub(crate) fn take_cost_status() -> Option { + COST_STATUS.with(|slot| slot.borrow_mut().take()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index bdd4ba21ec..d41ffba179 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -13,6 +13,7 @@ use sqlbuild_core::cost_report::models::BreakdownOptions; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::cost_report::attribution::report_cost_status; use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; use crate::bindings::_helpers::cost_telemetry::values::cost_aware; @@ -219,6 +220,7 @@ fn collection_failure<'py>( )?; let status: Bound<'py, PyAny> = host::entry(py, ExecutorHost::CostStatus)?.getattr("COLLECTION_FAILED")?; + report_cost_status(&status)?; let message: Bound<'py, PyAny> = PyString::new( py, &format!("Cost telemetry failed during {stage} ({error_type})."), @@ -307,6 +309,7 @@ pub(crate) fn finalize_build_cost<'py>( native_field(finalization, "cost_message")?, )? }; + report_cost_status(&native_field(&cost, "status")?)?; let record: Bound<'py, PyAny> = cost_record(finalization, cost)?; persist(finalization, &record, "artifact persistence")?; if !native_field(finalization, "render")?.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index f734d355cf..854d8e9fb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod attribution; pub(crate) mod degradation; pub(crate) mod exception_types; pub(crate) mod functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 80fbc37771..3eb528de1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -6,7 +6,7 @@ use pyo3::exceptions::{PyTimeoutError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyCFunction, PyDict, PyList, PyListMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::snowflake_cost::main::metadata_only_statement::metadata_only_statement; use sqlbuild_analysis::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; @@ -14,14 +14,16 @@ use sqlbuild_analysis::snowflake_cost::models::QueryTag; use sqlbuild_core::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; use sqlbuild_core::cost_allocation::main::query_history_sql::query_history_sql; use sqlbuild_core::cost_allocation::models::{QueryHistoryWindow, RunAllocation}; +use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported, within}; use crate::bindings::_helpers::cost_telemetry::allocation::{ SummaryState, allocate_observations, priced, summary_object, }; use crate::bindings::_helpers::cost_telemetry::health::consume_ledger_failure; use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; @@ -85,9 +87,43 @@ fn check_deadline(history: &History<'_>, py: Python<'_>) -> PyResult<()> { Ok(()) } +/// Whether a read's statement timeout is recovered from by reading smaller windows, so its failed statement is recorded as handled. +#[derive(Clone, Copy, PartialEq, Eq)] +enum TimeoutRetry { + Split, + None, +} + +/// Run `execute`, marking a statement timeout inside it as handled when the caller retries it. +fn with_timeout_retry<'py, T>( + py: Python<'py>, + retry: TimeoutRetry, + execute: impl FnOnce() -> PyResult, +) -> PyResult { + if retry == TimeoutRetry::None { + return execute(); + } + let is_timeout = PyCFunction::new_closure( + py, + None, + None, + |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult { + Ok(arguments + .get_item(0)? + .str()? + .to_string_lossy() + .contains(STATEMENT_TIMEOUT_CODE)) + }, + )?; + let scope: Bound<'py, PyAny> = host::entry(py, ExecutorHost::StatementLifecycle)? + .call_method1("handled_failure_scope", (is_timeout,))?; + within(&scope, |_| execute()) +} + fn history_rows<'py>( history: &History<'py>, (start, end): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), + retry: TimeoutRetry, ) -> PyResult>> { let py: Python<'py> = start.py(); check_deadline(history, py)?; @@ -107,10 +143,11 @@ fn history_rows<'py>( parameters.set_item("STATEMENT_TIMEOUT_IN_SECONDS", QUERY_TIMEOUT_SECONDS)?; let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item("statement_params", parameters)?; - let cursor: Bound<'py, PyAny> = + let cursor: Bound<'py, PyAny> = with_timeout_retry(py, retry, || { history .connection - .call_method("execute", (sql,), Some(&options))?; + .call_method("execute", (sql,), Some(&options)) + })?; check_deadline(history, py)?; let rows: Vec> = cursor .call_method0("fetchall")? @@ -157,12 +194,14 @@ fn bounded_rows<'py>( ) -> PyResult<(Vec>, bool)> { let py: Python<'py> = start.py(); let width: Bound<'py, PyAny> = end.sub(start)?; - let rows: Vec> = match history_rows(history, (start, Some(end))) { + let retry: TimeoutRetry = if width.gt(timedelta(py, ("seconds", MIN_TIMEOUT_SPLIT_SECONDS))?)? { + TimeoutRetry::Split + } else { + TimeoutRetry::None + }; + let rows: Vec> = match history_rows(history, (start, Some(end)), retry) { Ok(rows) => rows, - Err(error) - if statement_timeout(py, &error) - && width.gt(timedelta(py, ("seconds", MIN_TIMEOUT_SPLIT_SECONDS))?)? => - { + Err(error) if retry == TimeoutRetry::Split && statement_timeout(py, &error) => { return split_rows(history, (start, end), &width); } Err(error) => return Err(error), @@ -328,13 +367,14 @@ fn open_rows<'py>( (start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult<(Vec>, bool)> { let py: Python<'py> = start.py(); - let open_rows: Vec> = match history_rows(history, (start, None)) { - Ok(rows) => rows, - Err(error) if statement_timeout(py, &error) => { - return bounded_rows(history, (start, window_end)); - } - Err(error) => return Err(error), - }; + let open_rows: Vec> = + match history_rows(history, (start, None), TimeoutRetry::Split) { + Ok(rows) => rows, + Err(error) if statement_timeout(py, &error) => { + return bounded_rows(history, (start, window_end)); + } + Err(error) => return Err(error), + }; if open_rows.len() < history.result_limit { return Ok((open_rows, false)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs index 369db30e07..b4f6b07b65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs @@ -10,6 +10,11 @@ pub(crate) fn stdout_supports_color(py: Python<'_>) -> PyResult { if std::env::var_os(NO_COLOR_VARIABLE).is_some() { return Ok(false); } + stdout_is_tty(py) +} + +/// Whether `sys.stdout` is a terminal. +pub(crate) fn stdout_is_tty(py: Python<'_>) -> PyResult { let stdout = py.import("sys")?.getattr("stdout")?; if !stdout.hasattr("isatty")? { return Ok(false); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs index 7b60b819a0..61669cbe11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod project; pub(crate) mod query; pub(crate) mod query_values; pub(crate) mod registration; +pub(crate) mod text_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index a77de9d6a1..e659c3c734 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -4,13 +4,10 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyInt, PyString, PyStringMethods}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; -use sqlbuild_core::query_command::main::render_query_csv::render_query_csv; use sqlbuild_core::query_command::main::render_query_json::render_query_json; -use sqlbuild_core::query_command::main::render_query_long::render_query_long; -use sqlbuild_core::query_command::main::render_query_table::render_query_table; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_host::entries::models::AdapterHost; @@ -23,16 +20,14 @@ use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use crate::bindings::_helpers::query_command::query_values::{ - csv_text, display_text, json_cell, python_json_rows, strict_width, + decimal_instance, display_text, json_cell, python_json_rows, strict_width, }; +use crate::bindings::_helpers::query_command::text_output::{LONG_FORMAT, render_text}; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; -const LONG_FORMAT: &str = "long"; -const TABLE_FORMAT: &str = "table"; const JSON_FORMAT: &str = "json"; -const CSV_FORMAT: &str = "csv"; const STATEMENT_OK: &str = "OK\n"; const DEFAULT_LIMIT: i64 = 20; @@ -248,8 +243,51 @@ fn render_json( } } +/// Whether every value of the column at `index` is a number or NULL, and at least one is a number. +fn numeric_column(rows: &[Vec>], index: usize) -> PyResult { + let mut seen: bool = false; + for row in rows { + let Some(value) = row.get(index) else { + continue; + }; + if value.is_none() { + continue; + } + if !number(value)? { + return Ok(false); + } + seen = true; + } + Ok(seen) +} + +fn number(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Ok(false); + } + Ok(value.is_instance_of::() + || value.is_instance_of::() + || decimal_instance(value)?) +} + +/// The column types the adapter reported, where it knows them. +fn column_types(result: &Bound<'_, PyAny>, width: usize) -> PyResult>> { + let types: Vec> = native_field(result, "column_types")? + .extract() + .unwrap_or_default(); + Ok(if types.len() == width { + types + } else { + Vec::new() + }) +} + /// The result in the requested format; a statement without columns prints `OK`. -fn render(result: &Bound<'_, PyAny>, output_format: &str, limit: Option) -> PyResult { +fn render( + result: &Bound<'_, PyAny>, + output_format: Option<&str>, + limit: Option, +) -> PyResult { let py: Python<'_> = result.py(); let columns: Vec = native_field(result, "columns")? .try_iter()? @@ -259,27 +297,27 @@ fn render(result: &Bound<'_, PyAny>, output_format: &str, limit: Option) -> return Ok(STATEMENT_OK.to_owned()); } let rows: Vec>> = rows(result)?; - let display = |strict: bool| -> PyResult { - if strict { + match output_format { + Some(JSON_FORMAT) => render_json(result, &columns, &rows), + None | Some(LONG_FORMAT) => { for row in &rows { strict_width(columns.len(), row.len())?; } + let display = QueryDisplay { + types: column_types(result, columns.len())?, + numeric: (0..columns.len()) + .map(|index| numeric_column(&rows, index)) + .collect::>()?, + columns, + rows: texts(&rows, display_text)?, + truncated: native_field(result, "truncated")?.is_truthy()?, + limit, + }; + render_text(py, &display, output_format) } - Ok(QueryDisplay { - columns: columns.clone(), - rows: texts(&rows, display_text)?, - truncated: native_field(result, "truncated")?.is_truthy()?, - limit, - }) - }; - match output_format { - LONG_FORMAT => Ok(render_query_long(&display(true)?)), - TABLE_FORMAT => Ok(render_query_table(&display(false)?)), - JSON_FORMAT => render_json(result, &columns, &rows), - CSV_FORMAT => Ok(render_query_csv(&columns, &texts(&rows, csv_text)?)), - _ => Err(user_error( + Some(other) => Err(user_error( py, - &format!("unsupported query output format '{output_format}'"), + &format!("unsupported query output format '{other}'"), None, None, )), @@ -302,10 +340,10 @@ pub(crate) fn run_query_command( &["query_file", "selected_target", "output_format", "limit"], ), )?; - let output_format: String = if arguments.passed("output_format")? { + let output_format: Option = if arguments.passed("output_format")? { arguments.value("output_format")?.extract()? } else { - LONG_FORMAT.to_owned() + None }; let limit: Option = if arguments.passed("limit")? { arguments.value("limit")?.extract()? @@ -336,7 +374,7 @@ pub(crate) fn run_query_command( sql, limit, )?; - let rendered: String = render(&result, &output_format, limit)?; + let rendered: String = render(&result, output_format.as_deref(), limit)?; let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; stdout.call_method1("write", (PyString::new(py, &rendered),))?; stdout.call_method0("flush")?; @@ -348,7 +386,7 @@ pub(crate) fn run_query_command( #[pyo3(signature = (*, result, output_format, limit))] fn render_query_result( result: &Bound<'_, PyAny>, - output_format: &str, + output_format: Option<&str>, limit: Option, ) -> PyResult { render(result, output_format, limit) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs index 65f0682dc9..716b4b38b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs @@ -1,4 +1,4 @@ -//! Query cell values as display text, CSV fields and JSON values. +//! Query cell values as display text and JSON values. use pyo3::exceptions::{PyUnicodeEncodeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -18,7 +18,8 @@ fn python_text(value: &Bound<'_, PyAny>) -> PyResult { Ok(value.str()?.to_string_lossy().into_owned()) } -fn decimal_instance(value: &Bound<'_, PyAny>) -> PyResult { +/// Whether `value` is a `decimal.Decimal`. +pub(crate) fn decimal_instance(value: &Bound<'_, PyAny>) -> PyResult { value.is_instance(&imported(value.py(), "decimal", "Decimal")?) } @@ -40,17 +41,6 @@ pub(crate) fn display_text(value: &Bound<'_, PyAny>) -> PyResult { python_text(value) } -/// The text `csv.writer` writes: empty for `None`, `repr` for floats, else `str`. -pub(crate) fn csv_text(value: &Bound<'_, PyAny>) -> PyResult { - if value.is_none() { - return Ok(String::new()); - } - if value.is_instance_of::() { - return Ok(value.repr()?.to_string_lossy().into_owned()); - } - python_text(value) -} - /// The JSON `default` hook: Decimal in fixed-point notation, anything else as its `str`. #[pyfunction] pub(crate) fn query_json_default(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs new file mode 100644 index 0000000000..114ac23eb3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs @@ -0,0 +1,37 @@ +//! The text layout `sqb query` and `sqb recall query` share: a box when the result fits the output width, records otherwise. + +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_core::query_command::main::render_query_text::render_query_text; +use sqlbuild_core::query_command::models::{QueryDisplay, QueryLayout}; + +use crate::bindings::_helpers::build_progress::terminal::terminal_columns; +use crate::bindings::_helpers::presentation_text::terminal::stdout_is_tty; + +/// The format that always prints one record per row. +pub(crate) const LONG_FORMAT: &str = "long"; +/// The width a box may take when stdout is not a terminal. +const PIPED_WIDTH: usize = 120; + +/// The terminal's width when stdout is one, else [`PIPED_WIDTH`]. +fn output_width(py: Python<'_>) -> PyResult { + if !stdout_is_tty(py).unwrap_or(false) { + return Ok(PIPED_WIDTH); + } + terminal_columns(py) +} + +/// `display` as text: records for `--format long`, otherwise a box when it fits the output. +pub(crate) fn render_text( + py: Python<'_>, + display: &QueryDisplay, + format: Option<&str>, +) -> PyResult { + let layout: QueryLayout = if format == Some(LONG_FORMAT) { + QueryLayout::Long + } else { + QueryLayout::Fitted { + width: output_width(py)?, + } + }; + Ok(render_query_text(display, layout)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs index 3278baa93c..d2b239e961 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs @@ -31,9 +31,9 @@ enum Cell { Bytes(Vec), } -/// A query reply: column names, rows of cells, and rows read, bytes read and query ID. +/// A query reply: column names and types, rows of cells, and rows read, bytes read and query ID. type QueryReply = ( - Vec, + (Vec, Vec>), Vec>>, (Option, Option, Option), ); @@ -175,11 +175,12 @@ pub(crate) fn query( ], action, )?; - let (columns, rows, (rows_read, bytes_read, query_id)): QueryReply = + let ((columns, types), rows, (rows_read, bytes_read, query_id)): QueryReply = value.extract().map_err(|error| raised(action, &error))?; Ok(( QueryResult { columns, + types, rows: rows .into_iter() .map(|row| row.into_iter().map(cell_value).collect()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index 2e4b45435b..ad12342ac4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -30,10 +30,9 @@ use crate::bindings::_helpers::query_command::query::run_query_command; use crate::bindings::_helpers::recall::clickhouse_store::{ClickHouseLocation, ClickHouseReader}; use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_bytes}; use crate::bindings::_helpers::recall::constants::{ - DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, LONG_FORMAT, + DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, RECALL_AMBIGUOUS_CODE, RECALL_DISABLED_CODE, RECALL_NOT_FOUND_CODE, RECALL_OPTIONS, - RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, TABLE_FORMAT, - TABLE_FORMAT_DEPRECATION, WAREHOUSE_BACKEND, + RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, WAREHOUSE_BACKEND, }; use crate::bindings::_helpers::recall::rendering::{QueryContext, render_result, render_schema}; use crate::bindings::_helpers::records::construct::new_error; @@ -60,7 +59,8 @@ struct RecallRequest { selected_target: Option, sql: Option, backend: Option, - format: String, + /// `long`, `json`, or `None` for the default layout. + format: Option, limit: Option, no_limit: bool, max_cell_bytes: Option, @@ -89,7 +89,7 @@ fn recall_request(py: Python<'_>, options: Option>) -> PyResul selected_target: text("selected_target")?, sql: text("sql")?, backend: text("backend")?, - format: text("output_format")?.unwrap_or_else(|| LONG_FORMAT.to_owned()), + format: text("output_format")?, limit: number("limit")?, no_limit: arguments.value("no_limit")?.is_truthy()?, max_cell_bytes: number("max_cell_bytes")?, @@ -101,14 +101,6 @@ fn recall_request(py: Python<'_>, options: Option>) -> PyResul }) } -/// Warn once that `--format table` goes away in a later release; it still works in this one. -fn warn_table_format(py: Python<'_>, recall: &RecallRequest) -> PyResult<()> { - if recall.format == TABLE_FORMAT { - write_stderr(py, TABLE_FORMAT_DEPRECATION)?; - } - Ok(()) -} - /// The project directory, target and settings, or the coded failure. fn settings(py: Python<'_>, recall: &RecallRequest) -> PyResult<(PathBuf, RecallSettings)> { let raw: PathBuf = match &recall.project_dir { @@ -280,7 +272,6 @@ fn limit(recall: &RecallRequest) -> Option { } fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { - warn_table_format(py, recall)?; let (project_dir, settings) = settings(py, recall)?; let sql: String = recall.sql.clone().unwrap_or_default(); if sql.trim().is_empty() { @@ -315,17 +306,17 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { let (adapter, label) = (store.adapter(), store.label()); store.close(); let (result, statistics) = queried.map_err(|error| store_failure(py, &error))?; - let format: &str = &recall.format; + let format: Option<&str> = recall.format.as_deref(); let max_bytes: Option = match recall.max_cell_bytes { Some(bytes) if bytes <= 0 => None, Some(bytes) => Some(usize::try_from(bytes).unwrap_or(usize::MAX)), - None if format == JSON_FORMAT => None, + None if format == Some(JSON_FORMAT) => None, None => Some(DEFAULT_MAX_CELL_BYTES), }; let rendered = render_result( + py, &result, - format, - max_bytes, + (format, max_bytes), &QueryContext { target: settings.target.as_deref(), store_path: &label, @@ -334,7 +325,7 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { elapsed_ms: started.elapsed().as_secs_f64() * 1000.0, statistics: statistics.as_ref(), }, - ); + )?; write_stdout_bytes(py, rendered.as_bytes())?; Ok(0) } @@ -390,7 +381,6 @@ fn run_schema_ddl(py: Python<'_>, settings: &RecallSettings) -> PyResult { } fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { - warn_table_format(py, recall)?; let (project_dir, settings) = settings(py, recall)?; if recall.ddl { if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { @@ -426,7 +416,7 @@ fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { let (dialect, label) = (store.dialect(), store.label()); store.close(); let relations = described.map_err(|error| store_failure(py, &error))?; - let json_output: bool = recall.format == JSON_FORMAT; + let json_output: bool = recall.format.as_deref() == Some(JSON_FORMAT); write_stdout_bytes( py, render_schema( @@ -496,7 +486,7 @@ fn run_output(py: Python<'_>, recall: &RecallRequest) -> PyResult { problems.push("delivery did not report a complete flush".to_owned()); } match transcript.status.as_deref() { - Some("completed" | "failed") | None => {} + Some("completed" | "failed" | "cancelled") | None => {} Some(status) => problems.push(format!("the invocation is {status}")), } if !problems.is_empty() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 004722f9f0..2a1af6996c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -28,12 +28,9 @@ pub(crate) const RECALL_SETTINGS_CODE: &str = "C975"; pub(crate) const WAREHOUSE_BACKEND: &str = "warehouse"; /// Default cell width in text formats, in bytes. pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; -pub(crate) const LONG_FORMAT: &str = "long"; -pub(crate) const TABLE_FORMAT: &str = "table"; -/// What `--format table` prints in the release before it is removed. -pub(crate) const TABLE_FORMAT_DEPRECATION: &str = "warning: --format table is deprecated and will be removed; use the default long format, or --json/--format csv\n"; -pub(crate) const CSV_FORMAT: &str = "csv"; pub(crate) const JSON_FORMAT: &str = "json"; +/// How text output shows SQL NULL. +pub(crate) const NULL_TEXT: &str = "NULL"; /// The standard base64 alphabet binary JSON cells are encoded with. pub(crate) const BASE64_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -118,3 +115,9 @@ pub(crate) const UTC_OFFSET: &str = "+00:00"; pub(crate) const DEBUG_COMMAND: &str = "debug"; /// The namespace attribute of `sqb debug --no-connection`. pub(crate) const NO_CONNECTION_ATTRIBUTE: &str = "no_connection"; +/// The statement terminal events, at which the sink settles a statement's SQL. +pub(crate) const INVOCATION_STARTED: &str = "invocation_started"; +pub(crate) const INVOCATION_COMPLETED: &str = "invocation_completed"; +pub(crate) const INVOCATION_FAILED: &str = "invocation_failed"; +pub(crate) const STATEMENT_COMPLETED: &str = "statement_completed"; +pub(crate) const STATEMENT_FAILED: &str = "statement_failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index 84b0e96a13..f2a5cd4abc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -1,17 +1,16 @@ -//! `sqb recall query` and `schema` output: text cells, CSV and the JSON document. +//! `sqb recall query` and `schema` output: text cells, the shared text layout and the JSON document. +use pyo3::prelude::{PyResult, Python}; use serde_json::{Map, Value, json}; -use sqlbuild_core::query_command::main::render_query_csv::render_query_csv; -use sqlbuild_core::query_command::main::render_query_long::render_query_long; -use sqlbuild_core::query_command::main::render_query_table::render_query_table; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_recall::records::models::RecallValue; use sqlbuild_recall::schema::models::Dialect; use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryResult}; +use crate::bindings::_helpers::query_command::text_output::render_text; use crate::bindings::_helpers::recall::clickhouse_calls::QueryStatistics; use crate::bindings::_helpers::recall::constants::{ - BASE64_ALPHABET, CSV_FORMAT, JOIN_KEYS, JSON_FORMAT, TABLE_FORMAT, + BASE64_ALPHABET, JOIN_KEYS, JSON_FORMAT, NULL_TEXT, }; fn base64(data: &[u8]) -> String { @@ -102,14 +101,27 @@ pub(crate) struct QueryContext<'a> { pub(crate) statistics: Option<&'a QueryStatistics>, } -/// The rendered result in `format`: `long`, `table`, `csv` or `json`. +/// Whether every non-NULL value of the column at `index` is a number, and at least one is. +fn numeric_column(result: &QueryResult, index: usize) -> bool { + let mut seen: bool = false; + for row in &result.rows { + match row.get(index) { + Some(RecallValue::Integer(_) | RecallValue::Real(_)) => seen = true, + Some(RecallValue::Null) | None => {} + Some(_) => return false, + } + } + seen +} + +/// The rendered result in `format`: `json`, `long`, or `None` for the default layout; text cells are cut at `max_bytes`. pub(crate) fn render_result( + py: Python<'_>, result: &QueryResult, - format: &str, - max_bytes: Option, + (format, max_bytes): (Option<&str>, Option), context: &QueryContext<'_>, -) -> String { - if format == JSON_FORMAT { +) -> PyResult { + if format == Some(JSON_FORMAT) { let rows: Vec = result .rows .iter() @@ -131,42 +143,39 @@ pub(crate) fn render_result( "query_id": context.statistics.and_then(|statistics| statistics.query_id.clone()), }, }); - return format!("{document}\n"); - } - let csv: bool = format == CSV_FORMAT; - let null: &str = if csv { "" } else { "NULL" }; - let cells: Vec> = result - .rows - .iter() - .map(|row| display_row(row, null, max_bytes)) - .collect(); - if csv { - return render_query_csv(&result.columns, &cells); + return Ok(format!("{document}\n")); } let display = QueryDisplay { columns: result.columns.clone(), - rows: cells, + types: result.types.clone(), + numeric: (0..result.columns.len()) + .map(|index| numeric_column(result, index)) + .collect(), + rows: result + .rows + .iter() + .map(|row| display_row(row, NULL_TEXT, max_bytes)) + .collect(), truncated: result.truncated, limit: context .limit .map(|limit| i64::try_from(limit).unwrap_or(i64::MAX)), }; - if format == TABLE_FORMAT { - render_query_table(&display) - } else { - render_query_long(&display) - } + render_text(py, &display, format) } fn relation_note(name: &str) -> &'static str { match name { "invocations" => { - "one row per invocation; status is completed, failed, running, abandoned or unknown; \ - origin is integration, ci or local; integration, integration_job, \ + "one row per invocation; status is completed, failed, cancelled, running, abandoned \ + or unknown; origin is integration, ci or local; integration, integration_job, \ integration_run_id and integration_retry come from external_context" } - "statements" => "one row per statement; outcome no_terminal means no terminal fact arrived", - "audit_results" => "one row per evaluated audit", + "statements" => { + "one row per statement; outcome handled means SQLBuild recovered from the failure; \ + no_terminal means no terminal fact arrived" + } + "audit_results" => "one row per evaluated audit; its evidence rows are in artifacts", "output_invocations" => { "output completeness per invocation: gaps, drops, filtering, truncation, flush" } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 7ab0b0644e..85714984a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -8,9 +8,11 @@ use std::time::{Duration, Instant, SystemTime}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_cli::dispatch::main::closing_status::closing_status; +use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; use sqlbuild_cli::dispatch::models::CommandFacts; use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_recall::records::main::bounded_selection::bounded_selection; use sqlbuild_recall::records::main::command_mode::command_mode; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::invocation_origin::invocation_origin; @@ -22,7 +24,7 @@ use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; use sqlbuild_recall::settings::models::{ - RecallSettings, SettingsRequest, StoreLocation, StorePurpose, + AuditEvidence, RecallSettings, SettingsRequest, StoreLocation, StorePurpose, }; use sqlbuild_recall::store::main::observe_store_file::observe_store_file; use sqlbuild_recall::store::main::quarantine_store::quarantine_store; @@ -48,6 +50,7 @@ use crate::bindings::_helpers::recall::constants::{ }; use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_replacement_sink}; +use crate::bindings::_helpers::recall::sinks::InvocationEvents; use crate::bindings::_helpers::records::fields::field as native_field; /// What the invocation records: its session, when enabled, and the resolved target and adapter. @@ -64,6 +67,10 @@ pub(crate) struct InvocationRecall { pub(crate) replacements: Arc, /// A Recall configuration mistake the command fails with before it runs. pub(crate) configuration_error: Option, + /// The invocation's own lifecycle events the session has recorded. + pub(crate) invocation_events: Arc, + /// Which failing audits collect evidence rows, as `recall.audit_evidence` sets it. + pub(crate) audit_evidence: AuditEvidence, } /// The text of an optional attribute value. @@ -90,6 +97,24 @@ fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> Ok(items) } +/// The effective selection: positional selectors and `--select`, then what every `--select-file` lists, with the files as passed. A file that cannot be read adds no selectors here; the command reports it. +pub(crate) fn effective_selection(args: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec)> { + let mut selectors: Vec = list_attribute(args, "select")?; + let files: Vec = match args.getattr_opt("select_file")? { + Some(value) if !value.is_none() => value + .try_iter()? + .map(|item| escaped_text(&item?.str()?)) + .collect::>()?, + _ => Vec::new(), + }; + for file in &files { + if let Some(text) = file_text(Path::new(file)) { + selectors.extend(selector_file_selectors(&text)); + } + } + Ok((selectors, files)) +} + fn environment(name: &str) -> Option { std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) } @@ -239,6 +264,9 @@ fn invocation_facts( settings.destination_schema.as_deref().unwrap_or_default() ); let external_context: Option = external_context(py)?; + let (selectors, selector_files) = effective_selection(args)?; + let selection_count: u64 = selectors.len() as u64; + let (selection, selection_truncated) = bounded_selection(selectors); let no_connection: bool = facts.command.as_deref() == Some(DEBUG_COMMAND) && attribute(args, NO_CONNECTION_ATTRIBUTE)?.is_truthy()?; Ok(InvocationContext { @@ -246,7 +274,10 @@ fn invocation_facts( command_mode: command_mode(facts.command.as_deref(), no_connection).map(str::to_owned), command: facts.command.clone(), subcommand: facts.subcommand.clone(), - selection: list_attribute(args, "select")?, + selection, + selection_count: Some(selection_count), + selection_truncated, + selector_files, exclusion: list_attribute(args, "exclude")?, project_name: settings.project_name.clone(), project_dir_hash: Some(sha256_hex(project_dir.to_string_lossy().as_bytes())), @@ -433,6 +464,7 @@ fn started_session( setup_budget: Duration::ZERO, maintenance: true, defer_context: true, + audit_evidence: settings.audit_evidence, }); Ok(Some(Arc::new(session))) } @@ -466,6 +498,7 @@ pub(crate) fn start_invocation_recall( } else { None }; + let audit_evidence: AuditEvidence = settings.audit_evidence; Ok(InvocationRecall { session, target: settings.target.clone(), @@ -474,6 +507,8 @@ pub(crate) fn start_invocation_recall( settings: Some((settings, project_dir)), replacements: Arc::default(), configuration_error: None, + invocation_events: Arc::default(), + audit_evidence, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index 21d3360b94..29a01a6a17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -1,6 +1,8 @@ //! The native subscriber that publishes every lifecycle fact and artifact into Recall. -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; @@ -11,6 +13,10 @@ use sqlbuild_recall::records::main::parse_event::parse_event; use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::constants::{ + INVOCATION_COMPLETED, INVOCATION_FAILED, INVOCATION_STARTED, STATEMENT_COMPLETED, + STATEMENT_FAILED, +}; use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; use crate::bindings::_helpers::recall::replacements::RecallReplacements; @@ -42,11 +48,43 @@ pub(crate) fn publish_lifecycle_event( Ok(()) } -/// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. +/// The SQL and resource of a statement that has not reached its terminal event yet. +type RunningSql = HashMap)>; + +/// Which of an invocation's own lifecycle events the sink has recorded, so an interrupted run records each exactly once. +#[derive(Debug, Default)] +pub(crate) struct InvocationEvents { + started: AtomicBool, + ended: AtomicBool, +} + +impl InvocationEvents { + /// Whether `invocation_started` was recorded. + pub(crate) fn started(&self) -> bool { + self.started.load(Ordering::SeqCst) + } + + /// Whether `invocation_completed` or `invocation_failed` was recorded. + pub(crate) fn ended(&self) -> bool { + self.ended.load(Ordering::SeqCst) + } + + fn note(&self, event_type: &str) { + if event_type == INVOCATION_STARTED { + self.started.store(true, Ordering::SeqCst); + } else if event_type == INVOCATION_COMPLETED || event_type == INVOCATION_FAILED { + self.ended.store(true, Ordering::SeqCst); + } + } +} + +/// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. The SQL of a statement that fails or is cancelled is queued again with its terminal event's priority. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct RecallLifecycleSink { pub(crate) session: Arc, pub(crate) replacements: Arc, + pub(crate) running_sql: Arc>, + pub(crate) invocation_events: Arc, } #[pymethods] @@ -77,20 +115,57 @@ impl RecallLifecycleSink { texts.map(|(sql, statement_id, resource_id)| { self.session .publish_sql(&sql, statement_id.as_deref(), resource_id.as_deref()); + if let Some(statement_id) = statement_id { + self.running().insert(statement_id, (sql, resource_id)); + } }), ) } } impl RecallLifecycleSink { + fn running(&self) -> std::sync::MutexGuard<'_, RunningSql> { + self.running_sql + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// Forget a statement's SQL at its terminal event; a failed one's SQL is queued again first. + fn settle_statement(&self, event: &Bound<'_, PyAny>, event_type: &str) -> PyResult<()> { + if event_type != STATEMENT_COMPLETED && event_type != STATEMENT_FAILED { + return Ok(()); + } + let Some(statement_id) = event.getattr_opt("statement_id")? else { + return Ok(()); + }; + if statement_id.is_none() { + return Ok(()); + } + let statement_id: String = escaped_text(&statement_id.str()?)?; + let Some((sql, resource_id)) = self.running().remove(&statement_id) else { + return Ok(()); + }; + if event_type == STATEMENT_FAILED { + self.session + .publish_failed_sql(&sql, Some(&statement_id), resource_id.as_deref()); + } + Ok(()) + } + fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - if self + let event_type: String = match event.getattr_opt("event_type")? { + Some(event_type) => event_type.str()?.to_string(), + None => String::new(), + }; + self.settle_statement(event, &event_type)?; + if !self .replacements .deliver_lifecycle(py, &self.session, event)? { - return Ok(()); + publish_lifecycle_event(py, &self.session, event)?; } - publish_lifecycle_event(py, &self.session, event) + self.invocation_events.note(&event_type); + Ok(()) } } diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs index d30b2fe176..b4f54d98a5 100644 --- a/crates/sqlbuild-recall/src/records/constants.rs +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -10,6 +10,18 @@ pub const ARTIFACT_DIGEST_PREFIX: &str = "sha256:"; pub const SQL_ARTIFACT_KIND: &str = "sql"; pub const SQL_MEDIA_TYPE: &str = "text/x-sql"; pub const EXECUTED_SQL_ROLE: &str = "executed_sql"; +/// Evidence rows of a warning or failing audit, stored as a JSON array of row objects. +pub const AUDIT_EVIDENCE_KIND: &str = "audit_evidence"; +pub const AUDIT_EVIDENCE_MEDIA_TYPE: &str = "application/json"; +pub const AUDIT_EVIDENCE_ROLE: &str = "audit_evidence"; +/// The event whose payload may carry evidence rows. +pub const AUDIT_COMPLETED_EVENT: &str = "audit_completed"; +/// The payload field holding evidence rows, which Recall stores as an artifact instead. +pub const EVIDENCE_PAYLOAD_FIELD: &str = "evidence"; +/// The payload field naming the evidence artifact's digest, without its `sha256:` prefix. +pub const SAMPLE_DIGEST_PAYLOAD_FIELD: &str = "sample_digest"; +/// The most selector text an invocation's context row records; later selectors are counted but left out. +pub const MAX_RECORDED_SELECTION_BYTES: usize = 64 * 1024; /// Diagnostic severities, least severe first. pub const SEVERITY_ORDER: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; /// Event payload fields promoted to their own `recall_events` columns. diff --git a/crates/sqlbuild-recall/src/records/main/bounded_selection.rs b/crates/sqlbuild-recall/src/records/main/bounded_selection.rs new file mode 100644 index 0000000000..2bdfd968b9 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/bounded_selection.rs @@ -0,0 +1,20 @@ +//! `bounded_selection`: the selectors a context row records, within its bound. + +use crate::records::constants::MAX_RECORDED_SELECTION_BYTES; + +/// The leading selectors whose text fits [`MAX_RECORDED_SELECTION_BYTES`], and whether any were left out. +#[must_use] +pub fn bounded_selection(selectors: Vec) -> (Vec, bool) { + let mut used: usize = 0; + let mut kept: Vec = Vec::with_capacity(selectors.len()); + let total: usize = selectors.len(); + for selector in selectors { + used += selector.len(); + if used > MAX_RECORDED_SELECTION_BYTES { + break; + } + kept.push(selector); + } + let truncated: bool = kept.len() < total; + (kept, truncated) +} diff --git a/crates/sqlbuild-recall/src/records/main/invocation_row.rs b/crates/sqlbuild-recall/src/records/main/invocation_row.rs index dc5d54213b..e9d533309c 100644 --- a/crates/sqlbuild-recall/src/records/main/invocation_row.rs +++ b/crates/sqlbuild-recall/src/records/main/invocation_row.rs @@ -18,6 +18,12 @@ pub fn invocation_row(envelope: &RowEnvelope, context: &InvocationContext) -> Re ("command", optional_text(context.command.as_deref())), ("subcommand", optional_text(context.subcommand.as_deref())), ("selection", json_list(&context.selection)), + ( + "selection_count", + context.selection_count.map_or(RecallValue::Null, integer), + ), + ("selection_truncated", boolean(context.selection_truncated)), + ("selector_files", json_list(&context.selector_files)), ("exclusion", json_list(&context.exclusion)), ( "project_name", diff --git a/crates/sqlbuild-recall/src/records/main/mod.rs b/crates/sqlbuild-recall/src/records/main/mod.rs index 3ccedf2f77..4c8e37f114 100644 --- a/crates/sqlbuild-recall/src/records/main/mod.rs +++ b/crates/sqlbuild-recall/src/records/main/mod.rs @@ -1,4 +1,5 @@ pub mod artifact_rows; +pub mod bounded_selection; pub mod command_mode; pub mod conflict_row; pub mod delivery_row; diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index de2e666efb..e9da820c87 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -103,7 +103,14 @@ pub struct OmittedOutput { pub struct InvocationContext { pub command: Option, pub subcommand: Option, + /// The effective selectors: positional, `--select` and the contents of every selector file, at most [`MAX_RECORDED_SELECTION_BYTES`](crate::records::constants::MAX_RECORDED_SELECTION_BYTES). pub selection: Vec, + /// How many effective selectors there were, recorded ones and truncated ones alike. + pub selection_count: Option, + /// Selectors were left out of `selection` to keep it within its bound. + pub selection_truncated: bool, + /// The selector files as passed, in order. + pub selector_files: Vec, pub exclusion: Vec, pub project_name: Option, pub project_dir_hash: Option, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index ccf1b80752..638fe61891 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -6,7 +6,7 @@ use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Table}; /// The columns the `invocations` pass reads from its three sources, NULL (the terminal flag 0) where a source does not give them. -const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 30] = [ +const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 35] = [ ("ctx_command", "Nullable(String)", "NULL"), ("ctx_subcommand", "Nullable(String)", "NULL"), ("ctx_target", "Nullable(String)", "NULL"), @@ -23,6 +23,9 @@ const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 30] = [ ("ctx_recall_filter_active", "Nullable(UInt8)", "NULL"), ("ctx_origin", "Nullable(String)", "NULL"), ("ctx_command_mode", "Nullable(String)", "NULL"), + ("ctx_selection_count", "Nullable(Int64)", "NULL"), + ("ctx_selection_truncated", "Nullable(UInt8)", "NULL"), + ("ctx_selector_files", "Nullable(String)", "NULL"), ("heartbeat_at", "Nullable(DateTime64(6, 'UTC'))", "NULL"), ("beat_stale_at", "Nullable(DateTime64(6, 'UTC'))", "NULL"), ("beat_command", "Nullable(String)", "NULL"), @@ -40,11 +43,13 @@ const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 30] = [ ("event_error_type", "Nullable(String)", "NULL"), ("event_error_code", "Nullable(String)", "NULL"), ("event_error_message", "Nullable(String)", "NULL"), + ("event_signal_number", "Nullable(Int64)", "NULL"), + ("event_cost_status", "Nullable(String)", "NULL"), ("event_is_terminal", "UInt8", "0"), ]; /// The statement columns each statement event carries into the second fold, in tuple order. -const STATEMENT_FACT_COLUMNS: [&str; 17] = [ +const STATEMENT_FACT_COLUMNS: [&str; 18] = [ "statement_id", "event_type", "occurred_at", @@ -62,6 +67,7 @@ const STATEMENT_FACT_COLUMNS: [&str; 17] = [ "duration_ms", "row_count", "affected_rows", + "outcome", ]; /// The columns the `output_invocations` pass reads: the chunk's own values default to what adds nothing to a sum, and the delivery and context values to NULL. @@ -176,6 +182,9 @@ fn invocations(options: &DdlOptions) -> String { ("ctx_recall_filter_active", "recall_filter_active"), ("ctx_origin", "origin"), ("ctx_command_mode", "command_mode"), + ("ctx_selection_count", "selection_count"), + ("ctx_selection_truncated", "selection_truncated"), + ("ctx_selector_files", "selector_files"), ], &context, ); @@ -206,6 +215,14 @@ fn invocations(options: &DdlOptions) -> String { "event_error_message", "JSONExtract(payload, 'error_message', 'Nullable(String)')", ), + ( + "event_signal_number", + "JSONExtract(payload, 'signal_number', 'Nullable(Int64)')", + ), + ( + "event_cost_status", + "JSONExtract(payload, 'cost_status', 'Nullable(String)')", + ), ( "event_is_terminal", "event_type IN ('invocation_completed', 'invocation_failed')", @@ -244,6 +261,8 @@ folded AS ( {terminal_error_type} {terminal_error_code} {terminal_error_message} +{terminal_signal_number} +{terminal_cost_status} maxIf(heartbeat_at, source = 'beat') AS last_heartbeat_at, maxIf(beat_stale_at, source = 'beat') AS stale_at, minIf(beat_command, source = 'beat') AS first_beat_command, @@ -257,6 +276,7 @@ SELECT coalesce(started_occurred_at, invocation_started_at) AS started_at, multiIf( terminal_type = 'invocation_completed', 'completed', + terminal_type = 'invocation_failed' AND terminal_signal_number IS NOT NULL, 'cancelled', terminal_type = 'invocation_failed', 'failed', stale_at IS NULL, 'unknown', stale_at >= now64(6), 'running', @@ -268,10 +288,14 @@ SELECT if(stale_at < now64(6), toFloat64(dateDiff('millisecond', started_at, last_heartbeat_at)), NULL)) AS duration_ms, + multiIf(terminal_type IS NOT NULL, toUInt8(0), stale_at < now64(6), toUInt8(1), NULL) + AS ended_at_estimated, terminal_exit_code AS exit_code, terminal_error_type AS error_type, terminal_error_code AS error_code, terminal_error_message AS error_message, + terminal_signal_number AS signal_number, + terminal_cost_status AS cost_status, coalesce(context_command, started_command, first_beat_command) AS command, coalesce(context_subcommand, first_beat_subcommand) AS subcommand, context_origin AS origin, @@ -280,6 +304,11 @@ SELECT context_adapter AS adapter, context_project_name AS project_name, context_selection AS selection, + coalesce(context_selection_count, + if(isValidJSON(context_selection), toInt64(JSONLength(context_selection)), NULL)) + AS selection_count, + context_selection_truncated AS selection_truncated, + context_selector_files AS selector_files, context_exclusion AS exclusion, context_sqlbuild_version AS sqlbuild_version, context_git_revision AS git_revision, @@ -309,13 +338,16 @@ FROM folded", terminal_error_type = terminal("event_error_type", "terminal_error_type"), terminal_error_code = terminal("event_error_code", "terminal_error_code"), terminal_error_message = terminal("event_error_message", "terminal_error_message"), + terminal_signal_number = terminal("event_signal_number", "terminal_signal_number"), + terminal_cost_status = terminal("event_cost_status", "terminal_cost_status"), ) } -/// Two folds over one read of the events: by resource, which names the resource and gathers its statement events, then by statement. Both group by `invocation_id` and `invocation_started_at`, so a filter on either reaches the read; `sql_available` looks digests up in the stored SQL, which is shared across invocations. +/// Two folds over one read of the events: by resource, which names the resource and gathers its statement events, then by statement. Both group by `invocation_id` and `invocation_started_at`, so a filter on either reaches the read; `sql_available` needs a reference to the digest in the statement's own invocation and the stored SQL, which is shared across invocations. The references and the stored SQL are each read once. fn statements(options: &DdlOptions) -> String { let events = retained(Table::Events, options, ""); let artifacts = clickhouse_table(table_spec(Table::Artifacts).name, options); + let refs = retained(Table::ArtifactRefs, options, ""); let order = "tuple(invocation_sequence, occurred_at)"; let terminal = "event_type IN ('statement_completed', 'statement_failed')"; let statement_event = "statement_id IS NOT NULL AND startsWith(event_type, 'statement_')"; @@ -365,7 +397,8 @@ folded AS ( argMaxIf(error_code, {order}, {terminal}) AS last_error_code, argMaxIf(duration_ms, {order}, {terminal}) AS last_duration_ms, argMaxIf(row_count, {order}, {terminal}) AS last_row_count, - argMaxIf(affected_rows, {order}, {terminal}) AS last_affected_rows + argMaxIf(affected_rows, {order}, {terminal}) AS last_affected_rows, + argMaxIf(outcome, {order}, {terminal}) AS last_outcome FROM statement_events GROUP BY invocation_id, invocation_started_at, statement_id ) SELECT @@ -388,6 +421,7 @@ SELECT multiIf( NOT has_terminal, 'no_terminal', last_event_type = 'statement_completed', 'completed', + last_outcome = 'handled', 'handled', 'failed' ) AS outcome, if(has_terminal, last_error_type, NULL) AS error_type, @@ -396,8 +430,20 @@ SELECT if(has_terminal, last_row_count, NULL) AS row_count, if(has_terminal, last_affected_rows, NULL) AS affected_rows, first_sql_digest AS sql_digest, - concat('sha256:', first_sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available -FROM folded" + sql_referenced + AND concat('sha256:', first_sql_digest) IN (SELECT digest FROM {artifacts}) + AS sql_available, + multiIf( + sql_available, NULL, + sql_referenced, 'expired_or_not_captured', + 'not_captured' + ) AS sql_unavailable_reason +FROM ( + SELECT *, + (invocation_id, concat('sha256:', first_sql_digest)) + IN (SELECT invocation_id, digest FROM {refs}) AS sql_referenced + FROM folded +)" ) } @@ -428,6 +474,9 @@ fn audit_results(options: &DdlOptions) -> String { JSONExtract(payload, 'evaluation_mode', 'Nullable(String)') AS evaluation_mode, JSONExtract(payload, 'execution_error', 'Nullable(String)') AS execution_error, sql_digest, + JSONExtract(payload, 'evidence_count', 'Nullable(Int64)') AS evidence_count, + if(JSONHas(payload, 'evidence_truncated'), + toUInt8(JSONExtractBool(payload, 'evidence_truncated')), NULL) AS evidence_truncated, JSONExtract(payload, 'sample_digest', 'Nullable(String)') AS sample_digest FROM {events} WHERE event_type = 'audit_completed'" @@ -597,8 +646,9 @@ fn artifacts(options: &DdlOptions) -> String { artifact.stored_size AS size, artifact.stored_digest IS NOT NULL AS available, if(artifact.stored_digest IS NULL, 'expired_or_not_captured', NULL) AS unavailable_reason, - if(startsWith(artifact.stored_media_type, 'text/'), toValidUTF8(artifact.stored_content), NULL) - AS text, + if(startsWith(artifact.stored_media_type, 'text/') + OR artifact.stored_media_type = 'application/json', + toValidUTF8(artifact.stored_content), NULL) AS text, artifact.stored_content AS content FROM {refs} AS refs LEFT JOIN ( diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index c679a80e08..8d4e1c2f81 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -31,7 +31,7 @@ query_ids AS ( ), terminals AS ( SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, - row_count, affected_rows, + row_count, affected_rows, outcome, ROW_NUMBER() OVER ( PARTITION BY statement_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -39,6 +39,17 @@ terminals AS ( FROM facts WHERE event_type IN ('statement_completed', 'statement_failed') ), +captures AS ( + SELECT firsts.statement_id, ( + SELECT MAX(artifact.digest IS NOT NULL) + FROM recall_artifact_refs AS ref + LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest + WHERE ref.invocation_id = firsts.invocation_id + AND ref.digest = 'sha256:' || firsts.sql_digest + ) AS stored + FROM firsts + WHERE firsts.position = 1 +), resources AS ( SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, MIN(resource_kind) AS resource_kind @@ -65,7 +76,8 @@ SELECT terminals.occurred_at AS ended_at, CASE terminals.event_type WHEN 'statement_completed' THEN 'completed' - WHEN 'statement_failed' THEN 'failed' + WHEN 'statement_failed' THEN + CASE WHEN terminals.outcome = 'handled' THEN 'handled' ELSE 'failed' END ELSE 'no_terminal' END AS outcome, terminals.error_type, @@ -74,16 +86,19 @@ SELECT terminals.row_count, terminals.affected_rows, firsts.sql_digest, - EXISTS ( - SELECT 1 FROM recall_artifacts AS artifact - WHERE artifact.digest = 'sha256:' || firsts.sql_digest - ) AS sql_available + COALESCE(captures.stored, 0) AS sql_available, + CASE captures.stored + WHEN 1 THEN NULL + WHEN 0 THEN 'expired_or_not_captured' + ELSE 'not_captured' + END AS sql_unavailable_reason FROM firsts LEFT JOIN started ON started.statement_id = firsts.statement_id LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id LEFT JOIN terminals ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN captures ON captures.statement_id = firsts.statement_id LEFT JOIN resources ON resources.invocation_id = firsts.invocation_id AND resources.resource_id = firsts.resource_id @@ -114,6 +129,8 @@ const AUDIT_RESULTS: &str = "SELECT json_extract(payload, '$.evaluation_mode') AS evaluation_mode, json_extract(payload, '$.execution_error') AS execution_error, sql_digest, + json_extract(payload, '$.evidence_count') AS evidence_count, + json_extract(payload, '$.evidence_truncated') AS evidence_truncated, json_extract(payload, '$.sample_digest') AS sample_digest FROM recall_events WHERE event_type = 'audit_completed'"; @@ -225,7 +242,8 @@ const ARTIFACTS: &str = "SELECT artifact.size, artifact.digest IS NOT NULL AS available, CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, - CASE WHEN artifact.media_type LIKE 'text/%' THEN CAST(artifact.content AS TEXT) END AS text, + CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' + THEN CAST(artifact.content AS TEXT) END AS text, artifact.content FROM recall_artifact_refs AS refs LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; @@ -253,6 +271,8 @@ started AS ( terminals AS ( SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, json_extract(payload, '$.error_message') AS error_message, + json_extract(payload, '$.signal_number') AS signal_number, + json_extract(payload, '$.cost_status') AS cost_status, ROW_NUMBER() OVER ( PARTITION BY invocation_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -273,6 +293,8 @@ projected AS ( COALESCE(started.started_at, ids.invocation_started_at) AS started_at, CASE WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' + AND terminals.signal_number IS NOT NULL THEN 'cancelled' WHEN terminals.event_type = 'invocation_failed' THEN 'failed' WHEN beats.stale_at IS NULL THEN 'unknown' WHEN beats.stale_at >= {NOW} THEN 'running' @@ -285,6 +307,8 @@ projected AS ( terminals.error_type, terminals.error_code, terminals.error_message, + terminals.signal_number, + terminals.cost_status, COALESCE(context.command, started.command, beats.command) AS command, COALESCE(context.subcommand, beats.subcommand) AS subcommand, context.origin, @@ -293,6 +317,10 @@ projected AS ( context.adapter, context.project_name, context.selection, + COALESCE(context.selection_count, json_array_length(context.selection)) + AS selection_count, + context.selection_truncated, + context.selector_files, context.exclusion, context.sqlbuild_version, context.git_revision, @@ -336,10 +364,16 @@ SELECT (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 ) END AS duration_ms, + CASE + WHEN terminal_event IS NOT NULL THEN 0 + WHEN status = 'abandoned' THEN 1 + END AS ended_at_estimated, exit_code, error_type, error_code, error_message, + signal_number, + cost_status, command, subcommand, origin, @@ -348,6 +382,9 @@ SELECT adapter, project_name, selection, + selection_count, + selection_truncated, + selector_files, exclusion, sqlbuild_version, git_revision, diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 3e846c7b5d..48e5a69a36 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -3,7 +3,7 @@ use crate::schema::models::{ColumnSpec, ColumnType, Table, TableSpec}; /// The Recall schema version this build writes. Raise it when a column or view is added. -pub const RECALL_SCHEMA_VERSION: i64 = 4; +pub const RECALL_SCHEMA_VERSION: i64 = 5; /// Text that only older schema versions wrote into ClickHouse view definitions, with the last version that wrote it. Version 3 changed only view bodies, so view columns cannot tell version 2 views, which set `join_use_nulls`, from current ones. pub const SUPERSEDED_CLICKHOUSE_VIEW_MARKERS: [(&str, i64); 1] = [("join_use_nulls", 2)]; /// The oldest SQLite library the store's views run on (ordered `group_concat`). @@ -39,7 +39,7 @@ const CORRELATIONS: [ColumnSpec; 5] = [ ColumnSpec::optional("statement_id", ColumnType::Text), ]; -const INVOCATION_COLUMNS: [ColumnSpec; 21] = [ +const INVOCATION_COLUMNS: [ColumnSpec; 24] = [ ColumnSpec::optional("command", ColumnType::Text), ColumnSpec::optional("subcommand", ColumnType::Text), ColumnSpec::optional("selection", ColumnType::Json), @@ -61,6 +61,9 @@ const INVOCATION_COLUMNS: [ColumnSpec; 21] = [ ColumnSpec::optional("argv_digest", ColumnType::Text), ColumnSpec::optional("origin", ColumnType::Text), ColumnSpec::optional("command_mode", ColumnType::Text), + ColumnSpec::optional("selection_count", ColumnType::Integer), + ColumnSpec::optional("selection_truncated", ColumnType::Boolean), + ColumnSpec::optional("selector_files", ColumnType::Json), ]; const OUTPUT_COLUMNS: [ColumnSpec; 11] = [ diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index d43869eb41..588e30a678 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -154,7 +154,7 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh .collect(), required_fragments: &[ "INSERT INTO `observability`.`recall_meta`", - "SELECT 'schema', 'schema_version', '4'", + "SELECT 'schema', 'schema_version', '5'", ], expected_count: 1, }, diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs index 570122df90..e20b285870 100644 --- a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -37,6 +37,7 @@ pub(crate) fn overlay(base: RecallSection, layer: RecallSection) -> RecallSectio .diagnostics_min_severity .or(base.diagnostics_min_severity), max_output_bytes: layer.max_output_bytes.or(base.max_output_bytes), + audit_evidence: layer.audit_evidence.or(base.audit_evidence), } } diff --git a/crates/sqlbuild-recall/src/settings/constants.rs b/crates/sqlbuild-recall/src/settings/constants.rs index 97a4ea7b67..f9776e2a04 100644 --- a/crates/sqlbuild-recall/src/settings/constants.rs +++ b/crates/sqlbuild-recall/src/settings/constants.rs @@ -19,3 +19,7 @@ pub const PROJECT_FILE_NAME: &str = "sqlbuild_project.toml"; pub const LOCAL_FILE_NAME: &str = "sqlbuild_local.toml"; /// Bytes of each output stream the local SQLite store keeps per invocation: the first and last MiB. pub const DEFAULT_SQLITE_MAX_OUTPUT_BYTES: u64 = 2 * 1024 * 1024; +/// The audit outcome of a warning-severity failure, as `audit_completed` events name it. +pub const AUDIT_WARN_OUTCOME: &str = "warn"; +/// The audit outcome of an error-severity failure, as `audit_completed` events name it. +pub const AUDIT_ERROR_OUTCOME: &str = "error"; diff --git a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs index 41852edf74..fb04f0b3d4 100644 --- a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs +++ b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs @@ -10,7 +10,7 @@ use crate::settings::constants::{ LOCAL_FILE_NAME, PROJECT_FILE_NAME, }; use crate::settings::errors::SettingsError; -use crate::settings::models::{RecallSettings, SettingsRequest}; +use crate::settings::models::{AuditEvidence, RecallSettings, SettingsRequest}; /// The effective settings: `targets..recall` over `[recall]` over the built-in local store, local configuration over project configuration at each level. Only the connection of the request's purpose is resolved, so a machine holding only the writer's or only the reader's credentials resolves its own side. pub fn resolve_recall_settings(request: &SettingsRequest) -> Result { @@ -73,5 +73,6 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result, pub destination_database: Option, pub destination_schema: Option, + /// Which audit outcomes keep their evidence rows as an artifact. + pub audit_evidence: AuditEvidence, +} + +/// Which audit outcomes keep their evidence rows in Recall, as `recall.audit_evidence` sets it; evidence holds data values from the audited relation. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AuditEvidence { + /// Warnings and errors: `warn-and-error`. + #[default] + WarnAndError, + /// Errors only: `error`. + Error, + /// None: `off`. + Off, +} + +impl AuditEvidence { + /// The mode `recall.audit_evidence` names; anything else is the default. + #[must_use] + pub fn from_setting(setting: Option<&str>) -> Self { + match setting { + Some("error") => Self::Error, + Some("off") => Self::Off, + _ => Self::WarnAndError, + } + } + + /// The `recall.audit_evidence` value naming this mode. + #[must_use] + pub fn setting(self) -> &'static str { + match self { + Self::WarnAndError => "warn-and-error", + Self::Error => "error", + Self::Off => "off", + } + } + + /// Whether an audit with `outcome` keeps its evidence. + #[must_use] + pub fn keeps(self, outcome: &str) -> bool { + match self { + Self::WarnAndError => [AUDIT_WARN_OUTCOME, AUDIT_ERROR_OUTCOME].contains(&outcome), + Self::Error => outcome == AUDIT_ERROR_OUTCOME, + Self::Off => false, + } + } } /// Which connection an invocation uses, so only that one is resolved and its environment references expanded. diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs index 9c20bf9474..9594516108 100644 --- a/crates/sqlbuild-recall/src/settings/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -3,7 +3,9 @@ use std::path::PathBuf; use sqlbuild_config::models::ConfigValue; use crate::settings::main::resolve_recall_settings::resolve_recall_settings; -use crate::settings::models::{RecallSettings, SettingsRequest, StoreLocation, StorePurpose}; +use crate::settings::models::{ + AuditEvidence, RecallSettings, SettingsRequest, StoreLocation, StorePurpose, +}; /// The placeholder project directory resolved paths are reported under. pub(crate) const PROJECT: &str = ""; @@ -74,6 +76,7 @@ pub(crate) fn defaults() -> RecallSettings { project_adapter: Some("duckdb".to_owned()), destination_database: None, destination_schema: None, + audit_evidence: AuditEvidence::WarnAndError, } } diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index fea7a4fcf9..b5cd7052c4 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -1,4 +1,4 @@ -use crate::settings::models::{RecallSettings, StoreLocation, StorePurpose}; +use crate::settings::models::{AuditEvidence, RecallSettings, StoreLocation, StorePurpose}; use crate::settings::tests::helpers::{ central_store, defaults, local_store, resolution_error, resolved, }; @@ -23,7 +23,7 @@ const LAYERED_PROJECT: &[(&str, &str)] = &[ [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ [targets.prod]\n[targets.prod.recall]\nwrite_connection = \"central\"\nread_connection = \"central\"\n\ [targets.capped]\n[targets.capped.recall]\nwrite_connection = \"central\"\nread_connection = \"central\"\n\ - max_output_bytes = 4096\n\ + max_output_bytes = 4096\naudit_evidence = \"error\"\n\ [targets.qa]\n[targets.qa.recall]\nwrite_connection = \"other\"\nread_connection = \"other\"\n\ [targets.laptop]\n[targets.laptop.recall]\nread_connection = \"central\"\n", ), @@ -136,7 +136,7 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d }, }, ResolveTestCase { - description: "a configured output limit applies to a ClickHouse store", + description: "a configured output limit and audit evidence mode apply to one target", files: LAYERED_PROJECT, target: Some("capped"), purpose: StorePurpose::Record, @@ -148,6 +148,7 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d retention_days: 10, shutdown_timeout_ms: 1000, max_output_bytes: Some(4096), + audit_evidence: AuditEvidence::Error, ..defaults() }, }, diff --git a/crates/sqlbuild-recall/src/store/_helpers/column_types.rs b/crates/sqlbuild-recall/src/store/_helpers/column_types.rs new file mode 100644 index 0000000000..55211da6b7 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/column_types.rs @@ -0,0 +1,51 @@ +//! The types `query_store` reports for a result's columns. + +use crate::records::models::RecallValue; + +/// Each column's name and declared type, lowercased; `None` for a computed column. +pub(crate) fn column_facts( + statement: &rusqlite::Statement<'_>, +) -> (Vec, Vec>) { + statement + .columns() + .iter() + .map(|column| { + ( + column.name().to_owned(), + column.decl_type().map(str::to_lowercase), + ) + }) + .unzip() +} + +/// Each column's declared type, or for a computed column the storage class its values share. +pub(crate) fn column_types( + declared: Vec>, + rows: &[Vec], +) -> Vec> { + declared + .into_iter() + .enumerate() + .map(|(index, declared)| declared.or_else(|| stored_type(rows, index))) + .collect() +} + +/// The storage class every non-NULL value of a computed column holds, as SQLite names it; `None` when they differ or all are NULL. +fn stored_type(rows: &[Vec], index: usize) -> Option { + let mut found: Option<&'static str> = None; + for value in rows.iter().filter_map(|row| row.get(index)) { + let class: &'static str = match value { + RecallValue::Null => continue, + RecallValue::Integer(_) => "integer", + RecallValue::Real(_) => "real", + RecallValue::Text(_) => "text", + RecallValue::Blob(_) => "blob", + }; + match found { + None => found = Some(class), + Some(previous) if previous == class => {} + Some(_) => return None, + } + } + found.map(str::to_owned) +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index c1f9469a7f..3344c9a183 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod column_types; pub(crate) mod identity; pub(crate) mod quarantine; pub(crate) mod rows; diff --git a/crates/sqlbuild-recall/src/store/main/query_store.rs b/crates/sqlbuild-recall/src/store/main/query_store.rs index 26461a3ac5..801e4bcb10 100644 --- a/crates/sqlbuild-recall/src/store/main/query_store.rs +++ b/crates/sqlbuild-recall/src/store/main/query_store.rs @@ -1,5 +1,6 @@ //! `query_store`: run one read-only query against a store. +use crate::store::_helpers::column_types::{column_facts, column_types}; use crate::store::_helpers::sqlite::{checked, from_sql}; use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::models::{QueryResult, SqliteStore}; @@ -22,15 +23,12 @@ pub fn query_store( "Recall queries must be read-only SELECT statements", )); } - let columns: Vec = statement - .column_names() - .into_iter() - .map(str::to_owned) - .collect(); + let (columns, declared): (Vec, Vec>) = column_facts(&statement); let width: usize = columns.len(); let mut rows = checked(statement.query([]), "run Recall query")?; let mut result = QueryResult { columns, + types: Vec::new(), rows: Vec::new(), truncated: false, }; @@ -50,5 +48,6 @@ pub fn query_store( } result.rows.push(values); } + result.types = column_types(declared, &result.rows); Ok(result) } diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index 3e3d6ec05d..a92d4ead30 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -91,6 +91,8 @@ pub struct MaintenanceOutcome { #[derive(Clone, Debug, Default, PartialEq)] pub struct QueryResult { pub columns: Vec, + /// Each column's type as the store names it, where known. + pub types: Vec>, pub rows: Vec>, /// More rows existed beyond the limit. pub truncated: bool, diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index dc9f69ab2f..65baa52389 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -344,7 +344,7 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { }; let mut next_heartbeat: SystemTime = SystemTime::now(); let mut first_heartbeat: bool = true; - let mut materializer = Materializer::new(config.envelope.clone()); + let mut materializer = Materializer::new(config.envelope.clone(), config.audit_evidence); let batching: Batching = batching(config); loop { let (batch, closing) = next_batch(shared, next_heartbeat, batching); @@ -354,9 +354,11 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { || !built.unbuildable.is_empty() || !built.filtered.is_empty() || built.extra_events > 0 + || built.extra_artifacts > 0 { let mut state = locked(&shared.state); state.counts_mut(DeliveryStream::Events).accepted += built.extra_events; + state.counts_mut(DeliveryStream::Artifacts).accepted += built.extra_artifacts; state.account(&built.settled, Ok(&InsertOutcome::default())); state.account_unbuildable(&built.unbuildable); state.account_filtered(&built.filtered); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index dd17ee103a..54c22db4f2 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -2,8 +2,12 @@ use std::collections::HashSet; +use serde_json::{Map, Value}; + use crate::records::constants::{ - ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, + ARTIFACT_DIGEST_PREFIX, AUDIT_COMPLETED_EVENT, AUDIT_EVIDENCE_KIND, AUDIT_EVIDENCE_MEDIA_TYPE, + AUDIT_EVIDENCE_ROLE, EVIDENCE_PAYLOAD_FIELD, EXECUTED_SQL_ROLE, SAMPLE_DIGEST_PAYLOAD_FIELD, + SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, }; use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::event_object_row::event_object_row; @@ -11,6 +15,7 @@ use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; use crate::records::main::sha256_hex::sha256_hex; use crate::records::models::{ArtifactInput, OutputChunk, OutputStream, RecallRow, RowEnvelope}; +use crate::settings::models::AuditEvidence; use crate::writer::_helpers::output_chunk::output_chunk; use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; use crate::writer::models::{DeliveryStream, ReplacedEvents, ReplacedOutput}; @@ -30,22 +35,54 @@ pub(crate) struct Built { pub(crate) filtered: Vec, /// Events a lifecycle replacement published beyond the one it was handed, accepted as they are built. pub(crate) extra_events: u64, + /// Audit evidence artifacts taken out of their events, accepted as they are built. + pub(crate) extra_artifacts: u64, } -/// Builds rows for one session; remembers which SQL it has stored. +impl Built { + /// Account for one event's row and the artifact rows of its evidence. + fn add_event( + &mut self, + stream: DeliveryStream, + (row, evidence_rows): ( + Result, + Option>, + ), + ) { + if let Some(rows) = evidence_rows { + self.rows.extend(rows); + self.written.push(DeliveryStream::Artifacts); + self.extra_artifacts += 1; + } + match row { + Ok(row) => { + self.rows.push(row); + self.written.push(stream); + } + Err(_) => self.unbuildable.push(stream), + } + } +} + +/// Builds rows for one session; remembers which SQL and references it has stored. #[derive(Debug)] pub(crate) struct Materializer { envelope: RowEnvelope, artifacts_seen: HashSet, + /// Digests and the references already written for them. + refs_seen: HashSet<(String, String)>, /// Output bytes stored per stream, for the offsets of replaced chunks. output_bytes: [u64; 2], + audit_evidence: AuditEvidence, } impl Materializer { - pub(crate) fn new(envelope: RowEnvelope) -> Self { + pub(crate) fn new(envelope: RowEnvelope, audit_evidence: AuditEvidence) -> Self { Self { + audit_evidence, envelope, artifacts_seen: HashSet::new(), + refs_seen: HashSet::new(), output_bytes: [0; 2], } } @@ -92,26 +129,95 @@ impl Materializer { Ok(stored) } + /// The content row the first time a digest is seen, and a reference row for each statement that ran it; `None` when both are already written. fn sql_rows(&mut self, input: SqlInput) -> Option> { - let digest = format!( - "{ARTIFACT_DIGEST_PREFIX}{}", - sha256_hex(input.sql.as_bytes()) + let artifact: ArtifactInput = artifact_input( + input.sql.into_bytes(), + (SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, EXECUTED_SQL_ROLE), + (input.statement_id, input.resource_id), + &format_timestamp(input.seen_at), ); - if !self.artifacts_seen.insert(digest.clone()) { + self.artifact_rows(artifact) + } + + /// `artifact`'s content row the first time its digest is seen and its reference row the first time it is named; `None` when both are written. + fn artifact_rows(&mut self, artifact: ArtifactInput) -> Option> { + if !self + .refs_seen + .insert((artifact.digest.clone(), artifact.ref_id.clone())) + { return None; } - let artifact = ArtifactInput { - ref_id: input.statement_id.clone().unwrap_or_else(|| digest.clone()), - digest, - kind: SQL_ARTIFACT_KIND.to_owned(), - media_type: SQL_MEDIA_TYPE.to_owned(), - content: input.sql.into_bytes(), - seen_at: format_timestamp(input.seen_at), - role: EXECUTED_SQL_ROLE.to_owned(), - statement_id: input.statement_id, - resource_id: input.resource_id, + let first_sighting: bool = self.artifacts_seen.insert(artifact.digest.clone()); + let [content, reference] = artifact_rows(&self.envelope, &artifact); + Some(if first_sighting { + vec![content, reference] + } else { + vec![reference] + }) + } + + /// The row of `event`, then the artifact rows of its evidence: an `audit_completed` event's evidence rows leave its payload and, when the outcome keeps them, become an artifact its `sample_digest` names. + fn event_rows( + &mut self, + event: Map, + ) -> ( + Result, + Option>, + ) { + let (event, artifact): (Map, Option) = + self.take_evidence(event); + let evidence_rows: Option> = + artifact.and_then(|artifact| self.artifact_rows(artifact)); + (event_object_row(&self.envelope, &event), evidence_rows) + } + + /// `event` without its evidence rows, and the artifact holding them when its outcome keeps them. + fn take_evidence( + &self, + mut event: Map, + ) -> (Map, Option) { + let evidence: Option = + if event.get("event_type").and_then(Value::as_str) == Some(AUDIT_COMPLETED_EVENT) { + event + .get_mut("payload") + .and_then(Value::as_object_mut) + .and_then(|payload| payload.remove(EVIDENCE_PAYLOAD_FIELD)) + } else { + None + }; + let text = |field: &str| -> Option { + event.get(field).and_then(Value::as_str).map(str::to_owned) + }; + let outcome: Option<&str> = event + .get("payload") + .and_then(|payload| payload.get("outcome")) + .and_then(Value::as_str); + let Some(evidence) = + evidence.filter(|_| self.audit_evidence.keeps(outcome.unwrap_or_default())) + else { + return (event, None); }; - Some(artifact_rows(&self.envelope, &artifact).to_vec()) + let mut artifact: ArtifactInput = artifact_input( + evidence.to_string().into_bytes(), + ( + AUDIT_EVIDENCE_KIND, + AUDIT_EVIDENCE_MEDIA_TYPE, + AUDIT_EVIDENCE_ROLE, + ), + (None, text("resource_id")), + &text("occurred_at").unwrap_or_default(), + ); + artifact.ref_id = text("event_id").unwrap_or_default(); + let digest: String = artifact + .digest + .strip_prefix(ARTIFACT_DIGEST_PREFIX) + .unwrap_or(&artifact.digest) + .to_owned(); + if let Some(payload) = event.get_mut("payload").and_then(Value::as_object_mut) { + payload.insert(SAMPLE_DIGEST_PAYLOAD_FIELD.to_owned(), Value::from(digest)); + } + (event, Some(artifact)) } /// The rows of `batch`, with each record's stream sorted by how it is accounted; lifecycle replacements see their events in publication order. @@ -139,13 +245,7 @@ impl Materializer { built.rows.extend(rows); built.written.push(stream); } - Payload::Event(event) => match event_object_row(&self.envelope, &event) { - Ok(row) => { - built.rows.push(row); - built.written.push(stream); - } - Err(_) => built.unbuildable.push(stream), - }, + Payload::Event(event) => built.add_event(stream, self.event_rows(event)), Payload::Pending(sequence, pending) => pending_events.push((sequence, pending)), Payload::Sql(input) => match self.sql_rows(input) { Some(rows) => { @@ -166,13 +266,7 @@ impl Materializer { ReplacedEvents::Published(events) => { built.extra_events += events.len() as u64 - 1; for event in events { - match event_object_row(&self.envelope, &event) { - Ok(row) => { - built.rows.push(row); - built.written.push(stream); - } - Err(_) => built.unbuildable.push(stream), - } + built.add_event(stream, self.event_rows(event)); } } ReplacedEvents::Failed => built.unbuildable.push(stream), @@ -181,3 +275,24 @@ impl Materializer { built } } + +/// An artifact of `content`, referenced by its statement or, without one, by its digest. +fn artifact_input( + content: Vec, + (kind, media_type, role): (&str, &str, &str), + (statement_id, resource_id): (Option, Option), + seen_at: &str, +) -> ArtifactInput { + let digest = format!("{ARTIFACT_DIGEST_PREFIX}{}", sha256_hex(&content)); + ArtifactInput { + ref_id: statement_id.clone().unwrap_or_else(|| digest.clone()), + digest, + kind: kind.to_owned(), + media_type: media_type.to_owned(), + content, + seen_at: seen_at.to_owned(), + role: role.to_owned(), + statement_id, + resource_id, + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index 240150540a..ee88de2643 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -146,8 +146,27 @@ impl RecallSession { Ok(()) } - /// Queue the SQL a statement executed, once per digest per invocation. + /// Queue the SQL a statement executed: its text once per digest per invocation, and a reference per statement. pub fn publish_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { + self.queue_sql(sql, (statement_id, resource_id), Priority::Artifact); + } + + /// Queue the SQL of a statement that failed or was cancelled again, above output and diagnostics, so a full queue or a spent shutdown budget keeps it; it is stored once if it already was. + pub fn publish_failed_sql( + &self, + sql: &str, + statement_id: Option<&str>, + resource_id: Option<&str>, + ) { + self.queue_sql(sql, (statement_id, resource_id), Priority::FailedTerminal); + } + + fn queue_sql( + &self, + sql: &str, + (statement_id, resource_id): (Option<&str>, Option<&str>), + priority: Priority, + ) { if sql.len() > MAX_ARTIFACT_BYTES { let mut state = locked(&self.shared.state); let counts = state.counts_mut(DeliveryStream::Artifacts); @@ -166,7 +185,7 @@ impl RecallSession { stream: DeliveryStream::Artifacts, }; let mut state = locked(&self.shared.state); - state.push_record(record, Priority::Artifact); + state.push_record(record, priority); self.shared.wake.notify_one(); } diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs index bd59ae3060..9492fc4928 100644 --- a/crates/sqlbuild-recall/src/writer/models.rs +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -6,6 +6,7 @@ use std::thread::JoinHandle; use std::time::Duration; use crate::records::models::{DeliveryCounts, InvocationContext, RowEnvelope}; +use crate::settings::models::AuditEvidence; use crate::store::types::RecallStoreOpener; use crate::writer::_helpers::delivery::Shared; @@ -115,6 +116,8 @@ pub struct SessionConfig { /// Hold the context row until `RecallSession::publish_context` says whether a replacement filters history. pub defer_context: bool, pub maintenance_owner: String, + /// Which audit outcomes keep their evidence rows as an artifact; the rest are left out of the stored event. + pub audit_evidence: AuditEvidence, } /// How table setup went. diff --git a/crates/sqlbuild-recall/src/writer/tests.rs b/crates/sqlbuild-recall/src/writer/tests.rs index 81c66254ff..119759d076 100644 --- a/crates/sqlbuild-recall/src/writer/tests.rs +++ b/crates/sqlbuild-recall/src/writer/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_artifacts.rs"] +mod artifacts; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_queue.rs"] diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 22b2ed1912..97a319ea81 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -7,6 +7,7 @@ use crate::records::models::{ DeliveryCounts, InvocationContext, OutputStream, RecallValue, RowEnvelope, }; use crate::schema::models::Table; +use crate::settings::models::AuditEvidence; use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::query_store::query_store; use crate::store::main::read_output_transcript::read_output_transcript; @@ -94,6 +95,7 @@ pub(crate) fn config(path: PathBuf, invocation_id: &str) -> SessionConfig { maintenance: true, maintenance_owner: "test:42".to_owned(), defer_context: false, + audit_evidence: AuditEvidence::WarnAndError, } } @@ -583,7 +585,7 @@ pub(crate) fn replacement_order(priorities: &[Priority]) -> Vec { ); } let batch = state.take_batch(usize::MAX); - drop(Materializer::new(envelope("ordered")).build(batch, None)); + drop(Materializer::new(envelope("ordered"), AuditEvidence::WarnAndError).build(batch, None)); std::mem::take(&mut *seen.lock().expect("the record is not poisoned")) } @@ -605,3 +607,110 @@ pub(crate) fn creation_notices( matches!(setup, Some(SetupOutcome::PermissionDenied { .. })), ) } + +/// One `audit_completed` event of `outcome` carrying one evidence row. +fn audit_event(index: usize, outcome: &str) -> serde_json::Map { + let event: serde_json::Value = serde_json::json!({ + "event_id": format!("audit-{index}"), + "event_type": "audit_completed", + "schema_version": 2, + "producer": "sqlbuild", + "producer_version": "1", + "occurred_at": "2026-10-19T00:00:01.000000Z", + "invocation_id": "evidence", + "invocation_sequence": index, + "resource_id": format!("audit:orders_{outcome}"), + "payload": { + "audit_name": format!("orders_{outcome}"), + "outcome": outcome, + "evidence_count": 1, + "evidence_truncated": false, + "evidence": [{"order_id": index}], + }, + }); + serde_json::from_value(event).expect("the audit event is an object") +} + +/// Per audit outcome: whether the stored event still holds its evidence, its `sample_digest`, and the artifact text that digest names, after a session keeping `audit_evidence`. +pub(crate) fn stored_evidence(audit_evidence: AuditEvidence) -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + let session = start_recall_session(SessionConfig { + audit_evidence, + ..config(path.clone(), "evidence") + }); + for (index, outcome) in ["warn", "error"].iter().enumerate() { + session.publish_event_object(audit_event(index, outcome)); + } + drop(session.close()); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + query_store( + &store, + "SELECT r.outcome, json_extract(e.payload, '$.evidence') IS NOT NULL, r.sample_digest, \ + r.evidence_count, a.kind, a.text \ + FROM audit_results AS r JOIN recall_events AS e USING (event_id) \ + LEFT JOIN artifacts AS a ON a.digest = 'sha256:' || r.sample_digest \ + ORDER BY r.outcome", + None, + ) + .expect("the query is valid") + .rows +} + +/// Content rows, reference rows and conflicts after two statements ran the same SQL and the second failed. +pub(crate) fn failed_statement_sql() -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "failed")); + session.publish_sql("select 1", Some("s1"), Some("orders")); + session.publish_sql("select 1", Some("s2"), Some("orders")); + session.publish_failed_sql("select 1", Some("s2"), Some("orders")); + session.publish_failed_sql("select 2", Some("s3"), None); + drop(session.close()); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + query_store( + &store, + "SELECT (SELECT COUNT(*) FROM recall_artifacts), \ + (SELECT COUNT(*) FROM recall_artifact_refs), (SELECT COUNT(*) FROM recall_conflicts), \ + (SELECT group_concat(statement_id, ',' ORDER BY statement_id) FROM artifacts)", + None, + ) + .expect("the query is valid") + .rows +} + +pub(crate) fn recall_text(value: &str) -> RecallValue { + RecallValue::Text(value.to_owned()) +} + +/// The digest of `[{"order_id":}]` without its prefix. +fn evidence_digest(index: u8) -> RecallValue { + let content: String = format!("[{{\"order_id\":{index}}}]"); + recall_text(&crate::records::main::sha256_hex::sha256_hex( + content.as_bytes(), + )) +} + +/// The [`stored_evidence`] row of an audit whose one evidence row was kept as an artifact. +pub(crate) fn kept_evidence(outcome: &str, index: u8) -> Vec { + vec![ + recall_text(outcome), + RecallValue::Integer(0), + evidence_digest(index), + RecallValue::Integer(1), + recall_text("audit_evidence"), + recall_text(&format!("[{{\"order_id\":{index}}}]")), + ] +} + +/// The [`stored_evidence`] row of an audit whose evidence was not kept. +pub(crate) fn left_out_evidence(outcome: &str) -> Vec { + vec![ + recall_text(outcome), + RecallValue::Integer(0), + RecallValue::Null, + RecallValue::Integer(1), + RecallValue::Null, + RecallValue::Null, + ] +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs new file mode 100644 index 0000000000..9b236a0322 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs @@ -0,0 +1,60 @@ +use crate::records::models::RecallValue; +use crate::settings::models::AuditEvidence; +use crate::writer::tests::helpers::{ + failed_statement_sql, kept_evidence, left_out_evidence, recall_text, stored_evidence, +}; +use crate::writer::tests::test_types::{AuditEvidenceTestCase, FailedStatementTestCase}; + +#[test] +fn given_audit_evidence_settings_when_audits_complete_then_kept_evidence_is_an_artifact_and_none_stays_in_the_event() + { + let test_cases = [ + AuditEvidenceTestCase { + description: "warn_and_error keeps both outcomes' rows", + audit_evidence: AuditEvidence::WarnAndError, + expected_rows: vec![kept_evidence("error", 1), kept_evidence("warn", 0)], + }, + AuditEvidenceTestCase { + description: "error keeps only the failing audit's rows", + audit_evidence: AuditEvidence::Error, + expected_rows: vec![kept_evidence("error", 1), left_out_evidence("warn")], + }, + AuditEvidenceTestCase { + description: "off keeps no rows", + audit_evidence: AuditEvidence::Off, + expected_rows: vec![left_out_evidence("error"), left_out_evidence("warn")], + }, + ]; + + for test_case in test_cases { + assert_eq!( + stored_evidence(test_case.audit_evidence), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_statements_sharing_sql_and_a_failed_one_when_recorded_then_text_is_stored_once_and_every_statement_is_referenced() + { + let test_cases = [FailedStatementTestCase { + description: "two texts, three references, no conflict", + expected_rows: vec![vec![ + RecallValue::Integer(2), + RecallValue::Integer(3), + RecallValue::Integer(0), + recall_text("s1,s2,s3"), + ]], + }]; + + for test_case in test_cases { + assert_eq!( + failed_statement_sql(), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 67ab3e837c..ad776499f5 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -3,6 +3,7 @@ use std::time::{Duration, Instant}; use crate::records::models::{OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope}; use crate::schema::models::Table; +use crate::settings::models::AuditEvidence; use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::models::InsertOutcome; use crate::store::types::{RecallStore, RecallStoreOpener}; @@ -349,3 +350,14 @@ impl RecallStoreOpener for CreatingStore { Duration::from_millis(10) } } + +pub(crate) struct AuditEvidenceTestCase { + pub(crate) description: &'static str, + pub(crate) audit_evidence: AuditEvidence, + pub(crate) expected_rows: Vec>, +} + +pub(crate) struct FailedStatementTestCase { + pub(crate) description: &'static str, + pub(crate) expected_rows: Vec>, +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index a0c52b771b..57ad766e7a 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -10,9 +10,10 @@ use crate::lint_text::_helpers::sites::{byte_offset, compiled, lint_policy, site use crate::lint_text::constants::{ GENERATED_FIX, GENERATED_SQL_SUFFIX, HARNESS_CTE_PREFIXES, INVALID_DIAGNOSTIC, INVALID_FIX_EDIT, INVALID_FIX_OBJECT, INVALID_FIX_REASON, LINT_API_VERSION, LONG_LITERAL_CODE, - MISSING_DIAGNOSTICS, NATIVE_LINT_ERROR_CODE, NON_CONTIGUOUS_FIX, NON_OBJECT_RESPONSE, - NOT_AN_OBJECT, PARSE_ERROR_POSITION_PATTERN, PARSE_FAILURE_PREFIX, PARSE_FAILURE_REMEDIATION, - UNSUPPORTED_VERSION, UNUSED_CTE_CODE, + MACRO_MIXED_QUALIFICATION_REMEDIATION, MISSING_DIAGNOSTICS, MIXED_QUALIFICATION_CODE, + NATIVE_LINT_ERROR_CODE, NON_CONTIGUOUS_FIX, NON_OBJECT_RESPONSE, NOT_AN_OBJECT, + PARSE_ERROR_POSITION_PATTERN, PARSE_FAILURE_PREFIX, PARSE_FAILURE_REMEDIATION, + SQL_MACRO_PARAMETER_SENTINEL_PREFIX, UNSUPPORTED_VERSION, UNUSED_CTE_CODE, }; use crate::lint_text::models::{AuthoredDiagnostic, AuthoredEdit, ExpansionSpan, LintSqlBody}; use crate::sql_lint::main::backtick_identifiers::backtick_identifiers; @@ -199,16 +200,70 @@ pub(crate) fn without_generated_frame( } /// Whether lint text offset `offset` lies in SQL a `@sql.` call generated in the authored file. -fn from_sql_macro(body: &LintSqlBody, (text, dialect): (&str, &str), offset: i64) -> bool { +fn from_sql_macro(body: &LintSqlBody, file: (&str, &str), offset: i64) -> bool { + sql_macro_call_at(body, file, offset).is_some() +} + +/// The `@sql.` call in the authored file that generated lint text offset `offset`, if one did. +fn sql_macro_call_at( + body: &LintSqlBody, + (text, dialect): (&str, &str), + offset: i64, +) -> Option { let (mapped, generated) = map_offset(offset, &body.passes); if !generated { - return false; + return None; } match usize::try_from(body.body_start + mapped) { Ok(start) => interpolation_text_at(text, start, dialect) - .is_some_and(|token| token.starts_with(SQL_CALL_PREFIX)), - Err(_) => false, + .filter(|token| token.starts_with(SQL_CALL_PREFIX)), + Err(_before_the_file) => None, + } +} + +/// A pass's input offset where that pass's output holds it: inside a replaced range, the start or end of its replacement. +fn forward_pass(offset: i64, spans: &[ExpansionSpan], end: bool) -> i64 { + let mut shift: i64 = 0; + for span in spans { + if offset < span.source_start || (end && offset == span.source_start) { + break; + } + if offset < span.source_end || (end && offset == span.source_end) { + return if end { + span.output_end + } else { + span.output_start + }; + } + shift += (span.output_end - span.output_start) - (span.source_end - span.source_start); } + offset + shift +} + +/// Character ranges of the lint text that SQL macro calls generated, each with the text of the call that generated it. +pub(crate) fn sql_macro_output_spans( + body: &LintSqlBody, + file: (&str, &str), +) -> Vec<(usize, usize, String)> { + let mut spans: Vec<(usize, usize, String)> = Vec::new(); + for (index, pass) in body.passes.iter().enumerate() { + for span in pass { + let (mut start, mut end): (i64, i64) = (span.output_start, span.output_end); + for later in &body.passes[index + 1..] { + start = forward_pass(start, later, false); + end = forward_pass(end, later, true); + } + if end <= start { + continue; + } + if let Some(call) = sql_macro_call_at(body, file, start) { + spans.push((start as usize, end as usize, call)); + } + } + } + spans.sort_unstable(); + spans.dedup(); + spans } /// Keep diagnostics starting inside a framed expression, whose frame lint generated. @@ -393,6 +448,13 @@ pub(crate) fn authored_diagnostic( } else { reason }; + let remediation: String = if code == MIXED_QUALIFICATION_CODE + && body.lint_text.contains(SQL_MACRO_PARAMETER_SENTINEL_PREFIX) + { + MACRO_MIXED_QUALIFICATION_REMEDIATION.to_owned() + } else { + remediation + }; let message: String = if generated { match file.token_at(absolute) { Some(token) => format!("{message} (in SQL generated by {token})"), diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index 32f38b4e83..7835d03b1d 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -7,9 +7,11 @@ use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitte use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::lint_text::_helpers::sql_diagnostics::without_generated_comments; +use crate::lint_text::_helpers::sql_diagnostics::{ + sql_macro_output_spans, without_generated_comments, +}; use crate::lint_text::constants::{ - CROSS_KEYWORD, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, + CROSS_KEYWORD, CTE_ARGUMENT, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, }; use crate::lint_text::models::{LintSqlBody, LintSqlOptions}; @@ -37,6 +39,63 @@ fn mentions_cross_join(text: &str) -> bool { .any(|window| window.eq_ignore_ascii_case(CROSS_KEYWORD)) } +/// The names a `@sql.` call passes as whole arguments, such as `orders` in `@sql.net_amount(orders)`: the relations a plain slot read in its output may qualify with. Slot mappings and the reserved `__cte` name pass none. +fn call_relations(call: &str) -> Vec { + let Some(open) = call.find('(') else { + return Vec::new(); + }; + let close: usize = call + .rfind(')') + .filter(|&close| close > open) + .unwrap_or(call.len()); + let mut names: Vec = Vec::new(); + for argument in top_level_arguments(&call[open + 1..close]) { + if let Some(name) = passed_name(argument) { + names.push(name.to_owned()); + } + } + names +} + +/// The bare name one call argument passes, if it passes one: the value of `name := value` or the whole argument. +fn passed_name(argument: &str) -> Option<&str> { + let value: &str = match argument.split_once(":=") { + Some((name, _)) if name.contains('.') || name.trim().eq_ignore_ascii_case(CTE_ARGUMENT) => { + return None; + } + Some((_, value)) => value, + None => argument, + }; + let name: &str = value.trim().trim_matches('"'); + let bare: bool = !name.is_empty() + && name + .chars() + .all(|character| character.is_alphanumeric() || character == '_'); + bare.then_some(name) +} + +/// `arguments` split at its top-level commas, outside brackets and quotes. +fn top_level_arguments(arguments: &str) -> Vec<&str> { + let mut parts: Vec<&str> = Vec::new(); + let (mut depth, mut quote, mut start): (usize, Option, usize) = (0, None, 0); + for (index, character) in arguments.char_indices() { + match (quote, character) { + (Some(open), _) if character == open => quote = None, + (Some(_), _) => {} + (None, '\'' | '"') => quote = Some(character), + (None, '(' | '[') => depth += 1, + (None, ')' | ']') => depth = depth.saturating_sub(1), + (None, ',') if depth == 0 => { + parts.push(&arguments[start..index]); + start = index + 1; + } + _ => {} + } + } + parts.push(&arguments[start..]); + parts +} + fn strings(values: &[String]) -> Value { Value::from(values.to_vec()) } @@ -48,6 +107,17 @@ pub(crate) fn lint_payload( options: &LintSqlOptions, ) -> Value { let lint_text: Cow<'_, str> = without_generated_comments(body, (file, &options.dialect)); + let inserted: usize = OMITTED_CEREMONIAL_SELECT_SQL.chars().count(); + let outputs: Vec = sql_macro_output_spans(body, (file, &options.dialect)) + .into_iter() + .map(|(start, end, call)| { + let (start, end): (usize, usize) = match omitted { + Some(offset) if start >= offset => (start + inserted, end + inserted), + _ => (start, end), + }; + serde_json::json!({"start": start, "end": end, "relations": call_relations(&call)}) + }) + .collect(); let sql: String = match omitted { Some(offset) => { let characters: Vec = lint_text.chars().collect(); @@ -61,6 +131,9 @@ pub(crate) fn lint_payload( let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); let _ = payload.insert("sql".to_string(), Value::from(sql)); let _ = payload.insert("dialect".to_string(), Value::from(options.dialect.clone())); + if !outputs.is_empty() { + let _ = payload.insert("sql_macro_outputs".to_string(), Value::from(outputs)); + } if let Some(enabled) = &options.enabled_rules { let _ = payload.insert("enabled_rules".to_string(), strings(enabled)); } diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 6bb802f943..f49086dad9 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -93,6 +93,11 @@ pub(crate) const GROUPED_NAMED_ROLES: [(&[&str], bool); 9] = [ pub(crate) const LINT_API_VERSION: i64 = 1; pub(crate) const UNUSED_CTE_CODE: &str = "SQBRSQL005"; pub(crate) const LONG_LITERAL_CODE: &str = "SQBRSQL044"; +pub(crate) const MIXED_QUALIFICATION_CODE: &str = "SQBRSQL028"; +/// The reserved argument naming a CTE-list call's CTEs; it passes no relation. +pub(crate) const CTE_ARGUMENT: &str = "__cte"; +/// The help for mixed qualification in a SQL macro body, where slot reads are qualified through an alias. +pub(crate) const MACRO_MIXED_QUALIFICATION_REMEDIATION: &str = "Use one consistent qualification style in this query scope. A plain slot read such as `@rel.column` counts as unqualified; to qualify it, read through an alias: `FROM @rel AS s`, then `s.@rel.column`."; pub(crate) const NATIVE_LINT_ERROR_CODE: &str = "L003"; pub(crate) const PARSE_ERROR_POSITION_PATTERN: &str = r"^Parse error at line (?P\d+), column (?P\d+):"; diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs index a23e97824f..f5649d280e 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs @@ -3,12 +3,13 @@ use std::collections::HashSet; use polyglot_sql::tokens::{Span, Token, TokenType}; use crate::constants::{EMPTY_FIXTURE_QUERY_TOKEN_COUNT, ENCLOSING_CTE_TOKEN_COUNT}; +use crate::lint_text::constants::SQL_MACRO_PARAMETER_SENTINEL_PREFIX; use crate::sql_lint::_helpers::engine::{ direct_indices, is_comment, is_layout, is_query_from, query_end, significant_after, significant_before, token_depths, }; use crate::sql_lint::constants::RELATION_MODIFIER_KEYWORDS; -use crate::sql_lint::models::{AdditionalFactOptions, AdditionalQueryFacts}; +use crate::sql_lint::models::{AdditionalFactOptions, AdditionalQueryFacts, SqlMacroOutput}; use rustc_hash::{FxHashMap, FxHashSet}; use sqlbuild_core::constants::{CLOSE_PAREN, OPEN_PAREN, SQL_WILDCARD}; @@ -35,6 +36,7 @@ struct ReferenceFactContext<'a> { query: &'a QuerySlice<'a>, outer_relations: &'a FxHashSet, external_identifiers: &'a HashSet, + sql_macro_outputs: &'a [SqlMacroOutput], query_start: usize, query_end: usize, } @@ -50,6 +52,7 @@ struct SelectFactContext<'a> { tokens: &'a [Token], depths: &'a [usize], external_identifiers: &'a HashSet, + sql_macro_outputs: &'a [SqlMacroOutput], dependency_identifiers: &'a HashSet, allows_ceremonial_select: bool, allows_dynamic_output_star: bool, @@ -171,6 +174,7 @@ pub(super) fn collect_additional_facts( tokens, depths: &depths, external_identifiers: options.external_identifiers, + sql_macro_outputs: options.sql_macro_outputs, dependency_identifiers: options.dependency_identifiers, allows_ceremonial_select: options.allows_ceremonial_select, allows_dynamic_output_star: options.allows_dynamic_output_star, @@ -338,6 +342,7 @@ fn collect_select_additional_facts(context: &SelectFactContext<'_>) -> Additiona query: &reference_slice, outer_relations: &outer_relations, external_identifiers: context.external_identifiers, + sql_macro_outputs: context.sql_macro_outputs, query_start: select_index, query_end: end, }; @@ -663,10 +668,14 @@ fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQuer facts .unqualified_multi_source_columns .extend(unqualified.iter().map(|&index| tokens[index].span)); - } else if !qualified.is_empty() && !unqualified.is_empty() { + } else { facts .inconsistent_single_source_qualification - .extend(unqualified.iter().map(|&index| tokens[index].span)); + .extend(inconsistent_qualification( + tokens, + (&qualified, &unqualified), + context.sql_macro_outputs, + )); } facts.unknown_relation_qualifiers.extend( qualified @@ -685,6 +694,68 @@ fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQuer facts } +/// The unqualified references of a single-relation scope that also qualifies some, each judged as it was written. In a macro file a plain slot read such as `@orders.amount` lints as a parameter sentinel pair and counts as unqualified. In a call's output, a read qualified by a relation the call passes takes neither style: the slot decides how it expands, not the caller or the macro author. A scope whose every reference a macro call generated is judged once, in the macro file. +fn inconsistent_qualification( + tokens: &[Token], + (qualified, unqualified): (&[usize], &[usize]), + outputs: &[SqlMacroOutput], +) -> Vec { + let output_of = |index: usize| -> Option { + let start: usize = tokens[index].span.start; + outputs + .iter() + .position(|output| output.start <= start && start < output.end) + }; + let mut references = qualified.iter().chain(unqualified).copied(); + if let Some(first) = references.next() + && let Some(output) = output_of(first) + && references.all(|index| output_of(index) == Some(output)) + { + return Vec::new(); + } + let passed_relation_read = |index: &usize| { + output_of(*index).is_some_and(|output| { + let qualifier: String = unquoted_name(&tokens[*index].text); + outputs[output] + .relations + .iter() + .any(|relation| relation.eq_ignore_ascii_case(&qualifier)) + }) + }; + let (plain_reads, qualified): (Vec, Vec) = qualified + .iter() + .copied() + .filter(|index| !passed_relation_read(index)) + .partition(|&index| is_plain_slot_read(tokens, index)); + if qualified.is_empty() || (unqualified.is_empty() && plain_reads.is_empty()) { + return Vec::new(); + } + let mut unqualified: Vec = unqualified.to_vec(); + unqualified.extend(plain_reads); + unqualified.sort_unstable(); + unqualified + .iter() + .map(|&index| tokens[index].span) + .collect() +} + +/// An identifier's name without its quotes. +fn unquoted_name(text: &str) -> String { + text.trim_matches(|character| matches!(character, '"' | '`' | '[' | ']')) + .to_owned() +} + +/// Whether the qualified reference at `index` is a plain SQL macro slot read, both its relation and its slot a macro parameter. +fn is_plain_slot_read(tokens: &[Token], index: usize) -> bool { + let is_parameter = |token: &Token| { + token + .text + .to_ascii_lowercase() + .starts_with(SQL_MACRO_PARAMETER_SENTINEL_PREFIX) + }; + is_parameter(&tokens[index]) && tokens.get(index + 2).is_some_and(is_parameter) +} + fn simple_qualifier( tokens: &[Token], references: &[usize], diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs index 34c1ce5695..18a586fc5f 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs @@ -12,7 +12,7 @@ use polyglot_sql::{ use crate::sql_lint::constants::LINT_API_VERSION; use crate::sql_lint::models::{ AdditionalFactOptions, CrossJoinRequest, LintDiagnostic, LintEdit, LintRequest, LintResponse, - LintRuleMetadata, QueryFacts, + LintRuleMetadata, QueryFacts, SqlMacroOutput, }; use crate::sql_lint::_helpers::additional::collect_additional_facts; @@ -337,6 +337,7 @@ struct DiagnosticContext<'a> { struct FactBuildOptions<'a> { external_identifiers: &'a HashSet, + sql_macro_outputs: &'a [SqlMacroOutput], dependency_identifiers: &'a HashSet, externally_referenced_ctes: &'a HashSet, allows_ceremonial_select: bool, @@ -406,6 +407,7 @@ pub(crate) fn lint(request: LintRequest) -> Result { .collect(); let options = FactBuildOptions { external_identifiers: &external_identifiers, + sql_macro_outputs: &request.sql_macro_outputs, dependency_identifiers: &dependency_identifiers, externally_referenced_ctes: &externally_referenced_ctes, allows_ceremonial_select: request.allows_ceremonial_select, @@ -516,6 +518,7 @@ fn build_facts( tokens, &AdditionalFactOptions { external_identifiers: options.external_identifiers, + sql_macro_outputs: options.sql_macro_outputs, dependency_identifiers: options.dependency_identifiers, allows_ceremonial_select: options.allows_ceremonial_select, allows_dynamic_output_star: options.allows_dynamic_output_star, diff --git a/crates/sqlbuild-rules/src/sql_lint/models.rs b/crates/sqlbuild-rules/src/sql_lint/models.rs index ea4fbc58b2..59ea49ceee 100644 --- a/crates/sqlbuild-rules/src/sql_lint/models.rs +++ b/crates/sqlbuild-rules/src/sql_lint/models.rs @@ -83,12 +83,25 @@ pub(crate) struct LintRequest { pub relation_keys: HashMap>>, #[serde(default)] pub relation_names: HashMap, + /// The SQL each `@sql.` call in the query generated, so column qualification is judged as the macro wrote it. + #[serde(default)] + pub sql_macro_outputs: Vec, +} + +/// The character range one `@sql.` call's output fills, and the names its arguments pass, which a plain slot read such as `@rel.amount` expands to qualify with. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub(crate) struct SqlMacroOutput { + pub start: usize, + pub end: usize, + #[serde(default)] + pub relations: Vec, } #[derive(Debug)] pub(crate) struct AdditionalFactOptions<'a> { pub external_identifiers: &'a HashSet, pub dependency_identifiers: &'a HashSet, + pub sql_macro_outputs: &'a [SqlMacroOutput], pub allows_ceremonial_select: bool, pub allows_dynamic_output_star: bool, pub allows_empty_fixture_star: bool, diff --git a/crates/sqlbuild-rules/src/sql_lint/tests.rs b/crates/sqlbuild-rules/src/sql_lint/tests.rs index d1ff27056d..dde04c7550 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests.rs @@ -20,6 +20,8 @@ mod idempotency_reprint; mod layout_rules; #[path = "tests/test_line_wrap.rs"] mod line_wrap; +#[path = "tests/test_lint_exemptions.rs"] +mod lint_exemptions; #[path = "tests/test_set_operation_formatter.rs"] mod set_operation_formatter; #[path = "tests/test_sql_lint.rs"] diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs new file mode 100644 index 0000000000..46f0ca78e0 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs @@ -0,0 +1,130 @@ +use serde_json::{Value, json}; + +use crate::sql_lint::main::engine::lint_json; +use crate::sql_lint::tests::test_types::{ + DynamicOutputStarLintTestCase, SqlMacroOutputLintTestCase, +}; + +#[test] +fn given_sql_macro_output_when_linting_qualification_then_each_column_counts_as_written() +-> Result<(), String> { + let test_cases = [ + SqlMacroOutputLintTestCase { + description: "a bare column from an expression macro beside a qualified column fails", + sql: "SELECT o.order_id, status AS s FROM orders AS o", + output: "status", + relations: &[], + expected_count: 1, + }, + SqlMacroOutputLintTestCase { + description: "reads through a passed relation pass beside a bare caller column", + sql: "SELECT order_id, o.amount - o.discount AS net FROM orders AS o", + output: "o.amount - o.discount", + relations: &["o"], + expected_count: 0, + }, + SqlMacroOutputLintTestCase { + description: "reads through a passed relation pass beside a hand-qualified column", + sql: "SELECT o.order_id, o.amount - o.discount AS net FROM orders AS o", + output: "o.amount - o.discount", + relations: &["o"], + expected_count: 0, + }, + SqlMacroOutputLintTestCase { + description: "the caller's own mix beside reads through a passed relation fails", + sql: "SELECT o.order_id, status, o.amount AS a FROM orders AS o", + output: "o.amount", + relations: &["o"], + expected_count: 1, + }, + SqlMacroOutputLintTestCase { + description: "an aliased read counts as qualified beside a bare caller column", + sql: "SELECT order_id, s.amount AS a FROM orders AS s", + output: "s.amount", + relations: &["orders"], + expected_count: 1, + }, + SqlMacroOutputLintTestCase { + description: "a query scope the macro generated whole is judged in the macro file", + sql: "WITH c AS (SELECT a, w.b FROM w) SELECT * FROM c", + output: "SELECT a, w.b FROM w", + relations: &[], + expected_count: 0, + }, + ]; + for test_case in test_cases { + let start: usize = test_case + .sql + .find(test_case.output) + .ok_or("the output should be in the SQL")?; + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": ["SQBRSQL028"], + "sql_macro_outputs": [{ + "start": start, + "end": start + test_case.output.len(), + "relations": test_case.relations, + }], + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or("diagnostics should be an array")?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_compiler_proven_dynamic_output_when_linting_then_only_terminal_star_is_exempt() +-> Result<(), String> { + let test_cases = [DynamicOutputStarLintTestCase { + description: "unrelated CTE star remains controlled", + sql: "WITH pivot_input AS (SELECT customer_id, category, amount FROM orders), \ +final AS (SELECT * FROM pivot_input PIVOT(MAX(amount) FOR category IN (ANY))), \ +unrelated AS (SELECT * FROM pivot_input) SELECT * FROM final", + expected_count: 1, + expected_prefix: "unrelated AS (SELECT ", + }]; + for test_case in test_cases { + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "snowflake", + "enabled_rules": ["SQBRSQL021"], + "allows_dynamic_output_star": true, + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or("diagnostics should be an array")?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + let start = diagnostics[0]["start"].as_u64().unwrap_or_default() as usize; + assert!( + test_case.sql[..start].ends_with(test_case.expected_prefix), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs index 141ff9c8b3..a71418df82 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs @@ -1114,6 +1114,20 @@ fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> R expected_anchor: Some("name"), expected_replacement: None, }, + test_types::AdditionalLintRuleTestCase { + description: "plain macro slot reads count as unqualified beside a plain group read", + sql: "SELECT __sqlbuild_macro_parameter_1__, __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_2__ AS orders FROM __sqlbuild_macro_parameter_0__", + rule: "SQBRSQL028", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "a plain macro slot read beside a hand-qualified column is mixed", + sql: "SELECT __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_1__ AS orders, s.refunds FROM __sqlbuild_macro_parameter_0__ AS s", + rule: "SQBRSQL028", + expected_anchor: Some("__sqlbuild_macro_parameter_0__"), + expected_replacement: None, + }, test_types::AdditionalLintRuleTestCase { description: "unknown relation qualifier", sql: "SELECT missing.id FROM items AS present", @@ -1582,49 +1596,6 @@ fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> R Ok(()) } -#[test] -fn given_compiler_proven_dynamic_output_when_linting_then_only_terminal_star_is_exempt() --> Result<(), String> { - let test_cases = [test_types::DynamicOutputStarLintTestCase { - description: "unrelated CTE star remains controlled", - sql: "WITH pivot_input AS (SELECT customer_id, category, amount FROM orders), \ -final AS (SELECT * FROM pivot_input PIVOT(MAX(amount) FOR category IN (ANY))), \ -unrelated AS (SELECT * FROM pivot_input) SELECT * FROM final", - expected_count: 1, - expected_prefix: "unrelated AS (SELECT ", - }]; - for test_case in test_cases { - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "snowflake", - "enabled_rules": ["SQBRSQL021"], - "allows_dynamic_output_star": true, - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or("diagnostics should be an array")?; - - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - let start = diagnostics[0]["start"].as_u64().unwrap_or_default() as usize; - assert!( - test_case.sql[..start].ends_with(test_case.expected_prefix), - "{}", - test_case.description - ); - } - Ok(()) -} - #[test] fn given_ceremonial_cte_contexts_when_linting_then_only_harness_dependencies_are_reachable() -> Result<(), String> { diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs index 30f7edda4a..6a3348bde3 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs @@ -55,6 +55,15 @@ pub(crate) struct PlainSqlLintTestCase { pub expected_count: usize, } +pub(crate) struct SqlMacroOutputLintTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + /// The text a `@sql.` call generated, and the names the call passed. + pub(crate) output: &'static str, + pub(crate) relations: &'static [&'static str], + pub(crate) expected_count: usize, +} + pub(crate) struct DynamicOutputStarLintTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, diff --git a/fensu.toml b/fensu.toml index b554e3acbe..d64899dc9b 100644 --- a/fensu.toml +++ b/fensu.toml @@ -797,6 +797,16 @@ modules = ["sqlbuild.runtime.execution_limits.main.execution_deadline_scope"] symbols = ["execution_deadline_scope"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.auditing.main._audit_evidence_scope"] +symbols = ["audit_evidence_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.main._cancel_running_statements"] +symbols = ["cancel_running_statements"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.runtime.execution_limits.models"] symbols = ["RunItemTimeouts"] @@ -812,6 +822,11 @@ modules = ["sqlbuild.runtime.observability.classes.operation_lifecycle"] symbols = ["OperationLifecycle"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.statement_lifecycle"] +symbols = ["StatementLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.runtime.observability.exceptions"] symbols = ["ObservabilityValidationError"] @@ -1644,7 +1659,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1680,12 +1695,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", - "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", @@ -1779,7 +1795,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=22", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs=2", @@ -2650,6 +2666,7 @@ module_layers = [ "project_assembly=compile", "python_values=infra", "query_command=commands", + "query_command/text_output=output", "recall=output", "recall/command=commands", "record_behaviours=registry", diff --git a/pyproject.toml b/pyproject.toml index f9f3c7a72c..34baca77ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.27.0", + "fensu>=0.28.0", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index db4f9017dc..960d4d018d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -94,10 +94,12 @@ Check that target's `read_connection` in `[recall]` or `[targets..recall]` ClickHouse SQL, and `--json` reports `rows_read` and `bytes_read`; filter on `invocation_id` or `invocation_started_at` and keep a `LIMIT` on large stores. -The default `long` format is fine in most cases: read it as it is. If it is too verbose, select -fewer columns or add a `LIMIT` rather than changing the format or reshaping the output. Use -`--json` only when a program does something meaningful with the result, such as feeding it into -further processing, not just to print rows more compactly. +Keep the default output format for anything a person reads: it prints a table when the result fits +and one record per row otherwise, so nothing is cut. If a result is too long, reshape the question, +not the output: aggregate with `GROUP BY`, select only the columns you need, or add a `LIMIT`. A +long listing usually means the query is not precise enough yet. Never switch format to shorten +output or save context; `--json` is for programs that process the result. If you change the +format, say why. Every command except `sqb recall` is recorded in the store of the target it runs against, including `sqb debug` and failed commands. To check a shared target, prefer `sqb recall` reads: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 2a7c01186c..1a4af4ad95 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -82,7 +82,9 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb 4. **Column lineage.** Traces column-level dependencies across models. 5. **Contracts.** Checks declared columns against the inferred query output. 6. **Rules.** Runs the selected built-in Rules, then the selected custom Rules. -7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. +7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. When they fail, it + writes only the models' SQL, so you can read what the Rules checked; see + [When Rules fail](#when-rules-fail). Rules that need inferred types, such as `SQBRCONTRACT105`, are skipped when SQL analysis is off, and compile prints one note naming them. @@ -91,6 +93,23 @@ and compile prints one note naming them. findings block the artifacts. To check one Rule code or family without changing project configuration, use [`sqb rules run`](rules.md). +### When Rules fail + +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a +`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each +model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros +generated, and the compiled file of a model you removed from the project is deleted. Other +compiled files stay as the last passing compile left them. A model with Rule errors starts with a +header that marks it not buildable and lists its findings: + +```sql +-- sqb: compile failed for this model; not buildable +-- SQBRSQL028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns +SELECT o.order_id, status FROM __ref("orders") AS o +``` + +The next compile that passes writes the files again without the header. + ## Compiling a selection Selectors limit analysis and reporting. Discovery still covers the whole project, so every diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md index c2dbb5f2fa..994aa9bbff 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md @@ -10,7 +10,7 @@ Online: https://sqlbuild.com/docs/cli/query/ - Usage - Flags -- Output formats +- Output - Errors - Examples @@ -38,68 +38,70 @@ File paths are resolved from the current working directory. |------|-------------| | `sql` | SQL to execute (positional argument) | | `--file` | Read SQL from a file instead of the command line | -| `--format` | Output format: `long` (default), `table`, `json`, or `csv` | +| `--format` | `long` prints one record per row; `json` prints a JSON array. Without it, the [default output](#output) applies | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 20) | | `--no-limit` | Disable the row limit | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | -## Output formats +## Output -### long (default) +By default `sqb query` prints a table with each column's name and warehouse type: -Vertical record format, one field per line: +``` +┌──────────┬─────────────┬───────────┐ +│ order_id │ customer_id │ status │ +│ INTEGER │ INTEGER │ VARCHAR │ +├──────────┼─────────────┼───────────┤ +│ 1 │ 100 │ completed │ +│ 2 │ 200 │ completed │ +└──────────┴─────────────┴───────────┘ +2 rows ``` --[ RECORD 1 ]---------------------------+ -order_id | 1 -customer_id | 100 -status | completed --[ RECORD 2 ]---------------------------+ -order_id | 2 -customer_id | 200 -status | completed +The table must fit the terminal width, or 120 characters when output is redirected. When it is +wider, or a value spans several lines, `sqb query` prints one record per row instead, so no column +or value is ever cut. Numbers are right-aligned. Types show where the adapter reports them. -2 rows +The last line counts the rows. When the limit cut rows off, it says so and how to see more: + +``` +20 rows shown (limit 20; use --limit or --no-limit) ``` -### table +Above 50 rows it suggests reading fewer: `60 rows; to read fewer, aggregate with GROUP BY or add a +LIMIT`. -Horizontal table format: +### long + +`--format long` always prints one record per row: ``` -order_id | customer_id | status --------- | ----------- | --------- -1 | 100 | completed -2 | 200 | completed +-[ RECORD 1 ]---------------------------+ +order_id | 1 +customer_id | 100 +status | completed 2 rows ``` ### json -JSON array of objects: +`--format json`, or `--json`, prints a JSON array of objects for programs: ```json [{"order_id": 1, "customer_id": 100, "status": "completed"}] ``` -### csv - -Standard CSV with headers: - -```csv -order_id,customer_id,status -1,100,completed -2,200,completed -``` +`--format table` and `--format csv` are not accepted. The default output already prints a table +when the result fits; use `--json` for programs. ## Errors When the warehouse rejects the SQL, `sqb query` prints the warehouse message as `error[C108]: ...` on stderr and exits with code `2`. Nothing is written to stdout, so -redirected `json` or `csv` output is never partial. Invalid command input, such as a missing +redirected `json` output is never partial. Invalid command input, such as a missing query file, exits with code `1`. ## Examples @@ -108,8 +110,8 @@ query file, exits with code `1`. # Quick inspection with default format sqb query "SELECT * FROM dev.fact_orders" -# Table format with higher limit -sqb query "SELECT * FROM dev.dim_customers" --format table --limit 50 +# One record per row with a higher limit +sqb query "SELECT * FROM dev.dim_customers" --format long --limit 50 # Export to JSON sqb query "SELECT * FROM dev.daily_revenue" --json --no-limit diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md index dfb94e424f..ee013c108d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -35,7 +35,7 @@ Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| | `sql` | The query (positional argument) | -| `--format` | `long` (default) for reading, `json` for programs, or `csv` for export. `table` still works in this release but prints a deprecation warning and will be removed | +| `--format` | `long` prints one record per row; `json` is for programs. Without it, a table when the result fits, as [`sqb query`](query.md#output) prints it | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 100) | | `--no-limit` | Disable the row limit | @@ -52,15 +52,21 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -command | build -status | completed -exit_code | 0 -duration_ms | 752.0 +┌─────────┬───────────┬───────────┬─────────────┐ +│ command │ status │ exit_code │ duration_ms │ +│ text │ text │ integer │ real │ +├─────────┼───────────┼───────────┼─────────────┤ +│ build │ completed │ 0 │ 752.0 │ +└─────────┴───────────┴───────────┴─────────────┘ 1 row ``` +The second header line is each column's type in the store. A result wider than the terminal (120 +characters when output is redirected), or with a value spanning lines, prints one record per row +instead, so nothing is cut. `--format table` and `--format csv` are not accepted: the default +output already prints a table when it fits, and `--json` is for programs. + The slowest statements of past builds, with their SQL: ```bash @@ -71,15 +77,13 @@ sqb recall query --max-cell-bytes 60 \ ``` ``` --[ RECORD 1 ]---------------------------+ -resource_name | orders -ms | 3.6 -sql | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] - --[ RECORD 2 ]---------------------------+ -resource_name | NULL -ms | 3.2 -sql | CREATE SCHEMA IF NOT EXISTS dev +┌───────────────┬──────┬──────────────────────────────────────────────────────────────────────────┐ +│ resource_name │ ms │ sql │ +│ text │ real │ text │ +├───────────────┼──────┼──────────────────────────────────────────────────────────────────────────┤ +│ orders │ 5.1 │ CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 AS o…[+81 bytes] │ +│ NULL │ 3.2 │ CREATE SCHEMA IF NOT EXISTS dev │ +└───────────────┴──────┴──────────────────────────────────────────────────────────────────────────┘ 2 rows ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index 9fe8b1386d..a1495fdc8e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/concepts/audits/ - How audits work - Measurement audits +- Evidence - Audit factories - Audit metadata - Result history @@ -81,8 +82,19 @@ MODEL ( ``` `minimum_samples` keeps low-volume measurements distinct as `insufficient` rather than inventing a -pass or failure. Evidence is diagnostic and bounded by `evidence_limit`; the measurement and -threshold determine the outcome. +pass or failure. The measurement and threshold determine the outcome. + +## Evidence + +When an audit warns or fails, SQLBuild keeps some of the rows behind the failure as evidence: a +violation audit keeps the rows its query returned, and a measurement audit runs its `EVIDENCE` +query. `evidence_limit` caps the rows kept (default 20), and at most 64 KB of them are kept. Set it +on any attachment, for example `not_null (evidence_limit 5)`. + +Evidence is diagnostic and never changes the outcome. It holds data values from the audited +relation, and is recorded in the `evidence_json` column of `_sqlbuild_audit_results`, as +`evidence` in the `audit_completed` lifecycle event, and in +[Recall](observability/recall.md#audit-evidence), which `audit_evidence` controls. ## Audit factories diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index fa1c77ee06..28bf4feea9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -316,6 +316,16 @@ fully typed. - Every read inside `USING (...)` stands unqualified, as `USING` needs, including `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. +- Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and + unqualified columns, judges a query the macro writes whole, such as a CTE of a CTE-list macro, + once, in the macro file. There every plain read counts as unqualified and every + `alias.@weekly...` read as qualified. A step written only in plain reads, such as + `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, passes; to qualify, read + every slot through an alias. +- Where a call's output joins the caller's query, as with expression and column-list macros, the + query is judged at the call. A read through the relation the call passes, such as `o.created_at` + above, fits either style; any other column counts as the macro writes it, so a bare `status` + beside a hand-qualified `o.order_id` fails. A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, could bind its bare columns to the subquery's relations instead of the one the slot reads. The @@ -504,8 +514,10 @@ A `* REPLACE` item takes its source column's type rather than the call's. - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. - [Rules](../rules.md) and [column lineage](../column-lineage.md) read the expanded - SQL, so they see through every call. A rule finding inside an expansion is reported at its - position in the expanded query, not in the macro file. + SQL, so they see through every call. A Rule finding inside an expansion is reported at the call, + marked `(in SQL generated by @sql.name(...))`. The model's expanded SQL is in + `target/compiled/models/`, also when the Rules fail; see + [When Rules fail](../../cli/compile.md#when-rules-fail). - Editing a macro recompiles its callers. Test SQL macros with [macro-mode unit tests](../testing.md#macro-tests), and replace one diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index d4ee10a7ed..4c7cd9a531 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -10,6 +10,9 @@ Online: https://sqlbuild.com/docs/concepts/observability/recall/ - An investigation, start to finish - What is recorded +- How a run ended +- Selection +- Audit evidence - Git revision - Where a run came from - Settings @@ -41,32 +44,30 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -run | 2026-10-20T00:01:44.784683Z -outcome | warn -violation_count | 1 -status | completed -exit_code | 0 - --[ RECORD 2 ]---------------------------+ -run | 2026-10-20T00:00:34.652468Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 - --[ RECORD 3 ]---------------------------+ -run | 2026-10-19T23:59:23.827831Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 +┌─────────────────────────────┬─────────┬─────────────────┬───────────┬───────────┐ +│ run │ outcome │ violation_count │ status │ exit_code │ +│ text │ text │ integer │ text │ integer │ +├─────────────────────────────┼─────────┼─────────────────┼───────────┼───────────┤ +│ 2026-10-20T00:01:44.784683Z │ warn │ 1 │ completed │ 0 │ +│ 2026-10-20T00:00:34.652468Z │ pass │ 0 │ completed │ 0 │ +│ 2026-10-19T23:59:23.827831Z │ pass │ 0 │ completed │ 0 │ +└─────────────────────────────┴─────────┴─────────────────┴───────────┴───────────┘ 3 rows ``` The audit started warning on the latest build, and that build still exited 0, so nothing alerted. +See the rows the audit flagged, which Recall keeps as the audit's evidence: + +```bash +sqb recall query \ + "SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + WHERE a.audit_name = 'revenue_is_non_negative' AND a.outcome <> 'pass' + ORDER BY a.invocation_started_at DESC LIMIT 1" +``` + See what that build printed, exactly as the scheduler saw it: ```bash @@ -88,11 +89,12 @@ sqb recall query --backend warehouse \ ``` ``` --[ RECORD 1 ]---------------------------+ -payment_id | 9 -order_id | 8 -amount_cents | -2200 -paid_at | 2026-04-03 15:42:00 +┌────────────┬──────────┬──────────────┬─────────────────────┐ +│ payment_id │ order_id │ amount_cents │ paid_at │ +│ INTEGER │ INTEGER │ INTEGER │ TIMESTAMP │ +├────────────┼──────────┼──────────────┼─────────────────────┤ +│ 9 │ 8 │ -2200 │ 2026-04-03 15:42:00 │ +└────────────┴──────────┴──────────────┴─────────────────────┘ 1 row ``` @@ -109,21 +111,25 @@ For every recorded invocation, Recall keeps: for a failed invocation its cause as `error_type`, `error_code` and a one-line `error_message`, also when the command reported the failure only through its exit code (for example `sqb debug` failing to connect records `ConnectionError` and `C980`); +- the selectors it ran with, see [Selection](#selection); +- whether it finished attributing warehouse cost, as `cost_status`: `complete`, `partial`, + `pending`, `unavailable` or `collection_failed`, or `NULL` for an adapter without cost + attribution; - the commit it ran from (`git_revision`), see [Git revision](#git-revision); - where it came from (`origin`) and, for an orchestrated run, the job and run that started it, see [Where a run came from](#where-a-run-came-from); - its [lifecycle events](../observability.md): runs, resource attempts, statements, retries, and audit results; -- the SQL text of every executed statement, stored once per distinct statement and shared across - invocations; +- the SQL text of every executed statement, including statements that failed or were cancelled, + stored once per distinct statement and shared across invocations; +- the rows a failing audit flagged, see [Audit evidence](#audit-evidence); - its stdout and stderr, byte for byte, in the order they were written, up to [`max_output_bytes`](#output-limit) of each stream; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all of its history still succeeds, prints a warning followed by its own final status, and -`output_invocations` reports the gap. An invocation whose process died without finishing has -status `abandoned`. +`output_invocations` reports the gap. `sqb recall` only reads history: it publishes no lifecycle events, runs no project sinks, and keeps no compute logs, so a query starts quickly. Every command is recorded except `sqb recall` itself, commands that create a project (such as @@ -132,7 +138,59 @@ recorded either: a `sqb compile` answered from the launcher's warm cache, and `s `sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store -can contain sensitive SQL and output. +can contain sensitive SQL, output and audit evidence. + +## How a run ended + +`invocations.status` is one of: + +| Status | Meaning | +|--------|---------| +| `completed` | The command exited 0 | +| `failed` | The command exited with an error | +| `cancelled` | The command was stopped by SIGTERM or Ctrl-C; `signal_number` is `15` or `2` | +| `running` | The command is still running: its heartbeats are recent | +| `abandoned` | Its heartbeats stopped without an end: the process died, for example after SIGKILL or running out of memory | +| `unknown` | Recall has neither an end nor a heartbeat for it | + +A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it +cancels the statements it is running, records its real end time, its output and exit code 143, +prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process +at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run +stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. + +## Selection + +`invocations.selection` is the JSON array of selectors the command ran with: positional selectors, +`--select`, and the selectors read from each `--select-file`, whose paths are in `selector_files`. +`selection_count` counts them. At most 64 KiB of selectors are kept; beyond that the list is cut +and `selection_truncated` is `1`, while `selection_count` still counts all of them. + +## Audit evidence + +When an audit warns or fails, Recall keeps its [evidence](../audits.md#evidence): up to +`evidence_limit` of the rows it flagged. `audit_results` counts them in `evidence_count`, sets +`evidence_truncated` to `1` when the limit cut rows off, and names the stored rows by +`sample_digest`. Read them from `artifacts`, joined on the audit's `event_id`: + +```sql +SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id +WHERE a.outcome <> 'pass' +``` + +The evidence holds data values from your warehouse. Choose which failures keep it with +`audit_evidence`: + +| Value | Keeps evidence for | +|-------|--------------------| +| `warn-and-error` (default) | Audits that warn or fail | +| `error` | Audits that fail at error severity | +| `off` | No audit | + +`evidence_count` always counts the rows the audit kept as evidence; `sample_digest` is `NULL` when +Recall did not store them. ## Git revision @@ -179,6 +237,7 @@ retention = "30d" # days of history to keep; also accepts hours, suc shutdown_timeout = "3s" # longest delay Recall may add when a command exits diagnostics_min_severity = "info" max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything +audit_evidence = "warn-and-error" [targets.ci.recall] enabled = false # do not record runs against the ci target @@ -193,6 +252,7 @@ enabled = false # do not record runs against the ci target | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | | `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | +| `audit_evidence` | `"warn-and-error"` | Which failing audits keep the rows they flagged: `warn-and-error`, `error`, or `off`. See [Audit evidence](#audit-evidence) | To turn Recall off for the whole project: @@ -398,8 +458,8 @@ in the store's SQL dialect, against these views: | View | One row per | |------|-------------| | `invocations` | Invocation, with status, exit code, and timing | -| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | -| `audit_results` | Evaluated audit | +| `statements` | Executed statement, with resource, kind, purpose, outcome, duration, and row counts | +| `audit_results` | Evaluated audit, with its evidence count | | `events` | Lifecycle event | | `diagnostics` | Retained diagnostic | | `artifacts` | Reference from an invocation to captured SQL, with its text | @@ -416,7 +476,16 @@ invocation fails and lists the matches. In SQL, match a prefix with Run `sqb recall schema` for every column, and check the values a column holds before filtering on it, for example `SELECT DISTINCT resource_kind, purpose FROM statements`: a model's `resource_kind` is its materialization, such as `table` or `view`, and its build statements have -`purpose = 'model_build'`. +`purpose = 'model_build'`. On Snowflake, the statements that read query history for cost +attribution have `purpose = 'cost_telemetry'`. + +`statements.outcome` is `completed`, `failed`, `handled` or `no_terminal`. A `handled` statement +failed, and SQLBuild recovered from it: for example, a cost history read that timed out and was +read again in smaller windows. `no_terminal` means the statement never reported an end, usually +because the process died. `sql_available` is `1` when the statement's SQL is in `artifacts`; when it +is `0`, `sql_unavailable_reason` says why: `not_captured` when Recall never received the SQL, or +`expired_or_not_captured` when the statement references SQL text that is no longer stored, because +[retention](#retention) removed it or it was dropped before it was written. Every view has `invocation_id` and `invocation_started_at`. Filter on them to keep a query cheap: on a ClickHouse store, a filter on either reaches the table reads, so a query about one invocation or diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index 6e1d004192..6c7648ec1e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -31,11 +31,17 @@ what they printed. `sqb recall query` runs one read-only `SELECT` over a set of `invocation_started_at` and the filter reaches the table reads; without one, a view folds the whole retention window. - **Writing history never fails the command,** so a missing row means "not recorded", not "did not - happen": `invocations.status = 'abandoned'` (the process died), `statements.outcome = - 'no_terminal'` (no terminal fact arrived), and `output_invocations.complete = 0` with a - `gap_reason` when output is incomplete. + happen": `invocations.status = 'abandoned'` (the process died; `ended_at` is its last heartbeat + and `ended_at_estimated = 1`), `statements.outcome = 'no_terminal'` (no terminal fact arrived), + and `output_invocations.complete = 0` with a `gap_reason` when output is incomplete. A command + stopped by SIGTERM or Ctrl-C is `cancelled`, with `signal_number` 15 or 2 and its real end time. - **SQL is stored once by digest.** `statements.sql_digest` joins `artifacts` on - `(invocation_id, statement_id)` for the SQL text (`artifacts.text`). + `(invocation_id, statement_id)` for the SQL text (`artifacts.text`), also for failed and + cancelled statements. `statements.sql_available = 0` means the text is missing, and + `sql_unavailable_reason` says why. +- **Audit evidence is an artifact.** A warning or failing audit's flagged rows (up to + `evidence_limit`) are in `artifacts` with `ref_id = audit_results.event_id`; `sample_digest` is + `NULL` when `[recall] audit_evidence` kept none. Evidence holds warehouse data values. - **Output is a byte-for-byte copy** of stdout and stderr, up to `max_output_bytes` per stream (the local store keeps the first and last MiB). Use `sqb recall output` rather than querying `output_transcripts` for long output. @@ -50,12 +56,15 @@ what they printed. `sqb recall query` runs one read-only `SELECT` over a set of | Column | Values | |--------|--------| -| `invocations.status` | `completed`, `failed`, `running`, `abandoned`, `unknown` | +| `invocations.status` | `completed`, `failed`, `cancelled`, `running`, `abandoned`, `unknown` | +| `invocations.selection`, `selection_count`, `selector_files` | The selectors the command ran with, including those read from `--select-file` (paths in `selector_files`); `selection_truncated = 1` when the list exceeded 64 KiB | +| `invocations.cost_status` | `complete`, `partial`, `pending`, `unavailable`, `collection_failed`; `NULL` when the adapter attributes no cost | | `invocations.origin` | `integration` (an orchestrator such as Dagster started it), `ci` (CI environment variables were set), `local`; `NULL` on runs older SQLBuild versions recorded | | `invocations.command_mode` | What a diagnostic checked: `sqb debug` records `config_only` (`--no-connection`) or `config_and_connection`; `NULL` for other commands | | `invocations.integration`, `integration_job`, `integration_run_id`, `integration_retry` | The orchestrator, job, run and retry from `external_context`, for example `dagster` and its job name; `NULL` when there is none or an older version stored it as non-JSON | | `invocations.error_type`, `error_code`, `error_message` | The cause of a failed invocation, also when it failed only through its exit code (`sqb debug`: `C980` connection, `C981` query, `C982` other check) | -| `statements.outcome` | `completed`, `failed`, `no_terminal` | +| `statements.outcome` | `completed`, `failed`, `handled` (failed, and SQLBuild recovered, such as a timed-out cost history read retried in smaller windows), `no_terminal` | +| `statements.purpose` | `model_build` for a model's build statements, `cost_telemetry` for cost history reads, and others; check with `SELECT DISTINCT purpose` | | `audit_results.outcome` | `pass`, `warn`, `error`, `insufficient`; `execution_error` is set when the audit query itself failed | | `output_invocations.gap_reason` | `NULL` (complete), `truncated`, `filtered`, `dropped`, `unexplained` | | `events.event_type` | `invocation_started`, `invocation_completed`, `invocation_failed`, `run_started`, `run_completed`, `run_failed`, `resource_attempt_started`, `resource_attempt_completed`, `resource_attempt_failed`, `resource_attempt_skipped`, `operation_*`, `statement_started`, `statement_submitted`, `statement_completed`, `statement_failed`, `statement_heartbeat`, `microbatch_*`, `retry_scheduled`, `audit_completed` | @@ -88,13 +97,15 @@ what they printed. `sqb recall query` runs one read-only `SELECT` over a set of ## Output format -- The default `long` format is fine in most cases: read it as it is, every value in full, one - record at a time. -- If it is too verbose, select fewer columns or add a `LIMIT` (or aggregate) rather than changing - the format or reshaping the output. -- Use `--json` only when a program does something meaningful with the result, for example to chain - an `invocation_id` into the next query, or when you need `statistics` (`rows_read`, - `bytes_read`, elapsed time). Not just to print rows more compactly. +- Keep the default format for anything a person reads. It prints a table with column types when + the result fits and one record per row otherwise, every value in full. +- If a result is too long, reshape the question, not the output: aggregate with `GROUP BY`, select + only the columns you need, or add a `LIMIT`. A long listing usually means the query is not + precise enough yet. +- Never switch format to shorten output or save context. `--format long` always prints records; + `--json` is for programs, for example to chain an `invocation_id` into the next query, or when + you need `statistics` (`rows_read`, `bytes_read`, elapsed time). If you change the format, say + why. ## Worked investigations @@ -201,7 +212,7 @@ artifacts (view): captured SQL and other artifacts, by reference text ANY content BLOB -audit_results (view): one row per evaluated audit +audit_results (view): one row per evaluated audit; its evidence rows are in artifacts invocation_id TEXT join key invocation_started_at TEXT event_id TEXT @@ -222,6 +233,8 @@ audit_results (view): one row per evaluated audit evaluation_mode ANY execution_error ANY sql_digest TEXT join key + evidence_count ANY + evidence_truncated ANY sample_digest ANY conflicts (view): record keys reused with different content; the first row is kept @@ -296,17 +309,20 @@ events (view): every lifecycle fact ingested_at TEXT content_hash TEXT -invocations (view): one row per invocation; status is completed, failed, running, abandoned or unknown; origin is integration, ci or local; integration, integration_job, integration_run_id and integration_retry come from external_context +invocations (view): one row per invocation; status is completed, failed, cancelled, running, abandoned or unknown; origin is integration, ci or local; integration, integration_job, integration_run_id and integration_retry come from external_context invocation_id TEXT join key invocation_started_at ANY started_at ANY status ANY ended_at ANY duration_ms ANY + ended_at_estimated ANY exit_code INTEGER error_type TEXT error_code TEXT error_message ANY + signal_number ANY + cost_status ANY command ANY subcommand ANY origin TEXT @@ -315,6 +331,9 @@ invocations (view): one row per invocation; status is completed, failed, running adapter TEXT project_name TEXT selection TEXT + selection_count ANY + selection_truncated INTEGER + selector_files TEXT exclusion TEXT sqlbuild_version TEXT git_revision TEXT @@ -363,7 +382,7 @@ output_transcripts (view): exact stdout, stderr and combined bytes per invocatio stderr BLOB combined BLOB -statements (view): one row per statement; outcome no_terminal means no terminal fact arrived +statements (view): one row per statement; outcome handled means SQLBuild recovered from the failure; no_terminal means no terminal fact arrived invocation_id TEXT join key invocation_started_at TEXT statement_id TEXT join key @@ -388,6 +407,7 @@ statements (view): one row per statement; outcome no_terminal means no terminal affected_rows INTEGER sql_digest TEXT join key sql_available ANY + sql_unavailable_reason ANY ``` diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8c941e3403..eaf5e10e58 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3145,7 +3145,7 @@ class CliNamespace: max_columns: int | None query_sql: str | None query_file: str | None - query_format: str + query_format: str | None query_limit: int | None query_no_limit: bool cost_selector: str @@ -4271,6 +4271,7 @@ def scope_expected_model_names( def supports_color() -> bool: ... def parse_cli_invocation(argv: Sequence[str], *, use_color: bool) -> ParsedCliInvocation: ... def run_cli(argv: Sequence[str] | None = None, handlers: object | None = None) -> int: ... +def termination_requested() -> bool: ... def scope_test_ctes( texts: list[tuple[str, str]], syntax: dict[str, object] ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... @@ -4818,7 +4819,9 @@ def run_query_command( output_format: str = "long", limit: int | None = 20, ) -> int: ... -def render_query_result(*, result: QueryResult, output_format: str, limit: int | None) -> str: ... +def render_query_result( + *, result: QueryResult, output_format: str | None, limit: int | None +) -> str: ... def run_dag_command( *, project_dir: Path | None, @@ -6237,6 +6240,7 @@ def list_source_candidates( ) -> tuple[RelationInfo, ...]: ... class CostArtifactError(ValueError): ... +class CommandTerminated(KeyboardInterrupt): ... class SourceFreshnessInputError(ValueError): ... class SourceFreshnessObservationError(ValueError): ... diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 93ffba817f..94b3745544 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -573,6 +573,14 @@ "sqlbuild.runtime.execution_limits.main.execution_deadline_scope", "execution_deadline_scope", ), + "audit_evidence_scope": ( + "sqlbuild.executor.auditing.main._audit_evidence_scope", + "audit_evidence_scope", + ), + "cancel_running_statements": ( + "sqlbuild.runtime.execution_limits.main._cancel_running_statements", + "cancel_running_statements", + ), "EventDispatcher": ( "sqlbuild.runtime.observability.classes.event_dispatcher", "EventDispatcher", @@ -581,6 +589,10 @@ "sqlbuild.runtime.observability.classes.operation_lifecycle", "OperationLifecycle", ), + "StatementLifecycle": ( + "sqlbuild.runtime.observability.classes.statement_lifecycle", + "StatementLifecycle", + ), "ObservabilityValidationError": ( "sqlbuild.runtime.observability.exceptions", "ObservabilityValidationError", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 7d3105f8d3..c68e33d507 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -214,6 +214,9 @@ from sqlbuild.diagnostics.main.process_resource_reporting import ( from sqlbuild.diagnostics.models import ProcessResourceUsage as ProcessResourceUsage from sqlbuild.errors.contracts.exceptions import ExecutorInputError as ExecutorInputError from sqlbuild.errors.contracts.exceptions import SharedInputError as SharedInputError +from sqlbuild.executor.auditing.main._audit_evidence_scope import ( + audit_evidence_scope as audit_evidence_scope, +) from sqlbuild.executor.auditing.main._current_audit_result_projection import ( current_audit_result_projection as current_audit_result_projection, ) @@ -550,6 +553,9 @@ from sqlbuild.runtime.execution_history.types import RunStatus as RunStatus from sqlbuild.runtime.execution_limits.exceptions import ( ExecutionDurationLimitError as ExecutionDurationLimitError, ) +from sqlbuild.runtime.execution_limits.main._cancel_running_statements import ( + cancel_running_statements as cancel_running_statements, +) from sqlbuild.runtime.execution_limits.main.execution_deadline_scope import ( execution_deadline_scope as execution_deadline_scope, ) @@ -560,6 +566,9 @@ from sqlbuild.runtime.observability.classes.event_dispatcher import ( from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationLifecycle as OperationLifecycle, ) +from sqlbuild.runtime.observability.classes.statement_lifecycle import ( + StatementLifecycle as StatementLifecycle, +) from sqlbuild.runtime.observability.exceptions import ( ObservabilityValidationError as ObservabilityValidationError, ) diff --git a/src/sqlbuild/adapter/contract/classes/connection.py b/src/sqlbuild/adapter/contract/classes/connection.py index 56afb59516..ecd94b1f40 100644 --- a/src/sqlbuild/adapter/contract/classes/connection.py +++ b/src/sqlbuild/adapter/contract/classes/connection.py @@ -7,6 +7,7 @@ from collections.abc import Generator from typing import Any, ClassVar +from sqlbuild._native import CommandTerminated, termination_requested from sqlbuild.adapter.contract.main._attribute_failed_sql import attribute_failed_sql from sqlbuild.adapter.contract.models import QueryResult from sqlbuild.runtime.execution_limits.exceptions import ( @@ -24,6 +25,8 @@ ) from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +_TERMINATED_MESSAGE: str = "terminated by SIGTERM" + class ConnectionMixin(ABC): """Manages adapter connection lifecycle and transaction control.""" @@ -44,6 +47,8 @@ def execute(self, *, connection: Any, sql: str) -> Any: with tracked_statement_connection(adapter=self, connection=connection): result: Any = self._execute(connection=connection, sql=sql) except Exception as error: + if termination_requested(): + raise CommandTerminated(_TERMINATED_MESSAGE) from error try: enforce_execution_deadline(after_failure=True) except (ExecutionDurationLimitError, ItemTimeoutError) as deadline_error: diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 2a0c9d48a7..4909d3869c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1127,16 +1127,21 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + column_types: tuple[str | None, ...] = tuple( + None if column[1] is None else str(column[1]) for column in description + ) if limit is None: return QueryResult( columns=columns, rows=tuple(tuple(row) for row in cursor.fetchall()), + column_types=column_types, ) fetched_rows: list[tuple[object, ...]] = [tuple(row) for row in cursor.fetchmany(limit + 1)] return QueryResult( columns=columns, rows=tuple(fetched_rows[:limit]), truncated=len(fetched_rows) > limit, + column_types=column_types, ) def describe_relation(self, *, connection: Any, relation: str) -> tuple[ColumnInfo, ...]: diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index a66147d3ad..5c0a97c92a 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -335,6 +335,7 @@ class QueryResult: columns: tuple[str, ...] = field(default_factory=tuple) rows: tuple[tuple[object, ...], ...] = field(default_factory=tuple) truncated: bool = False + column_types: tuple[str | None, ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py index ae367239ac..7662f86184 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py @@ -116,8 +116,11 @@ def _query_settings( def run_recall_query( *, handle: int, sql: str, settings: dict[str, int], row_cap: int | None -) -> tuple[RecallFailure, tuple[list[str], list[list[RecallCell]], RecallStatistics] | None]: - """Columns, at most `row_cap` rows of plain values, and server statistics.""" +) -> tuple[ + RecallFailure, + tuple[tuple[list[str], list[str | None]], list[list[RecallCell]], RecallStatistics] | None, +]: + """Columns and their types, at most `row_cap` rows of plain values, and server statistics.""" try: client: Any = SharedRecallConnections.connections.get(handle=handle).client @@ -134,6 +137,9 @@ def run_recall_query( if row_cap is not None and len(rows) >= row_cap: break columns: list[str] = list(result.column_names) + types: list[str | None] = [ + getattr(column_type, "name", None) for column_type in result.column_types + ] summary: dict[str, Any] = dict(result.summary or {}) except Exception as error: return _failure(error), None @@ -142,7 +148,7 @@ def run_recall_query( _number(summary=summary, key="read_bytes"), summary.get("query_id"), ) - return None, (columns, rows[:row_cap] if row_cap is not None else rows, statistics) + return None, ((columns, types), rows[:row_cap] if row_cap is not None else rows, statistics) def run_recall_command(*, handle: int, sql: str) -> tuple[RecallFailure, None]: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index a92876ca3d..0820e86f25 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -230,11 +230,19 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + column_types: tuple[str | None, ...] = tuple( + None if column[1] is None else str(column[1]) for column in description + ) if limit is None: - return QueryResult(columns=columns, rows=tuple(cursor.fetchall())) + return QueryResult( + columns=columns, rows=tuple(cursor.fetchall()), column_types=column_types + ) fetched: list[tuple[object, ...]] = cursor.fetchmany(limit + 1) return QueryResult( - columns=columns, rows=tuple(fetched[:limit]), truncated=len(fetched) > limit + columns=columns, + rows=tuple(fetched[:limit]), + truncated=len(fetched) > limit, + column_types=column_types, ) def supports_zero_copy_clone(self) -> bool: diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 5dd568d615..3232567671 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -360,7 +360,12 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: ] truncated = len(fetched_rows) > limit rows = tuple(fetched_rows[:limit]) - return QueryResult(columns=columns, rows=rows, truncated=truncated) + column_types: tuple[str | None, ...] = tuple( + getattr(column, "type_display", None) for column in description + ) + return QueryResult( + columns=columns, rows=rows, truncated=truncated, column_types=column_types + ) def relation_exists( self, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 5122de425b..fb44ee1b78 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -149,6 +149,7 @@ BASE_TABLE_METADATA_TYPE, CURRENT_DATABASE_ATTRIBUTE, EXTERNAL_BROWSER_AUTHENTICATOR, + FIXED_POINT_RESULT_TYPE, MAX_STATEMENT_TIMEOUT_SECONDS, MFA_AUTHENTICATOR, OAUTH_AUTHORIZATION_CODE_AUTHENTICATOR, @@ -166,6 +167,7 @@ from sqlbuild.cost.types import CostCapability, CostStatus from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +from sqlbuild.runtime.observability.constants import COST_TELEMETRY_PURPOSE from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS _EXACT_COLUMN_INSPECTION_LIMIT: int = 32 @@ -363,6 +365,32 @@ def collect_run_cost( usd_per_credit: Decimal, rate_source: str, budget_seconds: float | None = None, + ) -> RunCostSummary: + with StatementLifecycle.purpose_scope(COST_TELEMETRY_PURPOSE): + return self._collect_run_cost( + connection_config=connection_config, + target_database=target_database, + run_id=run_id, + started_at=started_at, + completed_at=completed_at, + statement_ledger_path=statement_ledger_path, + usd_per_credit=usd_per_credit, + rate_source=rate_source, + budget_seconds=budget_seconds, + ) + + def _collect_run_cost( + self, + *, + connection_config: dict[str, object], + target_database: str | None, + run_id: str, + started_at: datetime, + completed_at: datetime, + statement_ledger_path: Path, + usd_per_credit: Decimal, + rate_source: str, + budget_seconds: float | None = None, ) -> RunCostSummary: telemetry_connection_config: dict[str, object] = dict(connection_config) telemetry_connection_config["login_timeout"] = _COST_LOGIN_TIMEOUT_SECONDS @@ -1863,13 +1891,16 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + column_types: tuple[str | None, ...] = tuple( + self._result_column_type(column) for column in description + ) if limit is None: rows: tuple[tuple[object, ...], ...] = tuple( tuple(row) for row in cursor.fetchall() ) if self._is_status_only_non_row_result(columns=columns, rows=rows): return QueryResult() - return QueryResult(columns=columns, rows=rows) + return QueryResult(columns=columns, rows=rows, column_types=column_types) fetched_rows: list[tuple[object, ...]] = [ tuple(row) for row in cursor.fetchmany(limit + 1) ] @@ -1880,10 +1911,28 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: columns=columns, rows=rows, truncated=len(fetched_rows) > limit, + column_types=column_types, ) finally: cursor.close() + @staticmethod + def _result_column_type(column: Any) -> str | None: + """Name a result column's Snowflake type from its cursor description entry.""" + + from snowflake.connector.constants import FIELD_ID_TO_NAME + + type_name: str | None = FIELD_ID_TO_NAME.get(column[1]) + if type_name is None: + return None + if type_name != FIXED_POINT_RESULT_TYPE: + return type_name + precision: int | None = column[4] + scale: int | None = column[5] + if precision is None: + return "NUMBER" + return f"NUMBER({precision},{scale or 0})" + @staticmethod def _is_status_only_non_row_result( *, columns: tuple[str, ...], rows: tuple[tuple[object, ...], ...] diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 1ae48c46af..e5f457a610 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -46,3 +46,4 @@ DEPENDENCY_VIEW_DOMAINS: tuple[str, ...] = ("VIEW", "MATERIALIZED VIEW", "SECURE VIEW") QUOTED_IDENTIFIER_MARK: str = '"' NAME_SEPARATOR: str = "." +FIXED_POINT_RESULT_TYPE: str = "FIXED" diff --git a/src/sqlbuild/executor/auditing/_helpers/evidence_setting.py b/src/sqlbuild/executor/auditing/_helpers/evidence_setting.py new file mode 100644 index 0000000000..0bb7f3f1eb --- /dev/null +++ b/src/sqlbuild/executor/auditing/_helpers/evidence_setting.py @@ -0,0 +1,27 @@ +"""The `recall.audit_evidence` setting the current invocation's audits collect evidence rows by.""" + +from __future__ import annotations + +from contextvars import ContextVar + +from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.executor.auditing.constants import ( + AUDIT_EVIDENCE_ERROR_ONLY, + AUDIT_EVIDENCE_OFF, + DEFAULT_AUDIT_EVIDENCE, +) + +current_audit_evidence: ContextVar[str] = ContextVar( + "sqlbuild_audit_evidence", default=DEFAULT_AUDIT_EVIDENCE +) + + +def keeps_audit_evidence(*, outcome: AuditOutcome) -> bool: + """Whether a failing audit with `outcome` collects the rows it flagged.""" + + mode: str = current_audit_evidence.get() + if mode == AUDIT_EVIDENCE_OFF: + return False + if mode == AUDIT_EVIDENCE_ERROR_ONLY: + return outcome == AuditOutcome.ERROR + return outcome in {AuditOutcome.WARN, AuditOutcome.ERROR} diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index 5ad6cdb1ce..d22010c7c1 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -25,6 +25,7 @@ AuditRunScope, ) from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context +from sqlbuild.executor.auditing._helpers.evidence_setting import keeps_audit_evidence from sqlbuild.executor.auditing.constants import ( DEFAULT_EVIDENCE_ROW_LIMIT, MAX_EVIDENCE_SERIALIZED_BYTES, @@ -107,6 +108,13 @@ def execute_audit_impl( # noqa: PLR0913 executed_sql=executed_sql, run_scope_phase=run_scope_phase, ) + if keeps_audit_evidence(outcome=outcome): + evidence_rows, evidence_truncated = _bounded_evidence( + cursor=cursor, raw_rows=rows, row_limit=_evidence_row_limit(audit) + ) + result = replace( + result, evidence_rows=evidence_rows, evidence_truncated=evidence_truncated + ) lifecycle.completed() return result @@ -218,13 +226,14 @@ def _measurement_result( # noqa: PLR0913 connection=context.connection, run_scope_phase=context.run_scope_phase, ) + row_limit: int = _evidence_row_limit(audit) + fetchmany: object = getattr(evidence_cursor, "fetchmany", None) evidence_rows, evidence_truncated = _bounded_evidence( cursor=evidence_cursor, - row_limit=( - DEFAULT_EVIDENCE_ROW_LIMIT - if audit.evidence_limit is None - else audit.evidence_limit + raw_rows=list( + fetchmany(row_limit + 1) if callable(fetchmany) else evidence_cursor.fetchall() ), + row_limit=row_limit, ) except Exception as caught_error: evidence_error = f"{type(caught_error).__name__}: {caught_error}" @@ -383,27 +392,31 @@ def _sample_count(*, row: Mapping[str, object], column: str) -> tuple[int | None return int(numeric), None +def _evidence_row_limit(audit: AuditPlanEntry) -> int: + return DEFAULT_EVIDENCE_ROW_LIMIT if audit.evidence_limit is None else audit.evidence_limit + + def _bounded_evidence( - *, cursor: Any, row_limit: int + *, cursor: Any, raw_rows: list[Any], row_limit: int ) -> tuple[tuple[Mapping[str, object], ...], bool]: - fetchmany: object = getattr(cursor, "fetchmany", None) - raw_rows: list[Any] = list( - fetchmany(row_limit + 1) if callable(fetchmany) else cursor.fetchall() - ) + """The leading rows within the row limit and serialized byte cap, and whether rows were cut.""" + truncated: bool = len(raw_rows) > row_limit retained: list[Mapping[str, object]] = [] + serialized_bytes: int = len(b"[]") for raw_row in raw_rows[:row_limit]: safe_row: Mapping[str, object] = { key: _json_safe(value) for key, value in _row_mapping(cursor=cursor, row=raw_row).items() } - candidate: list[Mapping[str, object]] = [*retained, safe_row] - serialized: bytes = json.dumps(candidate, sort_keys=True, separators=(",", ":")).encode( - "utf-8" + row_bytes: int = len( + json.dumps(safe_row, sort_keys=True, separators=(",", ":")).encode("utf-8") ) - if len(serialized) > MAX_EVIDENCE_SERIALIZED_BYTES: + separator_bytes: int = len(b",") if retained else 0 + if serialized_bytes + separator_bytes + row_bytes > MAX_EVIDENCE_SERIALIZED_BYTES: truncated = True break + serialized_bytes += separator_bytes + row_bytes retained.append(safe_row) return tuple(retained), truncated diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index 0c350d4400..a6a8f91cb6 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -40,6 +40,7 @@ from sqlbuild.runtime.observability.types import JSONValue _LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") +_MAX_FAILURE_REASON_LENGTH: int = 500 _LAST_AUDIT_RESULT_PROJECTION: ContextVar[AuditResultProjection | None] = ContextVar( "sqlbuild_last_audit_result_projection", default=None ) @@ -267,14 +268,15 @@ def project_audit_result_batch_impl( ), ) except AuditResultStorageError as error: + reason: str = _storage_failure_reason(error) _LOGGER.warning( "Audit result persistence degraded: attempted=%d written=0 failed=%d (%s)", len(executed), len(executed), - error, + reason, ) projection = AuditResultProjection( - attempted_count=len(executed), failed_count=len(executed) + attempted_count=len(executed), failed_count=len(executed), failure_reason=reason ) _LAST_AUDIT_RESULT_PROJECTION.set(projection) return projection @@ -437,6 +439,15 @@ def _render_bound(bound: MeasurementThresholdBound) -> dict[str, object]: return {"operator": bound.operator.value, "limit": bound.limit} +def _storage_failure_reason(error: AuditResultStorageError) -> str: + """The storage error with the warehouse error that caused it, on one bounded line.""" + + cause: BaseException | None = error.__cause__ + text: str = str(error) if cause is None else f"{error}: {cause}" + line: str = " ".join(text.split()) + return line[:_MAX_FAILURE_REASON_LENGTH] + + def _json_or_none(value: object | None) -> str | None: return None if value is None else json.dumps(value, sort_keys=True, separators=(",", ":")) diff --git a/src/sqlbuild/executor/auditing/constants.py b/src/sqlbuild/executor/auditing/constants.py index 89edc8e7d6..6bad0ef9c8 100644 --- a/src/sqlbuild/executor/auditing/constants.py +++ b/src/sqlbuild/executor/auditing/constants.py @@ -2,3 +2,6 @@ DEFAULT_EVIDENCE_ROW_LIMIT: int = 20 MAX_EVIDENCE_SERIALIZED_BYTES: int = 65_536 +DEFAULT_AUDIT_EVIDENCE: str = "warn-and-error" +AUDIT_EVIDENCE_ERROR_ONLY: str = "error" +AUDIT_EVIDENCE_OFF: str = "off" diff --git a/src/sqlbuild/executor/auditing/main/_audit_evidence_scope.py b/src/sqlbuild/executor/auditing/main/_audit_evidence_scope.py new file mode 100644 index 0000000000..d2ead16424 --- /dev/null +++ b/src/sqlbuild/executor/auditing/main/_audit_evidence_scope.py @@ -0,0 +1,20 @@ +"""Audit evidence setting scope entrypoint.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import Token + +from sqlbuild.executor.auditing._helpers.evidence_setting import current_audit_evidence + + +@contextmanager +def audit_evidence_scope(*, mode: str) -> Iterator[None]: + """Collect failing audits' evidence rows as the `recall.audit_evidence` `mode` says.""" + + token: Token[str] = current_audit_evidence.set(mode) + try: + yield + finally: + current_audit_evidence.reset(token) diff --git a/src/sqlbuild/executor/auditing/models.py b/src/sqlbuild/executor/auditing/models.py index 39458e8b8b..eaf0f4dd21 100644 --- a/src/sqlbuild/executor/auditing/models.py +++ b/src/sqlbuild/executor/auditing/models.py @@ -38,6 +38,7 @@ class AuditResultProjection: attempted_count: int = 0 written_count: int = 0 failed_count: int = 0 + failure_reason: str | None = None @property def degraded(self) -> bool: diff --git a/src/sqlbuild/runtime/execution_limits/_helpers/running.py b/src/sqlbuild/runtime/execution_limits/_helpers/running.py new file mode 100644 index 0000000000..40197a4247 --- /dev/null +++ b/src/sqlbuild/runtime/execution_limits/_helpers/running.py @@ -0,0 +1,5 @@ +"""The process-wide record of connections running a statement, so a termination can cancel them.""" + +from sqlbuild.runtime.execution_limits.classes.active_connections import ActiveConnections + +running_statements: ActiveConnections = ActiveConnections() diff --git a/src/sqlbuild/runtime/execution_limits/main/_cancel_running_statements.py b/src/sqlbuild/runtime/execution_limits/main/_cancel_running_statements.py new file mode 100644 index 0000000000..d7a605f903 --- /dev/null +++ b/src/sqlbuild/runtime/execution_limits/main/_cancel_running_statements.py @@ -0,0 +1,14 @@ +"""Cancel every statement the process is running, as a termination request does.""" + +from __future__ import annotations + +from sqlbuild.runtime.execution_limits._helpers.cancellation import cancel_connection +from sqlbuild.runtime.execution_limits._helpers.running import running_statements + + +def cancel_running_statements() -> None: + """Ask each connection with a statement in flight to cancel it.""" + + _ = running_statements.cancel_running( + cancel=lambda adapter, connection: cancel_connection(adapter=adapter, connection=connection) + ) diff --git a/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py b/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py index 4d54c446df..abeaaee289 100644 --- a/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py +++ b/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py @@ -10,15 +10,16 @@ current_invocation_connections_context, current_item_deadline_context, ) +from sqlbuild.runtime.execution_limits._helpers.running import running_statements from sqlbuild.runtime.execution_limits.classes.active_connections import ActiveConnections from sqlbuild.runtime.execution_limits.classes.item_deadline import ItemDeadline @contextmanager def tracked_statement_connection(*, adapter: Any, connection: Any) -> Generator[None]: - """Record that a statement runs on `connection` so a deadline can cancel it there.""" + """Record that a statement runs on `connection` so a deadline or termination can cancel it.""" - registries: list[ActiveConnections] = [] + registries: list[ActiveConnections] = [running_statements] invocation: ActiveConnections | None = current_invocation_connections_context.get() if invocation is not None: registries.append(invocation) diff --git a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py index 85d0901508..eb794a9f6d 100644 --- a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py @@ -44,6 +44,10 @@ "sqlbuild_statement_lifecycle_owner", default=None ) _STATEMENT_PURPOSE: ContextVar[str | None] = ContextVar("sqlbuild_statement_purpose", default=None) +_HANDLED_FAILURE: ContextVar[Callable[[BaseException], bool] | None] = ContextVar( + "sqlbuild_statement_handled_failure", default=None +) +_HANDLED_OUTCOME: str = "handled" type _ExecutionOwnerKey = tuple[int, int | None] @@ -86,6 +90,7 @@ def __init__( self._pending_failure: tuple[BaseException, str | None, str | None] | None = None self._statement_facts: tuple[str, str] | None = None self._purpose: str | None = _STATEMENT_PURPOSE.get() + self._is_handled: Callable[[BaseException], bool] | None = _HANDLED_FAILURE.get() self._item_deadline: ItemDeadline | None = None self._terminal_query_id: str | None = None @@ -108,6 +113,17 @@ def purpose_scope(purpose: str) -> Iterator[None]: finally: _STATEMENT_PURPOSE.reset(token) + @staticmethod + @contextmanager + def handled_failure_scope(is_handled: Callable[[BaseException], bool]) -> Iterator[None]: + """Mark failures the caller recovers from, such as a timeout it retries in parts.""" + + token: Token[Callable[[BaseException], bool] | None] = _HANDLED_FAILURE.set(is_handled) + try: + yield + finally: + _HANDLED_FAILURE.reset(token) + @staticmethod def discard_failed_attempt() -> None: """Forget a failed attempt deferred to the active statement before an adapter retry.""" @@ -271,6 +287,8 @@ def failed( payload["query_id"] = query_id if job_id is not None: payload["job_id"] = job_id + if self._failure_handled(error=error): + payload["outcome"] = _HANDLED_OUTCOME self._publish_terminal(event_type="statement_failed", payload=payload) def __exit__( @@ -329,6 +347,12 @@ def _terminal_payload(self) -> dict[str, JSONValue]: payload["duration_ms"] = max(0.0, (self._clock() - started) * 1000.0) return payload + def _failure_handled(self, *, error: BaseException) -> bool: + is_handled: Callable[[BaseException], bool] | None = self._is_handled + if is_handled is None: + return False + return bool(is_handled(error)) + def _stop_monitor(self) -> str | None: monitor: StatementMonitor | None = self._monitor if monitor is None: diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index df9f0d299d..46a00a0f79 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -10,6 +10,7 @@ STATEMENT_HEARTBEAT_THRESHOLD_SECONDS: float = 30.0 MODEL_BUILD_PURPOSE: str = "model_build" AUDIT_PURPOSE: str = "audit" +COST_TELEMETRY_PURPOSE: str = "cost_telemetry" LIFECYCLE_ENVELOPE_FIELDS_V1: frozenset[str] = frozenset( { "event_id", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py b/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py index 14c807538c..7d1d0b2a5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py @@ -21,7 +21,7 @@ [ ProjectLocalAdapterCliTestCase( description="query uses nested project local adapter", - command=("query", "SELECT 'ignored' AS value", "--format", "csv"), + command=("query", "SELECT 'ignored' AS value", "--format", "long"), expected_stdout_fragment="duckdb_plus", ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index ede85b5f93..d2b3d50036 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -289,7 +289,7 @@ def test_given_physical_source_without_freshness_when_running_on_bigquery_then_u BigQueryCliTestCase( description="query command uses bigquery local override", command=("query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID"), - expected_stdout_fragments=("ID | 1", "NAME | alice", "ID | 2", "NAME | bob"), + expected_stdout_fragments=("┌", "│ alice │", "│ bob │", "2 rows"), ), BigQueryCliTestCase( description="query command renders json output", @@ -302,14 +302,14 @@ def test_given_physical_source_without_freshness_when_running_on_bigquery_then_u expected_stdout_fragments=('"ID": 1', '"NAME": "alice"'), ), BigQueryCliTestCase( - description="query command renders csv output", + description="query command renders long output", command=( "query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID LIMIT 1", "--format", - "csv", + "long", ), - expected_stdout_fragments=("ID,NAME", "1,alice"), + expected_stdout_fragments=("ID | 1", "NAME | alice"), ), BigQueryCliTestCase( description="query command prints ok for ddl statements", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index a665d15cf1..20fb52540d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -341,7 +341,7 @@ def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abando running_id, recorded = killed_build(project_dir, heartbeat_seconds=test_case.heartbeat_seconds) - assert recorded == [[running_id, test_case.expected_status, 1, 0]] + assert recorded == [[running_id, test_case.expected_status, 1, 1, 0]] @pytest.mark.parametrize( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py index cb76fc1447..58c19acd4e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py @@ -211,14 +211,20 @@ def test_given_a_reader_when_querying_then_pinned_settings_apply_unless_it_may_n ], expected_added_columns=( ( + ("ended_at_estimated", "Nullable(UInt8)"), + ("signal_number", "Nullable(Int64)"), + ("cost_status", "Nullable(String)"), ("origin", "Nullable(String)"), ("command_mode", "Nullable(String)"), + ("selection_count", "Nullable(Int64)"), + ("selection_truncated", "Nullable(UInt8)"), + ("selector_files", "Nullable(String)"), ("integration", "Nullable(String)"), ("integration_job", "Nullable(String)"), ("integration_run_id", "Nullable(String)"), ("integration_retry", "Nullable(Int64)"), ), - (), + (("sql_unavailable_reason", "Nullable(String)"),), (), (), ), @@ -345,9 +351,9 @@ def test_given_representative_invocations_when_views_are_rewritten_then_rows_and expected_primary_key_reads=3, ), ClickHouseRecallTableReadsTestCase( - description="statements reads the events once and the stored SQL once", + description="statements reads the events, SQL references and stored SQL once each", view="statements", - expected_tables=("recall_artifacts", "recall_events"), + expected_tables=("recall_artifact_refs", "recall_artifacts", "recall_events"), expected_primary_key_reads=1, ), ClickHouseRecallTableReadsTestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 074f14f4a7..c8ca0e1e46 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -460,7 +460,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_databricks_th DatabricksCliTestCase( description="query command uses databricks local override", command=("query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID"), - expected_stdout_fragments=("ID | 1", "NAME | alice", "ID | 2", "NAME | bob"), + expected_stdout_fragments=("┌", "│ alice │", "│ bob │", "2 rows"), ), DatabricksCliTestCase( description="query command renders json output", @@ -473,14 +473,14 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_databricks_th expected_stdout_fragments=('"ID": 1', '"NAME": "alice"'), ), DatabricksCliTestCase( - description="query command renders csv output", + description="query command renders long output", command=( "query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID LIMIT 1", "--format", - "csv", + "long", ), - expected_stdout_fragments=("ID,NAME", "1,alice"), + expected_stdout_fragments=("ID | 1", "NAME | alice"), ), DatabricksCliTestCase( description="query command prints ok for ddl statements", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py index b87431f535..9c53a27ef0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py @@ -21,16 +21,24 @@ "test_case", [ QueryCliTestCase( - description="renders long output with default limit", + description="renders a table with column types when the result fits", command=("query", "SELECT 1 AS id, 'alice' AS name"), expected_stdout_fragment=( - "-[ RECORD 1 ]---------------------------+\nid | 1\nname | alice\n\n1 row\n" + "┌─────────┬─────────┐\n" + "│ id │ name │\n" + "│ INTEGER │ VARCHAR │\n" + "├─────────┼─────────┤\n" + "│ 1 │ alice │\n" + "└─────────┴─────────┘\n\n" + "1 row\n" ), ), QueryCliTestCase( - description="renders table output when requested", - command=("query", "SELECT 1 AS id", "--format", "table"), - expected_stdout_fragment="id\n--\n1\n\n1 row\n", + description="renders one record per row when requested", + command=("query", "SELECT 1 AS id, 'alice' AS name", "--format", "long"), + expected_stdout_fragment=( + "-[ RECORD 1 ]---------------------------+\nid | 1\nname | alice\n\n1 row\n" + ), ), ], ids=lambda case: case.description, @@ -67,9 +75,12 @@ def test_given_query_command_when_running_then_outputs_result( query_file_path="queries/orders.sql", query_sql="SELECT 7 AS order_id, 'gaufre' AS label\n", expected_stdout_fragment=( - "-[ RECORD 1 ]---------------------------+\n" - "order_id | 7\n" - "label | gaufre\n\n" + "┌──────────┬─────────┐\n" + "│ order_id │ label │\n" + "│ INTEGER │ VARCHAR │\n" + "├──────────┼─────────┤\n" + "│ 7 │ gaufre │\n" + "└──────────┴─────────┘\n\n" "1 row\n" ), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py index dcac52db7c..25872dca8c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py @@ -149,8 +149,8 @@ def test_given_project_when_running_command_then_project_discovery_runs_only_whe ( QueryTargetConnectionTestCase( description="default target connection", - command=("query", "SELECT label FROM destination", "--format", "csv"), - expected_stdout_fragment="label\ndev\n", + command=("query", "SELECT label FROM destination", "--format", "json"), + expected_stdout_fragment='[{"label": "dev"}]\n', ), QueryTargetConnectionTestCase( description="explicit target connection", @@ -158,11 +158,11 @@ def test_given_project_when_running_command_then_project_discovery_runs_only_whe "query", "SELECT label FROM destination", "--format", - "csv", + "json", "--target", "prod", ), - expected_stdout_fragment="label\nprod\n", + expected_stdout_fragment='[{"label": "prod"}]\n', ), ), ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 63f13f6223..3a5a7c0790 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -300,6 +300,8 @@ def test_given_physical_source_without_freshness_when_running_on_snowflake_then_ command=( "query", "SELECT CURRENT_DATABASE() AS database_name, CURRENT_SCHEMA() AS schema_name", + "--format", + "long", ), expected_stdout_fragments=("DATABASE_NAME | SQB_DB", "SCHEMA_NAME |"), expected_schema_fragment="SQLBUILD_E2E_", @@ -310,9 +312,9 @@ def test_given_physical_source_without_freshness_when_running_on_snowflake_then_ expected_stdout_fragments=('"ID": 1', '"NAME": "alice"'), ), SnowflakeCliTestCase( - description="query command renders csv output", - command=("query", "SELECT 1 AS id, 'alice' AS name", "--format", "csv"), - expected_stdout_fragments=("ID,NAME", "1,alice"), + description="query command renders a table with Snowflake types by default", + command=("query", "SELECT 1 AS id, 'alice' AS name"), + expected_stdout_fragments=("┌", "NUMBER(1,0)", "TEXT", "alice", "1 row"), ), SnowflakeCliTestCase( description="query command prints ok for ddl statements", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py index 26bdae45dc..0c5409bfe8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py @@ -99,10 +99,14 @@ def test_given_orders_project_when_building_twice_then_sqlite_matches_duckdb( "--no-color", "query", "--format", - "csv", + "long", "SELECT customer_id, total_cents FROM analytics__customer_totals ORDER BY 1", ), - expected_fragments=("customer_id,total_cents", "1,1975", "2,3000", "3,500"), + expected_fragments=( + "customer_id | 1\ntotal_cents | 1975", + "customer_id | 2\ntotal_cents | 3000", + "customer_id | 3\ntotal_cents | 500", + ), ), SqliteCommandE2ETestCase( description="debug connects and runs a query", diff --git a/tests/goldens/compiler/failure/rule-custom-finding.json b/tests/goldens/compiler/failure/rule-custom-finding.json index fae7d107f4..1f2b7f5aa9 100644 --- a/tests/goldens/compiler/failure/rule-custom-finding.json +++ b/tests/goldens/compiler/failure/rule-custom-finding.json @@ -22,6 +22,20 @@ ] } ], - "compiled": {}, + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "-- sqb: compile failed for this model; not buildable", + "-- XSQBRDIFF001 at models/staging/stg_orders.sql:1:1: Models avoid the stg_ prefix", + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, "manifest": null } diff --git a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json index 38d48563d4..4c5cbf2d49 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json +++ b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json @@ -25,6 +25,21 @@ ] } ], - "compiled": {}, + "compiled": { + "models/marts/customer_totals.sql": [ + "-- sqb: compile failed for this model; not buildable", + "-- SQBRSQL005 at models/marts/customer_totals.sql:5:117: CTE is unreachable from the final query", + "WITH totals AS (SELECT customer_id, (amount * 100) AS cents, 2 AS floor_cents FROM __ref(\"stg_orders\")), spare AS (SELECT 1 AS one)", + "SELECT customer_id, SUM(cents) AS total_amount", + "FROM totals WHERE cents > floor_cents", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, "manifest": null } diff --git a/tests/goldens/compiler/failure/rule-unused-cte.json b/tests/goldens/compiler/failure/rule-unused-cte.json index 7bf21cd0f3..6eb544614e 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte.json +++ b/tests/goldens/compiler/failure/rule-unused-cte.json @@ -25,6 +25,21 @@ ] } ], - "compiled": {}, + "compiled": { + "models/marts/customer_totals.sql": [ + "-- sqb: compile failed for this model; not buildable", + "-- SQBRSQL005 at models/marts/customer_totals.sql:5:6: CTE is unreachable from the final query", + "WITH spare AS (SELECT 1 AS one)", + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, "manifest": null } diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index d79ab7d5b3..42f5af9db4 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -259,13 +259,16 @@ def test_given_config_with_settings_when_connecting_then_applies_settings( columns=("id", "name"), rows=((1, "alice"),), truncated=True, + column_types=("INTEGER", "VARCHAR"), ), ), QueryTestCase( description="returns all rows without limit", sql="SELECT * FROM (VALUES (1), (2)) AS t(id) ORDER BY id", limit=None, - expected_result=QueryResult(columns=("id",), rows=((1,), (2,))), + expected_result=QueryResult( + columns=("id",), rows=((1,), (2,)), column_types=("INTEGER",) + ), ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py index 18a0f9c776..44e11aa80f 100644 --- a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py @@ -81,6 +81,7 @@ def test_given_table_when_describing_then_postgres_returns_columns_in_order( expected_result=QueryResult( columns=("id", "name"), rows=((1, "hello"),), + column_types=("int4", "text"), ), ) ], diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 84f3707c59..d5825b942e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1,5 +1,6 @@ from __future__ import annotations +import signal from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -1310,3 +1311,36 @@ class EnumLiteralCompileTestCase: expected_exit_code: int expected_diagnostics: tuple[tuple[str, str, int | None], ...] contract: str = "enforced" + + +@dataclass(frozen=True) +class PreservedTargetAuditTestCase: + description: str + target_toml: str + expected_history_schema: str + + +@dataclass(frozen=True) +class MacroSlotQualificationTestCase: + description: str + files: dict[str, str] + expected_exit_code: int + expected_fragments: tuple[str, ...] + absent_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class RuleFailureArtifactsTestCase: + description: str + open_orders_sql: str + expected_exit_code: int + expected_open_orders: str + expected_manifest: bool + + +@dataclass(frozen=True) +class SignalledScenarioTestCase: + description: str + sent_signal: signal.Signals + expected_exit_code: int + expected_last_line: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index e69977b35f..0a3476eb66 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -4,10 +4,14 @@ import json import os import shutil +import signal +import subprocess import sys import tempfile import threading +import time from collections.abc import Callable +from itertools import takewhile from pathlib import Path from types import ModuleType from typing import Any, cast @@ -38,6 +42,19 @@ STAGE_CAPTURE_SUFFIX: str = ".json" E2E_FIXTURES_DIR: Path = Path(__file__).resolve().parents[6] / "e2e" / "fixtures" +_WEEKLY_ORDERS: str = ( + 'MODEL (\n description "Weekly orders per product",\n materialized table,\n);\n\n' + "SELECT 1 AS product_id, CAST(10 AS BIGINT) AS orders_placed, " + "CAST(1 AS BIGINT) AS refund_count\n" +) +_REFUND_RATES: str = ( + 'MODEL (\n description "Weekly refund rate per product",\n materialized table,\n);\n\n' + 'WITH weekly AS (SELECT * FROM __ref("weekly_orders")),\n' + "@sql.weekly_rates(weekly, weekly.keys := (product_id), " + "weekly.order_count := orders_placed, __cte := by_product)\n" + "final AS (SELECT * FROM by_product)\n" + "SELECT * FROM final\n" +) def staging_directories(target_dir: Path) -> list[str]: @@ -1214,3 +1231,113 @@ def write_enum_project(*, root: Path, model_sql: str, contract: str) -> None: " columns (\n order_id (type INTEGER),\n status (type order_status),\n ),\n);\n\n" f"{model_sql}" ) + + +def _weekly_rates(first_step: str) -> str: + return ( + 'MACRO (\n description "Weekly refund rate per grain",\n' + " arguments (weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)),\n" + ");\n\n" + f"WITH counted AS (\n{first_step}\n),\n\n" + "rated AS (\n" + " SELECT\n" + " @weekly.keys,\n" + " SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n" + " FROM counted\n" + " GROUP BY @weekly.keys\n" + ")\n" + ) + + +def refund_rates_project(first_step: str) -> dict[str, str]: + """Weekly orders, refund-rate models and the SQL macro whose first CTE is `first_step`.""" + + return { + "models/weekly_orders.sql": _WEEKLY_ORDERS, + "models/refund_rates.sql": _REFUND_RATES, + "models/_macros/sql/weekly_rates.sql": _weekly_rates(first_step), + } + + +def open_orders_model(sql: str) -> str: + """An `open_orders` table model whose query is `sql`.""" + + return f'MODEL (\n description "Open orders",\n materialized table,\n);\n\n{sql}\n' + + +_ENDLESS_ORDER_TOTAL: str = "(SELECT SUM(big.n) FROM range(1000000000000) AS big(n)) * 0" + + +def write_endless_scenario_project(project_dir: Path) -> Path: + """A DuckDB project whose scenario builds a model that runs until it is cancelled.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "scenario_signals"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "scenario_signals.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n" + " description: Raw orders.\n" + " schema: main\n" + " table: raw_orders\n" + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table, sql_analysis false);\n\n" + f"SELECT SUM(o.amount) + {_ENDLESS_ORDER_TOTAL} AS total_amount " + 'FROM __source("raw_orders") AS o\n' + ), + "tests/scenarios/totals.sql": ( + "SCENARIO (description 'Order totals add up.');\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT 1 AS id, 10 AS amount),\n" + "__expected__order_totals AS (SELECT 10 AS total_amount)\n" + "SELECT 1\n" + ), + } + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + return project_dir + + +def signalled_scenario_run( + *, project_dir: Path, sent_signal: signal.Signals +) -> tuple[int, str, float]: + """Exit code and output of `sqb scenario test` sent `sent_signal` while its model runs, and + the seconds it took to stop.""" + + process: subprocess.Popen[str] = subprocess.Popen( + [ + str(Path(sys.executable).with_name("sqb")), + "--no-color", + "--project-dir", + str(project_dir), + "scenario", + "test", + "--concurrency", + "1", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + preexec_fn=_default_interrupt_handling, + ) + assert process.stdout is not None + started: str = "".join(takewhile(lambda line: "order_totals START" not in line, process.stdout)) + time.sleep(1.0) + process.send_signal(sent_signal) + sent: float = time.monotonic() + stdout, stderr = process.communicate(timeout=120) + return process.returncode, started + stdout + stderr, time.monotonic() - sent + + +def _default_interrupt_handling() -> None: + """Give the spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" + + _ = signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_audit_command_storage.py b/tests/integration/src/sqlbuild/cli/commands/main/test_audit_command_storage.py new file mode 100644 index 0000000000..4f9736e76c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_audit_command_storage.py @@ -0,0 +1,77 @@ +"""Real CLI coverage for where a standalone audit stores its history.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + PreservedTargetAuditTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import query_duckdb_rows + +_PROJECT: str = ( + 'name = "shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "shop.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' +) +_ORDERS_MODEL: str = ( + 'MODEL (\n description "One row per order",\n materialized table,\n schema sales,\n' + " columns (order_id (audits [not_null])),\n);\n\n" + "SELECT 1 AS order_id\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PreservedTargetAuditTestCase( + description="a preserved schema stores history beside the models", + target_toml='schema = "preserve"\n', + expected_history_schema="sales", + ), + PreservedTargetAuditTestCase( + description="a named schema stores history in it", + target_toml='schema = "dev"\n', + expected_history_schema="dev", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_target_schema_when_auditing_alone_then_history_is_stored_where_a_build_stores_it( + test_case: PreservedTargetAuditTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + _ = (tmp_path / "sqlbuild_project.toml").write_text( + _PROJECT + test_case.target_toml, encoding="utf-8" + ) + (tmp_path / "models").mkdir() + _ = (tmp_path / "models" / "orders.sql").write_text(_ORDERS_MODEL, encoding="utf-8") + assert main(["--no-color", "--project-dir", str(tmp_path), "build"]) == 0 + + exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "audit"]) + + output: str = "".join(capsys.readouterr()) + assert exit_code == 0, output + assert "warning:" not in output + db_path: Path = tmp_path / "shop.duckdb" + assert query_duckdb_rows( + db_path=db_path, + sql="SELECT table_schema FROM information_schema.tables " + "WHERE table_name = '_sqlbuild_audit_results'", + ) == ((test_case.expected_history_schema,),) + assert query_duckdb_rows( + db_path=db_path, + sql=f"SELECT COUNT(*) FROM {test_case.expected_history_schema}._sqlbuild_audit_results", + ) == ((2,),) + assert query_duckdb_rows( + db_path=db_path, + sql="SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = 'preserve'", + ) == ((0,),) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_command_termination.py b/tests/integration/src/sqlbuild/cli/commands/main/test_command_termination.py new file mode 100644 index 0000000000..7157e48cfc --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_command_termination.py @@ -0,0 +1,51 @@ +"""Real CLI coverage for how a scenario run stops on SIGINT and SIGTERM.""" + +from __future__ import annotations + +import signal +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + SignalledScenarioTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + signalled_scenario_run, + write_endless_scenario_project, +) + +_STOP_NOTICE: str = "Interrupted; cleaning up running scenarios..." + + +@pytest.mark.parametrize( + "test_case", + [ + SignalledScenarioTestCase( + description="Ctrl+C cancels the running statement", + sent_signal=signal.SIGINT, + expected_exit_code=-signal.SIGINT, + expected_last_line="KeyboardInterrupt", + ), + SignalledScenarioTestCase( + description="SIGTERM cancels the running statement the same way", + sent_signal=signal.SIGTERM, + expected_exit_code=143, + expected_last_line="error: terminated by SIGTERM", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_running_scenario_when_signalled_then_its_statement_is_cancelled_promptly( + test_case: SignalledScenarioTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_endless_scenario_project(tmp_path / "scenario_signals") + + exit_code, output, stop_seconds = signalled_scenario_run( + project_dir=project_dir, sent_signal=test_case.sent_signal + ) + + assert exit_code == test_case.expected_exit_code, output + assert stop_seconds < 15, output + assert output.count(_STOP_NOTICE) == 1, output + assert output.splitlines()[-1] == test_case.expected_last_line, output diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py index e7f1d438b3..a31ce8a271 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py @@ -233,7 +233,13 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu assert main(arguments) == 1 failed: dict[str, object] = json.loads(capsys.readouterr().out) assert failed["has_errors"] is True - assert {path: path.read_bytes() for path in before} == before + changed: Path = compiled / "models" / "orders_000.sql" + assert {path: path.read_bytes() for path in before} == { + **before, + changed: b"-- sqb: compile failed for this model; not buildable\n" + b"-- XSQBRARCH001 at models/orders_000.sql:1:1: Model rejected\n" + b"SELECT CAST(2 AS INTEGER) AS order_id\n", + } assert stale.exists() assert staging_directories(tmp_path / "target") == [] @@ -242,7 +248,6 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu final: dict[str, object] = json.loads(capsys.readouterr().out) assert final["has_errors"] is False assert not stale.exists() - changed: Path = compiled / "models" / "orders_000.sql" for path in modified_times.keys() - {changed}: assert path.stat().st_mtime_ns == modified_times[path] assert path.read_bytes() == before[path] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py new file mode 100644 index 0000000000..b39902a37e --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -0,0 +1,101 @@ +"""Real CLI coverage for the compiled SQL a compile writes when Rules fail.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + RuleFailureArtifactsTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import open_orders_model + +_PROJECT: str = ( + 'name = "shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "shop.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n\n' + "[rules]\n" + 'select = ["SQBRSQL028"]\n' +) +_ORDERS: str = ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" +) +_HEADER: str = "-- sqb: compile failed for this model; not buildable\n" + + +@pytest.mark.parametrize( + "test_case", + [ + RuleFailureArtifactsTestCase( + description="a model failing a Rule is written with a header listing its findings", + open_orders_sql='SELECT o.order_id, status FROM __ref("orders") AS o', + expected_exit_code=1, + expected_open_orders=( + _HEADER + "-- SQBRSQL028 at models/open_orders.sql:6:20: " + "Single-relation query mixes qualified and unqualified columns\n" + 'SELECT o.order_id, status FROM __ref("orders") AS o\n' + ), + expected_manifest=False, + ), + RuleFailureArtifactsTestCase( + description="a passing compile writes the same model without the header", + open_orders_sql='SELECT o.order_id, o.status FROM __ref("orders") AS o', + expected_exit_code=0, + expected_open_orders='SELECT o.order_id, o.status FROM __ref("orders") AS o\n', + expected_manifest=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failure_header( + test_case: RuleFailureArtifactsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT, + "models/orders.sql": _ORDERS, + "models/open_orders.sql": open_orders_model( + 'SELECT o.order_id, o.status FROM __ref("orders") AS o' + ), + "models/retired_orders.sql": _ORDERS, + } + for relative, contents in files.items(): + path: Path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + compile_args: list[str] = [ + "--no-color", + "--project-dir", + str(tmp_path), + "compile", + "--manifest", + ] + _ = main(compile_args) + _ = (tmp_path / "models" / "open_orders.sql").write_text( + open_orders_model(test_case.open_orders_sql), encoding="utf-8" + ) + assert (tmp_path / "target" / "manifest.json").exists() + (tmp_path / "models" / "retired_orders.sql").unlink() + + exit_code: int = main(compile_args) + + output: str = "".join(capsys.readouterr()) + compiled: Path = tmp_path / "target" / "compiled" / "models" + assert exit_code == test_case.expected_exit_code, output + assert (compiled / "open_orders.sql").read_text(encoding="utf-8") == ( + test_case.expected_open_orders + ) + assert (compiled / "orders.sql").read_text(encoding="utf-8") == ( + "SELECT 1 AS order_id, 'open' AS status\n" + ) + assert (tmp_path / "target" / "manifest.json").exists() is test_case.expected_manifest + assert sorted(path.name for path in compiled.iterdir()) == ["open_orders.sql", "orders.sql"] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py new file mode 100644 index 0000000000..3c739adeb2 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py @@ -0,0 +1,184 @@ +"""Real CLI coverage for how SQBRSQL028 judges the qualification of SQL macro slot reads.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + MacroSlotQualificationTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import refund_rates_project + +_PROJECT: str = ( + 'name = "shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "shop.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n\n' + "[rules]\n" + 'select = ["SQBRSQL028"]\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroSlotQualificationTestCase( + description="a first step written in plain slot reads passes", + files=refund_rates_project( + " SELECT\n" + " @weekly.keys,\n" + " @weekly.order_count AS orders,\n" + " @weekly.refund_count AS refunds\n" + " FROM @weekly" + ), + expected_exit_code=0, + expected_fragments=("Project compiled",), + absent_fragments=("SQBRSQL028",), + ), + MacroSlotQualificationTestCase( + description="aliased slot reads beside a plain group read fail once, at the macro", + files=refund_rates_project( + " SELECT\n" + " @weekly.keys,\n" + " s.@weekly.order_count AS orders,\n" + " s.@weekly.refund_count AS refunds\n" + " FROM @weekly AS s" + ), + expected_exit_code=1, + expected_fragments=( + "--> models/_macros/sql/weekly_rates.sql:8:5", + "to qualify it, read through an alias: `FROM @rel AS s`, then `s.@rel.column`", + ), + absent_fragments=("--> models/refund_rates.sql",), + ), + MacroSlotQualificationTestCase( + description="an expression macro's plain reads pass beside bare caller columns", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount, " + "CAST(1 AS DOUBLE) AS discount\n" + ), + "models/_macros/sql/net_amount.sql": ( + 'MACRO (\n description "Order amount after discount",\n' + " arguments (orders RELATION (amount DOUBLE, discount DOUBLE)),\n);\n\n" + "@orders.amount - @orders.discount\n" + ), + "models/order_totals.sql": ( + 'MODEL (\n description "Net amount per order",\n materialized table,\n);\n\n' + "SELECT order_id, @sql.net_amount(orders) AS net\n" + 'FROM __ref("orders") AS orders\n' + ), + }, + expected_exit_code=0, + expected_fragments=("Project compiled",), + absent_fragments=("SQBRSQL028",), + ), + MacroSlotQualificationTestCase( + description="an expression macro's plain reads pass beside hand-qualified columns", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount, " + "CAST(1 AS DOUBLE) AS discount\n" + ), + "models/_macros/sql/net_amount.sql": ( + 'MACRO (\n description "Order amount after discount",\n' + " arguments (orders RELATION (amount DOUBLE, discount DOUBLE)),\n);\n\n" + "@orders.amount - @orders.discount\n" + ), + "models/order_totals.sql": ( + 'MODEL (\n description "Net amount per order",\n materialized table,\n);\n\n' + "SELECT o.order_id, @sql.net_amount(o) AS net\n" + 'FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=0, + expected_fragments=("Project compiled",), + absent_fragments=("SQBRSQL028",), + ), + MacroSlotQualificationTestCase( + description="an expression macro's bare column beside a hand-qualified column fails", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" + ), + "models/_macros/sql/status_value.sql": ( + 'MACRO (\n description "The order status",\n returns VARCHAR,\n);\n\nstatus\n' + ), + "models/open_orders.sql": ( + 'MODEL (\n description "Open orders",\n materialized table,\n);\n\n' + 'SELECT o.order_id, @sql.status_value() AS s FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=1, + expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + absent_fragments=(), + ), + MacroSlotQualificationTestCase( + description="a column-list macro's bare output beside a hand-qualified column fails", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" + ), + "models/_macros/sql/order_status.sql": ( + 'MACRO (\n description "The order status column",\n);\n\nSELECT status\n' + ), + "models/open_orders.sql": ( + 'MODEL (\n description "Open orders",\n materialized table,\n);\n\n' + 'SELECT o.order_id, @sql.order_status() FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=1, + expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + absent_fragments=(), + ), + MacroSlotQualificationTestCase( + description="a caller that mixes styles itself still fails", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" + ), + "models/open_orders.sql": ( + 'MODEL (\n description "Open orders",\n materialized table,\n);\n\n' + 'SELECT o.order_id, status FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=1, + expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + absent_fragments=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_slot_reads_when_compiling_then_qualification_is_judged_as_written( + test_case: MacroSlotQualificationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = {"sqlbuild_project.toml": _PROJECT, **test_case.files} + for relative, contents in files.items(): + path: Path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "compile"]) + + output: str = "".join(capsys.readouterr()) + assert exit_code == test_case.expected_exit_code, output + assert [fragment in output for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ], output + assert [fragment in output for fragment in test_case.absent_fragments] == [ + False for _ in test_case.absent_fragments + ], output diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d2d435fb76..44e099df62 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -1440,7 +1440,11 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: assert cold["diagnostics"] == warm["diagnostics"] == first["diagnostics"] assert cold["compile_timings"]["rule_cache_hits"] == 0 assert warm["compile_timings"]["rule_cache_hits"] >= 1 - assert not (tmp_path / "target" / "compiled").exists() + assert ( + (tmp_path / "target" / "compiled" / "models" / "orders.sql") + .read_text(encoding="utf-8") + .startswith("-- sqb: compile failed for this model; not buildable\n-- XSQBRARCH001 at ") + ) host_inputs: Path = tmp_path / "target" / "rules-cache" / "host-inputs" assert not tuple(host_inputs.glob("*.payload")) diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 3852fd9470..62838d5545 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -233,10 +233,18 @@ class JsonErrorTestCase: @dataclass(frozen=True) -class DeprecatedFormatTestCase: +class RemovedFormatTestCase: description: str args: tuple[str, ...] - expected_warning: bool + expected_message: str + + +@dataclass(frozen=True) +class QueryLayoutTestCase: + description: str + args: tuple[str, ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] @dataclass(frozen=True) @@ -341,3 +349,52 @@ class InvocationPrefixTestCase: prefix_length: int expected_exit_code: int expected_error_code: str | None + + +@dataclass(frozen=True) +class RecordedSelectionTestCase: + description: str + selector_file: str + args: tuple[str, ...] + expected_selection: list[object] + + +@dataclass(frozen=True) +class TerminatedCommandTestCase: + description: str + orders_model: str + environment: dict[str, str] + wait_sql: str + expected_output: bytes + + +@dataclass(frozen=True) +class FailedStatementSqlTestCase: + description: str + model_sql: str + expected_sql_fragment: str + + +@dataclass(frozen=True) +class AuditEvidenceTestCase: + description: str + recall_toml: str + severity: str + expected_exit_code: int + expected_evidence: list[object] + expected_artifacts: list[list[object]] + expected_warehouse_evidence: list[tuple[object, ...]] + + +@dataclass(frozen=True) +class CostAttributionTestCase: + description: str + collected: str + expected_cost_status: str + + +@dataclass(frozen=True) +class ClosingTerminationTestCase: + description: str + shutdown_timeout: str + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 595fc40c59..7b4fbf0819 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -213,7 +213,8 @@ def killed_build(project_dir: Path, *, heartbeat_seconds: str) -> tuple[str, lis time.sleep(5 * float(heartbeat_seconds)) recorded: list[list[object]] = recall_rows( project_dir, - "SELECT i.invocation_id, i.status, i.ended_at IS NOT NULL, o.complete " + "SELECT i.invocation_id, i.status, i.ended_at IS NOT NULL, i.ended_at_estimated, " + "o.complete " "FROM invocations i JOIN output_invocations o USING (invocation_id)", ) return str(running[0][0]), recorded @@ -224,15 +225,8 @@ def killed_during_discovery( ) -> tuple[list[list[object]], subprocess.CompletedProcess[bytes]]: """Rows left by a build SIGKILLed while importing a slow sink, and `recall output last`.""" - sink: Path = project_dir / "sinks" / "slow_import.py" - sink.parent.mkdir(parents=True, exist_ok=True) - _ = sink.write_text( - "import os\nimport time\n\n" - "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n", - encoding="utf-8", - ) process: subprocess.Popen[bytes] = start( - project_dir, + write_slow_import_sink(project_dir), "build", SQLBUILD_RECALL_HEARTBEAT_SECONDS=heartbeat_seconds, ORDERS_IMPORT_SECONDS="60", @@ -251,6 +245,62 @@ def killed_during_discovery( return recorded, run(project_dir, "recall", "output", "last") +def write_slow_import_sink(project_dir: Path) -> Path: + """Add a sink module whose import sleeps for `ORDERS_IMPORT_SECONDS`, slowing discovery.""" + + sink: Path = project_dir / "sinks" / "slow_import.py" + sink.parent.mkdir(parents=True, exist_ok=True) + _ = sink.write_text( + "import os\nimport time\n\n" + "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n", + encoding="utf-8", + ) + return project_dir + + +def terminated_build( + project_dir: Path, *, wait_sql: str, **env: str +) -> tuple[subprocess.CompletedProcess[bytes], float]: + """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row, and the + seconds it took to stop.""" + + process: subprocess.Popen[bytes] = start(project_dir, "build", **env) + try: + _ = wait_for(wait_sql, project_dir, timeout=120) + process.send_signal(signal.SIGTERM) + sent: float = time.monotonic() + stdout, stderr = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + raise AssertionError(f"a terminated build kept running:\n{hang_dump(process)}") from None + completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( + process.args, process.returncode, stdout, stderr + ) + return completed, time.monotonic() - sent + + +def terminated_while_closing(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float]: + """A build sent SIGTERM while it waits for a locked Recall store at exit, and the seconds it + took to stop.""" + + with held_store_lock(project_dir): + process: subprocess.Popen[bytes] = start(project_dir, "build", PYTHONUNBUFFERED="1") + assert process.stdout is not None + printed: bytes = b"" + while b"Completed" not in printed and process.poll() is None: + printed += process.stdout.readline() + time.sleep(1) + process.send_signal(signal.SIGTERM) + sent: float = time.monotonic() + try: + stdout, stderr = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + raise AssertionError(f"a closing build kept running:\n{hang_dump(process)}") from None + completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( + process.args, process.returncode, printed + stdout, stderr + ) + return completed, time.monotonic() - sent + + def interrupted_build(project_dir: Path, **env: str) -> tuple[int | None, float]: """Exit code of a build sent SIGINT after its first logged event, and seconds to stop.""" diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 802145907f..2cb811acc8 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -245,7 +245,7 @@ def test_given_a_recall_query_when_rendered_as_json_then_it_matches_the_schema( "8", "SELECT 'first line' || char(10) || 'second line' AS note", ), - expected_fragments=("note | first li…[+14 bytes]", "1 row"), + expected_fragments=("│ first li…[+14 bytes] │", "1 row"), ), TextOutputTestCase( description="the schema lists views and join keys", @@ -390,7 +390,7 @@ def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abando running_id, recorded = killed_build(project_dir, heartbeat_seconds=test_case.heartbeat_seconds) assert recorded == [ - [running_id, test_case.expected_status, 1, test_case.expected_output_complete] + [running_id, test_case.expected_status, 1, 1, test_case.expected_output_complete] ] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index 4834953e71..d2def05b76 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -5,11 +5,12 @@ import pytest from tests.integration.src.sqlbuild.cli.recall._test_types import ( - DeprecatedFormatTestCase, JsonErrorTestCase, LeanRecallTestCase, + QueryLayoutTestCase, RecallDiagnosticsTestCase, RecallUsageErrorTestCase, + RemovedFormatTestCase, StatementStartTestCase, StoreBeingCreatedTestCase, ) @@ -22,8 +23,6 @@ write_sinks, ) -TABLE_DEPRECATION: str = "--format table is deprecated and will be removed" - @pytest.mark.parametrize( "test_case", @@ -81,35 +80,106 @@ def test_given_a_failing_recall_command_with_json_when_run_then_stdout_is_a_json @pytest.mark.parametrize( "test_case", [ - DeprecatedFormatTestCase( - description="table still renders and warns", + RemovedFormatTestCase( + description="recall query rejects table and points at the default", args=("recall", "query", "--format", "table", "SELECT 1 AS answer"), - expected_warning=True, + expected_message=( + "'table' was removed; the default output prints a table when the result fits" + ), ), - DeprecatedFormatTestCase( - description="the schema table warns too", + RemovedFormatTestCase( + description="recall schema rejects table too", args=("recall", "schema", "--format", "table"), - expected_warning=True, + expected_message="'table' was removed", + ), + RemovedFormatTestCase( + description="recall query rejects csv and points at json", + args=("recall", "query", "--format", "csv", "SELECT 1 AS answer"), + expected_message="'csv' was removed; use --json for programs", + ), + RemovedFormatTestCase( + description="query rejects table", + args=("query", "--format", "table", "SELECT 1 AS answer"), + expected_message="'table' was removed", + ), + RemovedFormatTestCase( + description="query rejects csv", + args=("query", "--format", "csv", "SELECT 1 AS answer"), + expected_message="'csv' was removed; use --json for programs", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_removed_format_when_querying_then_it_is_a_usage_error_naming_the_replacement( + test_case: RemovedFormatTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + rejected: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert rejected.returncode == 2 + assert rejected.stdout == b"" + assert test_case.expected_message in rejected.stderr.decode() + + +@pytest.mark.parametrize( + "test_case", + [ + QueryLayoutTestCase( + description="a narrow Recall result prints a box with column types", + args=("recall", "query", "SELECT 7 AS order_count, 'open' AS status"), + expected_fragments=("│ order_count │ status │", "│ integer │ text │", "1 row"), + unexpected_fragments=("RECORD",), + ), + QueryLayoutTestCase( + description="a Recall result wider than the output prints whole records", + args=("recall", "query", "SELECT replace(hex(zeroblob(70)), '0', 'x') AS note"), + expected_fragments=("-[ RECORD 1 ]", "note | " + "x" * 140, "1 row"), + unexpected_fragments=("┌",), + ), + QueryLayoutTestCase( + description="long always prints records", + args=("recall", "query", "--format", "long", "SELECT 7 AS order_count"), + expected_fragments=("-[ RECORD 1 ]", "order_count | 7"), + unexpected_fragments=("┌",), + ), + QueryLayoutTestCase( + description="a warehouse result shows the warehouse type names", + args=("query", "SELECT 7 AS order_id, CAST(12.5 AS DECIMAL(10,2)) AS amount"), + expected_fragments=("│ order_id │ amount │", "INTEGER", "DECIMAL(10,2)"), + unexpected_fragments=("RECORD",), + ), + QueryLayoutTestCase( + description="a limited result says how to see more", + args=("query", "SELECT range AS order_id FROM range(30)"), + expected_fragments=("20 rows shown (limit 20; use --limit or --no-limit)",), + unexpected_fragments=("GROUP BY",), ), - DeprecatedFormatTestCase( - description="the default long format does not warn", - args=("recall", "query", "SELECT 1 AS answer"), - expected_warning=False, + QueryLayoutTestCase( + description="many rows suggest reading fewer", + args=("query", "--no-limit", "SELECT range AS order_id FROM range(60)"), + expected_fragments=("60 rows; to read fewer, aggregate with GROUP BY or add a LIMIT",), + unexpected_fragments=(), ), ], ids=lambda case: case.description, ) -def test_given_the_table_format_when_querying_then_it_works_with_a_deprecation_warning( - test_case: DeprecatedFormatTestCase, tmp_path: Path +def test_given_a_query_result_when_printed_by_default_then_it_fits_the_output_without_hiding_data( + test_case: QueryLayoutTestCase, tmp_path: Path ) -> None: project_dir: Path = write_project(tmp_path / "orders_shop") assert run(project_dir, "build").returncode == 0 rendered: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) - assert rendered.returncode == 0 - assert rendered.stdout - assert (TABLE_DEPRECATION in rendered.stderr.decode()) is test_case.expected_warning + stdout: str = rendered.stdout.decode() + assert rendered.returncode == 0, rendered.stderr.decode(errors="replace") + assert [fragment in stdout for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ], stdout + assert [fragment in stdout for fragment in test_case.unexpected_fragments] == [ + False for _ in test_case.unexpected_fragments + ], stdout @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py new file mode 100644 index 0000000000..de0da599de --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -0,0 +1,335 @@ +import json +import subprocess +from pathlib import Path + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + AuditEvidenceTestCase, + ClosingTerminationTestCase, + CostAttributionTestCase, + FailedStatementSqlTestCase, + RecordedSelectionTestCase, + TerminatedCommandTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + EMITTED_TAIL, + ORDERS_MODEL, + recall_rows, + run, + terminated_build, + terminated_while_closing, + write_project, + write_slow_import_sink, +) + +CUSTOMERS_MODEL: str = ( + 'MODEL (\n description "One row per customer",\n materialized table,\n' + " columns (email (audits [not_null(severity warn)])),\n);\n\n" + "SELECT * FROM (VALUES (1, 'ada@example.com'), (2, NULL), (3, NULL)) " + "AS customers(customer_id, email)\n" +) + +COSTED_ADAPTER: str = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n" + "from sqlbuild.cost.models import RunCostSummary\n" + "from sqlbuild.cost.types import CostCapability, CostStatus\n\n\n" + "class DuckDbCostedAdapter(DuckDbAdapter):\n" + " adapter_name = 'duckdb_costed'\n\n" + " def cost_capability(self):\n" + " return CostCapability.SNOWFLAKE_QUERY_HISTORY\n\n" + " def collect_run_cost(self, *, usd_per_credit, rate_source, **kwargs):\n" + "{collected}" +) + + +@pytest.mark.parametrize( + "test_case", + [ + RecordedSelectionTestCase( + description="a selector file's selectors are recorded with the file", + selector_file="# nightly orders\norders\n\nemit_bytes\n", + args=("--select-file", "nightly.txt"), + expected_selection=['["orders","emit_bytes"]', 2, 0, '["nightly.txt"]'], + ), + RecordedSelectionTestCase( + description="positional, option and file selectors are recorded together", + selector_file="emit_bytes\n", + args=("orders", "--select-file", "nightly.txt"), + expected_selection=['["orders","emit_bytes"]', 2, 0, '["nightly.txt"]'], + ), + RecordedSelectionTestCase( + description="a command without selectors records none", + selector_file="", + args=(), + expected_selection=["[]", 0, 0, "[]"], + ), + ], + ids=lambda case: case.description, +) +def test_given_selectors_when_building_then_the_effective_selection_is_recorded( + test_case: RecordedSelectionTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + _ = (project_dir / "nightly.txt").write_text(test_case.selector_file, encoding="utf-8") + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", *test_case.args) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT selection, selection_count, selection_truncated, selector_files FROM invocations", + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert recorded == [test_case.expected_selection] + + +@pytest.mark.parametrize( + "test_case", + [ + TerminatedCommandTestCase( + description="during project discovery", + orders_model=ORDERS_MODEL, + environment={"ORDERS_IMPORT_SECONDS": "60"}, + wait_sql=( + "SELECT invocation_id FROM invocations WHERE status = 'running' " + "AND (julianday(last_heartbeat_at) - julianday(started_at)) * 86400 >= 2" + ), + expected_output=b"", + ), + TerminatedCommandTestCase( + description="while a task runs", + orders_model=ORDERS_MODEL, + environment={"ORDERS_TASK_SLEEP_SECONDS": "60"}, + wait_sql=( + "SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'" + ), + expected_output=EMITTED_TAIL, + ), + TerminatedCommandTestCase( + description="while a long warehouse statement runs", + orders_model=( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT sum(range % 7) AS order_id FROM range(80000000000)\n" + ), + environment={}, + wait_sql=( + "SELECT statement_id FROM statements " + "WHERE statement_kind = 'CREATE' AND ended_at IS NULL" + ), + expected_output=b"", + ), + ], + ids=lambda case: case.description, +) +def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_with_its_real_end( + test_case: TerminatedCommandTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_slow_import_sink(write_project(tmp_path / "orders_shop")) + _ = (project_dir / "models" / "orders.sql").write_text(test_case.orders_model, encoding="utf-8") + + terminated, stop_seconds = terminated_build( + project_dir, + wait_sql=test_case.wait_sql, + SQLBUILD_RECALL_HEARTBEAT_SECONDS="0.5", + **test_case.environment, + ) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.status, i.signal_number, i.exit_code, i.ended_at_estimated, " + "i.ended_at > i.started_at, o.flush_complete " + "FROM invocations i JOIN output_invocations o USING (invocation_id)", + ) + starts: list[list[object]] = recall_rows( + project_dir, "SELECT COUNT(*) FROM recall_events WHERE event_type = 'invocation_started'" + ) + output: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "output", "last") + + assert terminated.returncode == 143 + assert stop_seconds < 10 + assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") + assert b"Traceback" not in terminated.stderr + assert recorded == [["cancelled", 15, 143, 0, 1, 1]] + assert starts == [[1]] + assert test_case.expected_output in output.stdout + assert b"may be incomplete" not in output.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + ClosingTerminationTestCase( + description="a build waiting for a locked store at exit", + shutdown_timeout="60s", + expected_exit_code=143, + ), + ], + ids=lambda case: case.description, +) +def test_given_sigterm_while_recall_closes_when_a_build_has_finished_then_it_exits_143( + test_case: ClosingTerminationTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", + extra_toml=f'\n[recall]\nshutdown_timeout = "{test_case.shutdown_timeout}"\n', + ) + + terminated, stop_seconds = terminated_while_closing(project_dir) + + assert b"Completed" in terminated.stdout + assert terminated.returncode == test_case.expected_exit_code, terminated.stderr.decode( + errors="replace" + ) + assert stop_seconds < 10 + assert b"Traceback" not in terminated.stderr + assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") + + +@pytest.mark.parametrize( + "test_case", + [ + FailedStatementSqlTestCase( + description="a statement the warehouse rejects keeps its SQL", + model_sql="SELECT CAST('not a number' AS INTEGER) AS order_id", + expected_sql_fragment="CAST('not a number' AS INTEGER)", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failed_statement_when_recalled_then_its_sql_is_readable( + test_case: FailedStatementSqlTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + _ = (project_dir / "models" / "orders.sql").write_text( + f'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + f"{test_case.model_sql}\n", + encoding="utf-8", + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", "orders") + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT s.sql_available, s.sql_unavailable_reason, a.text FROM statements s " + "LEFT JOIN artifacts a USING (invocation_id, statement_id) WHERE s.outcome = 'failed'", + ) + + assert built.returncode == 1 + assert [row[:2] for row in recorded] == [[1, None]] + assert test_case.expected_sql_fragment in str(recorded[0][2]) + + +@pytest.mark.parametrize( + "test_case", + [ + AuditEvidenceTestCase( + description="a warning audit's violating rows are kept by default", + recall_toml="", + severity="warn", + expected_exit_code=0, + expected_evidence=["warn", 2, 2, 0, 1], + expected_artifacts=[["audit_evidence", "application/json"]], + expected_warehouse_evidence=[('[{"email":null},{"email":null}]',)], + ), + AuditEvidenceTestCase( + description="error-only evidence does not collect a warning's rows", + recall_toml='\n[recall]\naudit_evidence = "error"\n', + severity="warn", + expected_exit_code=0, + expected_evidence=["warn", 2, 0, 0, 0], + expected_artifacts=[], + expected_warehouse_evidence=[(None,)], + ), + AuditEvidenceTestCase( + description="error-only evidence collects a failing audit's rows", + recall_toml='\n[recall]\naudit_evidence = "error"\n', + severity="error", + expected_exit_code=1, + expected_evidence=["error", 2, 2, 0, 1], + expected_artifacts=[["audit_evidence", "application/json"]], + expected_warehouse_evidence=[('[{"email":null},{"email":null}]',)], + ), + AuditEvidenceTestCase( + description="evidence turned off collects no rows", + recall_toml='\n[recall]\naudit_evidence = "off"\n', + severity="error", + expected_exit_code=1, + expected_evidence=["error", 2, 0, 0, 0], + expected_artifacts=[], + expected_warehouse_evidence=[(None,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failing_audit_when_recalled_then_its_evidence_follows_the_setting( + test_case: AuditEvidenceTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.recall_toml) + _ = (project_dir / "models" / "customers.sql").write_text( + CUSTOMERS_MODEL.replace("severity warn", f"severity {test_case.severity}"), + encoding="utf-8", + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", "customers") + with duckdb.connect(str(project_dir / "warehouse.duckdb"), read_only=True) as warehouse: + warehouse_evidence: list[tuple[object, ...]] = warehouse.execute( + "SELECT evidence_json FROM dev._sqlbuild_audit_results" + ).fetchall() + evidence: list[list[object]] = recall_rows( + project_dir, + "SELECT outcome, violation_count, evidence_count, evidence_truncated, " + "sample_digest IS NOT NULL FROM audit_results", + ) + artifacts: list[list[object]] = recall_rows( + project_dir, + "SELECT a.kind, a.media_type, a.text FROM artifacts a JOIN audit_results r " + "ON a.invocation_id = r.invocation_id AND a.ref_id = r.event_id", + ) + + assert built.returncode == test_case.expected_exit_code, built.stderr.decode(errors="replace") + assert evidence == [test_case.expected_evidence] + assert warehouse_evidence == test_case.expected_warehouse_evidence + assert [row[:2] for row in artifacts] == test_case.expected_artifacts + assert [json.loads(str(row[2])) for row in artifacts] == [ + [{"email": None}, {"email": None}] for _ in test_case.expected_artifacts + ] + + +@pytest.mark.parametrize( + "test_case", + [ + CostAttributionTestCase( + description="a collected cost records its status", + collected=( + " return RunCostSummary(status=CostStatus.PARTIAL, " + "usd_per_credit=usd_per_credit, rate_source=rate_source)\n" + ), + expected_cost_status="partial", + ), + CostAttributionTestCase( + description="a failed collection is recorded as failed", + collected=" raise ConnectionError('query history unreachable')\n", + expected_cost_status="collection_failed", + ), + ], + ids=lambda case: case.description, +) +def test_given_cost_attribution_when_a_build_ends_then_the_invocation_records_how_far_it_got( + test_case: CostAttributionTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + toml: Path = project_dir / "sqlbuild_project.toml" + _ = toml.write_text( + toml.read_text(encoding="utf-8").replace('"duckdb"', '"duckdb_costed"', 1), + encoding="utf-8", + ) + adapter: Path = project_dir / "adapters" / "warehouse" / "duckdb_costed.py" + adapter.parent.mkdir(parents=True) + _ = adapter.write_text(COSTED_ADAPTER.format(collected=test_case.collected), encoding="utf-8") + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", "orders") + recorded: list[list[object]] = recall_rows( + project_dir, "SELECT status, cost_status FROM invocations" + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert recorded == [["completed", test_case.expected_cost_status]] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index be6cfb3c96..35713fd376 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -37,6 +37,12 @@ class SnowflakeCostCollectionTestCase: expected_limitation_fragment: str | None = None +@dataclass(frozen=True) +class SnowflakeCostPurposeTestCase: + description: str + expected_purposes: list[str | None] + + @dataclass(frozen=True) class SnowflakeMergeExclusionTestCase: description: str @@ -105,6 +111,13 @@ class SnowflakeRenderTableFunctionTestCase: expected_sql: str +@dataclass(frozen=True) +class SnowflakeQueryColumnTypesTestCase: + description: str + cursor_description: tuple[tuple[object, ...], ...] + expected_types: tuple[str | None, ...] + + @dataclass(frozen=True) class SnowflakeQueryColumnNamesTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index dd72a9d08d..f3424538b0 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -29,8 +29,8 @@ class FakeSnowflakeDescribeCursor: """Cursor double exposing Snowflake-style description metadata.""" - def __init__(self, description: tuple[tuple[str], ...]) -> None: - self.description: tuple[tuple[str], ...] = description + def __init__(self, description: tuple[tuple[object, ...], ...]) -> None: + self.description: tuple[tuple[object, ...], ...] = description self.executed_sql: str | None = None self.executemany_sql: str | None = None self.executemany_rows: list[tuple[object, ...]] = [] @@ -47,6 +47,22 @@ def close(self) -> None: self.closed = True +class FakeSnowflakeQueryCursor(FakeSnowflakeDescribeCursor): + """Cursor double returning rows under Snowflake result metadata.""" + + def __init__( + self, description: tuple[tuple[object, ...], ...], rows: list[tuple[object, ...]] + ) -> None: + super().__init__(description=description) + self._rows: list[tuple[object, ...]] = rows + + def fetchall(self) -> list[tuple[object, ...]]: + return self._rows + + def fetchmany(self, size: int) -> list[tuple[object, ...]]: + return self._rows[:size] + + class FakeSnowflakeDescribeConnection: """Connection double returning a preconfigured describe cursor.""" diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 80b645cce2..57f8ea7ea7 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -13,6 +13,7 @@ from sqlbuild.adapter.contract.models import ( ColumnInfo, ExpressionInferenceProfile, + QueryResult, RelationInfo, SchemaDiffResult, TableFreshnessMetadata, @@ -36,6 +37,7 @@ SnowflakePruneSqlTestCase, SnowflakeQualifiedColumnInspectionTestCase, SnowflakeQueryColumnNamesTestCase, + SnowflakeQueryColumnTypesTestCase, SnowflakeRelationAgeMetadataTestCase, SnowflakeRenderCloneTestCase, SnowflakeRenderCursorBoundLiteralTestCase, @@ -55,6 +57,7 @@ FakeSnowflakeMetadataConnection, FakeSnowflakeMetadataCursor, FakeSnowflakeMetadataSequenceConnection, + FakeSnowflakeQueryCursor, FakeSnowflakeRawConnection, build_bulk_columns_rows, build_bulk_sequence_connection, @@ -1155,6 +1158,43 @@ def test_given_equivalent_types_when_diffing_schema_then_snowflake_ignores_alias assert result == test_case.expected_result +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeQueryColumnTypesTestCase( + description="numbers carry precision and scale, other types their name", + cursor_description=( + ("ORDER_ID", 0, None, None, 38, 0, False), + ("AMOUNT", 0, None, None, 10, 2, True), + ("STATUS", 2, None, 16777216, None, None, True), + ("ORDERED_AT", 8, None, None, 0, 9, True), + ), + expected_types=("NUMBER(38,0)", "NUMBER(10,2)", "TEXT", "TIMESTAMP_NTZ"), + ), + SnowflakeQueryColumnTypesTestCase( + description="an unknown type code has no name", + cursor_description=(("PAYLOAD", 99, None, None, None, None, True),), + expected_types=(None,), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_result_metadata_when_querying_then_column_types_are_named( + test_case: SnowflakeQueryColumnTypesTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + cursor: FakeSnowflakeQueryCursor = FakeSnowflakeQueryCursor( + description=test_case.cursor_description, + rows=[tuple(None for _ in test_case.cursor_description)], + ) + + result: QueryResult = adapter.query( + connection=FakeSnowflakeDescribeConnection(cursor), sql="SELECT 1", limit=20 + ) + + assert result.column_types == test_case.expected_types + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py index afbc4b0c70..aa63188772 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py @@ -10,8 +10,16 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus +from sqlbuild.observability import ( + EventDispatcher, + LifecycleEvent, + dispatcher_scope, + invocation_scope, +) +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeCostCollectionTestCase, + SnowflakeCostPurposeTestCase, ) @@ -37,6 +45,21 @@ def __call__(self, config: dict[str, object]) -> _CostConnection: return self.connection +class _QueryHistoryCollect: + """Runs one query-history statement, as the real collector does.""" + + def __call__(self, **kwargs: Any) -> RunCostSummary: + with StatementLifecycle( + adapter="snowflake", sql="SELECT 1 FROM query_history", intent="query" + ) as lifecycle: + lifecycle.completed(row_count=0) + return RunCostSummary( + status=CostStatus.COMPLETE, + usd_per_credit=kwargs["usd_per_credit"], + rate_source=kwargs["rate_source"], + ) + + class _Collect: def __init__(self, *, error: bool) -> None: self.error: bool = error @@ -116,3 +139,46 @@ def test_given_telemetry_stage_failure_when_collecting_then_safe_status_is_retur assert connect.config["network_timeout"] == 15 assert (test_case.expected_message_fragment or "") in (summary.message or "") assert (test_case.expected_limitation_fragment or "") in " ".join(summary.limitations) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostPurposeTestCase( + description="query-history statements are recorded as cost telemetry", + expected_purposes=["cost_telemetry", "cost_telemetry"], + ), + ], + ids=lambda case: case.description, +) +def test_given_cost_collection_when_it_queries_history_then_its_statements_name_their_purpose( + test_case: SnowflakeCostPurposeTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + monkeypatch.setattr( + adapter, "connect", _Connect(connection=_CostConnection(close_error=False), error=False) + ) + monkeypatch.setattr( + "sqlbuild.adapters.snowflake.classes.snowflake_adapter.collect_snowflake_cost", + _QueryHistoryCollect(), + ) + events: list[LifecycleEvent] = [] + dispatcher: EventDispatcher = EventDispatcher() + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + now: datetime = datetime(2026, 8, 23, tzinfo=UTC) + + with invocation_scope("inv-cost-purpose"), dispatcher_scope(dispatcher): + _ = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id="run-1", + started_at=now, + completed_at=now, + statement_ledger_path=tmp_path / "statements.jsonl", + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert [event.payload.get("purpose") for event in events] == test_case.expected_purposes diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index 0c6990f34e..97f857f613 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -384,9 +384,9 @@ def test_given_positional_selectors_when_parsing_then_they_join_the_selection( ), OutputFormatAliasTestCase( description="query explicit format", - argv=("query", "SELECT 1", "--format", "csv"), + argv=("query", "SELECT 1", "--format", "long"), format_attribute="query_format", - expected_format="csv", + expected_format="long", ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py index 065d9b916a..a8dc4ef844 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py @@ -7,6 +7,6 @@ class QueryOutputTestCase: description: str result: QueryResult - output_format: str + output_format: str | None limit: int | None expected_output: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py index 74548abcd1..41cae555f3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py @@ -48,8 +48,7 @@ "-[ RECORD 1 ]---------------------------+\n" "id | 1\n" "\n" - "1 row\n" - "Showing 1 rows. Use --limit to show more or --no-limit to disable the limit.\n" + "1 row shown (limit 1; use --limit or --no-limit)\n" ), ), QueryOutputTestCase( @@ -60,11 +59,53 @@ expected_output='[{"id": 1, "name": "alice"}]\n', ), QueryOutputTestCase( - description="renders csv rows", - result=QueryResult(columns=("id", "name"), rows=((1, "alice"),)), - output_format="csv", + description="leaves a type cell empty where the adapter reports no type", + result=QueryResult( + columns=("order_id", "status"), + rows=((7, "open"),), + column_types=("INTEGER", None), + ), + output_format=None, + limit=20, + expected_output=( + "┌──────────┬────────┐\n" + "│ order_id │ status │\n" + "│ INTEGER │ │\n" + "├──────────┼────────┤\n" + "│ 7 │ open │\n" + "└──────────┴────────┘\n" + "\n" + "1 row\n" + ), + ), + QueryOutputTestCase( + description="renders a box with the warehouse types when the result fits", + result=QueryResult( + columns=("order_id", "amount"), + rows=((7, "12.50"),), + column_types=("INTEGER", "DECIMAL(10,2)"), + ), + output_format=None, limit=20, - expected_output="id,name\n1,alice\n", + expected_output=( + "┌──────────┬───────────────┐\n" + "│ order_id │ amount │\n" + "│ INTEGER │ DECIMAL(10,2) │\n" + "├──────────┼───────────────┤\n" + "│ 7 │ 12.50 │\n" + "└──────────┴───────────────┘\n" + "\n" + "1 row\n" + ), + ), + QueryOutputTestCase( + description="renders records when a value spans lines, so nothing is cut", + result=QueryResult(columns=("note",), rows=(("first\nsecond",),)), + output_format=None, + limit=20, + expected_output=( + "-[ RECORD 1 ]---------------------------+\nnote | first\nsecond\n\n1 row\n" + ), ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 2db7418051..c9186d4175 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1190,7 +1190,7 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: "query", "select 1 as id", "--format", - "table", + "long", "--limit", "5", ], @@ -1222,7 +1222,7 @@ def run_query( ) assert exit_code == test_case.expected_exit_code - assert received_args == [(None, "select 1 as id", None, "table", 5)] + assert received_args == [(None, "select 1 as id", None, "long", 5)] @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cost/_test_types.py b/tests/unit/src/sqlbuild/cost/_test_types.py index 158999812c..145f865885 100644 --- a/tests/unit/src/sqlbuild/cost/_test_types.py +++ b/tests/unit/src/sqlbuild/cost/_test_types.py @@ -104,6 +104,13 @@ class CostResourceScopeTestCase: expected_attempt: int +@dataclass(frozen=True) +class SnowflakeTimeoutOutcomeTestCase: + description: str + timeout_window: timedelta + expected_failure_outcomes: set[str | None] + + @dataclass(frozen=True) class SnowflakeScaleTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py index 7664da0998..04fadde417 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py @@ -22,12 +22,20 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus +from sqlbuild.observability import ( + EventDispatcher, + LifecycleEvent, + dispatcher_scope, + invocation_scope, +) +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from tests.unit.src.sqlbuild.cli.commands._helpers.cost.helpers import write_snowflake_project from tests.unit.src.sqlbuild.cost._test_types import ( SnowflakeCostBudgetTestCase, SnowflakeCostFailureMessageTestCase, SnowflakeLargeRunCostTestCase, SnowflakeScaleTestCase, + SnowflakeTimeoutOutcomeTestCase, ) from tests.unit.src.sqlbuild.cost.helpers import ( build_cost_run_record, @@ -215,6 +223,69 @@ def test_given_large_run_when_collecting_snowflake_cost_then_sql_stays_bounded( assert len(history.reads) <= test_case.expected_maximum_reads +class _ObservedHistory(_FakeQueryHistory): + """Query history read through statement lifecycles, as the Snowflake connection records them.""" + + def execute(self, sql: str, **kwargs: Any) -> _Cursor: + with StatementLifecycle(adapter="snowflake", sql=sql, intent="query") as lifecycle: + try: + cursor: _Cursor = super().execute(sql, **kwargs) + except Exception as error: + lifecycle.failed(error=error) + raise + lifecycle.completed(row_count=len(cursor.rows)) + return cursor + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeTimeoutOutcomeTestCase( + description="a timed-out read retried in smaller windows is handled", + timeout_window=timedelta(minutes=8), + expected_failure_outcomes={"handled"}, + ), + SnowflakeTimeoutOutcomeTestCase( + description="a timed-out read too narrow to split stays failed", + timeout_window=timedelta(0), + expected_failure_outcomes={"handled", None}, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_history_read_timeout_when_collecting_then_only_retried_reads_are_handled( + test_case: SnowflakeTimeoutOutcomeTestCase, tmp_path: Path +) -> None: + duration: timedelta = timedelta(minutes=42) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=6_500, duration=duration + ) + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id=_RUN_ID, rows=rows) + events: list[LifecycleEvent] = [] + dispatcher: EventDispatcher = EventDispatcher() + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + + with invocation_scope("inv-history-timeout"), dispatcher_scope(dispatcher): + _ = collect_snowflake_cost( + connection=_ObservedHistory(rows=rows, timeout_window=test_case.timeout_window), + run_id=_RUN_ID, + started_at=_ORIGIN, + completed_at=_ORIGIN + duration, + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + attempts=1, + ) + + failures: list[LifecycleEvent] = list( + filter(lambda event: event.event_type == "statement_failed", events) + ) + assert {event.payload.get("outcome") for event in failures} == ( + test_case.expected_failure_outcomes + ) + + class _ClosingHistory(_FakeQueryHistory): def close(self) -> None: return None diff --git a/tests/unit/src/sqlbuild/runtime/observability/_test_types.py b/tests/unit/src/sqlbuild/runtime/observability/_test_types.py index 5105567e38..64a82652bc 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/observability/_test_types.py @@ -374,3 +374,10 @@ class FailedMonitorStartCase: description: str threshold_seconds: float expected_event_types: tuple[str, ...] + + +@dataclass(frozen=True) +class HandledFailureCase: + description: str + error: BaseException + expected_outcome: str | None diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py b/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py index 4c6692bd76..257f95b52a 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py @@ -22,6 +22,7 @@ from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from tests.unit.src.sqlbuild.runtime.observability._test_types import ( ErrorCodePrivacyCase, + HandledFailureCase, StatementKindPrivacyCase, StatementLifecycleCase, ) @@ -659,3 +660,44 @@ def test_given_driver_error_code_when_failing_statement_then_only_safe_identifie assert events[-1].payload.get("error_code") == test_case.expected_error_code encoded_events: str = repr(events) assert all(fragment not in encoded_events for fragment in test_case.private_fragments) + + +class _StatementTimeoutError(Exception): + pass + + +@pytest.mark.parametrize( + "test_case", + [ + HandledFailureCase( + description="a failure the caller recovers from is marked handled", + error=_StatementTimeoutError("statement timed out"), + expected_outcome="handled", + ), + HandledFailureCase( + description="any other failure stays a failure", + error=ValueError("division by zero"), + expected_outcome=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_handled_failure_scope_when_a_statement_fails_then_only_recovered_failures_are_handled( + test_case: HandledFailureCase, +) -> None: + events: list[LifecycleEvent] = [] + dispatcher: EventDispatcher = EventDispatcher() + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + + with ( + invocation_scope("inv-handled-failure"), + dispatcher_scope(dispatcher), + StatementLifecycle.handled_failure_scope( + lambda error: isinstance(error, _StatementTimeoutError) + ), + StatementLifecycle(adapter="snowflake", sql="SELECT 1", intent="execute") as lifecycle, + ): + lifecycle.failed(error=test_case.error) + + assert events[-1].event_type == "statement_failed" + assert events[-1].payload.get("outcome") == test_case.expected_outcome diff --git a/uv.lock b/uv.lock index bf5d70d180..b10ee0c368 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.27.0" +version = "0.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6a/a3/d1eea20ab654f3c4b1b170cc8eeaf27d5092095cd74e6bc819bfff8d9312/fensu-0.27.0.tar.gz", hash = "sha256:023692a73c4ac2fdadbb167c59a58dc944c7a41b0d78a40fa260f6923a8b3e7b", size = 538895, upload-time = "2026-10-17T05:38:49.174Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/69/a132ffc407be97a0fdcfa052cb8b49408091db748e6afe0f7a6c77e6fa7d/fensu-0.28.0.tar.gz", hash = "sha256:11bcea3207f7fd92babc200f40d074e3da789ee11e7fc7024575a415de7c1176", size = 538990, upload-time = "2026-10-27T00:39:16.19Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/b7/7b4053d22df1307dbaebb4ed48280bb9c3663ee649005eb6ab5e2fec28b8/fensu-0.27.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d982c4fec0d2210b78fcf67098c32ca9f8b8592f338b2a327fcd1747b59bb310", size = 4154209, upload-time = "2026-10-17T05:38:28.74Z" }, - { url = "https://files.pythonhosted.org/packages/7a/0d/b7058d187e9908ec18f245c1b168c24b746633a0b13666d69052157da3eb/fensu-0.27.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:8c3cca4a4391028d7b07bb9ead34340f6ba0662c36ea4ef31f69b44a46847289", size = 4004244, upload-time = "2026-10-17T05:38:31.356Z" }, - { url = "https://files.pythonhosted.org/packages/f8/bc/b1eee94e617e954a2188c1802d3770b27624a97f775cbed310f54389c100/fensu-0.27.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5640ffe3cf8916b1533dfe51ca27d65bcea71c9fbd62ae5cfaa2079e8007bdaf", size = 4230195, upload-time = "2026-10-17T05:38:33.057Z" }, - { url = "https://files.pythonhosted.org/packages/7c/01/9f895e6a4427c8ee80ce5d779aceab5a2b994a156a4c1f3bf664108ab489/fensu-0.27.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7d26d2713a69f2e05b07c6f778e1e2f86896e26b74247e60952018288381c04", size = 4307270, upload-time = "2026-10-17T05:38:34.827Z" }, - { url = "https://files.pythonhosted.org/packages/08/61/2a380a704a49eea91a3539935af7b2d4ad09aa1d87d787f87c1de847ae4b/fensu-0.27.0-cp312-abi3-win_amd64.whl", hash = "sha256:15a9aef82221f25812d9c4f74e782dd677c73d566980eaed60b762da13ee2b3d", size = 3977639, upload-time = "2026-10-17T05:38:36.612Z" }, + { url = "https://files.pythonhosted.org/packages/59/e9/7434bd0565d54ff58796105728b3ccc8b838c7200997184978e6ebfdec4d/fensu-0.28.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:29df31aff900d20b0c845940af0cd91eead55e158a2511e682e3e887a22dada8", size = 4154207, upload-time = "2026-10-27T00:38:52.311Z" }, + { url = "https://files.pythonhosted.org/packages/7f/d8/e0875b269515d9a75643d5488bdc6f70a4b6c346995f12ce07304a3610f7/fensu-0.28.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:33a6d8b7d2f734c48f48087462c0e5a1ff94535c0e27e6a4820bc10303ff3653", size = 4004243, upload-time = "2026-10-27T00:38:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/e8/05/b87c0ca5cf3f724517d6958f1ebd68948f9399306bd6d6814783635bad9a/fensu-0.28.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de07b3fee5d908d2a766ca136753c8040d6fe25a1346d9b3b5a98483fc23d874", size = 4230192, upload-time = "2026-10-27T00:38:57.321Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d2/fdfc7fd322c3497c53c22754a33433e33678a6de84832fd41add09481a21/fensu-0.28.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f44293a2da6bf4558d00eb4f859b28f3fb69d396dad1c25ac466a92a7d1dd734", size = 4307267, upload-time = "2026-10-27T00:38:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/56/c0/2ab2c2c1af4ddd1b45c750679bd9f0b777d7eca3c0e00a233c991aaa13e7/fensu-0.28.0-cp312-abi3-win_amd64.whl", hash = "sha256:ffa88f1d259a1830d513ae930ff856d5b5e4941ba433889dea04d3db5753f281", size = 3977828, upload-time = "2026-10-27T00:39:01.761Z" }, ] [[package]] name = "fensu-cli" -version = "0.27.0" +version = "0.28.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/97/2540b6ea4bca959ac577502fe0c96b432c3b4e694daa1ca68df3a2a06b8a/fensu_cli-0.27.0.tar.gz", hash = "sha256:4276e7fcf7dddc4c36afc347536bf1fd7114fff246ad344544991a1edf7b4ee3", size = 760869, upload-time = "2026-10-17T05:38:50.61Z" } +sdist = { url = "https://files.pythonhosted.org/packages/79/d9/f347f514fb224f6a1f4ca6e616ac49b573f9dceba4ff8d9efcd334f48bea/fensu_cli-0.28.0.tar.gz", hash = "sha256:1feb379ed92e193cbd4d28ab3d7729798b88120aaee2f59fd7f341519b6ee945", size = 765638, upload-time = "2026-10-27T00:39:17.752Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/69/9491487602985731af4a7363754970bb35baece4e27cf2a7a0a95ebe4ae7/fensu_cli-0.27.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e90c8d298904d02c639e87541ba51a31ea3912af0bc354cdfea9d95f4428fd41", size = 7935880, upload-time = "2026-10-17T05:38:38.606Z" }, - { url = "https://files.pythonhosted.org/packages/de/4e/7e84f824b3902708ce4c8399f3402fb278cba9c48ed37877d67a4bad5245/fensu_cli-0.27.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1a9f3a2fb8ec8930af9b5cd59f3f1c027cd80a1ebae1a22c7d07133b80d7c65a", size = 7457913, upload-time = "2026-10-17T05:38:40.624Z" }, - { url = "https://files.pythonhosted.org/packages/05/9f/fc20897baafdac0039faef2cc3a324b059fdfe7f4a1522bac386f1e5f8ae/fensu_cli-0.27.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf73108b3778d0d74e11857e743042f4241174ec8f16aeee7e5c8c4769e41e7b", size = 7537447, upload-time = "2026-10-17T05:38:42.834Z" }, - { url = "https://files.pythonhosted.org/packages/46/be/1fc7c87cbf320ff215dcb204df808c592fefec6a0892a1103564a1c69b76/fensu_cli-0.27.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:435dad914ec0d061d5fceb10d8b049f75cdb5c031a156efe2f8309647c744b8c", size = 8035160, upload-time = "2026-10-17T05:38:45.074Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5b/53050980866370c7a41dc7e3e61de86cf67e5848d8560e6a9d166040679a/fensu_cli-0.27.0-py3-none-win_amd64.whl", hash = "sha256:68e91a38dab7e081c4ba504e22a7709169d8d6c470137834adcad116b52fa280", size = 7652792, upload-time = "2026-10-17T05:38:47.26Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b3/57f1de89e1cb7534b0ef6415822c8982a247b9d2efe8d2fddd85e35389c8/fensu_cli-0.28.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13ecc7a224a4b32eef19456f21659e682e636ef53f305919aa57cae27c8e3fec", size = 7978247, upload-time = "2026-10-27T00:39:04.644Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ff/2e887d727f82c61ea7cb6773429b61d4711b93f7f94814eb9a7e891384dc/fensu_cli-0.28.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:149e5e22c8895c0a3ed3956b30e693e78df7dfb67ce8e0c5682b84c77e811e7e", size = 7490546, upload-time = "2026-10-27T00:39:07.384Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/daa4c2d70e5b0398382f84c846aacc4a937c4d00b3938d00a67351b0f212/fensu_cli-0.28.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:99932d17956bb10da1ac677867567b504a961d0565fcda730c1e682434244113", size = 7570220, upload-time = "2026-10-27T00:39:09.852Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f9/c9e1095c507da3a53b3e7bf3f7c95f75824429ea45a72cb2c7af67a605f7/fensu_cli-0.28.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee51c5d6d0d5fa0b36c0f70d3d6fde429793b643f0075cdfdf3fc8b8d73e3ccb", size = 8086364, upload-time = "2026-10-27T00:39:12.052Z" }, + { url = "https://files.pythonhosted.org/packages/9c/89/ece32e452d2b4ade243dd66a2612619be7400f6acf59f854ce674bb78e30/fensu_cli-0.28.0-py3-none-win_amd64.whl", hash = "sha256:acca6ac39cbe0614d3a8e4ce7c2c12cc5fd3c800fc4c66b5484c54dc19ef4a66", size = 7700477, upload-time = "2026-10-27T00:39:14.15Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.27.0" }, + { name = "fensu", specifier = ">=0.28.0" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 0bb775514f..3fbe67637a 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -66,7 +66,9 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb 4. **Column lineage.** Traces column-level dependencies across models. 5. **Contracts.** Checks declared columns against the inferred query output. 6. **Rules.** Runs the selected built-in Rules, then the selected custom Rules. -7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. +7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. When they fail, it + writes only the models' SQL, so you can read what the Rules checked; see + [When Rules fail](#when-rules-fail). Rules that need inferred types, such as `SQBRCONTRACT105`, are skipped when SQL analysis is off, and compile prints one note naming them. @@ -75,6 +77,23 @@ and compile prints one note naming them. findings block the artifacts. To check one Rule code or family without changing project configuration, use [`sqb rules run`](/docs/cli/rules). +### When Rules fail + +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a +`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each +model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros +generated, and the compiled file of a model you removed from the project is deleted. Other +compiled files stay as the last passing compile left them. A model with Rule errors starts with a +header that marks it not buildable and lists its findings: + +```sql +-- sqb: compile failed for this model; not buildable +-- SQBRSQL028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns +SELECT o.order_id, status FROM __ref("orders") AS o +``` + +The next compile that passes writes the files again without the header. + ## Compiling a selection Selectors limit analysis and reporting. Discovery still covers the whole project, so every diff --git a/website/src/content/docs/docs/cli/query.mdx b/website/src/content/docs/docs/cli/query.mdx index 94128a39b7..b5805ed023 100644 --- a/website/src/content/docs/docs/cli/query.mdx +++ b/website/src/content/docs/docs/cli/query.mdx @@ -29,68 +29,70 @@ File paths are resolved from the current working directory. |------|-------------| | `sql` | SQL to execute (positional argument) | | `--file` | Read SQL from a file instead of the command line | -| `--format` | Output format: `long` (default), `table`, `json`, or `csv` | +| `--format` | `long` prints one record per row; `json` prints a JSON array. Without it, the [default output](#output) applies | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 20) | | `--no-limit` | Disable the row limit | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | -## Output formats +## Output -### long (default) +By default `sqb query` prints a table with each column's name and warehouse type: -Vertical record format, one field per line: +``` +┌──────────┬─────────────┬───────────┐ +│ order_id │ customer_id │ status │ +│ INTEGER │ INTEGER │ VARCHAR │ +├──────────┼─────────────┼───────────┤ +│ 1 │ 100 │ completed │ +│ 2 │ 200 │ completed │ +└──────────┴─────────────┴───────────┘ +2 rows ``` --[ RECORD 1 ]---------------------------+ -order_id | 1 -customer_id | 100 -status | completed --[ RECORD 2 ]---------------------------+ -order_id | 2 -customer_id | 200 -status | completed +The table must fit the terminal width, or 120 characters when output is redirected. When it is +wider, or a value spans several lines, `sqb query` prints one record per row instead, so no column +or value is ever cut. Numbers are right-aligned. Types show where the adapter reports them. -2 rows +The last line counts the rows. When the limit cut rows off, it says so and how to see more: + +``` +20 rows shown (limit 20; use --limit or --no-limit) ``` -### table +Above 50 rows it suggests reading fewer: `60 rows; to read fewer, aggregate with GROUP BY or add a +LIMIT`. -Horizontal table format: +### long + +`--format long` always prints one record per row: ``` -order_id | customer_id | status --------- | ----------- | --------- -1 | 100 | completed -2 | 200 | completed +-[ RECORD 1 ]---------------------------+ +order_id | 1 +customer_id | 100 +status | completed 2 rows ``` ### json -JSON array of objects: +`--format json`, or `--json`, prints a JSON array of objects for programs: ```json [{"order_id": 1, "customer_id": 100, "status": "completed"}] ``` -### csv - -Standard CSV with headers: - -```csv -order_id,customer_id,status -1,100,completed -2,200,completed -``` +`--format table` and `--format csv` are not accepted. The default output already prints a table +when the result fits; use `--json` for programs. ## Errors When the warehouse rejects the SQL, `sqb query` prints the warehouse message as `error[C108]: ...` on stderr and exits with code `2`. Nothing is written to stdout, so -redirected `json` or `csv` output is never partial. Invalid command input, such as a missing +redirected `json` output is never partial. Invalid command input, such as a missing query file, exits with code `1`. ## Examples @@ -99,8 +101,8 @@ query file, exits with code `1`. # Quick inspection with default format sqb query "SELECT * FROM dev.fact_orders" -# Table format with higher limit -sqb query "SELECT * FROM dev.dim_customers" --format table --limit 50 +# One record per row with a higher limit +sqb query "SELECT * FROM dev.dim_customers" --format long --limit 50 # Export to JSON sqb query "SELECT * FROM dev.daily_revenue" --json --no-limit diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index b7d15976ec..d219edfe46 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -26,7 +26,7 @@ Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| | `sql` | The query (positional argument) | -| `--format` | `long` (default) for reading, `json` for programs, or `csv` for export. `table` still works in this release but prints a deprecation warning and will be removed | +| `--format` | `long` prints one record per row; `json` is for programs. Without it, a table when the result fits, as [`sqb query`](/docs/cli/query#output) prints it | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 100) | | `--no-limit` | Disable the row limit | @@ -43,15 +43,21 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -command | build -status | completed -exit_code | 0 -duration_ms | 752.0 +┌─────────┬───────────┬───────────┬─────────────┐ +│ command │ status │ exit_code │ duration_ms │ +│ text │ text │ integer │ real │ +├─────────┼───────────┼───────────┼─────────────┤ +│ build │ completed │ 0 │ 752.0 │ +└─────────┴───────────┴───────────┴─────────────┘ 1 row ``` +The second header line is each column's type in the store. A result wider than the terminal (120 +characters when output is redirected), or with a value spanning lines, prints one record per row +instead, so nothing is cut. `--format table` and `--format csv` are not accepted: the default +output already prints a table when it fits, and `--json` is for programs. + The slowest statements of past builds, with their SQL: ```bash @@ -62,15 +68,13 @@ sqb recall query --max-cell-bytes 60 \ ``` ``` --[ RECORD 1 ]---------------------------+ -resource_name | orders -ms | 3.6 -sql | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] - --[ RECORD 2 ]---------------------------+ -resource_name | NULL -ms | 3.2 -sql | CREATE SCHEMA IF NOT EXISTS dev +┌───────────────┬──────┬──────────────────────────────────────────────────────────────────────────┐ +│ resource_name │ ms │ sql │ +│ text │ real │ text │ +├───────────────┼──────┼──────────────────────────────────────────────────────────────────────────┤ +│ orders │ 5.1 │ CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 AS o…[+81 bytes] │ +│ NULL │ 3.2 │ CREATE SCHEMA IF NOT EXISTS dev │ +└───────────────┴──────┴──────────────────────────────────────────────────────────────────────────┘ 2 rows ``` diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 06db321a40..5b39a4fd3d 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -61,8 +61,19 @@ MODEL ( ``` `minimum_samples` keeps low-volume measurements distinct as `insufficient` rather than inventing a -pass or failure. Evidence is diagnostic and bounded by `evidence_limit`; the measurement and -threshold determine the outcome. +pass or failure. The measurement and threshold determine the outcome. + +## Evidence + +When an audit warns or fails, SQLBuild keeps some of the rows behind the failure as evidence: a +violation audit keeps the rows its query returned, and a measurement audit runs its `EVIDENCE` +query. `evidence_limit` caps the rows kept (default 20), and at most 64 KB of them are kept. Set it +on any attachment, for example `not_null (evidence_limit 5)`. + +Evidence is diagnostic and never changes the outcome. It holds data values from the audited +relation, and is recorded in the `evidence_json` column of `_sqlbuild_audit_results`, as +`evidence` in the `audit_completed` lifecycle event, and in +[Recall](/docs/concepts/observability/recall#audit-evidence), which `audit_evidence` controls. ## Audit factories diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 7d87eff1b0..7948e1daeb 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -301,6 +301,16 @@ fully typed. - Every read inside `USING (...)` stands unqualified, as `USING` needs, including `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. +- Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and + unqualified columns, judges a query the macro writes whole, such as a CTE of a CTE-list macro, + once, in the macro file. There every plain read counts as unqualified and every + `alias.@weekly...` read as qualified. A step written only in plain reads, such as + `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, passes; to qualify, read + every slot through an alias. +- Where a call's output joins the caller's query, as with expression and column-list macros, the + query is judged at the call. A read through the relation the call passes, such as `o.created_at` + above, fits either style; any other column counts as the macro writes it, so a bare `status` + beside a hand-qualified `o.order_id` fails. A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, could bind its bare columns to the subquery's relations instead of the one the slot reads. The @@ -489,8 +499,10 @@ A `* REPLACE` item takes its source column's type rather than the call's. - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. - [Rules](/docs/concepts/rules) and [column lineage](/docs/concepts/column-lineage) read the expanded - SQL, so they see through every call. A rule finding inside an expansion is reported at its - position in the expanded query, not in the macro file. + SQL, so they see through every call. A Rule finding inside an expansion is reported at the call, + marked `(in SQL generated by @sql.name(...))`. The model's expanded SQL is in + `target/compiled/models/`, also when the Rules fail; see + [When Rules fail](/docs/cli/compile#when-rules-fail). - Editing a macro recompiles its callers. Test SQL macros with [macro-mode unit tests](/docs/concepts/testing#macro-tests), and replace one diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 401c4f772a..b459907887 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -23,32 +23,30 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -run | 2026-10-20T00:01:44.784683Z -outcome | warn -violation_count | 1 -status | completed -exit_code | 0 - --[ RECORD 2 ]---------------------------+ -run | 2026-10-20T00:00:34.652468Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 - --[ RECORD 3 ]---------------------------+ -run | 2026-10-19T23:59:23.827831Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 +┌─────────────────────────────┬─────────┬─────────────────┬───────────┬───────────┐ +│ run │ outcome │ violation_count │ status │ exit_code │ +│ text │ text │ integer │ text │ integer │ +├─────────────────────────────┼─────────┼─────────────────┼───────────┼───────────┤ +│ 2026-10-20T00:01:44.784683Z │ warn │ 1 │ completed │ 0 │ +│ 2026-10-20T00:00:34.652468Z │ pass │ 0 │ completed │ 0 │ +│ 2026-10-19T23:59:23.827831Z │ pass │ 0 │ completed │ 0 │ +└─────────────────────────────┴─────────┴─────────────────┴───────────┴───────────┘ 3 rows ``` The audit started warning on the latest build, and that build still exited 0, so nothing alerted. +See the rows the audit flagged, which Recall keeps as the audit's evidence: + +```bash +sqb recall query \ + "SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + WHERE a.audit_name = 'revenue_is_non_negative' AND a.outcome <> 'pass' + ORDER BY a.invocation_started_at DESC LIMIT 1" +``` + See what that build printed, exactly as the scheduler saw it: ```bash @@ -70,11 +68,12 @@ sqb recall query --backend warehouse \ ``` ``` --[ RECORD 1 ]---------------------------+ -payment_id | 9 -order_id | 8 -amount_cents | -2200 -paid_at | 2026-04-03 15:42:00 +┌────────────┬──────────┬──────────────┬─────────────────────┐ +│ payment_id │ order_id │ amount_cents │ paid_at │ +│ INTEGER │ INTEGER │ INTEGER │ TIMESTAMP │ +├────────────┼──────────┼──────────────┼─────────────────────┤ +│ 9 │ 8 │ -2200 │ 2026-04-03 15:42:00 │ +└────────────┴──────────┴──────────────┴─────────────────────┘ 1 row ``` @@ -91,21 +90,25 @@ For every recorded invocation, Recall keeps: for a failed invocation its cause as `error_type`, `error_code` and a one-line `error_message`, also when the command reported the failure only through its exit code (for example `sqb debug` failing to connect records `ConnectionError` and `C980`); +- the selectors it ran with, see [Selection](#selection); +- whether it finished attributing warehouse cost, as `cost_status`: `complete`, `partial`, + `pending`, `unavailable` or `collection_failed`, or `NULL` for an adapter without cost + attribution; - the commit it ran from (`git_revision`), see [Git revision](#git-revision); - where it came from (`origin`) and, for an orchestrated run, the job and run that started it, see [Where a run came from](#where-a-run-came-from); - its [lifecycle events](/docs/concepts/observability): runs, resource attempts, statements, retries, and audit results; -- the SQL text of every executed statement, stored once per distinct statement and shared across - invocations; +- the SQL text of every executed statement, including statements that failed or were cancelled, + stored once per distinct statement and shared across invocations; +- the rows a failing audit flagged, see [Audit evidence](#audit-evidence); - its stdout and stderr, byte for byte, in the order they were written, up to [`max_output_bytes`](#output-limit) of each stream; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all of its history still succeeds, prints a warning followed by its own final status, and -`output_invocations` reports the gap. An invocation whose process died without finishing has -status `abandoned`. +`output_invocations` reports the gap. `sqb recall` only reads history: it publishes no lifecycle events, runs no project sinks, and keeps no compute logs, so a query starts quickly. Every command is recorded except `sqb recall` itself, commands that create a project (such as @@ -114,7 +117,59 @@ recorded either: a `sqb compile` answered from the launcher's warm cache, and `s `sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store -can contain sensitive SQL and output. +can contain sensitive SQL, output and audit evidence. + +## How a run ended + +`invocations.status` is one of: + +| Status | Meaning | +|--------|---------| +| `completed` | The command exited 0 | +| `failed` | The command exited with an error | +| `cancelled` | The command was stopped by SIGTERM or Ctrl-C; `signal_number` is `15` or `2` | +| `running` | The command is still running: its heartbeats are recent | +| `abandoned` | Its heartbeats stopped without an end: the process died, for example after SIGKILL or running out of memory | +| `unknown` | Recall has neither an end nor a heartbeat for it | + +A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it +cancels the statements it is running, records its real end time, its output and exit code 143, +prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process +at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run +stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. + +## Selection + +`invocations.selection` is the JSON array of selectors the command ran with: positional selectors, +`--select`, and the selectors read from each `--select-file`, whose paths are in `selector_files`. +`selection_count` counts them. At most 64 KiB of selectors are kept; beyond that the list is cut +and `selection_truncated` is `1`, while `selection_count` still counts all of them. + +## Audit evidence + +When an audit warns or fails, Recall keeps its [evidence](/docs/concepts/audits#evidence): up to +`evidence_limit` of the rows it flagged. `audit_results` counts them in `evidence_count`, sets +`evidence_truncated` to `1` when the limit cut rows off, and names the stored rows by +`sample_digest`. Read them from `artifacts`, joined on the audit's `event_id`: + +```sql +SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id +WHERE a.outcome <> 'pass' +``` + +The evidence holds data values from your warehouse. Choose which failures keep it with +`audit_evidence`: + +| Value | Keeps evidence for | +|-------|--------------------| +| `warn-and-error` (default) | Audits that warn or fail | +| `error` | Audits that fail at error severity | +| `off` | No audit | + +`evidence_count` always counts the rows the audit kept as evidence; `sample_digest` is `NULL` when +Recall did not store them. ## Git revision @@ -161,6 +216,7 @@ retention = "30d" # days of history to keep; also accepts hours, suc shutdown_timeout = "3s" # longest delay Recall may add when a command exits diagnostics_min_severity = "info" max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything +audit_evidence = "warn-and-error" [targets.ci.recall] enabled = false # do not record runs against the ci target @@ -175,6 +231,7 @@ enabled = false # do not record runs against the ci target | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | | `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | +| `audit_evidence` | `"warn-and-error"` | Which failing audits keep the rows they flagged: `warn-and-error`, `error`, or `off`. See [Audit evidence](#audit-evidence) | To turn Recall off for the whole project: @@ -381,8 +438,8 @@ in the store's SQL dialect, against these views: | View | One row per | |------|-------------| | `invocations` | Invocation, with status, exit code, and timing | -| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | -| `audit_results` | Evaluated audit | +| `statements` | Executed statement, with resource, kind, purpose, outcome, duration, and row counts | +| `audit_results` | Evaluated audit, with its evidence count | | `events` | Lifecycle event | | `diagnostics` | Retained diagnostic | | `artifacts` | Reference from an invocation to captured SQL, with its text | @@ -399,7 +456,16 @@ invocation fails and lists the matches. In SQL, match a prefix with Run `sqb recall schema` for every column, and check the values a column holds before filtering on it, for example `SELECT DISTINCT resource_kind, purpose FROM statements`: a model's `resource_kind` is its materialization, such as `table` or `view`, and its build statements have -`purpose = 'model_build'`. +`purpose = 'model_build'`. On Snowflake, the statements that read query history for cost +attribution have `purpose = 'cost_telemetry'`. + +`statements.outcome` is `completed`, `failed`, `handled` or `no_terminal`. A `handled` statement +failed, and SQLBuild recovered from it: for example, a cost history read that timed out and was +read again in smaller windows. `no_terminal` means the statement never reported an end, usually +because the process died. `sql_available` is `1` when the statement's SQL is in `artifacts`; when it +is `0`, `sql_unavailable_reason` says why: `not_captured` when Recall never received the SQL, or +`expired_or_not_captured` when the statement references SQL text that is no longer stored, because +[retention](#retention) removed it or it was dropped before it was written. Every view has `invocation_id` and `invocation_started_at`. Filter on them to keep a query cheap: on a ClickHouse store, a filter on either reaches the table reads, so a query about one invocation or