feat!: harden Recall, unify query output and fix macro slot qualification

Cargo.toml

@@ -105,8 +105,13 @@ ruff_text_size = "=0.0.4"
 # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`.
 # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output.
 rustc-hash = "2"
+# The native SIGTERM handler: sets a flag the CLI watches, without unsafe code in the workspace.
+signal-hook = { version = "0.3", default-features = false }
+# Features signal-hook's registry enables, matched in the sqb build through sqlbuild-build-features.
+errno = "0.3"
+windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] }
 # Recall's local store: SQLite compiled in, so every build has the same engine version.
-rusqlite = { version = "0.37", features = ["bundled"] }
+rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] }
 rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] }
 saphyr-parser = "0.1"
 regex = "1"

crates/sqlbuild-build-features/Cargo.toml

@@ -22,6 +22,13 @@ once_cell.workspace = true
 regex-automata.workspace = true
 thiserror.workspace = true
 
+# signal-hook's registry, in the extension only, enables these on the platforms that use them.
+[target.'cfg(unix)'.dependencies]
+errno.workspace = true
+
+[target.'cfg(windows)'.dependencies]
+windows-sys.workspace = true
+
 # Proc-macro dependencies resolve with build dependencies, so the syn features PyO3's macros use
 # are enabled here. The empty build script makes Cargo resolve them.
 [build-dependencies]

crates/sqlbuild-cli/src/argv/_helpers/table.rs

@@ -131,6 +131,15 @@ impl TableReader {
                     help: help(choice.get("help").unwrap_or(&Json::Null)),
                 })
                 .collect(),
+            removed_choices: items(json, "removed_choices")
+                .iter()
+                .map(|removed| {
+                    (
+                        text(removed, "value").unwrap_or_default(),
+                        text(removed, "instead").unwrap_or_default(),
+                    )
+                })
+                .collect(),
         }
     }
 }

crates/sqlbuild-cli/src/argv/_helpers/values.rs

@@ -114,6 +114,16 @@ impl Run<'_> {
         if choices.iter().any(|choice| choice == text) {
             return Ok(());
         }
+        if let Some((_, instead)) = action
+            .removed_choices
+            .iter()
+            .find(|(removed, _)| removed == text)
+        {
+            return Err(argument_error(
+                action,
+                &format!("{} was removed; {instead}", self.host.repr(text)),
+            ));
+        }
         let listed: Vec<String> = choices
             .iter()
             .map(|choice| python_str_repr(choice))

crates/sqlbuild-cli/src/argv/command_table.json

@@ -4397,13 +4397,21 @@
          "--format"
         ],
         "dest": "query_format",
-        "default": "long",
         "choices": [
          "long",
-         "table",
-         "json",
-         "csv"
+         "json"
+        ],
+        "removed_choices": [
+         {
+          "value": "table",
+          "instead": "the default output prints a table when the result fits; use --format long for one record per row"
+         },
+         {
+          "value": "csv",
+          "instead": "use --json for programs, or the warehouse's own export for bulk data"
+         }
         ],
+        "help": "by default a table when the result fits, else one record per row; long always prints records; json for programs",
         "mutex_group": 0,
         "group": 1
        },
@@ -7183,14 +7191,21 @@
               "--format"
              ],
              "dest": "recall_format",
-             "default": "long",
              "choices": [
               "long",
-              "table",
-              "json",
-              "csv"
-             ],
-             "help": "long (default) for reading, json for programs, csv for export; table is deprecated",
+              "json"
+             ],
+             "removed_choices": [
+              {
+               "value": "table",
+               "instead": "the default output prints a table when the result fits; use --format long for one record per row"
+              },
+              {
+               "value": "csv",
+               "instead": "use --json for programs"
+              }
+             ],
+             "help": "by default a table when the result fits, else one record per row; long always prints records; json for programs",
              "mutex_group": 0,
              "group": 1
             },
@@ -7374,14 +7389,21 @@
               "--format"
              ],
              "dest": "recall_format",
-             "default": "long",
              "choices": [
               "long",
-              "table",
-              "json",
-              "csv"
-             ],
-             "help": "long (default) for reading, json for programs, csv for export; table is deprecated",
+              "json"
+             ],
+             "removed_choices": [
+              {
+               "value": "table",
+               "instead": "use the default long output"
+              },
+              {
+               "value": "csv",
+               "instead": "use --json for programs"
+              }
+             ],
+             "help": "long (default) for reading, json for programs",
              "mutex_group": 0,
              "group": 1
             },

crates/sqlbuild-cli/src/argv/constants.rs

@@ -159,7 +159,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [
     ("max_columns", NamespaceDefault::None),
     ("query_sql", NamespaceDefault::None),
     ("query_file", NamespaceDefault::None),
-    ("query_format", NamespaceDefault::Text("long")),
+    ("query_format", NamespaceDefault::None),
     ("query_limit", NamespaceDefault::Int(20)),
     ("query_no_limit", NamespaceDefault::Bool(false)),
     ("cost_selector", NamespaceDefault::Text("latest")),
@@ -228,7 +228,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [
     ("recall_command", NamespaceDefault::None),
     ("recall_sql", NamespaceDefault::None),
     ("recall_backend", NamespaceDefault::Text("logs")),
-    ("recall_format", NamespaceDefault::Text("long")),
+    ("recall_format", NamespaceDefault::None),
     ("recall_limit", NamespaceDefault::Int(100)),
     ("recall_no_limit", NamespaceDefault::Bool(false)),
     ("recall_max_cell_bytes", NamespaceDefault::None),

crates/sqlbuild-cli/src/argv/models.rs

@@ -98,6 +98,8 @@ pub struct ActionSpec {
     /// Sub-command names and their parsers, in order.
     pub parsers: Vec<(String, ParserSpec)>,
     pub choices_help: Vec<ChoiceHelp>,
+    /// Values no longer accepted, each with what to use instead; never listed in help.
+    pub removed_choices: Vec<(String, String)>,
 }
 
 /// One argument group's title and description.

crates/sqlbuild-cli/src/dispatch/constants.rs

@@ -12,6 +12,8 @@ pub(crate) const DBT_INIT_SUBCOMMAND: &str = "init";
 pub(crate) const DIFF_COMMAND_NAME: &str = "diff";
 /// `sqb recall` reads history and is never recorded itself.
 pub(crate) const RECALL_COMMAND_NAME: &str = "recall";
+/// A `--select-file` line starting with this is a comment.
+pub(crate) const SELECTOR_COMMENT_PREFIX: &str = "#";
 
 const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir);
 const SELECT: Field = field("select", Source::Select);

crates/sqlbuild-cli/src/dispatch/main/mod.rs

@@ -3,4 +3,5 @@ pub mod command_dispatch;
 pub mod event_export_warning;
 pub mod failure_report;
 pub mod invocation_policy;
+pub mod selector_file_selectors;
 pub mod unavailable_handler;

crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs

@@ -0,0 +1,17 @@
+//! The selectors a `--select-file` lists.
+
+use sqlbuild_core::text::main::python_splitlines::python_splitlines;
+use sqlbuild_core::text::main::python_strip::python_strip;
+
+use crate::dispatch::constants::SELECTOR_COMMENT_PREFIX;
+
+/// One selector per stripped line of `text`, without blank lines and `#` comments.
+#[must_use]
+pub fn selector_file_selectors(text: &str) -> Vec<String> {
+    python_splitlines(text)
+        .into_iter()
+        .map(python_strip)
+        .filter(|line| !line.is_empty() && !line.starts_with(SELECTOR_COMMENT_PREFIX))
+        .map(str::to_owned)
+        .collect()
+}

crates/sqlbuild-config/src/recall/constants.rs

@@ -2,7 +2,8 @@
 
 pub const RECALL_SECTION: &str = "recall";
 /// Keys `[recall]` and `targets.<name>.recall` accept, sorted.
-pub const RECALL_KEYS: [&str; 7] = [
+pub const RECALL_KEYS: [&str; 8] = [
+    "audit_evidence",
     "diagnostics_min_severity",
     "enabled",
     "max_output_bytes",
@@ -15,6 +16,8 @@ pub const RECALL_KEYS: [&str; 7] = [
 pub const REMOVED_CONNECTION_KEY: &str = "connection";
 /// The diagnostic severities a Recall store can keep from, least severe first.
 pub const RECALL_SEVERITIES: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"];
+/// Which audit outcomes keep their evidence rows in Recall, the default first.
+pub const AUDIT_EVIDENCE_MODES: [&str; 3] = ["warn-and-error", "error", "off"];
 /// The longest shutdown flush budget, in milliseconds.
 pub const MAX_SHUTDOWN_TIMEOUT_MS: u64 = 600_000;
 /// The longest retention, in days.

crates/sqlbuild-config/src/recall/main/recall_section.rs

@@ -3,7 +3,8 @@
 use crate::models::ConfigValue;
 use crate::recall::_helpers::values::{non_empty, output_bytes, retention_days, timeout_ms};
 use crate::recall::constants::{
-    MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES, REMOVED_CONNECTION_KEY,
+    AUDIT_EVIDENCE_MODES, MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES,
+    REMOVED_CONNECTION_KEY,
 };
 use crate::recall::models::RecallSection;
 
@@ -65,6 +66,16 @@ pub fn recall_section(value: &ConfigValue, label: &str) -> Result<RecallSection,
                 }
                 section.diagnostics_min_severity = Some(text.to_owned());
             }
+            "audit_evidence" => {
+                let text = non_empty(value, label, key)?;
+                if !AUDIT_EVIDENCE_MODES.contains(&text) {
+                    return Err(format!(
+                        "{label}.audit_evidence must be one of: {}",
+                        AUDIT_EVIDENCE_MODES.join(", ")
+                    ));
+                }
+                section.audit_evidence = Some(text.to_owned());
+            }
             other => {
                 return Err(format!(
                     "{label} contains unknown key(s): {other}. Allowed keys: {}",

crates/sqlbuild-config/src/recall/models.rs

@@ -13,4 +13,6 @@ pub struct RecallSection {
     pub diagnostics_min_severity: Option<String>,
     /// Bytes of each output stream to record; `Some(0)` records everything.
     pub max_output_bytes: Option<u64>,
+    /// Which audit outcomes keep their evidence rows: one of [`AUDIT_EVIDENCE_MODES`](crate::recall::constants::AUDIT_EVIDENCE_MODES).
+    pub audit_evidence: Option<String>,
 }

crates/sqlbuild-config/src/recall/tests/test_recall_section.rs

@@ -17,6 +17,7 @@ fn given_valid_sections_when_validated_then_settings_are_typed() {
                 ("shutdown_timeout", text("1500ms")),
                 ("diagnostics_min_severity", text("debug")),
                 ("max_output_bytes", ConfigValue::Integer(4_194_304)),
+                ("audit_evidence", text("error")),
             ]),
             expected_section: RecallSection {
                 enabled: Some(false),
@@ -26,6 +27,7 @@ fn given_valid_sections_when_validated_then_settings_are_typed() {
                 shutdown_timeout_ms: Some(1500),
                 diagnostics_min_severity: Some("debug".to_owned()),
                 max_output_bytes: Some(4_194_304),
+                audit_evidence: Some("error".to_owned()),
             },
         },
         ValidSectionTestCase {
@@ -64,6 +66,13 @@ fn given_valid_sections_when_validated_then_settings_are_typed() {
 #[test]
 fn given_invalid_sections_when_validated_then_the_message_names_the_key() {
     let test_cases = [
+        InvalidSectionTestCase {
+            description: "audit evidence names one of its modes",
+            value: table(&[("audit_evidence", text("always"))]),
+            label: "targets.prod.recall",
+            expected_message_prefix: "targets.prod.recall.audit_evidence must be one of: \
+                                      warn-and-error, error, off",
+        },
         InvalidSectionTestCase {
             description: "output bytes must not be negative",
             value: table(&[("max_output_bytes", ConfigValue::Integer(-1))]),

crates/sqlbuild-core/src/lifecycle_events/constants.rs

@@ -79,7 +79,7 @@ pub(crate) const FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: [&str; 8] = [
     "sql",
     "statement_sql",
 ];
-pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [
+pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 56] = [
     "adapter",
     "attached_column_name",
     "attached_target_kind",
@@ -92,6 +92,7 @@ pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [
     "binding_key",
     "command",
     "configured_batch_size",
+    "cost_status",
     "cursor_end_exclusive",
     "cursor_start",
     "definition_fingerprint",
@@ -653,7 +654,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [
         event_type: "invocation_completed",
         required_correlations: &[],
         required_payload: &[],
-        allowed_payload: &["command", "duration_ms", "exit_code"],
+        allowed_payload: &["command", "cost_status", "duration_ms", "exit_code"],
         terminal: true,
     },
     EventDefinition {
@@ -662,11 +663,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [
         required_payload: &[],
         allowed_payload: &[
             "command",
+            "cost_status",
             "duration_ms",
             "error_code",
             "error_message",
             "error_type",
             "exit_code",
+            "signal_number",
         ],
         terminal: true,
     },
@@ -1010,6 +1013,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [
             "intent",
             "job_id",
             "metadata",
+            "outcome",
             "purpose",
             "query_id",
             "row_count",

crates/sqlbuild-core/src/progress/constants.rs

@@ -113,7 +113,6 @@ pub(crate) const QUERY_COMMAND: &str = "query";
 pub(crate) const DBT_COMMAND: &str = "dbt";
 pub(crate) const DBT_JSON_OPTION: &str = "--json";
 pub(crate) const JSON_FORMAT: &str = "json";
-pub(crate) const CSV_FORMAT: &str = "csv";
 /// What replaces the cut tail of a transient status message.
 pub(crate) const STATUS_ELLIPSIS: &str = "...";
 /// The resource kind of audits and the prefix of their resource identifiers.