From 50dd720d3ce749eddd83fd9c45c74ddd45bcf558 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:57:08 +0100 Subject: [PATCH 01/18] feat!: separate Recall write and read connections `[recall]` and `[targets..recall]` now name `write_connection`, used by every command to record history, and `read_connection`, used by `sqb recall query`, `schema` and `output`. Readers never run DDL, so a SELECT-only user can investigate shared history. `read_connection` alone reads a shared store while the machine records locally. A writer that may not upgrade a store behind this version keeps writing the columns that exist and prints the statements an administrator should run once. `sqb recall schema --ddl` prints the DDL that creates the store or brings it up to date. BREAKING CHANGE: the Recall `connection` key is removed. A configuration that still sets it fails with an error naming `write_connection` and `read_connection`; set both to the old connection to keep the previous behaviour. --- .../sqlbuild-cli/src/argv/command_table.json | 12 ++ crates/sqlbuild-cli/src/argv/constants.rs | 3 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 +- .../sqlbuild-config/src/recall/constants.rs | 7 +- .../src/recall/main/recall_section.rs | 32 ++- crates/sqlbuild-config/src/recall/models.rs | 5 +- .../src/recall/tests/test_recall_section.rs | 28 ++- .../_helpers/recall/clickhouse_store.rs | 134 +++++++++---- .../src/bindings/_helpers/recall/command.rs | 9 +- .../src/bindings/_helpers/recall/constants.rs | 3 +- .../src/schema/main/ddl_script.rs | 43 ++++- crates/sqlbuild-recall/src/schema/main/mod.rs | 1 + .../src/schema/main/upgrade_statements.rs | 64 ++++++ crates/sqlbuild-recall/src/schema/models.rs | 9 + .../src/schema/tests/test_ddl.rs | 86 ++++++++- .../src/settings/_helpers/layers.rs | 7 +- .../settings/main/resolve_recall_settings.rs | 19 +- crates/sqlbuild-recall/src/settings/models.rs | 11 +- .../src/settings/tests/helpers.rs | 39 +++- .../src/settings/tests/test_resolve.rs | 114 ++++++----- crates/sqlbuild-recall/src/store/main/mod.rs | 1 + .../src/store/main/observe_sqlite_schema.rs | 41 ++++ .../commands/main/clickhouse/_test_types.py | 9 + .../cli/commands/main/clickhouse/helpers.py | 26 ++- .../main/clickhouse/test_clickhouse_recall.py | 90 +++++++++ .../src/sqlbuild/cli/recall/_test_types.py | 22 +++ .../cli/recall/test_recall_connections.py | 182 ++++++++++++++++++ website/src/content/docs/docs/cli/recall.mdx | 12 +- .../docs/concepts/adapters/clickhouse.mdx | 5 +- .../docs/docs/concepts/adapters/sqlite.mdx | 3 +- .../docs/concepts/observability/recall.mdx | 60 +++++- 31 files changed, 941 insertions(+), 139 deletions(-) create mode 100644 crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs create mode 100644 crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs create mode 100644 tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 2a81e8076a..6d8408e988 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -7017,6 +7017,18 @@ "help": "with --backend warehouse, describe one relation", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--ddl" + ], + "dest": "recall_ddl", + "nargs": 0, + "const": true, + "default": false, + "help": "print the SQL that creates the Recall store or brings it up to this version", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index dd249f7d96..050122eac5 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -63,7 +63,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 176] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 177] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -221,6 +221,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 176] = [ ("recall_max_cell_bytes", NamespaceDefault::None), ("recall_state", NamespaceDefault::Bool(false)), ("recall_relation", NamespaceDefault::None), + ("recall_ddl", NamespaceDefault::Bool(false)), ("recall_invocation", NamespaceDefault::Text("last")), ("recall_stream", NamespaceDefault::Text("combined")), ("recall_tail", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 678dcdc0cf..da56abafec 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -420,7 +420,7 @@ pub(crate) const QUERY_ROUTE: Route = routed( }, ); -const RECALL_ARGUMENTS: [Field; 13] = [ +const RECALL_ARGUMENTS: [Field; 14] = [ PROJECT_DIR, SELECTED_TARGET, same("recall_command"), @@ -434,6 +434,7 @@ const RECALL_ARGUMENTS: [Field; 13] = [ attr("invocation", "recall_invocation"), attr("stream", "recall_stream"), attr("tail", "recall_tail"), + attr("ddl", "recall_ddl"), ]; pub(crate) const RECALL_ROUTE: Route = routed( "run_recall", diff --git a/crates/sqlbuild-config/src/recall/constants.rs b/crates/sqlbuild-config/src/recall/constants.rs index 3d61cf1666..74a066bec1 100644 --- a/crates/sqlbuild-config/src/recall/constants.rs +++ b/crates/sqlbuild-config/src/recall/constants.rs @@ -2,14 +2,17 @@ pub const RECALL_SECTION: &str = "recall"; /// Keys `[recall]` and `targets..recall` accept, sorted. -pub const RECALL_KEYS: [&str; 6] = [ - "connection", +pub const RECALL_KEYS: [&str; 7] = [ "diagnostics_min_severity", "enabled", "max_output_bytes", + "read_connection", "retention", "shutdown_timeout", + "write_connection", ]; +/// The single store key `write_connection` and `read_connection` replaced. +pub const REMOVED_CONNECTION_KEY: &str = "connection"; /// The diagnostic severities a Recall store can keep from, least severe first. pub const RECALL_SEVERITIES: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; /// The longest shutdown flush budget, in milliseconds. diff --git a/crates/sqlbuild-config/src/recall/main/recall_section.rs b/crates/sqlbuild-config/src/recall/main/recall_section.rs index 536f3d4ac9..6901fbd9fb 100644 --- a/crates/sqlbuild-config/src/recall/main/recall_section.rs +++ b/crates/sqlbuild-config/src/recall/main/recall_section.rs @@ -2,7 +2,9 @@ use crate::models::ConfigValue; use crate::recall::_helpers::values::{non_empty, output_bytes, retention_days, timeout_ms}; -use crate::recall::constants::{MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES}; +use crate::recall::constants::{ + MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES, REMOVED_CONNECTION_KEY, +}; use crate::recall::models::RecallSection; /// The settings of `value`, or the message naming its first problem at `label`. @@ -20,7 +22,13 @@ pub fn recall_section(value: &ConfigValue, label: &str) -> Result section.enabled = Some(*flag), _ => return Err(format!("{label}.enabled must be true or false")), }, - "connection" => section.connection = Some(non_empty(value, label, key)?.to_owned()), + "write_connection" => { + section.write_connection = Some(non_empty(value, label, key)?.to_owned()); + } + "read_connection" => { + section.read_connection = Some(non_empty(value, label, key)?.to_owned()); + } + REMOVED_CONNECTION_KEY => return Err(removed_connection(value, label)), "retention" => { let text = non_empty(value, label, key)?; section.retention_days = Some(retention_days(text).ok_or_else(|| { @@ -65,5 +73,25 @@ pub fn recall_section(value: &ConfigValue, label: &str) -> Result String { + let name: String = match value { + ConfigValue::String(name) if !name.trim().is_empty() => name.trim().to_owned(), + _ => "".to_owned(), + }; + format!( + "{label}.connection was replaced by write_connection and read_connection: set \ + write_connection = \"{name}\" and read_connection = \"{name}\" (or a separate \ + read-only connection), or only read_connection to read shared history while this \ + machine records to the local store" + ) +} diff --git a/crates/sqlbuild-config/src/recall/models.rs b/crates/sqlbuild-config/src/recall/models.rs index 6c9e735663..e8960bcb99 100644 --- a/crates/sqlbuild-config/src/recall/models.rs +++ b/crates/sqlbuild-config/src/recall/models.rs @@ -4,7 +4,10 @@ #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct RecallSection { pub enabled: Option, - pub connection: Option, + /// The connection every command writes its history to; requires `read_connection`. + pub write_connection: Option, + /// The connection `sqb recall query`, `schema` and `output` read. + pub read_connection: Option, pub retention_days: Option, pub shutdown_timeout_ms: Option, pub diagnostics_min_severity: Option, diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs index b8c144918d..9ae44329bc 100644 --- a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -11,7 +11,8 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { description: "every key is typed and trimmed", value: table(&[ ("enabled", ConfigValue::Bool(false)), - ("connection", text(" history ")), + ("write_connection", text(" history_writer ")), + ("read_connection", text("history_reader")), ("retention", text("720h")), ("shutdown_timeout", text("1500ms")), ("diagnostics_min_severity", text("debug")), @@ -19,13 +20,22 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { ]), expected_section: RecallSection { enabled: Some(false), - connection: Some("history".to_owned()), + write_connection: Some("history_writer".to_owned()), + read_connection: Some("history_reader".to_owned()), retention_days: Some(30), shutdown_timeout_ms: Some(1500), diagnostics_min_severity: Some("debug".to_owned()), max_output_bytes: Some(4_194_304), }, }, + ValidSectionTestCase { + description: "a read connection alone reads shared history", + value: table(&[("read_connection", text("history_reader"))]), + expected_section: RecallSection { + read_connection: Some("history_reader".to_owned()), + ..RecallSection::default() + }, + }, ValidSectionTestCase { description: "zero output bytes means record everything", value: table(&[("max_output_bytes", ConfigValue::Integer(0))]), @@ -66,6 +76,20 @@ fn given_invalid_sections_when_validated_then_the_message_names_the_key() { label: "recall", expected_message_prefix: "recall.max_output_bytes must be", }, + InvalidSectionTestCase { + description: "the removed connection key names its replacements", + value: table(&[("connection", text("history"))]), + label: "targets.prod.recall", + expected_message_prefix: "targets.prod.recall.connection was replaced by write_connection \ + and read_connection: set write_connection = \"history\" and \ + read_connection = \"history\"", + }, + InvalidSectionTestCase { + description: "a write connection needs a read connection", + value: table(&[("write_connection", text("history"))]), + label: "recall", + expected_message_prefix: "recall.write_connection requires recall.read_connection", + }, InvalidSectionTestCase { description: "unknown keys are listed", value: table(&[("mode", text("x"))]), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 0ed9bb2e5c..d65b8b534c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -25,12 +25,14 @@ use sqlbuild_recall::schema::constants::{ }; use sqlbuild_recall::schema::main::add_column_statement::add_column_statement; use sqlbuild_recall::schema::main::clickhouse_relation::clickhouse_relation; +use sqlbuild_recall::schema::main::ddl_script::render_script; use sqlbuild_recall::schema::main::retention_fix_statements::retention_fix_statements; use sqlbuild_recall::schema::main::table_columns::table_columns; use sqlbuild_recall::schema::main::table_spec::table_spec; use sqlbuild_recall::schema::main::table_statements::table_statements; +use sqlbuild_recall::schema::main::upgrade_statements::upgrade_statements; use sqlbuild_recall::schema::main::view_statements::view_statements; -use sqlbuild_recall::schema::models::{ColumnSpec, DdlOptions, Dialect, Table}; +use sqlbuild_recall::schema::models::{ColumnSpec, DdlOptions, Dialect, ObservedSchema, Table}; use sqlbuild_recall::store::constants::LAST_INVOCATION_SELECTOR; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; use sqlbuild_recall::store::models::{ @@ -90,10 +92,10 @@ struct StoreState { version: i64, } -/// A store ready to write: its tables with their `CREATE` queries, and the column additions an administrator must run. +/// A store ready to write: its tables with their `CREATE` queries, and the upgrade an administrator must run when this user could not. struct ReadyStore { tables: Vec<(String, String)>, - refused_columns: Vec, + behind: Vec, } /// The store the delivery thread writes. @@ -224,7 +226,25 @@ impl ClickHouseStore { Ok(()) } - /// Create a store whose tables are missing; on existing tables add missing columns where allowed, create missing views, and replace views only on a version upgrade. + /// What the store holds, from its read columns, `views` and recorded `version`. + fn observed(&self, views: &HashSet, version: i64) -> ObservedSchema { + ObservedSchema { + tables: self + .columns + .iter() + .map(|(table, columns)| { + ( + *table, + columns.iter().map(|column| column.name.clone()).collect(), + ) + }) + .collect(), + views: views.iter().cloned().collect(), + version: (version > 0).then_some(version), + } + } + + /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade; a user who may not run that DDL writes the columns that exist and gets the statements an administrator must run. fn ensure_schema( &mut self, py: Python<'_>, @@ -239,26 +259,38 @@ impl ClickHouseStore { self.create(py, envelope, &views)?; return Ok(ReadyStore { tables: self.state(py)?.tables, - refused_columns: Vec::new(), + behind: Vec::new(), }); } self.read_columns(py)?; - let refused: Vec = self.add_columns(py)?; - let upgrade: bool = version > 0 && version < RECALL_SCHEMA_VERSION; - let views_due: bool = upgrade || views.len() < VIEW_NAMES.len(); - if views_due && !refused.is_empty() { - return Err(StoreError { - kind: StoreErrorKind::Permission, - message: "the Recall views need columns this user may not add".to_owned(), + let before: ObservedSchema = self.observed(&views, version); + let behind = |store: &Self| { + upgrade_statements(Dialect::ClickHouse, &store.location.options, &before) + }; + if self.add_columns(py)? { + return Ok(ReadyStore { + behind: behind(self), + tables, }); } + let upgrade: bool = version > 0 && version < RECALL_SCHEMA_VERSION; for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { - if upgrade { - self.command(py, &statement, "create the Recall views")?; + let statement: String = if upgrade { + statement } else if !views.contains(name) { - let statement = - statement.replacen("CREATE OR REPLACE VIEW", "CREATE VIEW IF NOT EXISTS", 1); - self.command(py, &statement, "create the Recall views")?; + statement.replacen("CREATE OR REPLACE VIEW", "CREATE VIEW IF NOT EXISTS", 1) + } else { + continue; + }; + match self.command(py, &statement, "create the Recall views") { + Ok(()) => {} + Err(error) if error.kind == StoreErrorKind::Permission => { + return Ok(ReadyStore { + behind: behind(self), + tables, + }); + } + Err(error) => return Err(error), } } if version < RECALL_SCHEMA_VERSION { @@ -266,7 +298,7 @@ impl ClickHouseStore { } Ok(ReadyStore { tables, - refused_columns: refused, + behind: Vec::new(), }) } @@ -282,8 +314,7 @@ impl ClickHouseStore { self.command(py, &statement, "create the Recall tables")?; } self.read_columns(py)?; - let refused: Vec = self.add_columns(py)?; - if !refused.is_empty() { + if self.add_columns(py)? { return Err(StoreError { kind: StoreErrorKind::Permission, message: "the Recall tables need columns this user may not add".to_owned(), @@ -297,8 +328,8 @@ impl ClickHouseStore { self.record_version(py, envelope) } - /// Add the additive columns the stored tables lack; when the user may not, the statements an administrator must run. - fn add_columns(&mut self, py: Python<'_>) -> Result, StoreError> { + /// Add the additive columns the stored tables lack; `true` when the user may not. + fn add_columns(&mut self, py: Python<'_>) -> Result { let missing: Vec<(Table, ColumnSpec)> = self.missing_columns(); let statements: Vec = missing .iter() @@ -310,11 +341,12 @@ impl ClickHouseStore { ) }) .collect(); - for (index, statement) in statements.iter().enumerate() { + for statement in &statements { match self.command(py, statement, "add a Recall column") { Ok(()) => {} Err(error) if error.kind == StoreErrorKind::Permission => { - return Ok(statements[index..].to_vec()); + self.read_columns(py)?; + return Ok(true); } Err(error) => return Err(error), } @@ -322,7 +354,7 @@ impl ClickHouseStore { if !missing.is_empty() { self.read_columns(py)?; } - Ok(Vec::new()) + Ok(false) } /// Every column this build writes that the stored tables lack. @@ -401,14 +433,15 @@ impl ClickHouseStore { .collect()) } - /// The notice for columns this user may not add: history is written without them until an administrator adds them. - fn columns_notice(&self, statements: &[String]) -> String { + /// The notice for a store behind this build that this user may not upgrade: history is written to the columns that exist until an administrator runs `statements`. + fn behind_notice(&self, statements: &[String]) -> String { format!( - "the ClickHouse store in `{}` lacks columns this version of SQLBuild writes, and this \ - user may not add them; history is still written without them. An administrator can \ - add them with:\n\n{};\n", + "the Recall store in ClickHouse database `{}` is behind this SQLBuild version, and \ + this user may not upgrade it; history is still written, without what the newer \ + version adds. An administrator should run:\n\n{}\n`sqb recall schema --ddl` \ + prints these statements again.\n", self.location.database(), - statements.join(";\n\n") + render_script(statements) ) } @@ -499,17 +532,14 @@ impl RecallStoreOpener for ClickHouseStoreOpener { notices: Vec::new(), }; match store.ensure_schema(py, envelope) { - Ok(ReadyStore { - tables, - refused_columns: refused, - }) => { - if !refused.is_empty() + Ok(ReadyStore { tables, behind }) => { + if !behind.is_empty() && (self.retention_notice)( &self.project_dir, - &format!("{}:columns", self.location.database()), + &format!("{}:upgrade", self.location.database()), ) { - store.notices.push(store.columns_notice(&refused)); + store.notices.push(store.behind_notice(&behind)); } if let Some(notice) = store.retention_conflict(&tables) && (self.retention_notice)( @@ -570,6 +600,34 @@ impl ClickHouseReader { Ok(Self { store }) } + /// The statements that create the store in `location`, or bring it up to this build's schema, read with `SELECT` only. + pub(crate) fn upgrade_statements( + py: Python<'_>, + location: &ClickHouseLocation, + ) -> Result, StoreError> { + let mut store = ClickHouseStore { + location: location.clone(), + connection: Some(calls::connect( + py, + (&location.config, false), + &location.connect_action(), + )?), + columns: HashMap::new(), + notices: Vec::new(), + }; + let observed = store.require_database(py).and_then(|()| { + let state: StoreState = store.state(py)?; + store.read_columns(py)?; + Ok(store.observed(&state.views, state.version)) + }); + store.close(); + Ok(upgrade_statements( + Dialect::ClickHouse, + &location.options, + &observed?, + )) + } + /// The database the store lives in. pub(crate) fn database(&self) -> &str { self.store.location.database() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index bd62a458f4..ceff59ae02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -6,11 +6,14 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; use sqlbuild_recall::records::models::OutputStream; -use sqlbuild_recall::schema::models::Dialect; +use sqlbuild_recall::schema::main::ddl_script::{ddl_statements, render_script}; +use sqlbuild_recall::schema::main::upgrade_statements::upgrade_statements; +use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; use sqlbuild_recall::store::main::describe_store::describe_store; +use sqlbuild_recall::store::main::observe_sqlite_schema::observe_sqlite_schema; use sqlbuild_recall::store::main::open_sqlite_store::open_sqlite_store; use sqlbuild_recall::store::main::query_store::query_store; use sqlbuild_recall::store::main::read_output_transcript::read_output_transcript; @@ -59,6 +62,7 @@ struct RecallRequest { invocation: Option, stream: Option, tail: Option, + ddl: bool, } fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { @@ -87,6 +91,7 @@ fn recall_request(py: Python<'_>, options: Option>) -> PyResul invocation: text("invocation")?, stream: text("stream")?, tail: number("tail")?, + ddl: arguments.value("ddl")?.is_truthy()?, }) } @@ -160,7 +165,7 @@ impl ReadStore { } fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult { - match &settings.store { + match &settings.read_store { StoreLocation::Sqlite { path } => { let store = open_sqlite_store(path, OpenMode::ReadOnly).map_err(|error| { if error.kind == StoreErrorKind::NotFound { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index a33d548588..97ffb18292 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -42,7 +42,7 @@ pub(crate) const DEFAULT_QUERY_LIMIT: i64 = 100; /// The invocation selector naming the most recently started invocation. pub(crate) const LAST_INVOCATION: &str = "last"; /// The keyword arguments the `sqb recall` dispatch route passes. -pub(crate) const RECALL_OPTIONS: [&str; 13] = [ +pub(crate) const RECALL_OPTIONS: [&str; 14] = [ "project_dir", "selected_target", "recall_command", @@ -56,6 +56,7 @@ pub(crate) const RECALL_OPTIONS: [&str; 13] = [ "invocation", "stream", "tail", + "ddl", ]; /// Required text envelope fields of a known lifecycle event. pub(crate) const EVENT_TEXT_FIELDS: [&str; 5] = [ diff --git a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs index 671d2d8449..51ba25f6f6 100644 --- a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs +++ b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs @@ -22,19 +22,52 @@ fn clickhouse_version_row(options: &DdlOptions) -> String { ) } -/// Every table, index and view statement, then for ClickHouse the schema version row, each terminated by `;`. +/// The SQLite `recall_meta` row recording the schema version. +fn sqlite_version_row() -> String { + let now = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + format!( + "INSERT OR IGNORE INTO \"{meta}\" (component, key, value, recorded_at, invocation_id, \ + invocation_started_at, schema_version, producer_version, ingested_at, content_hash)\n\ + VALUES ('{META_SCHEMA_COMPONENT}', '{META_SCHEMA_VERSION_KEY}', \ + '{RECALL_SCHEMA_VERSION}', {now}, '{DDL_SCRIPT_INVOCATION}', {now}, \ + {RECALL_SCHEMA_VERSION}, '{version}', {now}, '')", + meta = table_spec(Table::Meta).name, + version = env!("CARGO_PKG_VERSION"), + ) +} + +/// The insert recording this build's schema version, which a writer allowed only `SELECT` and `INSERT` may also run. #[must_use] -pub fn ddl_script(dialect: Dialect, options: &DdlOptions) -> String { +pub fn version_row_statement(dialect: Dialect, options: &DdlOptions) -> String { + match dialect { + Dialect::Sqlite => sqlite_version_row(), + Dialect::ClickHouse => clickhouse_version_row(options), + } +} + +/// Every table, index and view statement, then the schema version row. +#[must_use] +pub fn ddl_statements(dialect: Dialect, options: &DdlOptions) -> Vec { let mut statements: Vec = table_statements(dialect, options); statements.extend( view_statements(dialect, options) .into_iter() .map(|(_, statement)| statement), ); - if dialect == Dialect::ClickHouse { - statements.push(clickhouse_version_row(options)); - } + statements.push(version_row_statement(dialect, options)); + statements +} + +/// `statements` as one script, each terminated by `;`. +#[must_use] +pub fn render_script(statements: &[String]) -> String { let mut script: String = statements.join(";\n\n"); script.push_str(";\n"); script } + +/// The complete creation script of [`ddl_statements`]. +#[must_use] +pub fn ddl_script(dialect: Dialect, options: &DdlOptions) -> String { + render_script(&ddl_statements(dialect, options)) +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index 854c85f2d9..3f6cca0f1d 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -6,4 +6,5 @@ pub mod sqlite_retention_sql; pub mod table_columns; pub mod table_spec; pub mod table_statements; +pub mod upgrade_statements; pub mod view_statements; diff --git a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs new file mode 100644 index 0000000000..10b8806d4d --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs @@ -0,0 +1,64 @@ +//! `upgrade_statements`: what an administrator runs to bring an observed store up to this build's schema. + +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION}; +use crate::schema::main::add_column_statement::add_column_statement; +use crate::schema::main::ddl_script::{ddl_statements, version_row_statement}; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect, ObservedSchema}; + +/// The additive statements `observed` needs, in order: missing tables, missing columns, views, then the schema version row. Empty when the store is current; the whole creation script when it has no Recall tables. +#[must_use] +pub fn upgrade_statements( + dialect: Dialect, + options: &DdlOptions, + observed: &ObservedSchema, +) -> Vec { + if observed.tables.is_empty() { + return ddl_statements(dialect, options); + } + let mut statements: Vec = Vec::new(); + if ALL_TABLES + .iter() + .any(|table| !observed.tables.contains_key(table)) + { + statements.extend(table_statements(dialect, options)); + } + for (table, present) in &observed.tables { + for column in table_columns(*table) { + if !present.contains(column.name) { + statements.push(add_column_statement( + dialect, + options, + (table_spec(*table), &column), + )); + } + } + } + let upgrade: bool = observed + .version + .is_none_or(|version| version < RECALL_SCHEMA_VERSION); + for (name, statement) in view_statements(dialect, options) { + match (dialect, upgrade) { + (Dialect::Sqlite, true) => { + statements.push(format!("DROP VIEW IF EXISTS {}", sqlite_identifier(name))); + statements.push(statement); + } + (Dialect::ClickHouse, true) => statements.push(statement), + (_, false) if observed.views.contains(name) => {} + (Dialect::Sqlite, false) => statements.push(statement), + (Dialect::ClickHouse, false) => statements.push(statement.replacen( + "CREATE OR REPLACE VIEW", + "CREATE VIEW IF NOT EXISTS", + 1, + )), + } + } + if upgrade { + statements.push(version_row_statement(dialect, options)); + } + statements +} diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 526db74cf9..dbc4d55943 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -109,3 +109,12 @@ pub struct SqliteRetentionSql { /// `?1` cutoff: maintenance claims older than retention. pub delete_old_claims: String, } + +/// What a Recall store holds: its tables with their column names, its views, and the recorded schema version. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ObservedSchema { + pub tables: std::collections::BTreeMap>, + pub views: std::collections::BTreeSet, + /// The highest recorded schema version; `None` when none is recorded. + pub version: Option, +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index a924eb5c78..e00cb22302 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -1,10 +1,12 @@ -use crate::schema::constants::{ALL_TABLES, VIEW_NAMES}; +use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION, VIEW_NAMES}; use crate::schema::main::add_column_statement::add_column_statement; use crate::schema::main::ddl_script::ddl_script; +use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; use crate::schema::main::table_statements::table_statements; +use crate::schema::main::upgrade_statements::upgrade_statements; use crate::schema::main::view_statements::view_statements; -use crate::schema::models::{ColumnSpec, ColumnType, Dialect, Table}; +use crate::schema::models::{ColumnSpec, ColumnType, Dialect, ObservedSchema, Table}; use crate::schema::tests::helpers::{clickhouse_table, contained, options, sqlite_schema_after}; use crate::schema::tests::test_types::{ AddColumnTestCase, FragmentTestCase, SqliteScriptTestCase, StatementSetTestCase, @@ -200,3 +202,83 @@ fn given_a_new_column_when_upgraded_then_the_statement_is_additive_and_nullable( ); } } + +#[test] +fn given_an_observed_store_when_upgraded_then_only_the_missing_additive_statements_are_listed() { + let current = || ObservedSchema { + tables: ALL_TABLES + .iter() + .map(|table| { + ( + *table, + table_columns(*table) + .iter() + .map(|column| column.name.to_owned()) + .collect(), + ) + }) + .collect(), + views: VIEW_NAMES.iter().map(|name| (*name).to_owned()).collect(), + version: Some(RECALL_SCHEMA_VERSION), + }; + let mut behind = current(); + behind.version = Some(RECALL_SCHEMA_VERSION - 1); + if let Some(columns) = behind.tables.get_mut(&Table::Heartbeats) { + columns.remove("subcommand"); + } + let mut missing_view = current(); + missing_view.views.remove("statements"); + let test_cases: [(&str, ObservedSchema, Dialect, Vec<&str>); 4] = [ + ( + "a current store needs nothing", + current(), + Dialect::Sqlite, + Vec::new(), + ), + ( + "an empty store gets the whole creation script", + ObservedSchema::default(), + Dialect::ClickHouse, + vec!["CREATE TABLE IF NOT EXISTS `observability`.`recall_meta`"], + ), + ( + "a store a version behind adds its column, replaces views and records the version", + behind, + Dialect::ClickHouse, + vec![ + "ALTER TABLE `observability`.`recall_heartbeats` ADD COLUMN IF NOT EXISTS `subcommand` Nullable(String)", + "CREATE OR REPLACE VIEW `observability`.`invocations`", + "INSERT INTO `observability`.`recall_meta`", + ], + ), + ( + "a current store missing a view only creates it", + missing_view, + Dialect::ClickHouse, + vec!["CREATE VIEW IF NOT EXISTS `observability`.`statements`"], + ), + ]; + + for (description, observed, dialect, expected_prefixes) in test_cases { + let statements: Vec = upgrade_statements(dialect, &options(None), &observed); + let firsts: Vec = expected_prefixes + .iter() + .map(|prefix| { + statements + .iter() + .any(|statement| statement.starts_with(prefix)) + }) + .collect(); + assert_eq!( + firsts, + vec![true; expected_prefixes.len()], + "{description}: {statements:#?}" + ); + if expected_prefixes.is_empty() { + assert!(statements.is_empty(), "{description}: {statements:#?}"); + } + if description.starts_with("a current store missing") { + assert_eq!(statements.len(), 1, "{description}"); + } + } +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs index f18f117f25..d931d397e2 100644 --- a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -29,7 +29,8 @@ pub(crate) fn section( pub(crate) fn overlay(base: RecallSection, layer: RecallSection) -> RecallSection { RecallSection { enabled: layer.enabled.or(base.enabled), - connection: layer.connection.or(base.connection), + write_connection: layer.write_connection.or(base.write_connection), + read_connection: layer.read_connection.or(base.read_connection), retention_days: layer.retention_days.or(base.retention_days), shutdown_timeout_ms: layer.shutdown_timeout_ms.or(base.shutdown_timeout_ms), diagnostics_min_severity: layer @@ -120,7 +121,7 @@ fn clickhouse_location( pub(crate) fn store_location( project_dir: &Path, documents: [Option<&ConfigValue>; 2], - (connection, project_adapter): (Option<&str>, Option<&str>), + (key, connection, project_adapter): (&str, Option<&str>, Option<&str>), ) -> Result { let Some(name) = connection else { return Ok(StoreLocation::Sqlite { @@ -131,7 +132,7 @@ pub(crate) fn store_location( }; let entries = connection_entries(documents, name).ok_or_else(|| { SettingsError::new(format!( - "recall.connection names unknown connection '{name}'; define [connections.{name}]" + "recall.{key} names unknown connection '{name}'; define [connections.{name}]" )) })?; let value = |key: &str| { diff --git a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs index a85925af60..3346b36741 100644 --- a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs +++ b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs @@ -46,7 +46,20 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result = match effective.max_output_bytes { Some(0) => None, @@ -57,7 +70,9 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result, + /// Where this invocation's own history is written. pub store: StoreLocation, - /// The configured connection name, `None` for the built-in local store. - pub connection: Option, + /// Where `sqb recall query`, `schema` and `output` read history. + pub read_store: StoreLocation, + /// The configured write connection, `None` for the built-in local store. + pub write_connection: Option, + /// The configured read connection, `None` for the built-in local store. + pub read_connection: Option, pub retention_days: u32, pub shutdown_timeout_ms: u64, pub diagnostics_min_severity: String, diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs index 39ef48f7e4..6330cf615e 100644 --- a/crates/sqlbuild-recall/src/settings/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -1,5 +1,7 @@ use std::path::PathBuf; +use sqlbuild_config::models::ConfigValue; + use crate::settings::main::resolve_recall_settings::resolve_recall_settings; use crate::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; @@ -22,15 +24,44 @@ pub(crate) fn resolved(files: &[(&str, &str)], target: Option<&str>) -> Result StoreLocation { + StoreLocation::Sqlite { + path: PathBuf::from(PROJECT).join("logs").join("recall.sqlite3"), + } +} + +/// The `central` ClickHouse store of the layered test project. +pub(crate) fn central_store() -> StoreLocation { + StoreLocation::ClickHouse { + connection: "central".to_owned(), + config: vec![ + ( + "host".to_owned(), + ConfigValue::String("localhost".to_owned()), + ), + ( + "database".to_owned(), + ConfigValue::String("observability".to_owned()), + ), + ( + "password".to_owned(), + ConfigValue::String("secret".to_owned()), + ), + ], + database: "observability".to_owned(), + } +} + /// The default settings for project `shop` on the duckdb adapter. pub(crate) fn defaults() -> RecallSettings { RecallSettings { enabled: true, target: None, - store: StoreLocation::Sqlite { - path: PathBuf::from(PROJECT).join("logs").join("recall.sqlite3"), - }, - connection: None, + store: local_store(), + read_store: local_store(), + write_connection: None, + read_connection: None, retention_days: 30, shutdown_timeout_ms: 3000, diagnostics_min_severity: "info".to_owned(), diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index a5c11c43b3..62ab1d9d24 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -1,7 +1,5 @@ -use sqlbuild_config::models::ConfigValue; - use crate::settings::models::{RecallSettings, StoreLocation}; -use crate::settings::tests::helpers::{defaults, resolved}; +use crate::settings::tests::helpers::{central_store, defaults, local_store, resolved}; use crate::settings::tests::test_types::{ResolveErrorTestCase, ResolveTestCase}; const DEFAULT_PROJECT: &[(&str, &str)] = &[( @@ -13,16 +11,17 @@ const LAYERED_PROJECT: &[(&str, &str)] = &[ ( "sqlbuild_project.toml", "name = \"shop\"\nadapter = \"duckdb\"\n\ - [recall]\nretention = \"10d\"\nconnection = \"history\"\n\ + [recall]\nretention = \"10d\"\nwrite_connection = \"history\"\nread_connection = \"history\"\n\ [connections.history]\nadapter = \"sqlite\"\npath = \"state/history.sqlite3\"\n\ [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ database = \"observability\"\npassword = \"secret\"\n\ [connections.other]\nadapter = \"postgres\"\nhost = \"localhost\"\n\ [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ - [targets.prod]\n[targets.prod.recall]\nconnection = \"central\"\n\ - [targets.capped]\n[targets.capped.recall]\nconnection = \"central\"\n\ + [targets.prod]\n[targets.prod.recall]\nwrite_connection = \"central\"\nread_connection = \"central\"\n\ + [targets.capped]\n[targets.capped.recall]\nwrite_connection = \"central\"\nread_connection = \"central\"\n\ max_output_bytes = 4096\n\ - [targets.qa]\n[targets.qa.recall]\nconnection = \"other\"\n", + [targets.qa]\n[targets.qa.recall]\nwrite_connection = \"other\"\nread_connection = \"other\"\n\ + [targets.laptop]\n[targets.laptop.recall]\nread_connection = \"central\"\n", ), ( "sqlbuild_local.toml", @@ -30,6 +29,13 @@ const LAYERED_PROJECT: &[(&str, &str)] = &[ ), ]; +const READER_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n[recall]\nread_connection = \"central\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"secret\"\n", +)]; + const OUTPUT_LIMIT_PROJECT: &[(&str, &str)] = &[( "sqlbuild_project.toml", "name = \"shop\"\nadapter = \"duckdb\"\n[recall]\nmax_output_bytes = 0\n\ @@ -59,7 +65,11 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d store: StoreLocation::Sqlite { path: "/tmp/opencode/elsewhere.sqlite3".into(), }, - connection: Some("history".to_owned()), + read_store: StoreLocation::Sqlite { + path: "/tmp/opencode/elsewhere.sqlite3".into(), + }, + write_connection: Some("history".to_owned()), + read_connection: Some("history".to_owned()), retention_days: 10, shutdown_timeout_ms: 1000, ..defaults() @@ -71,25 +81,10 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d target: Some("prod"), expected_settings: RecallSettings { target: Some("prod".to_owned()), - store: StoreLocation::ClickHouse { - connection: "central".to_owned(), - config: vec![ - ( - "host".to_owned(), - ConfigValue::String("localhost".to_owned()), - ), - ( - "database".to_owned(), - ConfigValue::String("observability".to_owned()), - ), - ( - "password".to_owned(), - ConfigValue::String("secret".to_owned()), - ), - ], - database: "observability".to_owned(), - }, - connection: Some("central".to_owned()), + store: central_store(), + read_store: central_store(), + write_connection: Some("central".to_owned()), + read_connection: Some("central".to_owned()), retention_days: 10, shutdown_timeout_ms: 1000, max_output_bytes: None, @@ -102,25 +97,10 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d target: Some("capped"), expected_settings: RecallSettings { target: Some("capped".to_owned()), - store: StoreLocation::ClickHouse { - connection: "central".to_owned(), - config: vec![ - ( - "host".to_owned(), - ConfigValue::String("localhost".to_owned()), - ), - ( - "database".to_owned(), - ConfigValue::String("observability".to_owned()), - ), - ( - "password".to_owned(), - ConfigValue::String("secret".to_owned()), - ), - ], - database: "observability".to_owned(), - }, - connection: Some("central".to_owned()), + store: central_store(), + read_store: central_store(), + write_connection: Some("central".to_owned()), + read_connection: Some("central".to_owned()), retention_days: 10, shutdown_timeout_ms: 1000, max_output_bytes: Some(4096), @@ -157,13 +137,47 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d connection: "other".to_owned(), adapter: "postgres".to_owned(), }, - connection: Some("other".to_owned()), + read_store: StoreLocation::Unsupported { + connection: "other".to_owned(), + adapter: "postgres".to_owned(), + }, + write_connection: Some("other".to_owned()), + read_connection: Some("other".to_owned()), retention_days: 10, shutdown_timeout_ms: 1000, max_output_bytes: None, ..defaults() }, }, + ResolveTestCase { + description: "a target read connection replaces only the read side", + files: LAYERED_PROJECT, + target: Some("laptop"), + expected_settings: RecallSettings { + target: Some("laptop".to_owned()), + store: StoreLocation::Sqlite { + path: "/tmp/opencode/elsewhere.sqlite3".into(), + }, + read_store: central_store(), + write_connection: Some("history".to_owned()), + read_connection: Some("central".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + max_output_bytes: Some(2 * 1024 * 1024), + ..defaults() + }, + }, + ResolveTestCase { + description: "a read connection alone reads shared history and writes the local store", + files: READER_PROJECT, + target: None, + expected_settings: RecallSettings { + store: local_store(), + read_store: central_store(), + read_connection: Some("central".to_owned()), + ..defaults() + }, + }, ]; for test_case in test_cases { @@ -183,10 +197,10 @@ fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { description: "unknown connections are named", files: &[( "sqlbuild_project.toml", - "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nconnection = \"missing\"\n", + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nwrite_connection = \"local\"\nread_connection = \"missing\"\n[connections.local]\npath = \"a.sqlite3\"\nadapter = \"sqlite\"\n", )], target: None, - expected_message_fragment: "unknown connection 'missing'", + expected_message_fragment: "recall.read_connection names unknown connection 'missing'", }, ResolveErrorTestCase { description: "target settings are named with their path", @@ -201,7 +215,7 @@ fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { description: "an unset environment reference in a ClickHouse connection is named", files: &[( "sqlbuild_project.toml", - "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nconnection = \"central\"\n\ + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nread_connection = \"central\"\n\ [connections.central]\nadapter = \"clickhouse\"\n\ password = \"${env:SQLBUILD_RECALL_UNSET_FOR_TEST}\"\n", )], diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 584ad7dfc5..60a156dd05 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -3,6 +3,7 @@ pub mod describe_store; pub mod ensure_sqlite_schema; pub mod insert_rows; pub mod maintain_sqlite_store; +pub mod observe_sqlite_schema; pub mod observe_store_file; pub mod open_sqlite_store; pub mod quarantine_store; diff --git a/crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs b/crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs new file mode 100644 index 0000000000..94efbfe767 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs @@ -0,0 +1,41 @@ +//! `observe_sqlite_schema`: the tables, columns, views and schema version a SQLite store holds, read without writing. + +use std::collections::BTreeSet; + +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::ObservedSchema; +use crate::store::_helpers::schema::stored_columns; +use crate::store::_helpers::sqlite::{checked, stored_schema_version}; +use crate::store::errors::StoreError; +use crate::store::models::SqliteStore; + +/// The store's Recall tables with their columns, its views and its recorded schema version. +pub fn observe_sqlite_schema(store: &SqliteStore) -> Result { + let connection = &store.connection; + let mut statement = checked( + connection.prepare("SELECT type, name FROM sqlite_schema WHERE type IN ('table', 'view')"), + "inspect Recall store", + )?; + let objects: Vec<(String, String)> = checked( + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .and_then(Iterator::collect), + "inspect Recall store", + )?; + let mut observed = ObservedSchema::default(); + for (kind, name) in objects { + if kind == "view" { + observed.views.insert(name); + } else if let Some(table) = ALL_TABLES + .iter() + .find(|table| table_spec(**table).name == name) + { + let columns: BTreeSet = + stored_columns(connection, *table)?.into_iter().collect(); + observed.tables.insert(*table, columns); + } + } + observed.version = stored_schema_version(connection)?; + Ok(observed) +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index fd6ed9b0ae..58ed3c1736 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -198,3 +198,12 @@ class ClickHouseRecallUnlimitedOutputTestCase: description: str thread_lines: str expected_minimum_bytes: int + + +@dataclass(frozen=True) +class ClickHouseRecallBehindStoreTestCase: + description: str + expected_notice: str + expected_ddl_fragments: tuple[str, ...] + expected_recorded: list[list[object]] + expected_heartbeat_columns: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 76d02e003c..bd20eb141a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -361,7 +361,8 @@ def recall_store_toml( f'database = "{database}"\n' f"{connection_settings}" "\n[recall]\n" - 'connection = "history"\n' + 'write_connection = "history"\n' + 'read_connection = "history"\n' f"{recall_settings}" ) @@ -477,6 +478,19 @@ def apply_printed_retention_fix(*, config: dict[str, object], stderr: str) -> in ) +def apply_script(*, config: dict[str, object], script: str) -> None: + """Run, as an administrator, a script of `;`-terminated statements.""" + + execute_clickhouse_sql( + config=config, + statements=tuple( + statement.strip().removesuffix(";") + for statement in script.split(";\n\n") + if statement.strip() + ), + ) + + def apply_printed_ddl( *, config: dict[str, object], stderr: str, edits: tuple[tuple[str, str], ...] = () ) -> None: @@ -494,9 +508,13 @@ def apply_printed_ddl( @contextmanager def limited_user( - *, config: dict[str, object], database: str, revoked_inserts: tuple[str, ...] = () + *, + config: dict[str, object], + database: str, + revoked_inserts: tuple[str, ...] = (), + privileges: str = "SELECT, INSERT", ) -> Iterator[dict[str, object]]: - """A user with only `SELECT` and `INSERT` on `database`, less `revoked_inserts`.""" + """A user with only `privileges` on `database`, less `revoked_inserts`.""" name: str = build_unique_schema_name(prefix="recall_writer") password: str = uuid.uuid4().hex @@ -504,7 +522,7 @@ def limited_user( config=config, statements=( f"CREATE USER {name} IDENTIFIED BY '{password}'", - f"GRANT SELECT, INSERT ON {database}.* TO {name}", + f"GRANT {privileges} ON {database}.* TO {name}", *(f"REVOKE INSERT ON {database}.{table} FROM {name}" for table in revoked_inserts), ), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index af62819639..ee06b5fe4f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( ClickHouseRecallAbandonedTestCase, ClickHouseRecallArtifactAgeTestCase, + ClickHouseRecallBehindStoreTestCase, ClickHouseRecallConcurrencyTestCase, ClickHouseRecallDuplicateTestCase, ClickHouseRecallExpiryTestCase, @@ -29,6 +30,7 @@ age_stored_artifacts, apply_printed_ddl, apply_printed_retention_fix, + apply_script, duplicate_recall_rows, execute_clickhouse_sql, expire_recall_parts, @@ -730,3 +732,91 @@ def test_given_large_output_when_recorded_in_clickhouse_then_no_output_limit_app if __name__ == "__main__": pytest.main([__file__, "-vv"]) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallBehindStoreTestCase( + description="a SELECT, INSERT writer keeps recording and the printed DDL upgrades it", + expected_notice="is behind this SQLBuild version, and this user may not upgrade it", + expected_ddl_fragments=( + "ADD COLUMN IF NOT EXISTS `command` Nullable(String);", + "CREATE OR REPLACE VIEW `{database}`.`invocations`", + "INSERT INTO `{database}`.`recall_meta`", + ), + expected_recorded=[["build", "completed"], ["build", "completed"]], + expected_heartbeat_columns=((2,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_store_one_version_behind_when_a_select_insert_user_builds_then_it_warns_once( + test_case: ClickHouseRecallBehindStoreTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + readonly: Path = write_recall_project( + parent=tmp_path / "readonly", + config=clickhouse_e2e_config, + database=recall_database, + settings=("", "settings = { readonly = 2 }\n"), + ) + apply_printed_ddl( + config=clickhouse_e2e_config, + stderr=run(readonly, "build").stderr.decode(errors="replace"), + edits=FIRST_RELEASE_EDITS, + ) + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"ALTER TABLE {recall_database}.recall_meta DELETE WHERE component = 'schema' " + "SETTINGS mutations_sync = 2", + ), + ) + + with ( + limited_user(config=clickhouse_e2e_config, database=recall_database) as writer, + limited_user( + config=clickhouse_e2e_config, database=recall_database, privileges="SELECT" + ) as reader, + ): + project_dir: Path = write_recall_project( + parent=tmp_path / "writer", config=writer, database=recall_database + ) + reader_dir: Path = write_recall_project( + parent=tmp_path / "reader", config=reader, database=recall_database + ) + builds: list[subprocess.CompletedProcess[bytes]] = [ + run(project_dir, "build"), + run(project_dir, "build"), + ] + recorded: list[list[object]] = recall_rows( + reader_dir, "SELECT command, status FROM invocations ORDER BY invocation_started_at" + ) + printed: subprocess.CompletedProcess[bytes] = run(reader_dir, "recall", "schema", "--ddl") + apply_script(config=clickhouse_e2e_config, script=printed.stdout.decode()) + repeated: subprocess.CompletedProcess[bytes] = run(reader_dir, "recall", "schema", "--ddl") + + notices: list[int] = [ + built.stderr.decode(errors="replace").count(test_case.expected_notice) for built in builds + ] + ddl: str = printed.stdout.decode() + assert [built.returncode for built in builds] == [0, 0] + assert notices == [1, 0] + assert recorded == test_case.expected_recorded + assert printed.returncode == 0, printed.stderr + assert [ + fragment.format(database=recall_database) in ddl + for fragment in test_case.expected_ddl_fragments + ] == [True for _ in test_case.expected_ddl_fragments] + assert (repeated.returncode, repeated.stdout) == (0, b"") + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count() FROM system.columns WHERE database = '{recall_database}' " + "AND table = 'recall_heartbeats' AND name IN ('command', 'subcommand')", + ) + == test_case.expected_heartbeat_columns + ) diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 061f25f5ca..2ea4b310d6 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -200,3 +200,25 @@ class ThirdPartyLogTestCase: expected_present: tuple[str, ...] expected_absent: tuple[str, ...] expected_traceback_count: int + + +@dataclass(frozen=True) +class RecallConnectionErrorTestCase: + description: str + recall_toml: str + args: tuple[str, ...] + expected_message: str + + +@dataclass(frozen=True) +class ReadOnlyReaderTestCase: + description: str + expected_shared_invocations: list[list[object]] + + +@dataclass(frozen=True) +class BehindStoreTestCase: + description: str + expected_ddl_fragments: tuple[str, ...] + expected_up_to_date_note: str + diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py new file mode 100644 index 0000000000..7a8e521a2b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py @@ -0,0 +1,182 @@ +import sqlite3 +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + BehindStoreTestCase, + ReadOnlyReaderTestCase, + RecallConnectionErrorTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import recall_rows, run, write_project + +SHARED_STORE_TOML: str = ( + "\n[connections.history]\n" + 'adapter = "sqlite"\n' + 'path = "{path}"\n' + "\n[recall]\n" + "{keys}" +) + + +def _shared_project(parent: Path, *, store: Path, keys: str) -> Path: + return write_project( + parent / "orders_shop", + extra_toml=SHARED_STORE_TOML.format(path=store.as_posix(), keys=keys), + ) + + +def _schema(store: Path) -> list[tuple[object, ...]]: + with sqlite3.connect(store) as connection: + return connection.execute( + "SELECT m.type, m.name, p.name FROM sqlite_schema AS m " + "LEFT JOIN pragma_table_xinfo(m.name) AS p ORDER BY m.type, m.name, p.cid" + ).fetchall() + + +def _make_store_behind(store: Path) -> None: + """Turn a current store into one a previous SQLBuild version wrote: no newer column, views or version.""" + + with sqlite3.connect(store) as connection: + views: list[str] = [ + row[0] + for row in connection.execute("SELECT name FROM sqlite_schema WHERE type = 'view'") + ] + for view in views: + _ = connection.execute(f'DROP VIEW "{view}"') + _ = connection.execute('ALTER TABLE "recall_heartbeats" DROP COLUMN "subcommand"') + _ = connection.execute("DELETE FROM recall_meta WHERE component = 'schema'") + + +@pytest.mark.parametrize( + "test_case", + [ + RecallConnectionErrorTestCase( + description="the removed connection key names its replacements on a build", + recall_toml='[recall]\nconnection = "history"\n', + args=("build",), + expected_message=( + "recall.connection was replaced by write_connection and read_connection: " + 'set write_connection = "history" and read_connection = "history"' + ), + ), + RecallConnectionErrorTestCase( + description="the removed connection key fails recall queries too", + recall_toml='[targets.dev.recall]\nconnection = "history"\n', + args=("recall", "query", "SELECT 1"), + expected_message="targets.dev.recall.connection was replaced by write_connection", + ), + RecallConnectionErrorTestCase( + description="a write connection without a read connection is rejected", + recall_toml='[recall]\nwrite_connection = "history"\n', + args=("build",), + expected_message="recall.write_connection requires recall.read_connection", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_recall_connections_when_running_then_the_command_fails_naming_the_keys( + test_case: RecallConnectionErrorTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", + extra_toml=( + '\n[connections.history]\nadapter = "sqlite"\npath = "history.sqlite3"\n\n' + + test_case.recall_toml + ), + ) + + completed: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert completed.returncode != 0 + assert test_case.expected_message in completed.stderr.decode(errors="replace") + assert not (project_dir / "history.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ReadOnlyReaderTestCase( + description="a reader sees shared history and records its own runs locally", + expected_shared_invocations=[["build", "completed"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_only_a_read_connection_when_running_then_reads_are_shared_and_writes_local( + test_case: ReadOnlyReaderTestCase, tmp_path: Path +) -> None: + store: Path = tmp_path / "shared" / "history.sqlite3" + writer: Path = _shared_project( + tmp_path / "writer", + store=store, + keys='write_connection = "history"\nread_connection = "history"\n', + ) + assert run(writer, "build").returncode == 0 + reader: Path = _shared_project( + tmp_path / "reader", store=store, keys='read_connection = "history"\n' + ) + schema_before: list[tuple[object, ...]] = _schema(store) + + built: subprocess.CompletedProcess[bytes] = run(reader, "build") + shared: list[list[object]] = recall_rows(reader, "SELECT command, status FROM invocations") + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert b"Recall" not in built.stderr + assert shared == test_case.expected_shared_invocations + assert (reader / "logs" / "recall.sqlite3").exists() + assert _schema(store) == schema_before + + +@pytest.mark.parametrize( + "test_case", + [ + BehindStoreTestCase( + description="readers print the upgrade without applying it, and it applies cleanly", + expected_ddl_fragments=( + 'ALTER TABLE "recall_heartbeats" ADD COLUMN "subcommand" TEXT;', + 'DROP VIEW IF EXISTS "invocations";', + 'CREATE VIEW IF NOT EXISTS "invocations" AS', + "INSERT OR IGNORE INTO \"recall_meta\"", + ), + expected_up_to_date_note="is up to date with this SQLBuild version", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_store_one_version_behind_when_read_then_no_ddl_runs_and_the_printed_ddl_applies( + test_case: BehindStoreTestCase, tmp_path: Path +) -> None: + store: Path = tmp_path / "shared" / "history.sqlite3" + project_dir: Path = _shared_project( + tmp_path, + store=store, + keys='write_connection = "history"\nread_connection = "history"\n', + ) + assert run(project_dir, "build").returncode == 0 + _make_store_behind(store) + schema_before: list[tuple[object, ...]] = _schema(store) + + queried: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "SELECT count(*) FROM recall_invocations" + ) + printed: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "schema", "--ddl") + schema_after_reads: list[tuple[object, ...]] = _schema(store) + with sqlite3.connect(store) as connection: + connection.executescript(printed.stdout.decode()) + repeated: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "schema", "--ddl") + rebuilt: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + ddl: str = printed.stdout.decode() + assert (queried.returncode, printed.returncode, repeated.returncode) == (0, 0, 0) + assert schema_after_reads == schema_before + assert [fragment in ddl for fragment in test_case.expected_ddl_fragments] == [ + True for _ in test_case.expected_ddl_fragments + ] + assert "CREATE TABLE" not in ddl + assert repeated.stdout == b"" + assert test_case.expected_up_to_date_note in repeated.stderr.decode() + assert rebuilt.returncode == 0 + assert b"Recall" not in rebuilt.stderr + assert recall_rows(project_dir, "SELECT count(*) FROM invocations") == [[2]] diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index c436103d4f..88043b1ab1 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -5,8 +5,9 @@ sidebar: label: "recall" --- -Read the local [Recall](/docs/concepts/observability/recall) store: past invocations, the -statements they executed, and what they printed. `sqb recall` never records itself. +Read the [Recall](/docs/concepts/observability/recall) store that `read_connection` names, or the +local store: past invocations, the statements they executed, and what they printed. `sqb recall` +never records itself, and never creates or changes the store. ## Usage @@ -81,7 +82,12 @@ Timestamps are ISO-8601 UTC text, so they compare correctly as strings. Filter o ## sqb recall schema List every Recall view with its columns, types, and join keys. Accepts `--format`, `--json`, and -`--target`. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse +`--target`. + +With `--ddl`, print instead the SQL that creates the store, or brings an existing store up to this +SQLBuild version (missing columns, replaced views, and the schema version row), for an +administrator to run. It reads the store through `read_connection` and changes nothing; when the +store is up to date it prints nothing and says so on stderr. Run it again after every upgrade. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse instead, or describes one with `--relation `. ## sqb recall output diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 806e7e943e..c8759345e5 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -181,6 +181,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from A ClickHouse connection can also hold [Recall](/docs/concepts/observability/recall)'s command history for a whole team, whichever adapter the project builds with. Set -`[recall] connection` to it; Recall creates its tables on first use, keeps every output byte by -default, and needs only `SELECT` and `INSERT` once the tables exist. See +`[recall] write_connection` and `read_connection` to it, one connection or separate writer and +reader users. Recall creates its tables on first use, keeps every output byte by default, and needs +only `SELECT` and `INSERT` once the tables exist; readers need only `SELECT`. See [Sharing history in ClickHouse](/docs/concepts/observability/recall#sharing-history-in-clickhouse). diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx index d9f871d197..48bdd38330 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -122,4 +122,5 @@ as `datetime(ordered_at, 'start of day')`, have no DuckDB translation and fail l [Recall](/docs/concepts/observability/recall) keeps its local history in a SQLite file, `logs/recall.sqlite3`, for every project, whichever adapter it builds with. Point -`[recall] connection` at a `sqlite` connection to put the file somewhere else. +`[recall] write_connection` and `read_connection` at a `sqlite` connection to put the file somewhere +else. diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index ea29473b8a..565b79db3c 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -111,7 +111,8 @@ enabled = false # do not record runs against the ci target | Key | Default | Description | |-----|---------|-------------| | `enabled` | `true` | Set to `false` to stop recording | -| `connection` | (none) | A `sqlite` connection whose `path` names the store, or a `clickhouse` connection; defaults to `logs/recall.sqlite3` | +| `write_connection` | (none) | The connection every command writes its history to: a `sqlite` connection whose `path` names the store, or a `clickhouse` connection. Requires `read_connection`. Defaults to `logs/recall.sqlite3` | +| `read_connection` | (none) | The connection `sqb recall query`, `schema` and `output` read; may name the same connection as `write_connection`. Set alone to read shared history while this machine's own commands record to `logs/recall.sqlite3` | | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | @@ -162,12 +163,17 @@ repeats at most once a day. If the store file is not a SQLite database, Recall m `recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. -A `connection` on any adapter other than `sqlite` or `clickhouse` is not supported; Recall prints a -notice and records nothing. +A `write_connection` on any adapter other than `sqlite` or `clickhouse` is not supported; Recall +prints a notice and records nothing. + +`write_connection` and `read_connection` replaced the single `connection` key. A configuration that +still sets `connection` fails with an error naming the replacements: set both keys to the old +connection to keep the previous behaviour. ## Sharing history in ClickHouse -Point `connection` at a ClickHouse connection to keep one history for a whole team. Every machine +Point `write_connection` and `read_connection` at ClickHouse connections to keep one history for a +whole team. Every machine that writes to it, including CI and scheduled production runs, becomes queryable from anywhere: see what last night's production build ran and printed without access to the machine, find the first run where an audit started failing across everyone's runs, or compare a production run with @@ -185,15 +191,49 @@ password = "${env:CLICKHOUSE_PASSWORD}" database = "observability" [recall] -connection = "history" +write_connection = "history_writer" # every command writes history (INSERT, SELECT) +read_connection = "history_reader" # sqb recall query, schema and output (SELECT only) retention = "30d" ``` -- **Setup.** On first use Recall creates its `recall_*` MergeTree tables and the views above in that - database, and adds columns that a newer SQLBuild introduces. After that, writing needs only - `INSERT` and `SELECT` on the tables. If the user may not create them, the command still succeeds - and prints the complete DDL for an administrator to run; a store created from it accepts writers - limited to `SELECT` and `INSERT`. +`sqb recall query`, `schema` and `output` use only `read_connection` and never create or change +anything in the store, so someone investigating production runs needs no credentials that could +rewrite or delete its history. Both keys may name one connection when separate users are not +wanted. A machine that only reads shared history, such as a laptop investigating production runs, +sets only `read_connection`; its own commands keep recording to the local store. + +### Users and grants + +SQLBuild does not enforce grants; it only lets the roles be separated. A least-privilege setup: + +```sql +-- once, as an administrator: the users +CREATE USER recall_writer IDENTIFIED BY '...'; +CREATE USER recall_reader IDENTIFIED BY '...'; +GRANT SELECT, INSERT ON observability.* TO recall_writer; +GRANT SELECT ON observability.* TO recall_reader; +``` + +Then, as the administrator, create the store with the DDL that `sqb recall schema --ddl` prints +(any connection that can read the database works, so the reader's is enough): + +```bash +sqb recall schema --ddl --target prod > recall.sql # review, then run it with an admin user +``` + +Repeat the same step after upgrading SQLBuild: `sqb recall schema --ddl` reads the store and prints +only what this version adds (`ALTER TABLE … ADD COLUMN`, replaced views and the schema version row), +or says that the store is up to date. Changes are always additive, and older SQLBuild versions keep +writing to an upgraded store. + +- **Setup.** A writer allowed to run DDL creates the `recall_*` MergeTree tables and the views above + in that database on first use, and adds the columns and views a newer SQLBuild introduces. If the + user may not create the store, the command still succeeds and prints the complete DDL for an + administrator to run; a store created from it accepts writers limited to `SELECT` and `INSERT`. +- **Upgrades.** When a store is behind this SQLBuild version and the writer may not upgrade it, the + command still runs and exits as it otherwise would. History is still written, to the columns + that exist, and Recall prints one warning (at most once a day) with the exact statements an + administrator should run; `sqb recall schema --ddl` prints them again. - **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. From 657c744e12eaabac3a797c1de242173c9426fbda Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:57:21 +0100 Subject: [PATCH 02/18] fix: print sqb recall --json failures as a JSON error on stdout With --json or --format json, a failing recall query or schema still exits 1 and prints its coded error on stderr, and now also prints {"error": {"code", "message", "help"}} on stdout so callers that parse the output read the error. --- .../src/bindings/_helpers/cli/entry.rs | 37 ++++++++++++ .../src/sqlbuild/cli/recall/_test_types.py | 7 +++ .../src/sqlbuild/cli/recall/helpers.py | 2 +- .../sqlbuild/cli/recall/test_recall_cli.py | 59 +++++++++++++++++++ website/src/content/docs/docs/cli/recall.mdx | 8 +++ 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index f8cf896796..46b60dd6d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -199,6 +199,42 @@ fn write_compile_machine_error( print_object(&document, None, None) } +/// With `sqb recall … --json`, the failure as `{"error": {"code", "message", "help"}}` on stdout, so callers parsing the output read the error instead of failing to parse it. +fn write_recall_machine_error( + invocation: &Invocation<'_>, + error: &Bound<'_, PyAny>, + fallback_code: &str, +) -> PyResult<()> { + if !invocation.command_is("recall") { + return Ok(()); + } + let format: Option = invocation + .args + .getattr_opt("recall_format")? + .and_then(|format| format.extract().ok()); + if format.as_deref() != Some("json") { + return Ok(()); + } + let (code, message, help) = error_parts(error, fallback_code)?; + let help: Option = if help.is_none() { + None + } else { + Some(help.str()?.to_string()) + }; + let document = serde_json::json!({ + "error": { + "code": code.str()?.to_string(), + "message": message.str()?.to_string(), + "help": help, + } + }); + print_object( + &PyString::new(invocation.py(), &document.to_string()).into_any(), + None, + None, + ) +} + fn render_diff_error<'py>( py: Python<'py>, fields: &[(&str, Bound<'py, PyAny>)], @@ -345,6 +381,7 @@ fn report_failure<'py>( error }; write_compile_machine_error(invocation, &effective, fallback_code)?; + write_recall_machine_error(invocation, &effective, fallback_code)?; if effective.is_instance(&error_class(py, ErrorId::QueryDiffOutcome)?)? { let line: Bound<'py, PyAny> = PyString::new(py, DIFF_OUTCOME_PREFIX) .into_any() diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 2ea4b310d6..3bec1c7b06 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -222,3 +222,10 @@ class BehindStoreTestCase: expected_ddl_fragments: tuple[str, ...] expected_up_to_date_note: str + +@dataclass(frozen=True) +class JsonErrorTestCase: + description: str + args: tuple[str, ...] + expected_code: str + diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index c7028d75fa..0c76e4d683 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -156,7 +156,7 @@ def wait_for(predicate_sql: str, project_dir: Path, *, timeout: float) -> list[l completed: subprocess.CompletedProcess[bytes] = run( project_dir, "recall", "query", "--json", predicate_sql ) - rows = json.loads(completed.stdout or EMPTY_RESULT)["rows"] + rows = json.loads(completed.stdout or EMPTY_RESULT).get("rows", []) return rows diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py new file mode 100644 index 0000000000..e91e3ae2c5 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -0,0 +1,59 @@ +import json +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + JsonErrorTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import run, write_project + + +@pytest.mark.parametrize( + "test_case", + [ + JsonErrorTestCase( + description="an invalid query", + args=("recall", "query", "--json", "SELECT no_such_column FROM invocations"), + expected_code="C974", + ), + JsonErrorTestCase( + description="a write statement", + args=("recall", "query", "--format", "json", "DELETE FROM recall_output"), + expected_code="C974", + ), + JsonErrorTestCase( + description="a read store with no history", + args=("recall", "query", "--json", "SELECT 1", "--target", "missing"), + expected_code="C971", + ), + JsonErrorTestCase( + description="a schema request without a store", + args=("recall", "schema", "--json", "--target", "missing"), + expected_code="C971", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failing_recall_command_with_json_when_run_then_stdout_is_a_json_error( + test_case: JsonErrorTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", + extra_toml=( + '\n[connections.elsewhere]\nadapter = "sqlite"\npath = "missing/history.sqlite3"\n' + '\n[targets.missing]\nconnection = "local"\nschema = "dev"\n' + '[targets.missing.recall]\nread_connection = "elsewhere"\n' + ), + ) + assert run(project_dir, "build").returncode == 0 + + failed: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + document: dict[str, dict[str, object]] = json.loads(failed.stdout) + assert failed.returncode == 1 + assert document["error"]["code"] == test_case.expected_code + assert isinstance(document["error"]["message"], str) + assert f"error[{test_case.expected_code}]" in failed.stderr.decode() + diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index 88043b1ab1..da21e7e691 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -113,6 +113,14 @@ other reason is followed by a note on stderr naming the gap. ## Errors +With `--json` (or `--format json`), a failing `query` or `schema` still exits `1` and prints its +coded error on stderr, and prints the error as JSON on stdout, so a program parsing the output reads +the error instead of failing to parse it: + +```json +{"error": {"code": "C974", "message": "invalid Recall query: no such column: status ...", "help": null}} +``` + | Code | Meaning | |------|---------| | `C970` | Recall is disabled for this project or target | From d6321e56477b2914ffaab897820e05728341fe0b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:57:35 +0100 Subject: [PATCH 03/18] feat: record the cause of failed invocations in Recall A failed invocation now records error_type, error_code and a one-line error_message on its invocation_failed event and in the invocations view, also when the command reports the failure only through its exit code. sqb debug records C980 (connection), C981 (query) or C982 (other check). The Recall schema version rises to 2 for the changed view. --- .../src/debug_report/constants.rs | 8 +++ .../main/debug_connection_lines.rs | 17 +++--- .../src/debug_report/main/debug_failure.rs | 36 +++++++++++++ .../src/debug_report/main/mod.rs | 1 + .../sqlbuild-core/src/debug_report/models.rs | 8 +++ .../debug_report/tests/test_debug_report.rs | 50 ++++++++++++++++- .../src/lifecycle_events/constants.rs | 2 + .../src/bindings/_helpers/cli/entry.rs | 25 +++++++++ .../bindings/_helpers/cli/failure_cause.rs | 25 +++++++++ .../src/bindings/_helpers/cli/mod.rs | 1 + .../_helpers/debug_command/run_debug.rs | 9 ++++ .../src/schema/_helpers/clickhouse_views.rs | 5 +- .../src/schema/_helpers/sqlite_views.rs | 3 ++ .../sqlbuild-recall/src/schema/constants.rs | 2 +- .../src/schema/tests/test_ddl.rs | 2 +- .../src/sqlbuild/cli/recall/_test_types.py | 8 +++ .../recall/test_recall_invocation_facts.py | 53 +++++++++++++++++++ website/src/content/docs/docs/cli/debug.mdx | 8 +++ .../docs/concepts/observability/recall.mdx | 6 ++- 19 files changed, 256 insertions(+), 13 deletions(-) create mode 100644 crates/sqlbuild-core/src/debug_report/main/debug_failure.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs create mode 100644 tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py diff --git a/crates/sqlbuild-core/src/debug_report/constants.rs b/crates/sqlbuild-core/src/debug_report/constants.rs index b7afd7f981..d175ce0a1a 100644 --- a/crates/sqlbuild-core/src/debug_report/constants.rs +++ b/crates/sqlbuild-core/src/debug_report/constants.rs @@ -18,3 +18,11 @@ pub(crate) const PREVIEW_EDGE_CHARACTERS: usize = 4; pub(crate) const NOT_CONFIGURED: &str = "not configured"; pub(crate) const NOT_PRESENT: &str = "not present"; pub(crate) const HEADER: &str = "SQLBuild Diagnostics"; +pub(crate) const CONNECTION_TEST_LABEL: &str = "connection test"; +pub(crate) const QUERY_TEST_LABEL: &str = "query test"; +/// The code recorded when `sqb debug` cannot connect. +pub(crate) const CONNECTION_FAILED_CODE: &str = "C980"; +/// The code recorded when `sqb debug` connects but `SELECT 1` fails. +pub(crate) const QUERY_FAILED_CODE: &str = "C981"; +/// The code recorded when another `sqb debug` check fails. +pub(crate) const DEBUG_CHECK_FAILED_CODE: &str = "C982"; diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_connection_lines.rs b/crates/sqlbuild-core/src/debug_report/main/debug_connection_lines.rs index f0e8591b0e..d0877c25e2 100644 --- a/crates/sqlbuild-core/src/debug_report/main/debug_connection_lines.rs +++ b/crates/sqlbuild-core/src/debug_report/main/debug_connection_lines.rs @@ -1,7 +1,8 @@ //! The connection section of `sqb debug`. use crate::debug_report::constants::{ - MASKED_VALUE, PREVIEW_CHARACTER_LIMIT, PREVIEW_EDGE_CHARACTERS, SECRET_CONNECTION_KEYS, + CONNECTION_TEST_LABEL, MASKED_VALUE, PREVIEW_CHARACTER_LIMIT, PREVIEW_EDGE_CHARACTERS, + QUERY_TEST_LABEL, SECRET_CONNECTION_KEYS, }; use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugLine, DebugStatus}; @@ -32,21 +33,21 @@ fn check_lines(check: &ConnectionCheck) -> [DebugLine; 2] { match check { ConnectionCheck::Skipped => [ line( - "connection test", + CONNECTION_TEST_LABEL, DebugStatus::Skip, "skipped by --no-connection", ), - line("query test", DebugStatus::Skip, "connection skipped"), + line(QUERY_TEST_LABEL, DebugStatus::Skip, "connection skipped"), ], ConnectionCheck::ConnectFailed(error) => [ - line("connection test", DebugStatus::Error, error), - line("query test", DebugStatus::Skip, "connection failed"), + line(CONNECTION_TEST_LABEL, DebugStatus::Error, error), + line(QUERY_TEST_LABEL, DebugStatus::Skip, "connection failed"), ], ConnectionCheck::Connected { query_error } => [ - line("connection test", DebugStatus::Ok, "connected"), + line(CONNECTION_TEST_LABEL, DebugStatus::Ok, "connected"), match query_error { - Some(error) => line("query test", DebugStatus::Error, error), - None => line("query test", DebugStatus::Ok, "SELECT 1"), + Some(error) => line(QUERY_TEST_LABEL, DebugStatus::Error, error), + None => line(QUERY_TEST_LABEL, DebugStatus::Ok, "SELECT 1"), }, ], } diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_failure.rs b/crates/sqlbuild-core/src/debug_report/main/debug_failure.rs new file mode 100644 index 0000000000..860e8154f1 --- /dev/null +++ b/crates/sqlbuild-core/src/debug_report/main/debug_failure.rs @@ -0,0 +1,36 @@ +//! `debug_failure`: the structured cause of a failed `sqb debug`, for the invocation record. + +use crate::debug_report::constants::{ + CONNECTION_FAILED_CODE, CONNECTION_TEST_LABEL, DEBUG_CHECK_FAILED_CODE, QUERY_FAILED_CODE, + QUERY_TEST_LABEL, +}; +use crate::debug_report::models::{DebugFailure, DebugReport, DebugStatus}; + +/// The first failed check: its error type, code and message; `None` when every check passed. +#[must_use] +pub fn debug_failure(report: &DebugReport) -> Option { + let line = [ + &report.connection, + &report.configuration, + &report.runtime, + &report.providers, + ] + .into_iter() + .flatten() + .find(|line| line.status == Some(DebugStatus::Error))?; + let detail: String = line + .status_message + .clone() + .filter(|message| !message.is_empty()) + .unwrap_or_else(|| line.message.clone()); + let (error_type, error_code) = match line.label.as_str() { + CONNECTION_TEST_LABEL => ("ConnectionError", CONNECTION_FAILED_CODE), + QUERY_TEST_LABEL => ("QueryError", QUERY_FAILED_CODE), + _ => ("ConfigurationError", DEBUG_CHECK_FAILED_CODE), + }; + Some(DebugFailure { + error_type: error_type.to_owned(), + error_code: error_code.to_owned(), + message: format!("{}: {detail}", line.label), + }) +} diff --git a/crates/sqlbuild-core/src/debug_report/main/mod.rs b/crates/sqlbuild-core/src/debug_report/main/mod.rs index 764b8549c6..b8fe67b310 100644 --- a/crates/sqlbuild-core/src/debug_report/main/mod.rs +++ b/crates/sqlbuild-core/src/debug_report/main/mod.rs @@ -1,6 +1,7 @@ pub mod debug_config_files; pub mod debug_configuration_lines; pub mod debug_connection_lines; +pub mod debug_failure; pub mod debug_provider_lines; pub mod debug_runtime_lines; pub mod format_debug_json; diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs index 8740cbca71..c0ec5075af 100644 --- a/crates/sqlbuild-core/src/debug_report/models.rs +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -167,3 +167,11 @@ impl DebugLine { } } } + +/// Why `sqb debug` failed: the first failed check as an error type, code and message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DebugFailure { + pub error_type: String, + pub error_code: String, + pub message: String, +} diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index a2d2323351..ba6e9103e3 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -1,8 +1,11 @@ use crate::cli_text::models::CliStyle; use crate::debug_report::main::debug_connection_lines::debug_connection_lines; +use crate::debug_report::main::debug_failure::debug_failure; use crate::debug_report::main::format_debug_json::format_debug_json; use crate::debug_report::main::format_debug_text::format_debug_text; -use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugLine}; +use crate::debug_report::models::{ + ConnectionCheck, ConnectionValue, DebugFailure, DebugLine, DebugReport, DebugStatus, +}; use crate::debug_report::tests::helpers::{line_summary, sample_report, text}; use crate::debug_report::tests::test_types::{ConnectionLinesTestCase, FormatTestCase}; @@ -99,3 +102,48 @@ fn given_connection_settings_when_listing_then_values_are_sorted_masked_and_shor ); } } + +#[test] +fn given_a_debug_report_when_its_failure_is_read_then_the_first_failed_check_is_coded() { + let passed = DebugReport { + connection: vec![DebugLine::checked( + "connection test", + "", + (DebugStatus::Ok, Some("connected")), + )], + ..DebugReport::default() + }; + let invalid_config = DebugReport { + configuration: vec![DebugLine::checked( + "project file", + "/repo/sqlbuild_project.toml", + (DebugStatus::Error, Some("invalid TOML")), + )], + ..DebugReport::default() + }; + let test_cases: [(&str, DebugReport, Option); 3] = [ + ( + "a failed connection test is a connection error", + sample_report(), + Some(DebugFailure { + error_type: "ConnectionError".to_owned(), + error_code: "C980".to_owned(), + message: "connection test: authentication failed".to_owned(), + }), + ), + ( + "another failed check is a configuration error", + invalid_config, + Some(DebugFailure { + error_type: "ConfigurationError".to_owned(), + error_code: "C982".to_owned(), + message: "project file: invalid TOML".to_owned(), + }), + ), + ("a passing report has no failure", passed, None), + ]; + + for (description, report, expected) in test_cases { + assert_eq!(debug_failure(&report), expected, "{description}"); + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index f370a52fc6..f6448e6691 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -301,6 +301,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [EventDefinition; 20] = [ "command", "duration_ms", "error_code", + "error_message", "error_type", "exit_code", ], @@ -662,6 +663,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ "command", "duration_ms", "error_code", + "error_message", "error_type", "exit_code", ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 46b60dd6d7..d28fd6e0e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, key use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; +use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; use crate::bindings::_helpers::cli::launch_state::offer_launch_record; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, @@ -48,6 +49,7 @@ const UNKNOWN_COMMAND: &str = "unknown"; const DIFF_PUBLICATION_CODE: &str = "C257"; const DIFF_EXECUTION_STATUS: &str = "execution_failed"; const DIFF_OUTCOME_PREFIX: &str = "Query diff outcome "; +const ERROR_MESSAGE_CHARACTERS: usize = 500; /// Expected staying-Python command failures, loaded only once a command raised. const EXPECTED_COMMAND_FAILURES: [ExecutorHost; 4] = [ ExecutorHost::AuditExecutionError, @@ -483,6 +485,12 @@ fn add_invocation_context( Ok(()) } +/// The first line of `message`, at most [`ERROR_MESSAGE_CHARACTERS`] characters. +fn short_message(message: &str) -> String { + let line: &str = message.lines().next().unwrap_or_default().trim(); + line.chars().take(ERROR_MESSAGE_CHARACTERS).collect() +} + /// Dispatch once while publishing canonical invocation lifecycle facts. fn observed_dispatch<'py>( invocation: &Invocation<'py>, @@ -496,6 +504,7 @@ fn observed_dispatch<'py>( return dispatch_command(&invocation.args, invocation.handlers.clone()); } let command: String = invocation.command_name(); + drop(take_failure_cause()); let started: Instant = Instant::now(); let elapsed = |started: Instant| -> PyResult> { u64::try_from(started.elapsed().as_millis()) @@ -529,6 +538,11 @@ fn observed_dispatch<'py>( let terminal: &str = if exit_code.eq(0)? { "invocation_completed" } else { + if let Some(cause) = take_failure_cause() { + payload.set_item("error_type", cause.error_type)?; + payload.set_item("error_code", cause.error_code)?; + payload.set_item("error_message", short_message(&cause.message))?; + } "invocation_failed" }; publish(lifecycle_event(py, terminal, payload)?)?; @@ -539,6 +553,17 @@ fn observed_dispatch<'py>( payload.set_item("command", &command)?; payload.set_item("duration_ms", elapsed(started)?)?; payload.set_item("error_type", error.get_type(py).getattr("__name__")?)?; + let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if let Some(code) = value.getattr_opt("code")? + && let Ok(code) = code.extract::() + { + payload.set_item("error_code", code)?; + } + let message: String = match value.getattr_opt("message")? { + Some(message) if !message.is_none() => message.str()?.to_string(), + _ => value.str()?.to_string(), + }; + payload.set_item("error_message", short_message(&message))?; publish(lifecycle_event(py, "invocation_failed", payload)?)?; Err(error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs new file mode 100644 index 0000000000..1deae61649 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs @@ -0,0 +1,25 @@ +//! The cause a command reports when it fails through its exit code instead of raising, for the invocation's terminal event. + +use std::sync::Mutex; + +/// A failure's error type, code and short message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct FailureCause { + pub(crate) error_type: String, + pub(crate) error_code: String, + pub(crate) message: String, +} + +static FAILURE_CAUSE: Mutex> = Mutex::new(None); + +/// Record why the running command is about to return a failing exit code. +pub(crate) fn report_failure_cause(cause: FailureCause) { + if let Ok(mut slot) = FAILURE_CAUSE.lock() { + *slot = Some(cause); + } +} + +/// The cause the command reported, cleared so the next invocation starts without one. +pub(crate) fn take_failure_cause() -> Option { + FAILURE_CAUSE.lock().ok().and_then(|mut slot| slot.take()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 871fd065d2..ee447df5e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod argv; pub(crate) mod dispatch; pub(crate) mod entry; +pub(crate) mod failure_cause; pub(crate) mod launch_state; pub(crate) mod namespace; pub(crate) mod observability; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index e55c49a4c5..ae20cf9408 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -12,6 +12,7 @@ use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::debug_report::main::debug_config_files::debug_config_files; use sqlbuild_core::debug_report::main::debug_configuration_lines::debug_configuration_lines; use sqlbuild_core::debug_report::main::debug_connection_lines::debug_connection_lines; +use sqlbuild_core::debug_report::main::debug_failure::debug_failure; use sqlbuild_core::debug_report::main::debug_provider_lines::debug_provider_lines; use sqlbuild_core::debug_report::main::debug_runtime_lines::debug_runtime_lines; use sqlbuild_core::debug_report::main::format_debug_json::format_debug_json; @@ -25,6 +26,7 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; use crate::bindings::_helpers::command_runtime::runtime_errors::{ @@ -396,6 +398,13 @@ pub(crate) fn run_debug_command( let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; stdout.call_method1("write", (text,))?; stdout.call_method0("flush")?; + if let Some(failure) = debug_failure(&report) { + report_failure_cause(FailureCause { + error_type: failure.error_type, + error_code: failure.error_code, + message: failure.message, + }); + } Ok(if report.success() { 0 } else { 1 }) } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index fa774d6aeb..a90d6fa71c 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -52,7 +52,9 @@ terminals AS ( argMax(exit_code, tuple(invocation_sequence, occurred_at)) AS terminal_exit_code, argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS terminal_duration_ms, argMax(error_type, tuple(invocation_sequence, occurred_at)) AS terminal_error_type, - argMax(error_code, tuple(invocation_sequence, occurred_at)) AS terminal_error_code + argMax(error_code, tuple(invocation_sequence, occurred_at)) AS terminal_error_code, + argMax(JSONExtract(payload, 'error_message', 'Nullable(String)'), + tuple(invocation_sequence, occurred_at)) AS terminal_error_message FROM {events} WHERE event_type IN ('invocation_completed', 'invocation_failed') GROUP BY invocation_id @@ -82,6 +84,7 @@ SELECT terminals.terminal_exit_code AS exit_code, terminals.terminal_error_type AS error_type, terminals.terminal_error_code AS error_code, + terminals.terminal_error_message AS error_message, coalesce(context.command, started.command, beats.command) AS command, coalesce(context.subcommand, beats.subcommand) AS subcommand, context.target AS target, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 91d09d179f..3f46b6ef9d 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -251,6 +251,7 @@ started AS ( ), terminals AS ( SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, + json_extract(payload, '$.error_message') AS error_message, ROW_NUMBER() OVER ( PARTITION BY invocation_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -282,6 +283,7 @@ projected AS ( terminals.exit_code, terminals.error_type, terminals.error_code, + terminals.error_message, COALESCE(context.command, started.command, beats.command) AS command, COALESCE(context.subcommand, beats.subcommand) AS subcommand, context.target, @@ -322,6 +324,7 @@ SELECT exit_code, error_type, error_code, + error_message, command, subcommand, target, diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index d6306c1e9b..942caabe09 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -3,7 +3,7 @@ use crate::schema::models::{ColumnSpec, ColumnType, Table, TableSpec}; /// The Recall schema version this build writes. Raise it when a column or view is added. -pub const RECALL_SCHEMA_VERSION: i64 = 1; +pub const RECALL_SCHEMA_VERSION: i64 = 2; /// The oldest SQLite library the store's views run on (ordered `group_concat`). pub const MINIMUM_SQLITE_VERSION: (u32, u32) = (3, 44); /// The `recall_meta` component of schema rows. diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index e00cb22302..3a25189ea9 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -132,7 +132,7 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh .collect(), required_fragments: &[ "INSERT INTO `observability`.`recall_meta`", - "SELECT 'schema', 'schema_version', '1'", + "SELECT 'schema', 'schema_version', '2'", ], expected_count: 1, }, diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 3bec1c7b06..65b9965557 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -229,3 +229,11 @@ class JsonErrorTestCase: args: tuple[str, ...] expected_code: str + +@dataclass(frozen=True) +class FailureCauseTestCase: + description: str + args: tuple[str, ...] + expected_failure: list[object] + expected_message_prefix: str + diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py new file mode 100644 index 0000000000..1376ee8521 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py @@ -0,0 +1,53 @@ +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + FailureCauseTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import recall_rows, run, write_project + +BROKEN_TARGET: str = ( + "\n[connections.broken]\n" + 'database = "missing_directory/warehouse.duckdb"\n\n' + "[targets.broken]\n" + 'connection = "broken"\n' + 'schema = "dev"\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + FailureCauseTestCase( + description="a debug connection failure reported by exit code", + args=("debug", "--target", "broken"), + expected_failure=["debug", "failed", 1, "ConnectionError", "C980"], + expected_message_prefix="connection test: ", + ), + FailureCauseTestCase( + description="a raised coded error", + args=("build", "--select", "no_such_model"), + expected_failure=["build", "failed", None, "PlannerInputError", "S007"], + expected_message_prefix="unknown selector name 'no_such_model'", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failed_command_when_recorded_then_its_cause_is_structured( + test_case: FailureCauseTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=BROKEN_TARGET) + + failed: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT command, status, exit_code, error_type, error_code, error_message " + "FROM invocations", + ) + + assert failed.returncode != 0 + assert [row[:5] for row in recorded] == [test_case.expected_failure] + assert str(recorded[0][5]).startswith(test_case.expected_message_prefix) + diff --git a/website/src/content/docs/docs/cli/debug.mdx b/website/src/content/docs/docs/cli/debug.mdx index 09c8138b78..3039b125df 100644 --- a/website/src/content/docs/docs/cli/debug.mdx +++ b/website/src/content/docs/docs/cli/debug.mdx @@ -79,3 +79,11 @@ Connection: ## Exit codes Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). +A failed run is recorded in [Recall](/docs/concepts/observability/recall) with the first failed +check as its cause: + +| `error_code` | `error_type` | Meaning | +|------|------|---------| +| `C980` | `ConnectionError` | The connection test failed | +| `C981` | `QueryError` | The connection worked but `SELECT 1` failed | +| `C982` | `ConfigurationError` | Another check failed, such as an invalid configuration file | diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 565b79db3c..7acc6b97e3 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -70,7 +70,11 @@ return `--json`. For every recorded invocation, Recall keeps: -- the invocation itself: command, arguments, target, adapter, status, exit code, and timing; +- the invocation itself: command, arguments, target, adapter, status, exit code, and timing, and + for a failed invocation its cause as `error_type`, `error_code` and a one-line + `error_message`, also when the command reported the failure only through its exit code (for + example `sqb debug` failing to connect records `ConnectionError` and `C980`); +- the commit it ran from (`git_revision`), see [Git revision](#git-revision); - its [lifecycle events](/docs/concepts/observability): runs, resource attempts, statements, retries, and audit results; - the SQL text of every executed statement, stored once per distinct statement and shared across From 3a6c15771e0bf2682066d22c476798ee5397ffdf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:57:41 +0100 Subject: [PATCH 04/18] feat: record git_revision in deployments, worktrees and submodules SQLBUILD_GIT_REVISION and SQLBUILD_GIT_DIRTY set the recorded revision and dirty flag where the deployment has no .git. A .git file's gitdir pointer is followed, and a worktree reads its refs from the common directory. --- .../src/bindings/_helpers/recall/constants.rs | 4 + .../src/bindings/_helpers/recall/session.rs | 77 +++++++++++++++---- .../src/sqlbuild/cli/recall/_test_types.py | 7 ++ .../recall/test_recall_invocation_facts.py | 60 +++++++++++++++ .../docs/concepts/observability/recall.mdx | 15 ++++ 5 files changed, 149 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 97ffb18292..29b5ade02f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -2,6 +2,10 @@ /// Seconds between heartbeats; tests shorten it to observe abandoned invocations quickly. pub(crate) const HEARTBEAT_ENVIRONMENT: &str = "SQLBUILD_RECALL_HEARTBEAT_SECONDS"; +/// The commit a deployment without `.git` records as `git_revision`. +pub(crate) const GIT_REVISION_ENVIRONMENT: &str = "SQLBUILD_GIT_REVISION"; +/// Whether that deployment's tree had uncommitted changes, recorded as `git_dirty`. +pub(crate) const GIT_DIRTY_ENVIRONMENT: &str = "SQLBUILD_GIT_DIRTY"; /// Marker files under `target/` limiting store notices to one per interval. pub(crate) const DDL_NOTICE_PREFIX: &str = ".recall-notice-"; pub(crate) const NOTICE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index fa67fe81fa..7a1c43e8d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -33,9 +33,9 @@ use crate::bindings::_helpers::recall::clickhouse_store::{ }; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::constants::{ - CLOSE_POLL_MS, DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, - MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, RECALL_HISTORY, - STDOUT_STREAM_INDEX, + CLOSE_POLL_MS, DDL_NOTICE_PREFIX, GIT_DIRTY_ENVIRONMENT, GIT_REVISION_ENVIRONMENT, + HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, + QUARANTINE_START_WAIT_MS, RECALL_HISTORY, STDOUT_STREAM_INDEX, }; use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_replacement_sink}; @@ -97,24 +97,73 @@ fn project_dir(args: &Bound<'_, PyAny>) -> PyResult { Ok(std::fs::canonicalize(&raw).unwrap_or(raw)) } -/// The commit `HEAD` names, read from the repository files without running git. +/// The repository directory a `.git` entry names: the directory itself, or the `gitdir:` a worktree's or submodule's `.git` file points to. +fn git_directory(entry: &Path) -> Option { + if entry.is_dir() { + return Some(entry.to_path_buf()); + } + let pointer: String = file_text(entry)?; + let target: &str = pointer.trim().strip_prefix("gitdir:")?.trim(); + let target: PathBuf = PathBuf::from(target); + let resolved: PathBuf = if target.is_absolute() { + target + } else { + entry.parent()?.join(target) + }; + resolved.is_dir().then_some(resolved) +} + +/// The commit `HEAD` names, read from the repository files without running git; a worktree reads its refs from the common directory. fn git_revision(project_dir: &Path) -> Option { let git: PathBuf = project_dir .ancestors() .map(|directory| directory.join(".git")) - .find(|candidate| candidate.is_dir())?; + .find(|candidate| candidate.exists()) + .and_then(|entry| git_directory(&entry))?; + let common: PathBuf = file_text(&git.join("commondir")) + .map_or_else(|| git.clone(), |relative| git.join(relative.trim())); let head: String = file_text(&git.join("HEAD"))?; let Some(reference) = head.trim().strip_prefix("ref: ") else { return Some(head.trim().to_owned()); }; - if let Some(revision) = file_text(&git.join(reference)) { - return Some(revision.trim().to_owned()); + for directory in [&git, &common] { + if let Some(revision) = file_text(&directory.join(reference)) { + return Some(revision.trim().to_owned()); + } + } + [&git, &common] + .into_iter() + .filter_map(|directory| file_text(&directory.join("packed-refs"))) + .flat_map(|packed| { + packed + .lines() + .filter_map(|line| line.split_once(' ')) + .filter(|(_, name)| *name == reference) + .map(|(revision, _)| revision.to_owned()) + .collect::>() + }) + .next() +} + +/// The revision a deployment sets in [`GIT_REVISION_ENVIRONMENT`], else the one the repository's files name. +fn recorded_git_revision(project_dir: &Path) -> Option { + environment(GIT_REVISION_ENVIRONMENT) + .map(|revision| revision.trim().to_owned()) + .filter(|revision| !revision.is_empty()) + .or_else(|| git_revision(project_dir)) +} + +/// Whether the working tree had uncommitted changes, as a deployment sets it in [`GIT_DIRTY_ENVIRONMENT`]; unknown otherwise. +fn recorded_git_dirty() -> Option { + match environment(GIT_DIRTY_ENVIRONMENT)? + .trim() + .to_ascii_lowercase() + .as_str() + { + "1" | "true" | "yes" => Some(true), + "0" | "false" | "no" => Some(false), + _ => None, } - file_text(&git.join("packed-refs"))? - .lines() - .filter_map(|line| line.split_once(' ')) - .find(|(_, name)| *name == reference) - .map(|(revision, _)| revision.to_owned()) } fn host_token() -> Option { @@ -178,8 +227,8 @@ fn invocation_facts( destination_schema: settings.destination_schema.clone(), destination_fingerprint: Some(sha256_hex(fingerprint_source.as_bytes())), sqlbuild_version: Some(env!("CARGO_PKG_VERSION").to_owned()), - git_revision: git_revision(project_dir), - git_dirty: None, + git_revision: recorded_git_revision(project_dir), + git_dirty: recorded_git_dirty(), host_token: host_token(), pid: Some(std::process::id()), external_context: external_context(py)?, diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 65b9965557..e5eefd843a 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -237,3 +237,10 @@ class FailureCauseTestCase: expected_failure: list[object] expected_message_prefix: str + +@dataclass(frozen=True) +class GitRevisionTestCase: + description: str + layout: str + environment: dict[str, str] + expected_dirty: object diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py index 1376ee8521..e77ce07d8d 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py @@ -5,6 +5,7 @@ from tests.integration.src.sqlbuild.cli.recall._test_types import ( FailureCauseTestCase, + GitRevisionTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import recall_rows, run, write_project @@ -17,6 +18,17 @@ ) +def _git(directory: Path, *args: str) -> str: + completed: subprocess.CompletedProcess[str] = subprocess.run( + ["git", "-c", "user.name=Orders", "-c", "user.email=orders@example.com", *args], + cwd=directory, + capture_output=True, + check=True, + text=True, + ) + return completed.stdout.strip() + + @pytest.mark.parametrize( "test_case", [ @@ -51,3 +63,51 @@ def test_given_a_failed_command_when_recorded_then_its_cause_is_structured( assert [row[:5] for row in recorded] == [test_case.expected_failure] assert str(recorded[0][5]).startswith(test_case.expected_message_prefix) + +@pytest.mark.parametrize( + "test_case", + [ + GitRevisionTestCase( + description="the deployment's override wins", + layout="no_repository", + environment={"SQLBUILD_GIT_REVISION": "4f1c2e9", "SQLBUILD_GIT_DIRTY": "true"}, + expected_dirty=1, + ), + GitRevisionTestCase( + description="a git worktree reads HEAD through its .git file", + layout="worktree", + environment={}, + expected_dirty=None, + ), + GitRevisionTestCase( + description="a plain checkout reads HEAD from .git", + layout="checkout", + environment={}, + expected_dirty=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_deployment_or_checkout_when_recorded_then_the_git_revision_is_known( + test_case: GitRevisionTestCase, tmp_path: Path +) -> None: + repository: Path = tmp_path / "repository" + project_dir: Path = write_project(repository / "orders_shop") + expected_revision: str = test_case.environment.get("SQLBUILD_GIT_REVISION", "") + if test_case.layout != "no_repository": + _ = _git(repository, "init", "-q", "-b", "main") + _ = _git(repository, "add", ".") + _ = _git(repository, "commit", "-q", "-m", "orders") + expected_revision = _git(repository, "rev-parse", "HEAD") + if test_case.layout == "worktree": + _ = _git(repository, "worktree", "add", "-q", "-b", "orders", str(tmp_path / "linked")) + project_dir = tmp_path / "linked" / "orders_shop" + + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", **test_case.environment + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert recall_rows(project_dir, "SELECT git_revision, git_dirty FROM invocations") == [ + [expected_revision, test_case.expected_dirty] + ] diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 7acc6b97e3..7877c7de98 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -96,6 +96,21 @@ recorded either: a `sqb compile` answered from the launcher's warm cache, and `s The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. +## Git revision + +`invocations.git_revision` is the commit `HEAD` names in the repository above the project +directory, including git worktrees and submodules, whose `.git` is a file pointing at the +repository. Deployments usually have no `.git` at all, so set the revision when building the image +or starting the job, or comparing production runs by `git_revision` finds nothing: + +```bash +export SQLBUILD_GIT_REVISION="$(git rev-parse HEAD)" # recorded as git_revision +export SQLBUILD_GIT_DIRTY=false # optional: recorded as git_dirty (true/false) +``` + +`SQLBUILD_GIT_REVISION` takes precedence over the repository. Without `SQLBUILD_GIT_DIRTY`, +`git_dirty` is `NULL`. + ## Settings Configure Recall in `[recall]` in `sqlbuild_project.toml`. A `[targets..recall]` table From 351dc3bda23c0ec67811f635d67b427dd5b700c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:57:45 +0100 Subject: [PATCH 05/18] feat: deprecate sqb recall --format table --format table still renders in this release but prints a deprecation warning first; the default long format is for reading, --json for programs and --format csv for export. Docs examples now use the long format. --- .../sqlbuild-cli/src/argv/command_table.json | 2 + .../src/bindings/_helpers/recall/command.rs | 75 ++++++++++++++++++- .../src/bindings/_helpers/recall/constants.rs | 2 + .../src/sqlbuild/cli/recall/_test_types.py | 7 ++ .../sqlbuild/cli/recall/test_recall_cli.py | 36 +++++++++ website/src/content/docs/docs/cli/recall.mdx | 27 ++++--- .../docs/concepts/observability/recall.mdx | 48 +++++++++--- 7 files changed, 174 insertions(+), 23 deletions(-) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 6d8408e988..8fe116fcff 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -6853,6 +6853,7 @@ "json", "csv" ], + "help": "long (default) for reading, json for programs, csv for export; table is deprecated", "mutex_group": 0, "group": 1 }, @@ -7042,6 +7043,7 @@ "json", "csv" ], + "help": "long (default) for reading, json for programs, csv for export; table is deprecated", "mutex_group": 0, "group": 1 }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index ceff59ae02..5be2a9ab2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -27,7 +27,8 @@ use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_byte use crate::bindings::_helpers::recall::constants::{ DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, LONG_FORMAT, RECALL_AMBIGUOUS_CODE, RECALL_DISABLED_CODE, RECALL_NOT_FOUND_CODE, RECALL_OPTIONS, - RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, WAREHOUSE_BACKEND, + RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, TABLE_FORMAT, + TABLE_FORMAT_DEPRECATION, WAREHOUSE_BACKEND, }; use crate::bindings::_helpers::recall::rendering::{QueryContext, render_result, render_schema}; use crate::bindings::_helpers::records::construct::new_error; @@ -95,6 +96,14 @@ fn recall_request(py: Python<'_>, options: Option>) -> PyResul }) } +/// Warn once that `--format table` goes away in a later release; it still works in this one. +fn warn_table_format(py: Python<'_>, recall: &RecallRequest) -> PyResult<()> { + if recall.format == TABLE_FORMAT { + write_stderr(py, TABLE_FORMAT_DEPRECATION)?; + } + Ok(()) +} + /// The project directory, target and settings, or the coded failure. fn settings(py: Python<'_>, recall: &RecallRequest) -> PyResult<(PathBuf, RecallSettings)> { let raw: PathBuf = match &recall.project_dir { @@ -242,6 +251,7 @@ fn limit(recall: &RecallRequest) -> Option { } fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { + warn_table_format(py, recall)?; let (project_dir, settings) = settings(py, recall)?; let sql: String = recall.sql.clone().unwrap_or_default(); if sql.trim().is_empty() { @@ -300,8 +310,71 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { Ok(0) } +/// Print what creates the read store, or brings it up to this build's schema, without changing it. +fn run_schema_ddl(py: Python<'_>, settings: &RecallSettings) -> PyResult { + let (statements, place): (Result, StoreError>, String) = match &settings.read_store + { + StoreLocation::Sqlite { path } => { + let options = DdlOptions { + database: None, + retention_days: settings.retention_days, + cluster: None, + }; + let statements = if path.exists() { + open_sqlite_store(path, OpenMode::ReadOnly) + .and_then(|store| observe_sqlite_schema(&store)) + .map(|observed| upgrade_statements(Dialect::Sqlite, &options, &observed)) + } else { + Ok(ddl_statements(Dialect::Sqlite, &options)) + }; + (statements, format!("at {}", path.display())) + } + StoreLocation::ClickHouse { + config, database, .. + } => ( + ClickHouseReader::upgrade_statements( + py, + &ClickHouseLocation::new(config, database, settings.retention_days), + ), + format!("in ClickHouse database `{database}`"), + ), + StoreLocation::Unsupported { .. } => { + return open_store(py, settings).map(|store| { + store.close(); + 0 + }); + } + }; + let statements = + statements.map_err(|error| user_error(py, &error.message, RECALL_STORE_CODE))?; + if statements.is_empty() { + write_stderr( + py, + &format!( + "The Recall store {place} is up to date with this SQLBuild version; there is \ + nothing to run.\n" + ), + )?; + } else { + write_stdout_bytes(py, render_script(&statements).as_bytes())?; + } + Ok(0) +} + fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { + warn_table_format(py, recall)?; let (project_dir, settings) = settings(py, recall)?; + if recall.ddl { + if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { + return Err(user_error( + py, + "--ddl prints the Recall store's DDL and cannot be combined with --backend \ + warehouse", + RECALL_QUERY_CODE, + )); + } + return run_schema_ddl(py, &settings); + } if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { let sql = match &recall.relation { Some(relation) => format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 29b5ade02f..2ff17d07c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -28,6 +28,8 @@ pub(crate) const WAREHOUSE_BACKEND: &str = "warehouse"; pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; pub(crate) const LONG_FORMAT: &str = "long"; pub(crate) const TABLE_FORMAT: &str = "table"; +/// What `--format table` prints in the release before it is removed. +pub(crate) const TABLE_FORMAT_DEPRECATION: &str = "warning: --format table is deprecated and will be removed; use the default long format, or --json/--format csv\n"; pub(crate) const CSV_FORMAT: &str = "csv"; pub(crate) const JSON_FORMAT: &str = "json"; /// The standard base64 alphabet binary JSON cells are encoded with. diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index e5eefd843a..eed1121998 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -230,6 +230,13 @@ class JsonErrorTestCase: expected_code: str +@dataclass(frozen=True) +class DeprecatedFormatTestCase: + description: str + args: tuple[str, ...] + expected_warning: bool + + @dataclass(frozen=True) class FailureCauseTestCase: description: str diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index e91e3ae2c5..bbbaa4e107 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -5,10 +5,13 @@ import pytest from tests.integration.src.sqlbuild.cli.recall._test_types import ( + DeprecatedFormatTestCase, JsonErrorTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import run, write_project +TABLE_DEPRECATION: str = "--format table is deprecated and will be removed" + @pytest.mark.parametrize( "test_case", @@ -57,3 +60,36 @@ def test_given_a_failing_recall_command_with_json_when_run_then_stdout_is_a_json assert isinstance(document["error"]["message"], str) assert f"error[{test_case.expected_code}]" in failed.stderr.decode() + +@pytest.mark.parametrize( + "test_case", + [ + DeprecatedFormatTestCase( + description="table still renders and warns", + args=("recall", "query", "--format", "table", "SELECT 1 AS answer"), + expected_warning=True, + ), + DeprecatedFormatTestCase( + description="the schema table warns too", + args=("recall", "schema", "--format", "table"), + expected_warning=True, + ), + DeprecatedFormatTestCase( + description="the default long format does not warn", + args=("recall", "query", "SELECT 1 AS answer"), + expected_warning=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_the_table_format_when_querying_then_it_works_with_a_deprecation_warning( + test_case: DeprecatedFormatTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + assert run(project_dir, "build").returncode == 0 + + rendered: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert rendered.returncode == 0 + assert rendered.stdout + assert (TABLE_DEPRECATION in rendered.stderr.decode()) is test_case.expected_warning diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index da21e7e691..8a3f63d21e 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -26,7 +26,7 @@ Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| | `sql` | The query (positional argument) | -| `--format` | `long` (default), `table`, `json`, or `csv` | +| `--format` | `long` (default) for reading, `json` for programs, or `csv` for export. `table` still works in this release but prints a deprecation warning and will be removed | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 100) | | `--no-limit` | Disable the row limit | @@ -37,15 +37,17 @@ Statements that write, and multiple statements, are rejected. The latest invocation in a project named `orders_shop`: ```bash -sqb recall query --format table \ +sqb recall query \ "SELECT command, status, exit_code, duration_ms FROM invocations ORDER BY invocation_started_at DESC LIMIT 1" ``` ``` -command | status | exit_code | duration_ms -------- | --------- | --------- | ----------- -build | completed | 0 | 752.0 +-[ RECORD 1 ]---------------------------+ +command | build +status | completed +exit_code | 0 +duration_ms | 752.0 1 row ``` @@ -53,17 +55,22 @@ build | completed | 0 | 752.0 The slowest statements of past builds, with their SQL: ```bash -sqb recall query --format table --max-cell-bytes 60 \ +sqb recall query --max-cell-bytes 60 \ "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql FROM statements s JOIN artifacts a USING (invocation_id, statement_id) WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" ``` ``` -resource_name | ms | sql -------------- | --- | ------------------------------------------------------------------------ -orders | 3.6 | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] -NULL | 3.2 | CREATE SCHEMA IF NOT EXISTS dev +-[ RECORD 1 ]---------------------------+ +resource_name | orders +ms | 3.6 +sql | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] + +-[ RECORD 2 ]---------------------------+ +resource_name | NULL +ms | 3.2 +sql | CREATE SCHEMA IF NOT EXISTS dev 2 rows ``` diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 7877c7de98..7d41a6a0c6 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -16,18 +16,33 @@ still pass, because they run on mocked data. The build finishes with a warning t The next morning, ask when the audit started failing, and what each of those builds reported: ```bash -sqb recall query --format table \ +sqb recall query \ "SELECT a.invocation_started_at AS run, a.outcome, a.violation_count, i.status, i.exit_code FROM audit_results a JOIN invocations i USING (invocation_id) WHERE a.audit_name = 'revenue_is_non_negative' ORDER BY run DESC LIMIT 3" ``` ``` -run | outcome | violation_count | status | exit_code ---------------------------- | ------- | --------------- | --------- | --------- -2026-10-20T00:01:44.784683Z | warn | 1 | completed | 0 -2026-10-20T00:00:34.652468Z | pass | 0 | completed | 0 -2026-10-19T23:59:23.827831Z | pass | 0 | completed | 0 +-[ RECORD 1 ]---------------------------+ +run | 2026-10-20T00:01:44.784683Z +outcome | warn +violation_count | 1 +status | completed +exit_code | 0 + +-[ RECORD 2 ]---------------------------+ +run | 2026-10-20T00:00:34.652468Z +outcome | pass +violation_count | 0 +status | completed +exit_code | 0 + +-[ RECORD 3 ]---------------------------+ +run | 2026-10-19T23:59:23.827831Z +outcome | pass +violation_count | 0 +status | completed +exit_code | 0 3 rows ``` @@ -50,14 +65,16 @@ sqb recall output last --tail 4 Then check what is true in the warehouse now, with `sqb recall query` again: ```bash -sqb recall query --backend warehouse --format table \ +sqb recall query --backend warehouse \ "SELECT payment_id, order_id, amount_cents, paid_at FROM prod.stg_payments WHERE amount_cents < 0" ``` ``` -payment_id | order_id | amount_cents | paid_at ----------- | -------- | ------------ | ------------------- -9 | 8 | -2200 | 2026-04-03 15:42:00 +-[ RECORD 1 ]---------------------------+ +payment_id | 9 +order_id | 8 +amount_cents | -2200 +paid_at | 2026-04-03 15:42:00 1 row ``` @@ -202,11 +219,18 @@ The connection needs `adapter = "clickhouse"` when the project uses another adap `database` must already exist; Recall never creates databases. ```toml -[connections.history] +[connections.history_writer] adapter = "clickhouse" host = "${env:CLICKHOUSE_HOST}" user = "recall_writer" -password = "${env:CLICKHOUSE_PASSWORD}" +password = "${env:RECALL_WRITER_PASSWORD}" +database = "observability" + +[connections.history_reader] +adapter = "clickhouse" +host = "${env:CLICKHOUSE_HOST}" +user = "recall_reader" +password = "${env:RECALL_READER_PASSWORD}" database = "observability" [recall] From 3a1243f260e6deed6b5867f9c14a4d576a037f5e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:17:20 +0100 Subject: [PATCH 06/18] perf: make sqb recall query, schema and output start quickly sqb recall only reads history, so it no longer publishes lifecycle events, loads or runs project sinks, or keeps compute logs. The Recall store connects through a light ClickHouse client helper instead of importing the whole adapter, a query no longer inspects the store before running (it checks only when the query fails), and clickhouse-connect >= 0.15 keeps dataframe libraries out of the import. Same machine, 5 runs each of `sqb recall query --json "SELECT 1"`: SQLite store 568M -> 400M instructions, about 0.38 s -> 0.13 s; ClickHouse store 1,120M -> 845M instructions, about 0.68 s -> 0.29 s. --- .../src/dispatch/main/invocation_policy.rs | 8 +- crates/sqlbuild-cli/src/dispatch/models.rs | 2 + .../dispatch/tests/test_invocation_policy.rs | 9 +- .../src/bindings/_helpers/cli/entry.rs | 30 +++-- .../_helpers/recall/clickhouse_store.rs | 60 +++++++--- .../src/bindings/_helpers/recall/command.rs | 20 +++- .../src/bindings/_helpers/recall/constants.rs | 2 + pyproject.toml | 2 +- .../clickhouse/_helpers/connect_clickhouse.py | 112 ++++++++++++++++++ .../clickhouse/_helpers/recall_store.py | 18 ++- .../clickhouse/classes/clickhouse_adapter.py | 100 +--------------- .../clickhouse/classes/recall_connections.py | 3 + .../entrypoint/_dispatch_with_compute_logs.py | 3 - .../main/clickhouse/test_clickhouse_recall.py | 6 +- .../src/sqlbuild/cli/recall/_test_types.py | 6 + .../sqlbuild/cli/recall/test_recall_cli.py | 37 +++++- uv.lock | 2 +- .../docs/concepts/observability/recall.mdx | 3 +- 18 files changed, 262 insertions(+), 161 deletions(-) create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/connect_clickhouse.py diff --git a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs index e01102eb49..105b5f410c 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs @@ -6,7 +6,7 @@ use crate::dispatch::constants::{ }; use crate::dispatch::models::{CommandFacts, InvocationPolicy}; -/// Whether the command creates a project, is observed, checks skill freshness and is recorded; only observed commands are recorded, so every recorded invocation has terminal events. +/// Whether the command creates a project, is observed, checks skill freshness, is recorded and has its streams captured; only observed commands are recorded, so every recorded invocation has terminal events. `sqb recall` only reads history, so it publishes no lifecycle events, runs no sinks and keeps no compute logs. #[must_use] pub fn invocation_policy(facts: &CommandFacts) -> InvocationPolicy { let command: &str = facts.command.as_deref().unwrap_or_default(); @@ -14,11 +14,13 @@ pub fn invocation_policy(facts: &CommandFacts) -> InvocationPolicy { || (command == DBT_COMMAND_NAME && facts.subcommand.as_deref() == Some(DBT_INIT_SUBCOMMAND)); let raw_query_diff: bool = command == DIFF_COMMAND_NAME && facts.raw_query_diff; - let observed: bool = !creates_project && !raw_query_diff; + let reads_history: bool = command == RECALL_COMMAND_NAME; + let observed: bool = !creates_project && !raw_query_diff && !reads_history; InvocationPolicy { creates_project, observed, maintains_skills: !SKILL_EXEMPT_COMMANDS.contains(&command) && !raw_query_diff, - recalled: observed && !command.is_empty() && command != RECALL_COMMAND_NAME, + recalled: observed && !command.is_empty(), + captured: !creates_project && !reads_history, } } diff --git a/crates/sqlbuild-cli/src/dispatch/models.rs b/crates/sqlbuild-cli/src/dispatch/models.rs index ecca6454b4..1eb5c45b67 100644 --- a/crates/sqlbuild-cli/src/dispatch/models.rs +++ b/crates/sqlbuild-cli/src/dispatch/models.rs @@ -121,6 +121,8 @@ pub struct InvocationPolicy { pub maintains_skills: bool, /// Whether Recall records the invocation; `sqb recall` never records itself. pub recalled: bool, + /// Whether compute logs capture the process streams; read-only `sqb recall` skips them. + pub captured: bool, } /// What a failed command reports: its fallback error code and the debug log message. diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs index fe47e5f633..eac68f0057 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs @@ -14,6 +14,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() observed: true, maintains_skills: true, recalled: true, + captured: true, }, }, InvocationPolicyTestCase { @@ -24,6 +25,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() observed: false, maintains_skills: true, recalled: false, + captured: false, }, }, InvocationPolicyTestCase { @@ -34,6 +36,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() observed: false, maintains_skills: false, recalled: false, + captured: false, }, }, InvocationPolicyTestCase { @@ -47,16 +50,18 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() observed: false, maintains_skills: false, recalled: false, + captured: true, }, }, InvocationPolicyTestCase { - description: "recall reads history and is never recorded itself", + description: "recall only reads history: no events, sinks, recording or compute logs", facts: facts("recall", Some("query")), expected_policy: InvocationPolicy { creates_project: false, - observed: true, + observed: false, maintains_skills: true, recalled: false, + captured: false, }, }, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index d28fd6e0e8..cbe8f1b3da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -7,7 +7,7 @@ use pyo3::exceptions::PySystemExit; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBool, PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyTuple}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::dispatch::main::failure_report::failure_report; use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; @@ -671,21 +671,19 @@ fn parsed_invocation<'py>( Ok(handled_dispatch(&invocation)?.unbind()) }, )?; - let dispatched: PyResult> = call_with( - &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, - &[ - ("args", args), - ("identity", identity.clone()), - ("operation", operation.into_any()), - ( - "creates_project", - PyBool::new(py, policy.creates_project) - .to_owned() - .into_any(), - ), - ("recall", recall_sink), - ], - ); + let dispatched: PyResult> = if policy.captured { + call_with( + &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, + &[ + ("args", args), + ("identity", identity.clone()), + ("operation", operation.into_any()), + ("recall", recall_sink), + ], + ) + } else { + operation.call0() + }; let exit_status: Option = match &dispatched { Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), Ok(_) | Err(_) => None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index d65b8b534c..4b40395dea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -43,7 +43,7 @@ use sqlbuild_recall::store::types::{RecallStore, RecallStoreOpener}; use crate::bindings::_helpers::recall::clickhouse_calls::{ self as calls, QueryStatistics, StoreHandle, }; -use crate::bindings::_helpers::recall::constants::SECONDS_PER_DAY; +use crate::bindings::_helpers::recall::constants::{SECONDS_PER_DAY, UNKNOWN_TABLE_ERROR}; /// Where a ClickHouse store lives and how long it keeps history. #[derive(Clone, Debug)] @@ -575,7 +575,7 @@ pub(crate) struct ClickHouseReader { impl ClickHouseReader { pub(crate) fn open(py: Python<'_>, location: &ClickHouseLocation) -> Result { - let mut store = ClickHouseStore { + let store = ClickHouseStore { location: location.clone(), connection: Some(calls::connect( py, @@ -585,19 +585,27 @@ impl ClickHouseReader { columns: HashMap::new(), notices: Vec::new(), }; - let state: StoreState = store.state(py)?; + Ok(Self { store }) + } + + /// Fail with "no history yet" when the store's tables or views are missing; queries check only after they fail, so a successful one costs no extra round trip. + pub(crate) fn require_store(&self, py: Python<'_>) -> Result<(), StoreError> { + let state: StoreState = self.store.state(py)?; if state.tables.len() < ALL_TABLES.len() || state.views.len() < VIEW_NAMES.len() { - store.close(); return Err(StoreError { kind: StoreErrorKind::NotFound, - message: format!( - "no Recall history is recorded yet in ClickHouse database `{}`, or its tables \ - are still being created; run a command first", - location.database() - ), + message: self.not_created_message(), }); } - Ok(Self { store }) + Ok(()) + } + + fn not_created_message(&self) -> String { + format!( + "no Recall history is recorded yet in ClickHouse database `{}`, or its tables are \ + still being created; run a command first", + self.database() + ) } /// The statements that create the store in `location`, or bring it up to this build's schema, read with `SELECT` only. @@ -650,13 +658,31 @@ impl ClickHouseReader { (sql, &settings), "run the Recall query", ) - .map_err(|failure| StoreError { - kind: if failure.kind == StoreErrorKind::Busy { - StoreErrorKind::Other - } else { - StoreErrorKind::Query - }, - message: failure.message, + .map_err(|failure| { + // A store not created yet is a store failure (C971), not a missing invocation. + match self.require_store(py) { + Err(missing) if missing.kind == StoreErrorKind::NotFound => { + return StoreError { + kind: StoreErrorKind::Other, + message: missing.message, + }; + } + Ok(()) if failure.message.contains(UNKNOWN_TABLE_ERROR) => { + return StoreError { + kind: StoreErrorKind::Other, + message: self.not_created_message(), + }; + } + Ok(()) | Err(_) => {} + } + StoreError { + kind: if failure.kind == StoreErrorKind::Busy { + StoreErrorKind::Other + } else { + StoreErrorKind::Query + }, + message: failure.message, + } })?; if let Some(limit) = limit && result.rows.len() > limit diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index 5be2a9ab2e..6dc0291723 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -173,7 +173,8 @@ impl ReadStore { } } -fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult { +/// The read store; `checked` first confirms a ClickHouse store exists, which a query skips and confirms only if it fails. +fn open_store(py: Python<'_>, settings: &RecallSettings, checked: bool) -> PyResult { match &settings.read_store { StoreLocation::Sqlite { path } => { let store = open_sqlite_store(path, OpenMode::ReadOnly).map_err(|error| { @@ -201,6 +202,15 @@ fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult py, &ClickHouseLocation::new(config, database, settings.retention_days), ) + .and_then(|reader| { + if checked { + if let Err(error) = reader.require_store(py) { + reader.close(); + return Err(error); + } + } + Ok(reader) + }) .map(ReadStore::ClickHouse) .map_err(|error| { let error = if error.kind == StoreErrorKind::NotFound { @@ -272,7 +282,7 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { return warehouse_query(py, recall, &project_dir, (&sql, limit)); } - let store = open_store(py, &settings)?; + let store = open_store(py, &settings, false)?; let started = Instant::now(); let limit = limit.map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); let queried = match &store { @@ -339,7 +349,7 @@ fn run_schema_ddl(py: Python<'_>, settings: &RecallSettings) -> PyResult { format!("in ClickHouse database `{database}`"), ), StoreLocation::Unsupported { .. } => { - return open_store(py, settings).map(|store| { + return open_store(py, settings, true).map(|store| { store.close(); 0 }); @@ -390,7 +400,7 @@ fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { }; return warehouse_query(py, recall, &project_dir, (&sql, None)); } - let store = open_store(py, &settings)?; + let store = open_store(py, &settings, true)?; let described = match &store { ReadStore::Sqlite { store, .. } => describe_store(store), ReadStore::ClickHouse(reader) => reader.describe(py), @@ -429,7 +439,7 @@ fn tail_lines(data: &[u8], lines: usize) -> &[u8] { fn run_output(py: Python<'_>, recall: &RecallRequest) -> PyResult { let (_, settings) = settings(py, recall)?; - let store = open_store(py, &settings)?; + let store = open_store(py, &settings, true)?; let selector: &str = recall.invocation.as_deref().unwrap_or(LAST_INVOCATION); let stream = recall.stream.as_deref().and_then(OutputStream::from_name); let read = store diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 2ff17d07c5..f2765ac651 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -98,6 +98,8 @@ pub(crate) const UTC_ZONE_NAME: &str = "UTC"; pub(crate) const CLOSE_POLL_MS: u64 = 50; /// Longest wait at startup for another process moving a corrupt store aside; the writer thread retries within the shutdown budget. pub(crate) const QUARANTINE_START_WAIT_MS: u64 = 200; +/// The ClickHouse error name of a query naming a relation that does not exist yet. +pub(crate) const UNKNOWN_TABLE_ERROR: &str = "UNKNOWN_TABLE"; /// Seconds in a day, for the artifact refresh cutoff. pub(crate) const SECONDS_PER_DAY: u64 = 24 * 60 * 60; /// The parameter a lifecycle-event sink takes its event in. diff --git a/pyproject.toml b/pyproject.toml index 38b197f406..a60cb08d73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ sqlserver = [ "pymssql>=2.3.0", ] clickhouse = [ - "clickhouse-connect>=0.8", + "clickhouse-connect>=0.15", ] ingestr = [ "dbc>=0.3.1", diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/connect_clickhouse.py b/src/sqlbuild/adapters/clickhouse/_helpers/connect_clickhouse.py new file mode 100644 index 0000000000..cba2ff7730 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/connect_clickhouse.py @@ -0,0 +1,112 @@ +"""Open a ClickHouse client with SQLBuild's session settings, without loading the full adapter.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection +from sqlbuild.adapters.clickhouse.constants import ( + CONNECTION_OPTION_KEYS, + DEFAULT_DATABASE, + DEFAULT_HTTP_PORT, + DEFAULT_HTTPS_PORT, + DSN_OPTION, + MINIMUM_SERVER_VERSION, + SESSION_SETTINGS, + SETTING_CONFLICT_CODE, + TIMEOUT_OPTIONS, + VERIFY_OPTION, +) + + +def connect_clickhouse(config: dict[str, Any]) -> _ClickHouseConnection: + """A connection for `config`: validated options, required session settings, supported server.""" + + try: + import clickhouse_connect + except ImportError as error: + raise AdapterUserError( + message="ClickHouse adapter requires optional dependency clickhouse-connect. " + "Install with: pip install 'sqlbuild[clickhouse]'", + code="A701", + ) from error + unknown: tuple[str, ...] = tuple(sorted(set(config) - CONNECTION_OPTION_KEYS)) + if unknown: + raise AdapterUserError( + message=f"ClickHouse connection has unsupported options: {', '.join(unknown)}", + code="A702", + help="Supported options: " + ", ".join(sorted(CONNECTION_OPTION_KEYS)), + ) + secure: bool = bool(config.get("secure", False)) + settings: dict[str, object] = _session_settings(dict(config.get("settings") or {})) + client_options: dict[str, Any] = { + "host": str(config.get("host", "localhost")), + "port": int(config.get("port", DEFAULT_HTTPS_PORT if secure else DEFAULT_HTTP_PORT)), + "username": str(config.get("username", config.get("user", "default"))), + "password": str(config.get("password", "")), + "database": str(config.get("database", DEFAULT_DATABASE)), + "secure": secure, + "settings": settings, + "autogenerate_session_id": False, + } + if VERIFY_OPTION in config: + client_options[VERIFY_OPTION] = bool(config[VERIFY_OPTION]) + if DSN_OPTION in config: + client_options[DSN_OPTION] = str(config[DSN_OPTION]) + for timeout_key in TIMEOUT_OPTIONS: + if timeout_key in config: + client_options[timeout_key] = int(config[timeout_key]) + + def create_client() -> Any: + return clickhouse_connect.get_client(**client_options) + + client: Any = create_client() + version: str = str(client.server_version) + if _version_tuple(version) < MINIMUM_SERVER_VERSION: + client.close() + minimum: str = ".".join(str(part) for part in MINIMUM_SERVER_VERSION) + raise AdapterUserError( + message=f"ClickHouse server {version} is older than the supported {minimum}", + code="A703", + help=f"Upgrade the server to ClickHouse {minimum} or newer.", + ) + return _ClickHouseConnection(client=client, interrupt_client_factory=create_client) + + +def _version_tuple(version: str) -> tuple[int, ...]: + parts: list[int] = [] + for part in version.split(".")[:2]: + digits: str = "".join(character for character in part if character.isdigit()) + parts.append(int(digits or 0)) + return tuple(parts) + + +def _session_settings(user_settings: dict[str, object]) -> dict[str, object]: + """Merge user settings under the required ones, refusing a user value that differs.""" + + conflicts: tuple[str, ...] = tuple( + sorted( + name + for name, value in user_settings.items() + if name in SESSION_SETTINGS + and _setting_text(value) != _setting_text(SESSION_SETTINGS[name]) + ) + ) + if conflicts: + required: str = ", ".join(f"{name} = {SESSION_SETTINGS[name]!r}" for name in conflicts) + raise AdapterUserError( + message=( + "ClickHouse connection settings override settings SQLBuild requires: " + + ", ".join(conflicts) + ), + code=SETTING_CONFLICT_CODE, + help=f"Remove them from [connection.settings]; SQLBuild runs with {required}.", + ) + return {**user_settings, **SESSION_SETTINGS} + + +def _setting_text(value: object) -> str: + if isinstance(value, bool): + return "1" if value else "0" + return str(value).strip().lower() diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py index 818ed30e31..25c686a128 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py @@ -7,8 +7,9 @@ import threading from typing import Any -from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from sqlbuild.adapters.clickhouse._helpers.connect_clickhouse import connect_clickhouse from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection +from sqlbuild.adapters.clickhouse.classes.recall_connections import RECALL_CONNECTIONS from sqlbuild.adapters.clickhouse.types import RecallCell, RecallFailure, RecallStatistics _CLIENT_MODULE: str = "clickhouse_connect" @@ -67,11 +68,10 @@ def connect_recall_store( setattr(threading.current_thread(), quiet_thread_attribute, True) options: dict[str, object] = {_CONNECT_TIMEOUT_KEY: _DEFAULT_CONNECT_TIMEOUT_SECONDS, **config} try: - adapter: ClickHouseAdapter = ClickHouseAdapter() - connection: _ClickHouseConnection = adapter.connect(options) + connection: _ClickHouseConnection = connect_clickhouse(options) except Exception as error: return _failure(error), 0 - return None, adapter.recall_connections.add(connection=connection) + return None, RECALL_CONNECTIONS.add(connection=connection) def run_recall_query( @@ -80,7 +80,7 @@ def run_recall_query( """Columns, rows of plain values and server statistics; strings not in UTF-8 stay bytes.""" try: - result: Any = ClickHouseAdapter.recall_connections.get(handle=handle).client.query( + result: Any = RECALL_CONNECTIONS.get(handle=handle).client.query( sql, settings=settings, query_formats=_BYTES_FORMATS ) except Exception as error: @@ -99,7 +99,7 @@ def run_recall_command(*, handle: int, sql: str) -> tuple[RecallFailure, None]: """Run one DDL statement.""" try: - ClickHouseAdapter.recall_connections.get(handle=handle).client.command(sql) + RECALL_CONNECTIONS.get(handle=handle).client.command(sql) except Exception as error: return _failure(error), None return None, None @@ -111,7 +111,7 @@ def insert_recall_rows( """Insert one `RowBinary` block; a retried block with the same token is deduplicated.""" try: - ClickHouseAdapter.recall_connections.get(handle=handle).client.raw_insert( + RECALL_CONNECTIONS.get(handle=handle).client.raw_insert( table, column_names=column_names, insert_block=payload, @@ -126,9 +126,7 @@ def insert_recall_rows( def close_recall_store(*, handle: int) -> None: """Close and forget the connection; a failure to close is ignored.""" - connection: _ClickHouseConnection | None = ClickHouseAdapter.recall_connections.pop( - handle=handle - ) + connection: _ClickHouseConnection | None = RECALL_CONNECTIONS.pop(handle=handle) if connection is None: return try: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 158f0bf427..53cb57e93f 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -86,6 +86,7 @@ ) from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.clickhouse._helpers.connect_clickhouse import connect_clickhouse from sqlbuild.adapters.clickhouse._helpers.output_columns import ( alias_qualified_columns, key_identifiers, @@ -108,22 +109,16 @@ ) from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection from sqlbuild.adapters.clickhouse.classes.partition_locks import PartitionLocks -from sqlbuild.adapters.clickhouse.classes.recall_connections import RecallConnections from sqlbuild.adapters.clickhouse.classes.relation_write_locks import RelationWriteLocks from sqlbuild.adapters.clickhouse.constants import ( - CONNECTION_OPTION_KEYS, DEFAULT_COLUMN_DEFAULT_KIND, DEFAULT_DATABASE, - DEFAULT_HTTP_PORT, - DEFAULT_HTTPS_PORT, - DSN_OPTION, EXCHANGE_UNSUPPORTED_CODES, EXPECTED_OUTPUT_COLUMNS_CONFIG_KEY, FRESH_GENERATION_CLAUSE, GENERATION_COMMENT_PREFIX, MAX_IDENTIFIER_LENGTH, MAX_QUERY_SIZE_BYTES, - MINIMUM_SERVER_VERSION, MODEL_NAME_CONFIG_KEY, NULL_IN_ORDINARY_COLUMN_CODE, NULL_IN_ORDINARY_COLUMN_SERVER_CODE, @@ -134,14 +129,10 @@ PHYSICAL_STORAGE_CONFIG_KEY, RELATION_NOT_FOUND_CODES, RELATION_SEPARATOR, - SESSION_SETTINGS, - SETTING_CONFLICT_CODE, SNAPSHOT_REBUILD_INFIX, SNAPSHOT_WORK_INFIX, STATE_TABLE_ENGINE_KEYWORD, STATE_TIMESTAMP_TYPE, - TIMEOUT_OPTIONS, - VERIFY_OPTION, VIEW_ENGINES, ) from sqlbuild.adapters.clickhouse.models import ( @@ -182,58 +173,9 @@ class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): ) _partition_locks: ClassVar[PartitionLocks] = PartitionLocks() _relation_locks: ClassVar[RelationWriteLocks] = RelationWriteLocks() - recall_connections: ClassVar[RecallConnections] = RecallConnections() def connect(self, config: dict[str, Any]) -> _ClickHouseConnection: - try: - import clickhouse_connect - except ImportError as error: - raise AdapterUserError( - message="ClickHouse adapter requires optional dependency clickhouse-connect. " - "Install with: pip install 'sqlbuild[clickhouse]'", - code="A701", - ) from error - unknown: tuple[str, ...] = tuple(sorted(set(config) - CONNECTION_OPTION_KEYS)) - if unknown: - raise AdapterUserError( - message=f"ClickHouse connection has unsupported options: {', '.join(unknown)}", - code="A702", - help="Supported options: " + ", ".join(sorted(CONNECTION_OPTION_KEYS)), - ) - secure: bool = bool(config.get("secure", False)) - settings: dict[str, object] = _session_settings(dict(config.get("settings") or {})) - client_options: dict[str, Any] = { - "host": str(config.get("host", "localhost")), - "port": int(config.get("port", DEFAULT_HTTPS_PORT if secure else DEFAULT_HTTP_PORT)), - "username": str(config.get("username", config.get("user", "default"))), - "password": str(config.get("password", "")), - "database": str(config.get("database", DEFAULT_DATABASE)), - "secure": secure, - "settings": settings, - "autogenerate_session_id": False, - } - if VERIFY_OPTION in config: - client_options[VERIFY_OPTION] = bool(config[VERIFY_OPTION]) - if DSN_OPTION in config: - client_options[DSN_OPTION] = str(config[DSN_OPTION]) - for timeout_key in TIMEOUT_OPTIONS: - if timeout_key in config: - client_options[timeout_key] = int(config[timeout_key]) - - def create_client() -> Any: - return clickhouse_connect.get_client(**client_options) - - client: Any = create_client() - version: str = str(client.server_version) - if _version_tuple(version) < MINIMUM_SERVER_VERSION: - client.close() - minimum: str = ".".join(str(part) for part in MINIMUM_SERVER_VERSION) - raise AdapterUserError( - message=f"ClickHouse server {version} is older than the supported {minimum}", - code="A703", - help=f"Upgrade the server to ClickHouse {minimum} or newer.", - ) - return _ClickHouseConnection(client=client, interrupt_client_factory=create_client) + return connect_clickhouse(config) def _execute(self, *, connection: _ClickHouseConnection, sql: str) -> Any: log_sql(logger=logging.getLogger("sqlbuild.adapter.clickhouse"), sql=sql) @@ -2702,14 +2644,6 @@ def _state_table_ddl(sql: str) -> str: return f"{ddl} ENGINE = MergeTree ORDER BY tuple()" -def _version_tuple(version: str) -> tuple[int, ...]: - parts: list[int] = [] - for part in version.split(".")[:2]: - digits: str = "".join(character for character in part if character.isdigit()) - parts.append(int(digits or 0)) - return tuple(parts) - - def _error_code(error: BaseException) -> int | None: current: BaseException | None = error while current is not None: @@ -2809,33 +2743,3 @@ def _prune_history_sql( f"PARTITION BY {partition} ORDER BY {order}) AS __sqlbuild_history_rank FROM {table}) " f"WHERE __sqlbuild_history_rank > {retain_versions}) SETTINGS transform_null_in = 1" ) - - -def _session_settings(user_settings: dict[str, object]) -> dict[str, object]: - """Merge user settings under the required ones, refusing a user value that differs.""" - - conflicts: tuple[str, ...] = tuple( - sorted( - name - for name, value in user_settings.items() - if name in SESSION_SETTINGS - and _setting_text(value) != _setting_text(SESSION_SETTINGS[name]) - ) - ) - if conflicts: - required: str = ", ".join(f"{name} = {SESSION_SETTINGS[name]!r}" for name in conflicts) - raise AdapterUserError( - message=( - "ClickHouse connection settings override settings SQLBuild requires: " - + ", ".join(conflicts) - ), - code=SETTING_CONFLICT_CODE, - help=f"Remove them from [connection.settings]; SQLBuild runs with {required}.", - ) - return {**user_settings, **SESSION_SETTINGS} - - -def _setting_text(value: object) -> str: - if isinstance(value, bool): - return "1" if value else "0" - return str(value).strip().lower() diff --git a/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py b/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py index 0eeea1f566..e64abf6751 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py +++ b/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py @@ -35,3 +35,6 @@ def pop(self, *, handle: int) -> _ClickHouseConnection | None: with self._guard: return self._connections.pop(handle, None) + + +RECALL_CONNECTIONS: RecallConnections = RecallConnections() diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index 86aa0fb094..49ce8eab0a 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -32,13 +32,10 @@ def dispatch_with_compute_logs( args: CliNamespace, identity: ExecutionIdentity, operation: Callable[[], int], - creates_project: bool, recall: RecallLifecycleSink | None = None, ) -> int: """Run one composed dispatch while retaining exact process streams best-effort.""" - if creates_project: - return operation() project_dir: Path = Path(args.project_dir) if args.project_dir is not None else Path.cwd() project_roots: tuple[str, ...] = diagnostic_project_roots(project_dir=project_dir) routing_options: DiagnosticRoutingOptions = DiagnosticRoutingOptions( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index ee06b5fe4f..d1f2ee2697 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -617,14 +617,14 @@ def test_given_sql_older_than_retention_when_still_referenced_then_artifacts_sho "test_case", [ ClickHouseRecallQueryOutputSinkTestCase( - description="query results reach a command-output sink", + description="a read-only recall query runs no project sinks", sql="SELECT 'shipped_orders' AS answer", expected_fragment="shipped_orders", ), ], ids=lambda case: case.description, ) -def test_given_an_output_sink_when_querying_clickhouse_recall_then_the_sink_sees_the_results( +def test_given_an_output_sink_when_querying_clickhouse_recall_then_the_sink_is_not_run( test_case: ClickHouseRecallQueryOutputSinkTestCase, tmp_path: Path, clickhouse_e2e_config: dict[str, object], @@ -640,7 +640,7 @@ def test_given_an_output_sink_when_querying_clickhouse_recall_then_the_sink_sees assert queried.returncode == 0, queried.stderr assert test_case.expected_fragment.encode() in queried.stdout - assert test_case.expected_fragment in (project_dir / "output.log").read_text(encoding="utf-8") + assert not (project_dir / "output.log").exists() @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index eed1121998..879183d0de 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -251,3 +251,9 @@ class GitRevisionTestCase: layout: str environment: dict[str, str] expected_dirty: object + + +@dataclass(frozen=True) +class LeanRecallTestCase: + description: str + args: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index bbbaa4e107..f55de15781 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -7,8 +7,14 @@ from tests.integration.src.sqlbuild.cli.recall._test_types import ( DeprecatedFormatTestCase, JsonErrorTestCase, + LeanRecallTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + OUTPUT_LOG_SINK, + run, + write_project, + write_sinks, ) -from tests.integration.src.sqlbuild.cli.recall.helpers import run, write_project TABLE_DEPRECATION: str = "--format table is deprecated and will be removed" @@ -93,3 +99,32 @@ def test_given_the_table_format_when_querying_then_it_works_with_a_deprecation_w assert rendered.returncode == 0 assert rendered.stdout assert (TABLE_DEPRECATION in rendered.stderr.decode()) is test_case.expected_warning + + +@pytest.mark.parametrize( + "test_case", + [ + LeanRecallTestCase( + description="query", + args=("recall", "query", "SELECT count(*) FROM invocations"), + ), + LeanRecallTestCase(description="schema", args=("recall", "schema")), + LeanRecallTestCase(description="output", args=("recall", "output", "last")), + ], + ids=lambda case: case.description, +) +def test_given_project_sinks_when_reading_recall_then_no_sink_runs_and_no_compute_log_is_kept( + test_case: LeanRecallTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + _ = write_sinks(project_dir, {"output_log": OUTPUT_LOG_SINK}) + assert run(project_dir, "build").returncode == 0 + (project_dir / "output.log").unlink() + logs_before: set[Path] = set((project_dir / "logs").glob("*/*")) + + read: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert read.returncode == 0, read.stderr.decode(errors="replace") + assert read.stdout + assert not (project_dir / "output.log").exists() + assert set((project_dir / "logs").glob("*/*")) == logs_before diff --git a/uv.lock b/uv.lock index 8d170cf0e7..5d7d033357 100644 --- a/uv.lock +++ b/uv.lock @@ -4326,7 +4326,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "clickhouse-connect", marker = "extra == 'clickhouse'", specifier = ">=0.8" }, + { name = "clickhouse-connect", marker = "extra == 'clickhouse'", specifier = ">=0.15" }, { name = "dagster", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "dagster-webserver", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "databricks-sql-connector", extras = ["pyarrow"], marker = "extra == 'databricks'", specifier = ">=4.1.2" }, diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 7d41a6a0c6..3f37697816 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -105,7 +105,8 @@ of its history still succeeds, prints a warning followed by its own final status `output_invocations` reports the gap. An invocation whose process died without finishing has status `abandoned`. -Every command is recorded except `sqb recall` itself, commands that create a project (such as +`sqb recall` only reads history: it publishes no lifecycle events, runs no project sinks, and keeps +no compute logs, so a query starts quickly. Every command is recorded except `sqb recall` itself, commands that create a project (such as `sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, `sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. From aac65ea3b92d9e604e20faf0bb54edc63c895efc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:34:25 +0100 Subject: [PATCH 07/18] fix: print sqb plan --json failures as a JSON error on stdout A plan that fails before it can be produced now prints the same {"error": {...}} document as sqb recall --json, so callers parsing the output read the error. --- .../src/bindings/_helpers/cli/entry.rs | 24 ++++++++++--------- .../sqlbuild/cli/recall/test_recall_cli.py | 5 ++++ website/src/content/docs/docs/cli/plan.mdx | 4 ++++ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index cbe8f1b3da..ea80449cf9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -201,20 +201,22 @@ fn write_compile_machine_error( print_object(&document, None, None) } -/// With `sqb recall … --json`, the failure as `{"error": {"code", "message", "help"}}` on stdout, so callers parsing the output read the error instead of failing to parse it. -fn write_recall_machine_error( +/// With `sqb recall … --json` or `sqb plan --json`, the failure as `{"error": {"code", "message", "help"}}` on stdout, so callers parsing the output read the error instead of failing to parse it. +fn write_machine_error( invocation: &Invocation<'_>, error: &Bound<'_, PyAny>, fallback_code: &str, ) -> PyResult<()> { - if !invocation.command_is("recall") { - return Ok(()); - } - let format: Option = invocation - .args - .getattr_opt("recall_format")? - .and_then(|format| format.extract().ok()); - if format.as_deref() != Some("json") { + let json_output: bool = if invocation.command_is("recall") { + invocation + .args + .getattr_opt("recall_format")? + .and_then(|format| format.extract::().ok()) + .is_some_and(|format| format == "json") + } else { + invocation.command_is("plan") && invocation.flag("json")? + }; + if !json_output { return Ok(()); } let (code, message, help) = error_parts(error, fallback_code)?; @@ -383,7 +385,7 @@ fn report_failure<'py>( error }; write_compile_machine_error(invocation, &effective, fallback_code)?; - write_recall_machine_error(invocation, &effective, fallback_code)?; + write_machine_error(invocation, &effective, fallback_code)?; if effective.is_instance(&error_class(py, ErrorId::QueryDiffOutcome)?)? { let line: Bound<'py, PyAny> = PyString::new(py, DIFF_OUTCOME_PREFIX) .into_any() diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index f55de15781..e03d923482 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -37,6 +37,11 @@ args=("recall", "query", "--json", "SELECT 1", "--target", "missing"), expected_code="C971", ), + JsonErrorTestCase( + description="a plan with an unknown selector", + args=("plan", "--json", "--select", "no_such_model"), + expected_code="S007", + ), JsonErrorTestCase( description="a schema request without a store", args=("recall", "schema", "--json", "--target", "missing"), diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 9340cfa2ef..8fb761fc2a 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -130,6 +130,10 @@ when the plan has an error-severity finding that `sqb build` would refuse, such claim; the final line then reads `✗ Plan failed with errors`. `sqb plan --json` uses the same exit codes, so CI can gate on either form. +When `sqb plan --json` fails before it can produce a plan, for example on an unknown selector, it +still exits `1` and prints its coded error on stderr, and prints the error as JSON on stdout: +`{"error": {"code": "S007", "message": "...", "help": null}}`. + Release note: earlier versions printed `Plan complete with errors` and exited `0` for such plans. ## Missing upstream dependencies From 7cdf01ed8fe4491bac47d88c10563f6d6bc99c2a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:48:33 +0100 Subject: [PATCH 08/18] perf: let Recall view filters reach the ClickHouse table reads The ClickHouse `invocations` and `statements` views now fold each invocation by `invocation_id` and `invocation_started_at`, which its rows share, so a filter on either column is pushed into the table scans instead of folding the whole retention window first. `statements` gains `invocation_started_at` in both dialects. The views stay additive; no summary tables are written. Synthetic store of 12,004 invocations and 804,268 events, rows read: statements for one invocation 1,773,957 -> 49,014 (381 -> 56 ms); invocations started in the last hour 2,705,033 -> 908,070; invocations for one invocation 121,525 -> 121,537 (already bounded). --- .../src/schema/_helpers/clickhouse_views.rs | 136 +++++++++--------- .../src/schema/_helpers/sqlite_views.rs | 3 +- .../commands/main/clickhouse/_test_types.py | 8 ++ .../cli/commands/main/clickhouse/helpers.py | 29 ++++ .../main/clickhouse/test_clickhouse_recall.py | 69 +++++++++ .../sqlbuild/cli/recall/test_recall_cli.py | 17 +++ .../docs/concepts/observability/recall.mdx | 4 + 7 files changed, 200 insertions(+), 66 deletions(-) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index a90d6fa71c..e042973bfc 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -25,6 +25,7 @@ fn deduplicated(table: Table, options: &DdlOptions) -> String { ) } +/// Every fold groups by `invocation_id` and `invocation_started_at`, which an invocation's rows share, so a filter on either reaches the table reads instead of folding the whole retention window. fn invocations(options: &DdlOptions) -> String { let context = deduplicated(Table::Invocations, options); let events = deduplicated(Table::Events, options); @@ -32,21 +33,23 @@ fn invocations(options: &DdlOptions) -> String { format!( "WITH ids AS ( - SELECT invocation_id, min(invocation_started_at) AS invocation_started_at + SELECT invocation_id, invocation_started_at FROM ( SELECT invocation_id, invocation_started_at FROM {context} UNION ALL SELECT invocation_id, invocation_started_at FROM {beats} UNION ALL SELECT invocation_id, invocation_started_at FROM {events} WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') ) - GROUP BY invocation_id + GROUP BY invocation_id, invocation_started_at ), started AS ( - SELECT invocation_id, min(occurred_at) AS started_at, any(command) AS command - FROM {events} WHERE event_type = 'invocation_started' GROUP BY invocation_id + SELECT invocation_id, invocation_started_at, min(occurred_at) AS started_at, + any(command) AS command + FROM {events} WHERE event_type = 'invocation_started' + GROUP BY invocation_id, invocation_started_at ), terminals AS ( - SELECT invocation_id, + SELECT invocation_id, invocation_started_at, argMax(event_type, tuple(invocation_sequence, occurred_at)) AS terminal_type, argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS terminal_at, argMax(exit_code, tuple(invocation_sequence, occurred_at)) AS terminal_exit_code, @@ -57,12 +60,12 @@ terminals AS ( tuple(invocation_sequence, occurred_at)) AS terminal_error_message FROM {events} WHERE event_type IN ('invocation_completed', 'invocation_failed') - GROUP BY invocation_id + GROUP BY invocation_id, invocation_started_at ), beats AS ( - SELECT invocation_id, max(beat_at) AS last_heartbeat_at, max(stale_at) AS stale_at, + SELECT invocation_id, invocation_started_at, max(beat_at) AS last_heartbeat_at, max(stale_at) AS stale_at, min(command) AS command, min(subcommand) AS subcommand - FROM {beats} GROUP BY invocation_id + FROM {beats} GROUP BY invocation_id, invocation_started_at ) SELECT ids.invocation_id AS invocation_id, @@ -102,91 +105,94 @@ SELECT beats.last_heartbeat_at AS last_heartbeat_at FROM ids LEFT JOIN {context} AS context ON context.invocation_id = ids.invocation_id + AND context.invocation_started_at = ids.invocation_started_at LEFT JOIN started ON started.invocation_id = ids.invocation_id + AND started.invocation_started_at = ids.invocation_started_at LEFT JOIN terminals ON terminals.invocation_id = ids.invocation_id + AND terminals.invocation_started_at = ids.invocation_started_at LEFT JOIN beats ON beats.invocation_id = ids.invocation_id + AND beats.invocation_started_at = ids.invocation_started_at SETTINGS join_use_nulls = 1" ) } +/// One fold per statement, grouped by its invocation as well, so a filter on `invocation_id` or `invocation_started_at` reaches the table reads. fn statements(options: &DdlOptions) -> String { let events = deduplicated(Table::Events, options); let artifacts = clickhouse_table(table_spec(Table::Artifacts).name, options); + let order = "tuple(invocation_sequence, occurred_at)"; + let terminal = "event_type IN ('statement_completed', 'statement_failed')"; format!( "WITH facts AS ( SELECT * FROM {events} WHERE statement_id IS NOT NULL AND startsWith(event_type, 'statement_') ), -firsts AS ( - SELECT statement_id, - argMin(invocation_id, tuple(invocation_sequence, occurred_at)) AS first_invocation_id, - argMin(run_id, tuple(invocation_sequence, occurred_at)) AS first_run_id, - argMin(resource_id, tuple(invocation_sequence, occurred_at)) AS first_resource_id, - argMin(resource_attempt_id, tuple(invocation_sequence, occurred_at)) AS first_resource_attempt_id, - argMin(operation_id, tuple(invocation_sequence, occurred_at)) AS first_operation_id, - argMin(intent, tuple(invocation_sequence, occurred_at)) AS first_intent, - argMin(statement_kind, tuple(invocation_sequence, occurred_at)) AS first_statement_kind, - argMin(purpose, tuple(invocation_sequence, occurred_at)) AS first_purpose, - argMin(sql_digest, tuple(invocation_sequence, occurred_at)) AS first_sql_digest, +folded AS ( + SELECT invocation_id, invocation_started_at, statement_id, + argMin(run_id, {order}) AS first_run_id, + argMin(resource_id, {order}) AS first_resource_id, + argMin(resource_attempt_id, {order}) AS first_resource_attempt_id, + argMin(operation_id, {order}) AS first_operation_id, + argMin(intent, {order}) AS first_intent, + argMin(statement_kind, {order}) AS first_statement_kind, + argMin(purpose, {order}) AS first_purpose, + argMin(sql_digest, {order}) AS first_sql_digest, minIf(occurred_at, event_type = 'statement_started') AS started_at, minIf(occurred_at, event_type = 'statement_submitted') AS submitted_at, - max(query_id) AS any_query_id - FROM facts GROUP BY statement_id -), -terminals AS ( - SELECT statement_id, - argMax(event_type, tuple(invocation_sequence, occurred_at)) AS last_event_type, - argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS ended_at, - argMax(query_id, tuple(invocation_sequence, occurred_at)) AS last_query_id, - argMax(error_type, tuple(invocation_sequence, occurred_at)) AS last_error_type, - argMax(error_code, tuple(invocation_sequence, occurred_at)) AS last_error_code, - argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS last_duration_ms, - argMax(row_count, tuple(invocation_sequence, occurred_at)) AS last_row_count, - argMax(affected_rows, tuple(invocation_sequence, occurred_at)) AS last_affected_rows - FROM facts - WHERE event_type IN ('statement_completed', 'statement_failed') - GROUP BY statement_id + max(query_id) AS any_query_id, + countIf({terminal}) AS terminal_count, + argMaxIf(event_type, {order}, {terminal}) AS last_event_type, + argMaxIf(occurred_at, {order}, {terminal}) AS last_ended_at, + argMaxIf(query_id, {order}, {terminal}) AS last_query_id, + argMaxIf(error_type, {order}, {terminal}) AS last_error_type, + argMaxIf(error_code, {order}, {terminal}) AS last_error_code, + argMaxIf(duration_ms, {order}, {terminal}) AS last_duration_ms, + argMaxIf(row_count, {order}, {terminal}) AS last_row_count, + argMaxIf(affected_rows, {order}, {terminal}) AS last_affected_rows + FROM facts GROUP BY invocation_id, invocation_started_at, statement_id ), resources AS ( - SELECT invocation_id, resource_id, min(resource_name) AS first_resource_name, - min(resource_kind) AS first_resource_kind + SELECT invocation_id, invocation_started_at, resource_id, + min(resource_name) AS first_resource_name, min(resource_kind) AS first_resource_kind FROM {events} WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL - GROUP BY invocation_id, resource_id + GROUP BY invocation_id, invocation_started_at, resource_id ) SELECT - firsts.first_invocation_id AS invocation_id, - firsts.statement_id AS statement_id, - firsts.first_run_id AS run_id, - firsts.first_resource_id AS resource_id, + folded.invocation_id AS invocation_id, + folded.invocation_started_at AS invocation_started_at, + folded.statement_id AS statement_id, + folded.first_run_id AS run_id, + folded.first_resource_id AS resource_id, resources.first_resource_name AS resource_name, resources.first_resource_kind AS resource_kind, - firsts.first_resource_attempt_id AS resource_attempt_id, - firsts.first_operation_id AS operation_id, - firsts.first_intent AS intent, - firsts.first_statement_kind AS statement_kind, - firsts.first_purpose AS purpose, - coalesce(terminals.last_query_id, firsts.any_query_id) AS query_id, - firsts.started_at AS started_at, - firsts.submitted_at AS submitted_at, - terminals.ended_at AS ended_at, + folded.first_resource_attempt_id AS resource_attempt_id, + folded.first_operation_id AS operation_id, + folded.first_intent AS intent, + folded.first_statement_kind AS statement_kind, + folded.first_purpose AS purpose, + if(folded.terminal_count > 0, coalesce(folded.last_query_id, folded.any_query_id), + folded.any_query_id) AS query_id, + folded.started_at AS started_at, + folded.submitted_at AS submitted_at, + if(folded.terminal_count > 0, folded.last_ended_at, NULL) AS ended_at, multiIf( - terminals.last_event_type = 'statement_completed', 'completed', - terminals.last_event_type = 'statement_failed', 'failed', - 'no_terminal' + folded.terminal_count = 0, 'no_terminal', + folded.last_event_type = 'statement_completed', 'completed', + 'failed' ) AS outcome, - terminals.last_error_type AS error_type, - terminals.last_error_code AS error_code, - terminals.last_duration_ms AS duration_ms, - terminals.last_row_count AS row_count, - terminals.last_affected_rows AS affected_rows, - firsts.first_sql_digest AS sql_digest, - concat('sha256:', firsts.first_sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available -FROM firsts -LEFT JOIN terminals ON terminals.statement_id = firsts.statement_id + if(folded.terminal_count > 0, folded.last_error_type, NULL) AS error_type, + if(folded.terminal_count > 0, folded.last_error_code, NULL) AS error_code, + if(folded.terminal_count > 0, folded.last_duration_ms, NULL) AS duration_ms, + if(folded.terminal_count > 0, folded.last_row_count, NULL) AS row_count, + if(folded.terminal_count > 0, folded.last_affected_rows, NULL) AS affected_rows, + folded.first_sql_digest AS sql_digest, + concat('sha256:', folded.first_sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available +FROM folded LEFT JOIN resources - ON resources.invocation_id = firsts.first_invocation_id - AND resources.resource_id = firsts.first_resource_id + ON resources.invocation_id = folded.invocation_id + AND resources.invocation_started_at = folded.invocation_started_at + AND resources.resource_id = folded.first_resource_id SETTINGS join_use_nulls = 1" ) } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 3f46b6ef9d..122a62434e 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -11,7 +11,7 @@ const STATEMENTS: &str = "WITH facts AS ( WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' ), firsts AS ( - SELECT invocation_id, statement_id, run_id, resource_id, resource_attempt_id, operation_id, + SELECT invocation_id, invocation_started_at, statement_id, run_id, resource_id, resource_attempt_id, operation_id, intent, statement_kind, purpose, sql_digest, ROW_NUMBER() OVER ( PARTITION BY statement_id ORDER BY invocation_sequence, occurred_at, event_id @@ -48,6 +48,7 @@ resources AS ( ) SELECT firsts.invocation_id, + firsts.invocation_started_at, firsts.statement_id, firsts.run_id, firsts.resource_id, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 58ed3c1736..56081c6069 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -207,3 +207,11 @@ class ClickHouseRecallBehindStoreTestCase: expected_ddl_fragments: tuple[str, ...] expected_recorded: list[list[object]] expected_heartbeat_columns: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRecallPushdownTestCase: + description: str + sql: str + expected_rows: list[list[object]] + max_read_fraction: float diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index bd20eb141a..777a44cd28 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -532,6 +532,35 @@ def limited_user( execute_clickhouse_sql(config=config, statements=(f"DROP USER IF EXISTS {name}",)) +def replicate_invocations( + *, config: dict[str, object], database: str, copies: int, spacing_minutes: int +) -> None: + """Store `copies` more invocations shaped like each recorded one, `spacing_minutes` apart.""" + + shift: str = f"toIntervalMinute(n * {spacing_minutes})" + shifted: dict[str, str] = { + "recall_events": ( + ", if(statement_id IS NULL, NULL, concat(statement_id, '-', toString(n))) " + "AS statement_id, concat(event_id, '-', toString(n)) AS event_id, " + f"occurred_at - {shift} AS occurred_at" + ), + "recall_invocations": "", + "recall_heartbeats": f", beat_at - {shift} AS beat_at, stale_at - {shift} AS stale_at", + } + execute_clickhouse_sql( + config=config, + statements=tuple( + f"INSERT INTO {database}.{table} SELECT * EXCEPT (n) REPLACE (" + "concat(invocation_id, '-', toString(n)) AS invocation_id, " + f"invocation_started_at - {shift} AS invocation_started_at{extra}) " + f"FROM (SELECT * FROM {database}.{table}) AS source " + f"CROSS JOIN (SELECT number + 1 AS n FROM numbers({copies})) AS copies " + "SETTINGS insert_deduplicate = 0, max_partitions_per_insert_block = 1000" + for table, extra in shifted.items() + ), + ) + + def duplicate_recall_rows(*, config: dict[str, object], database: str) -> None: """Deliver every stored Recall row a second time, as a retried insert would.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index d1f2ee2697..18350c48df 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -19,6 +19,7 @@ ClickHouseRecallOutputTestCase, ClickHouseRecallPartialInsertTestCase, ClickHouseRecallPermissionTestCase, + ClickHouseRecallPushdownTestCase, ClickHouseRecallQueryOutputSinkTestCase, ClickHouseRecallRetentionConflictTestCase, ClickHouseRecallRetentionTestCase, @@ -37,6 +38,7 @@ fetch_clickhouse_rows, limited_user, physical_rows, + replicate_invocations, stored_ttl_days, write_recall_project, ) @@ -820,3 +822,70 @@ def test_given_a_store_one_version_behind_when_a_select_insert_user_builds_then_ ) == test_case.expected_heartbeat_columns ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallPushdownTestCase( + description="statements of one invocation read only that invocation's granules", + sql="SELECT count() > 0 FROM statements WHERE invocation_id = '{invocation_id}'", + expected_rows=[[1]], + max_read_fraction=0.25, + ), + ClickHouseRecallPushdownTestCase( + description="statements expose invocation_started_at to bound a scan", + sql=( + "SELECT count() > 0 FROM statements " + "WHERE invocation_started_at >= '{invocation_started_at}'" + ), + expected_rows=[[1]], + max_read_fraction=0.25, + ), + ClickHouseRecallPushdownTestCase( + description="invocations of one invocation read a bounded part of the store", + sql="SELECT count() FROM invocations WHERE invocation_id = '{invocation_id}'", + expected_rows=[[1]], + max_read_fraction=0.25, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_large_store_when_filtering_a_view_then_the_filter_reaches_the_table_reads( + test_case: ClickHouseRecallPushdownTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + assert run(project_dir, "build").returncode == 0 + replicate_invocations( + config=clickhouse_e2e_config, database=recall_database, copies=3000, spacing_minutes=10 + ) + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=(f"OPTIMIZE TABLE {recall_database}.recall_events FINAL",), + ) + (invocation_id, started_at), *_ = fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT invocation_id, toString(invocation_started_at) " + f"FROM {recall_database}.recall_invocations WHERE position(invocation_id, '-') = 0", + ) + stored_events: int = physical_rows( + config=clickhouse_e2e_config, database=recall_database, where="1" + ) + + queried: subprocess.CompletedProcess[bytes] = run( + project_dir, + "recall", + "query", + "--json", + test_case.sql.format(invocation_id=invocation_id, invocation_started_at=started_at), + ) + + document: dict[str, object] = json.loads(queried.stdout) + statistics: dict[str, int] = cast(dict[str, int], document["statistics"]) + assert document["rows"] == test_case.expected_rows + assert statistics["rows_read"] < stored_events * test_case.max_read_fraction diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index e03d923482..977d555ef8 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -11,6 +11,7 @@ ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( OUTPUT_LOG_SINK, + recall_rows, run, write_project, write_sinks, @@ -133,3 +134,19 @@ def test_given_project_sinks_when_reading_recall_then_no_sink_runs_and_no_comput assert read.stdout assert not (project_dir / "output.log").exists() assert set((project_dir / "logs").glob("*/*")) == logs_before + + +def test_given_recorded_statements_when_filtered_by_start_time_then_the_view_exposes_it( + tmp_path: Path, +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + assert run(project_dir, "build").returncode == 0 + + rows: list[list[object]] = recall_rows( + project_dir, + "SELECT COUNT(*) > 0, COUNT(DISTINCT s.invocation_started_at = i.invocation_started_at) " + "FROM statements s JOIN invocations i USING (invocation_id) " + "WHERE s.invocation_started_at >= (SELECT MIN(invocation_started_at) FROM invocations)", + ) + + assert rows == [[1, 1]] diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 3f37697816..43c9c0ba68 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -350,6 +350,10 @@ in the store's SQL dialect, against these views: Run `sqb recall schema` for every column. +Every view has `invocation_id` and `invocation_started_at`. Filter on them to keep a query cheap: +on a ClickHouse store, a filter on either reaches the table reads, so a query about one invocation or +the last day reads that part of the store instead of folding the whole retention window. + ## Relationship to compute logs Compute logs (`logs///stdout.log`, `stderr.log`, and `diagnostics.jsonl`) are From 54f6e0ad0b64b387fdd0e6ead48abde39b742c24 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:08:42 +0100 Subject: [PATCH 09/18] fix: report a Recall store still being created as not recorded yet A query that ran while a concurrent command was still creating the local store's views failed with C974 (no such table), which made the concurrent-builds test flaky under load. It now reports C971, like a store that does not exist yet. --- .../src/bindings/_helpers/recall/command.rs | 20 ++++++++++++++--- .../sqlbuild/cli/recall/test_recall_cli.py | 22 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index 6dc0291723..e51ef28f48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -6,6 +6,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; use sqlbuild_recall::records::models::OutputStream; +use sqlbuild_recall::schema::constants::VIEW_NAMES; use sqlbuild_recall::schema::main::ddl_script::{ddl_statements, render_script}; use sqlbuild_recall::schema::main::upgrade_statements::upgrade_statements; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; @@ -237,6 +238,19 @@ fn open_store(py: Python<'_>, settings: &RecallSettings, checked: bool) -> PyRes } } +/// The "no history yet" failure when a query failed because a writer is still creating the store's views, so it reads like the store not existing yet rather than an invalid query. +fn not_created(store: &SqliteStore, path: &str) -> Option { + let observed = observe_sqlite_schema(store).ok()?; + let complete: bool = VIEW_NAMES.iter().all(|view| observed.views.contains(*view)); + (!complete).then(|| StoreError { + kind: StoreErrorKind::Other, + message: format!( + "no Recall history is recorded yet at {path}, or its tables are still being created; \ + run a command first" + ), + }) +} + /// `sqb recall query --backend warehouse` and `schema --backend warehouse` run through `sqb query`. fn warehouse_query( py: Python<'_>, @@ -286,9 +300,9 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { let started = Instant::now(); let limit = limit.map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); let queried = match &store { - ReadStore::Sqlite { store, .. } => { - query_store(store, &sql, limit).map(|result| (result, None)) - } + ReadStore::Sqlite { store, path } => query_store(store, &sql, limit) + .map(|result| (result, None)) + .map_err(|failure| not_created(store, path).unwrap_or(failure)), ReadStore::ClickHouse(reader) => reader .query(py, &sql, limit) .map(|(result, statistics)| (result, Some(statistics))), diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index 977d555ef8..a4483176e8 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -1,4 +1,5 @@ import json +import sqlite3 import subprocess from pathlib import Path @@ -150,3 +151,24 @@ def test_given_recorded_statements_when_filtered_by_start_time_then_the_view_exp ) assert rows == [[1, 1]] + + +def test_given_a_store_whose_views_are_still_being_created_when_queried_then_it_is_not_recorded_yet( + tmp_path: Path, +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + assert run(project_dir, "build").returncode == 0 + with sqlite3.connect(project_dir / "logs" / "recall.sqlite3") as connection: + views: list[str] = [ + row[0] + for row in connection.execute("SELECT name FROM sqlite_schema WHERE type = 'view'") + ] + for view in views: + _ = connection.execute(f'DROP VIEW "{view}"') + + queried: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "SELECT COUNT(*) FROM invocations" + ) + + assert queried.returncode == 1 + assert "error[C971]: no Recall history is recorded yet" in queried.stderr.decode() From c2ff593e5097c7783df9259fb8e7ad5ab72d6329 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:11:57 +0100 Subject: [PATCH 10/18] docs: add a Recall investigation guide to the agent skill references/investigating-runs.md describes how the store works, values worth knowing, traps, output-format guidance and worked investigations checked against SQLite and ClickHouse stores, and ends with every view and column. That section is generated from `sqb recall schema` by the docs-skill generator, and check-docs-skill fails when it drifts. SKILL.md points to the guide and now names read_connection. The bundled docs pages are regenerated. --- scripts/docs_skill/_helpers/recall_schema.py | 60 +++ scripts/docs_skill/constants.py | 5 + scripts/docs_skill/main/check.py | 18 +- scripts/docs_skill/main/generate.py | 19 +- src/sqlbuild/.agents/skills/sqlbuild/SKILL.md | 11 +- .../sqlbuild/references/docs/cli/debug.md | 8 + .../sqlbuild/references/docs/cli/plan.md | 4 + .../sqlbuild/references/docs/cli/recall.md | 47 ++- .../docs/concepts/adapters/clickhouse.md | 5 +- .../docs/concepts/adapters/sqlite.md | 3 +- .../docs/concepts/observability/recall.md | 137 +++++-- .../sqlbuild/references/investigating-runs.md | 366 ++++++++++++++++++ .../cli/recall/test_recall_connections.py | 8 +- .../recall/test_recall_invocation_facts.py | 7 +- .../docs_skill/_helpers/_test_types.py | 8 + .../docs_skill/_helpers/test_recall_schema.py | 41 ++ .../scripts/docs_skill/main/test_check.py | 9 +- 17 files changed, 697 insertions(+), 59 deletions(-) create mode 100644 scripts/docs_skill/_helpers/recall_schema.py create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md create mode 100644 tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py diff --git a/scripts/docs_skill/_helpers/recall_schema.py b/scripts/docs_skill/_helpers/recall_schema.py new file mode 100644 index 0000000000..abb44e05d2 --- /dev/null +++ b/scripts/docs_skill/_helpers/recall_schema.py @@ -0,0 +1,60 @@ +"""The Recall views section of the investigation guide, rendered by `sqb recall schema` itself.""" + +import os +import sqlite3 +import subprocess +import sys +import tempfile +from pathlib import Path + +from scripts.docs_skill.constants import SCHEMA_SECTION_END, SCHEMA_SECTION_START + +_PROJECT_TOML: str = 'name = "orders_shop"\nadapter = "duckdb"\n' +_TABLE_MARKER: str = " (table)" + + +def _sqb(project_dir: Path, *args: str) -> str: + home: Path = project_dir / "home" + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(Path(sys.executable).with_name("sqlbuild-python")), *args], + cwd=project_dir, + capture_output=True, + check=True, + text=True, + env={ + **os.environ, + "HOME": str(home), + "XDG_CACHE_HOME": str(home / ".cache"), + "XDG_CONFIG_HOME": str(home / ".config"), + "NO_COLOR": "1", + }, + ) + return completed.stdout + + +def render_recall_views() -> str: + """Every Recall view, its columns and join keys, as `sqb recall schema` prints a new store.""" + + with tempfile.TemporaryDirectory() as directory: + project_dir: Path = Path(directory) + (project_dir / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") + script: str = _sqb(project_dir, "recall", "schema", "--ddl") + (project_dir / "logs").mkdir() + with sqlite3.connect(project_dir / "logs" / "recall.sqlite3") as connection: + connection.executescript(script) + connection.close() + described: list[str] = _sqb(project_dir, "recall", "schema").splitlines() + views: list[str] = [] + for line in described[1:]: + if _TABLE_MARKER in line and not line.startswith(" "): + break + views.append(line) + return "\n".join(views).strip() + "\n" + + +def with_recall_views(guide: str, views: str) -> str: + """`guide` with its generated schema section replaced by `views`.""" + + start: int = guide.index(SCHEMA_SECTION_START) + len(SCHEMA_SECTION_START) + end: int = guide.index(SCHEMA_SECTION_END) + return f"{guide[:start]}\n\n```text\n{views}```\n\n{guide[end:]}" diff --git a/scripts/docs_skill/constants.py b/scripts/docs_skill/constants.py index 9e230cf9b3..40d5571059 100644 --- a/scripts/docs_skill/constants.py +++ b/scripts/docs_skill/constants.py @@ -15,3 +15,8 @@ FRONTMATTER_DELIMITER: str = "---" TABLE_OF_CONTENTS_MIN_LINES: int = 100 UNGROUPED_LABEL: str = "Other pages" +INVESTIGATION_GUIDE: Path = Path( + "src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md" +) +SCHEMA_SECTION_START: str = "" +SCHEMA_SECTION_END: str = "" diff --git a/scripts/docs_skill/main/check.py b/scripts/docs_skill/main/check.py index 448ec6b068..115739bdcf 100644 --- a/scripts/docs_skill/main/check.py +++ b/scripts/docs_skill/main/check.py @@ -7,8 +7,9 @@ from pathlib import Path from scripts.docs_skill._helpers.output import outdated_reference_pages +from scripts.docs_skill._helpers.recall_schema import render_recall_views, with_recall_views from scripts.docs_skill._helpers.render import build_reference_pages -from scripts.docs_skill.constants import DEFAULT_DOCS_ROOT, DEFAULT_OUTPUT_DIR +from scripts.docs_skill.constants import DEFAULT_DOCS_ROOT, DEFAULT_OUTPUT_DIR, INVESTIGATION_GUIDE def check_docs_skill(argv: list[str] | None = None) -> int: @@ -19,6 +20,10 @@ def check_docs_skill(argv: list[str] | None = None) -> int: output_dir=args.output_dir, pages=build_reference_pages(docs_root=args.docs_root), ) + if args.investigation_guide is not None: + guide: str = args.investigation_guide.read_text(encoding="utf-8") + if with_recall_views(guide, render_recall_views()) != guide: + outdated = (*outdated, args.investigation_guide.as_posix()) if not outdated: print(f"Docs skill reference pages in {args.output_dir} are up to date") return 0 @@ -38,4 +43,13 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--docs-root", type=Path, default=DEFAULT_DOCS_ROOT) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) - return parser.parse_args(argv) + parser.add_argument( + "--investigation-guide", + type=Path, + default=INVESTIGATION_GUIDE, + help="guide whose Recall schema section is checked; '' skips it", + ) + args: argparse.Namespace = parser.parse_args(argv) + if args.investigation_guide == Path(): + args.investigation_guide = None + return args diff --git a/scripts/docs_skill/main/generate.py b/scripts/docs_skill/main/generate.py index 94d9deaac5..5ba6fd0a12 100644 --- a/scripts/docs_skill/main/generate.py +++ b/scripts/docs_skill/main/generate.py @@ -6,8 +6,9 @@ from pathlib import Path from scripts.docs_skill._helpers.output import write_reference_pages +from scripts.docs_skill._helpers.recall_schema import render_recall_views, with_recall_views from scripts.docs_skill._helpers.render import build_reference_pages -from scripts.docs_skill.constants import DEFAULT_DOCS_ROOT, DEFAULT_OUTPUT_DIR +from scripts.docs_skill.constants import DEFAULT_DOCS_ROOT, DEFAULT_OUTPUT_DIR, INVESTIGATION_GUIDE def generate_docs_skill(argv: list[str] | None = None) -> int: @@ -18,6 +19,11 @@ def generate_docs_skill(argv: list[str] | None = None) -> int: output_dir=args.output_dir, pages=build_reference_pages(docs_root=args.docs_root), ) + if args.investigation_guide is not None: + guide: str = args.investigation_guide.read_text(encoding="utf-8") + _ = args.investigation_guide.write_text( + with_recall_views(guide, render_recall_views()), encoding="utf-8" + ) return 0 @@ -29,4 +35,13 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--docs-root", type=Path, default=DEFAULT_DOCS_ROOT) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) - return parser.parse_args(argv) + parser.add_argument( + "--investigation-guide", + type=Path, + default=INVESTIGATION_GUIDE, + help="guide whose Recall schema section is regenerated; '' skips it", + ) + args: argparse.Namespace = parser.parse_args(argv) + if args.investigation_guide == Path(): + args.investigation_guide = None + return args diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 9d81f9037f..3f57f46e06 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -76,8 +76,11 @@ statements and SQL, audit results, errors, timing and exact output. `sqb recall views (`invocations`, `statements`, `audit_results`, `artifacts` and more); `sqb recall query "SELECT ..."` runs SQL over them; `sqb recall output last --tail 50` prints what the latest command printed. Add `--backend warehouse` to check live warehouse state with the same command. +Read [references/investigating-runs.md](references/investigating-runs.md) before writing recall +queries; `statements` and `audit_results` use `outcome`, not `status`. -Check `[recall] connection` in the project config to see which store you are reading: +Check `read_connection` in `[recall]` or `[targets..recall]` to see which store you are +reading: - **Local (the default).** A SQLite file, `logs/recall.sqlite3`, holding this checkout's own runs. Use it for "what did my last build do?" and to compare runs during your session. @@ -86,8 +89,8 @@ Check `[recall] connection` in the project config to see which store you are rea actually ran and printed last night without access to the machine, find the first run where an audit started failing or a statement slowed down across all runs, and compare a production run with yours by `target`, `git_revision`, `sqlbuild_version` or `host_token`. Queries use - ClickHouse SQL, and `--json` reports `rows_read` and `bytes_read`; filter by time and keep a - `LIMIT` on large stores. + ClickHouse SQL, and `--json` reports `rows_read` and `bytes_read`; filter on `invocation_id` or + `invocation_started_at` and keep a `LIMIT` on large stores. Read [references/docs/concepts/observability/recall.md](references/docs/concepts/observability/recall.md). @@ -207,6 +210,8 @@ diff. Rules. - [references/macros-and-declarations.md](references/macros-and-declarations.md): macros, enums, constants, placement and scopes. + - [references/investigating-runs.md](references/investigating-runs.md): Recall store design, + every view and column, traps and worked investigations of past runs. - Full documentation for this SQLBuild version: [references/docs/CONTENTS.md](references/docs/CONTENTS.md) lists every page; open only the page you need, for example `references/docs/cli/build.md` or `references/docs/concepts/snapshots.md`. The same pages are online at diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md index 08d13112a7..6360945e36 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md @@ -82,3 +82,11 @@ Connection: ## Exit codes Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). +A failed run is recorded in [Recall](../concepts/observability/recall.md) with the first failed +check as its cause: + +| `error_code` | `error_type` | Meaning | +|------|------|---------| +| `C980` | `ConnectionError` | The connection test failed | +| `C981` | `QueryError` | The connection worked but `SELECT 1` failed | +| `C982` | `ConfigurationError` | Another check failed, such as an invalid configuration file | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index fa6806eb22..38771ec77c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -140,6 +140,10 @@ when the plan has an error-severity finding that `sqb build` would refuse, such claim; the final line then reads `✗ Plan failed with errors`. `sqb plan --json` uses the same exit codes, so CI can gate on either form. +When `sqb plan --json` fails before it can produce a plan, for example on an unknown selector, it +still exits `1` and prints its coded error on stderr, and prints the error as JSON on stdout: +`{"error": {"code": "S007", "message": "...", "help": null}}`. + Release note: earlier versions printed `Plan complete with errors` and exited `0` for such plans. ## Missing upstream dependencies diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md index c243f37b4e..20c58781f3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -14,8 +14,9 @@ Online: https://sqlbuild.com/docs/cli/recall/ - sqb recall output - Errors -Read the local [Recall](../concepts/observability/recall.md) store: past invocations, the -statements they executed, and what they printed. `sqb recall` never records itself. +Read the [Recall](../concepts/observability/recall.md) store that `read_connection` names, or the +local store: past invocations, the statements they executed, and what they printed. `sqb recall` +never records itself, and never creates or changes the store. ## Usage @@ -34,7 +35,7 @@ Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| | `sql` | The query (positional argument) | -| `--format` | `long` (default), `table`, `json`, or `csv` | +| `--format` | `long` (default) for reading, `json` for programs, or `csv` for export. `table` still works in this release but prints a deprecation warning and will be removed | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 100) | | `--no-limit` | Disable the row limit | @@ -45,15 +46,17 @@ Statements that write, and multiple statements, are rejected. The latest invocation in a project named `orders_shop`: ```bash -sqb recall query --format table \ +sqb recall query \ "SELECT command, status, exit_code, duration_ms FROM invocations ORDER BY invocation_started_at DESC LIMIT 1" ``` ``` -command | status | exit_code | duration_ms -------- | --------- | --------- | ----------- -build | completed | 0 | 752.0 +-[ RECORD 1 ]---------------------------+ +command | build +status | completed +exit_code | 0 +duration_ms | 752.0 1 row ``` @@ -61,17 +64,22 @@ build | completed | 0 | 752.0 The slowest statements of past builds, with their SQL: ```bash -sqb recall query --format table --max-cell-bytes 60 \ +sqb recall query --max-cell-bytes 60 \ "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql FROM statements s JOIN artifacts a USING (invocation_id, statement_id) WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" ``` ``` -resource_name | ms | sql -------------- | --- | ------------------------------------------------------------------------ -orders | 3.6 | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] -NULL | 3.2 | CREATE SCHEMA IF NOT EXISTS dev +-[ RECORD 1 ]---------------------------+ +resource_name | orders +ms | 3.6 +sql | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] + +-[ RECORD 2 ]---------------------------+ +resource_name | NULL +ms | 3.2 +sql | CREATE SCHEMA IF NOT EXISTS dev 2 rows ``` @@ -90,7 +98,12 @@ Timestamps are ISO-8601 UTC text, so they compare correctly as strings. Filter o ## sqb recall schema List every Recall view with its columns, types, and join keys. Accepts `--format`, `--json`, and -`--target`. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse +`--target`. + +With `--ddl`, print instead the SQL that creates the store, or brings an existing store up to this +SQLBuild version (missing columns, replaced views, and the schema version row), for an +administrator to run. It reads the store through `read_connection` and changes nothing; when the +store is up to date it prints nothing and says so on stderr. Run it again after every upgrade. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse instead, or describes one with `--relation `. ## sqb recall output @@ -116,6 +129,14 @@ other reason is followed by a note on stderr naming the gap. ## Errors +With `--json` (or `--format json`), a failing `query` or `schema` still exits `1` and prints its +coded error on stderr, and prints the error as JSON on stdout, so a program parsing the output reads +the error instead of failing to parse it: + +```json +{"error": {"code": "C974", "message": "invalid Recall query: no such column: status ...", "help": null}} +``` + | Code | Meaning | |------|---------| | `C970` | Recall is disabled for this project or target | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index 5367571795..80a71cf9f5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -195,6 +195,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from A ClickHouse connection can also hold [Recall](../observability/recall.md)'s command history for a whole team, whichever adapter the project builds with. Set -`[recall] connection` to it; Recall creates its tables on first use, keeps every output byte by -default, and needs only `SELECT` and `INSERT` once the tables exist. See +`[recall] write_connection` and `read_connection` to it, one connection or separate writer and +reader users. Recall creates its tables on first use, keeps every output byte by default, and needs +only `SELECT` and `INSERT` once the tables exist; readers need only `SELECT`. See [Sharing history in ClickHouse](../observability/recall.md#sharing-history-in-clickhouse). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md index fc912192a9..1d0dcc04fa 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -136,4 +136,5 @@ as `datetime(ordered_at, 'start of day')`, have no DuckDB translation and fail l [Recall](../observability/recall.md) keeps its local history in a SQLite file, `logs/recall.sqlite3`, for every project, whichever adapter it builds with. Point -`[recall] connection` at a `sqlite` connection to put the file somewhere else. +`[recall] write_connection` and `read_connection` at a `sqlite` connection to put the file somewhere +else. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index e834bcc90c..7f875b4941 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/concepts/observability/recall/ - An investigation, start to finish - What is recorded +- Git revision - Settings - Output limit - Retention @@ -32,18 +33,33 @@ still pass, because they run on mocked data. The build finishes with a warning t The next morning, ask when the audit started failing, and what each of those builds reported: ```bash -sqb recall query --format table \ +sqb recall query \ "SELECT a.invocation_started_at AS run, a.outcome, a.violation_count, i.status, i.exit_code FROM audit_results a JOIN invocations i USING (invocation_id) WHERE a.audit_name = 'revenue_is_non_negative' ORDER BY run DESC LIMIT 3" ``` ``` -run | outcome | violation_count | status | exit_code ---------------------------- | ------- | --------------- | --------- | --------- -2026-10-20T00:01:44.784683Z | warn | 1 | completed | 0 -2026-10-20T00:00:34.652468Z | pass | 0 | completed | 0 -2026-10-19T23:59:23.827831Z | pass | 0 | completed | 0 +-[ RECORD 1 ]---------------------------+ +run | 2026-10-20T00:01:44.784683Z +outcome | warn +violation_count | 1 +status | completed +exit_code | 0 + +-[ RECORD 2 ]---------------------------+ +run | 2026-10-20T00:00:34.652468Z +outcome | pass +violation_count | 0 +status | completed +exit_code | 0 + +-[ RECORD 3 ]---------------------------+ +run | 2026-10-19T23:59:23.827831Z +outcome | pass +violation_count | 0 +status | completed +exit_code | 0 3 rows ``` @@ -66,14 +82,16 @@ sqb recall output last --tail 4 Then check what is true in the warehouse now, with `sqb recall query` again: ```bash -sqb recall query --backend warehouse --format table \ +sqb recall query --backend warehouse \ "SELECT payment_id, order_id, amount_cents, paid_at FROM prod.stg_payments WHERE amount_cents < 0" ``` ``` -payment_id | order_id | amount_cents | paid_at ----------- | -------- | ------------ | ------------------- -9 | 8 | -2200 | 2026-04-03 15:42:00 +-[ RECORD 1 ]---------------------------+ +payment_id | 9 +order_id | 8 +amount_cents | -2200 +paid_at | 2026-04-03 15:42:00 1 row ``` @@ -86,7 +104,11 @@ return `--json`. For every recorded invocation, Recall keeps: -- the invocation itself: command, arguments, target, adapter, status, exit code, and timing; +- the invocation itself: command, arguments, target, adapter, status, exit code, and timing, and + for a failed invocation its cause as `error_type`, `error_code` and a one-line + `error_message`, also when the command reported the failure only through its exit code (for + example `sqb debug` failing to connect records `ConnectionError` and `C980`); +- the commit it ran from (`git_revision`), see [Git revision](#git-revision); - its [lifecycle events](../observability.md): runs, resource attempts, statements, retries, and audit results; - the SQL text of every executed statement, stored once per distinct statement and shared across @@ -100,7 +122,8 @@ of its history still succeeds, prints a warning followed by its own final status `output_invocations` reports the gap. An invocation whose process died without finishing has status `abandoned`. -Every command is recorded except `sqb recall` itself, commands that create a project (such as +`sqb recall` only reads history: it publishes no lifecycle events, runs no project sinks, and keeps +no compute logs, so a query starts quickly. Every command is recorded except `sqb recall` itself, commands that create a project (such as `sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, `sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. @@ -108,6 +131,21 @@ recorded either: a `sqb compile` answered from the launcher's warm cache, and `s The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. +## Git revision + +`invocations.git_revision` is the commit `HEAD` names in the repository above the project +directory, including git worktrees and submodules, whose `.git` is a file pointing at the +repository. Deployments usually have no `.git` at all, so set the revision when building the image +or starting the job, or comparing production runs by `git_revision` finds nothing: + +```bash +export SQLBUILD_GIT_REVISION="$(git rev-parse HEAD)" # recorded as git_revision +export SQLBUILD_GIT_DIRTY=false # optional: recorded as git_dirty (true/false) +``` + +`SQLBUILD_GIT_REVISION` takes precedence over the repository. Without `SQLBUILD_GIT_DIRTY`, +`git_dirty` is `NULL`. + ## Settings Configure Recall in `[recall]` in `sqlbuild_project.toml`. A `[targets..recall]` table @@ -127,7 +165,8 @@ enabled = false # do not record runs against the ci target | Key | Default | Description | |-----|---------|-------------| | `enabled` | `true` | Set to `false` to stop recording | -| `connection` | (none) | A `sqlite` connection whose `path` names the store, or a `clickhouse` connection; defaults to `logs/recall.sqlite3` | +| `write_connection` | (none) | The connection every command writes its history to: a `sqlite` connection whose `path` names the store, or a `clickhouse` connection. Requires `read_connection`. Defaults to `logs/recall.sqlite3` | +| `read_connection` | (none) | The connection `sqb recall query`, `schema` and `output` read; may name the same connection as `write_connection`. Set alone to read shared history while this machine's own commands record to `logs/recall.sqlite3` | | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | @@ -178,12 +217,17 @@ repeats at most once a day. If the store file is not a SQLite database, Recall m `recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. -A `connection` on any adapter other than `sqlite` or `clickhouse` is not supported; Recall prints a -notice and records nothing. +A `write_connection` on any adapter other than `sqlite` or `clickhouse` is not supported; Recall +prints a notice and records nothing. + +`write_connection` and `read_connection` replaced the single `connection` key. A configuration that +still sets `connection` fails with an error naming the replacements: set both keys to the old +connection to keep the previous behaviour. ## Sharing history in ClickHouse -Point `connection` at a ClickHouse connection to keep one history for a whole team. Every machine +Point `write_connection` and `read_connection` at ClickHouse connections to keep one history for a +whole team. Every machine that writes to it, including CI and scheduled production runs, becomes queryable from anywhere: see what last night's production build ran and printed without access to the machine, find the first run where an audit started failing across everyone's runs, or compare a production run with @@ -193,23 +237,64 @@ The connection needs `adapter = "clickhouse"` when the project uses another adap `database` must already exist; Recall never creates databases. ```toml -[connections.history] +[connections.history_writer] adapter = "clickhouse" host = "${env:CLICKHOUSE_HOST}" user = "recall_writer" -password = "${env:CLICKHOUSE_PASSWORD}" +password = "${env:RECALL_WRITER_PASSWORD}" +database = "observability" + +[connections.history_reader] +adapter = "clickhouse" +host = "${env:CLICKHOUSE_HOST}" +user = "recall_reader" +password = "${env:RECALL_READER_PASSWORD}" database = "observability" [recall] -connection = "history" +write_connection = "history_writer" # every command writes history (INSERT, SELECT) +read_connection = "history_reader" # sqb recall query, schema and output (SELECT only) retention = "30d" ``` -- **Setup.** On first use Recall creates its `recall_*` MergeTree tables and the views above in that - database, and adds columns that a newer SQLBuild introduces. After that, writing needs only - `INSERT` and `SELECT` on the tables. If the user may not create them, the command still succeeds - and prints the complete DDL for an administrator to run; a store created from it accepts writers - limited to `SELECT` and `INSERT`. +`sqb recall query`, `schema` and `output` use only `read_connection` and never create or change +anything in the store, so someone investigating production runs needs no credentials that could +rewrite or delete its history. Both keys may name one connection when separate users are not +wanted. A machine that only reads shared history, such as a laptop investigating production runs, +sets only `read_connection`; its own commands keep recording to the local store. + +### Users and grants + +SQLBuild does not enforce grants; it only lets the roles be separated. A least-privilege setup: + +```sql +-- once, as an administrator: the users +CREATE USER recall_writer IDENTIFIED BY '...'; +CREATE USER recall_reader IDENTIFIED BY '...'; +GRANT SELECT, INSERT ON observability.* TO recall_writer; +GRANT SELECT ON observability.* TO recall_reader; +``` + +Then, as the administrator, create the store with the DDL that `sqb recall schema --ddl` prints +(any connection that can read the database works, so the reader's is enough): + +```bash +sqb recall schema --ddl --target prod > recall.sql # review, then run it with an admin user +``` + +Repeat the same step after upgrading SQLBuild: `sqb recall schema --ddl` reads the store and prints +only what this version adds (`ALTER TABLE … ADD COLUMN`, replaced views and the schema version row), +or says that the store is up to date. Changes are always additive, and older SQLBuild versions keep +writing to an upgraded store. + +- **Setup.** A writer allowed to run DDL creates the `recall_*` MergeTree tables and the views above + in that database on first use, and adds the columns and views a newer SQLBuild introduces. If the + user may not create the store, the command still succeeds and prints the complete DDL for an + administrator to run; a store created from it accepts writers limited to `SELECT` and `INSERT`. +- **Upgrades.** When a store is behind this SQLBuild version and the writer may not upgrade it, the + command still runs and exits as it otherwise would. History is still written, to the columns + that exist, and Recall prints one warning (at most once a day) with the exact statements an + administrator should run; `sqb recall schema --ddl` prints them again. - **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. @@ -281,6 +366,10 @@ in the store's SQL dialect, against these views: Run `sqb recall schema` for every column. +Every view has `invocation_id` and `invocation_started_at`. Filter on them to keep a query cheap: +on a ClickHouse store, a filter on either reaches the table reads, so a query about one invocation or +the last day reads that part of the store instead of folding the whole retention window. + ## Relationship to compute logs Compute logs (`logs///stdout.log`, `stderr.log`, and `diagnostics.jsonl`) are diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md new file mode 100644 index 0000000000..d9976dad03 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -0,0 +1,366 @@ +# Investigating past runs with Recall + +Use Recall to answer "what did that run do?" without rerunning anything: which commands ran and +how they ended, which statements they executed and with what SQL, which audits failed, and exactly +what they printed. `sqb recall query` runs one read-only `SELECT` over a set of views; +`sqb recall output` prints a run's output byte for byte. + +## Which store you are reading + +- **Local (the default).** `logs/recall.sqlite3` in the project: this checkout's own runs. Queries + use SQLite SQL. +- **Shared.** `read_connection` in `[recall]` or `[targets..recall]` names a ClickHouse (or + SQLite) connection holding every machine's runs: teammates, CI, scheduled production builds. + Queries on a ClickHouse store use ClickHouse SQL. `write_connection` is where this machine's own + commands record; it can differ from `read_connection`, so your own runs may be in the local store + while you read the shared one. +- `--target ` picks the target whose Recall settings apply. The first lines of + `sqb recall schema` name the store and dialect you are reading. + +## How the store works + +- **The views are the interface.** Query `invocations`, `statements`, `audit_results`, `events`, + `diagnostics`, `artifacts`, `output_invocations`, `output_transcripts`, `output_tail` and + `conflicts`. Never query the `recall_*` tables: they hold raw facts, and folding them correctly + (duplicates, terminal facts, retention) is what the views do. +- **One invocation, many facts.** Every view has `invocation_id` and `invocation_started_at`; join + on `invocation_id` (and on `statement_id`, `run_id`, `resource_id` or `sql_digest` where both + sides have it). +- **`invocation_started_at` bounds scans.** Retention is per invocation: all rows of an invocation + expire together. On a shared ClickHouse store, filter on `invocation_id` or + `invocation_started_at` and the filter reaches the table reads; without one, a view folds the + whole retention window. +- **Writing history never fails the command,** so a missing row means "not recorded", not "did not + happen": `invocations.status = 'abandoned'` (the process died), `statements.outcome = + 'no_terminal'` (no terminal fact arrived), and `output_invocations.complete = 0` with a + `gap_reason` when output is incomplete. +- **SQL is stored once by digest.** `statements.sql_digest` joins `artifacts` on + `(invocation_id, statement_id)` for the SQL text (`artifacts.text`). +- **Output is a byte-for-byte copy** of stdout and stderr, up to `max_output_bytes` per stream (the + local store keeps the first and last MiB). Use `sqb recall output` rather than querying + `output_transcripts` for long output. +- **The same views exist in both dialects.** Column names and meanings match; types and SQL + functions differ (SQLite stores timestamps as ISO-8601 UTC text, ClickHouse as + `DateTime64(6, 'UTC')`). +- **Not recorded:** `sqb recall` itself, commands that create a project (`sqb init`), raw-query + `sqb diff`, commands answered without starting Python (a warm-cache `sqb compile`, `sqb clean`), + and anything when `[recall] enabled = false`. + +## Values worth knowing + +| Column | Values | +|--------|--------| +| `invocations.status` | `completed`, `failed`, `running`, `abandoned`, `unknown` | +| `invocations.error_type`, `error_code`, `error_message` | The cause of a failed invocation, also when it failed only through its exit code (`sqb debug`: `C980` connection, `C981` query, `C982` other check) | +| `statements.outcome` | `completed`, `failed`, `no_terminal` | +| `audit_results.outcome` | `pass`, `warn`, `error`, `insufficient`; `execution_error` is set when the audit query itself failed | +| `output_invocations.gap_reason` | `NULL` (complete), `truncated`, `filtered`, `dropped`, `unexplained` | +| `events.event_type` | `invocation_started`, `invocation_completed`, `invocation_failed`, `run_started`, `run_completed`, `run_failed`, `resource_attempt_started`, `resource_attempt_completed`, `resource_attempt_failed`, `resource_attempt_skipped`, `operation_*`, `statement_started`, `statement_submitted`, `statement_completed`, `statement_failed`, `statement_heartbeat`, `microbatch_*`, `retry_scheduled`, `audit_completed` | + +## Traps + +- `invocations` has `status`; `statements` and `audit_results` have `outcome`. There is no + `statements.status`. +- On a shared ClickHouse store write ClickHouse SQL (`now() - INTERVAL 1 DAY`, `countIf`), not + SQLite SQL (`strftime`, `datetime('now', '-1 day')`). +- Keep queries few and targeted: select the columns you need, filter on `invocation_id` or + `invocation_started_at`, and add `LIMIT`. `SELECT *` over a view on a large store is slow. +- Filter on a literal `invocation_id`. On ClickHouse, `invocation_id = (SELECT ...)` does not reach + the table reads, so look the ID up first and paste it into the next query. +- `--json` returns `{"columns": [...], "rows": [[...], ...], "row_count", "truncated", + "statistics"}`: rows are arrays in column order, and `statistics` carries `rows_read`, + `bytes_read` and `elapsed_ms` on ClickHouse. A failing `--json` query prints + `{"error": {"code", "message", "help"}}` on stdout and exits `1`. +- The default `--limit` is 100 rows; `truncated: true` means there were more. + +## Output format + +- Read results in the default `long` format: every value in full, one record at a time. +- When there are too many rows to read, narrow the query (fewer columns, `LIMIT`, aggregation); + don't switch format. +- Use `--json` only when a program consumes the output, for example to chain an `invocation_id` + into the next query, or when you need `statistics` (`rows_read`, `bytes_read`, elapsed time). + Don't use `--json` by default "to be safe". +- Use `--format csv` for export. `--format table` is deprecated. + +## Worked investigations + +Each query below runs as written against a local SQLite store; the ClickHouse spelling is given +where it differs. + +Why did the last run fail? + +```bash +sqb recall query "SELECT invocation_id, command, status, exit_code, error_type, error_code, error_message + FROM invocations WHERE status = 'failed' ORDER BY invocation_started_at DESC LIMIT 1" +sqb recall output --stream stderr --tail 40 +``` + +For the last scheduled run, filter on the orchestrator's context or the target, for example +`WHERE target = 'prod'`. + +What did a model actually execute, with its SQL? Find the run, then its statements: + +```bash +sqb recall query "SELECT invocation_id, invocation_started_at, status FROM invocations + WHERE command = 'build' ORDER BY invocation_started_at DESC LIMIT 1" +sqb recall query "SELECT s.statement_kind, s.outcome, s.duration_ms, a.text + FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + WHERE s.invocation_id = '' AND s.resource_name = 'orders' + ORDER BY s.started_at" +``` + +The slowest statements in a run: + +```bash +sqb recall query "SELECT resource_name, statement_kind, duration_ms, row_count, outcome + FROM statements WHERE invocation_id = '' + ORDER BY duration_ms DESC LIMIT 10" +``` + +The first run where an audit started failing: + +```bash +sqb recall query "SELECT invocation_id, invocation_started_at, outcome, violation_count + FROM audit_results WHERE audit_name = 'revenue_is_non_negative' + ORDER BY invocation_started_at DESC LIMIT 20" +``` + +Read down until `outcome` changes back to `pass`; the row above it is the first failing run. + +A production run against mine: + +```bash +sqb recall query "SELECT invocation_id, target, git_revision, sqlbuild_version, status, duration_ms + FROM invocations WHERE command = 'build' AND target IN ('prod', 'dev') + ORDER BY invocation_started_at DESC LIMIT 4" +``` + +`git_revision` is empty in deployments without `.git` unless they set `SQLBUILD_GIT_REVISION`. + +Did a run record everything? + +```bash +sqb recall query "SELECT complete, gap_reason, missing_sequences, reported_drops, truncated_bytes + FROM output_invocations WHERE invocation_id = ''" +``` + +On ClickHouse, bound time-based questions the same way, for example +`WHERE invocation_started_at >= now() - INTERVAL 1 DAY`; on SQLite, +`WHERE invocation_started_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-1 day')`. + +## Running queries in parallel + +`sqb recall query` is read-only and starts quickly, so independent questions can be separate calls +run at the same time. When the answers relate to each other, prefer one query with CTEs, joins or +`UNION ALL`. + +## Every view and column + +`sqb recall schema` prints the same list for the store you are reading, with its dialect's types. + + + +```text +Timestamps are ISO-8601 UTC text with microseconds; filter invocation_started_at to bound scans; all rows of an invocation expire together. + +artifacts (view): captured SQL and other artifacts, by reference + invocation_id TEXT join key + invocation_started_at TEXT + digest TEXT + ref_id TEXT + role TEXT + statement_id TEXT join key + resource_id TEXT join key + kind TEXT + media_type TEXT + size INTEGER + available ANY + unavailable_reason ANY + text ANY + content BLOB + +audit_results (view): one row per evaluated audit + invocation_id TEXT join key + invocation_started_at TEXT + event_id TEXT + occurred_at TEXT + run_id TEXT join key + resource_id TEXT join key + resource_attempt_id TEXT + audit_name ANY + audit_definition_name ANY + target_relation ANY + attached_target_kind ANY + attached_target_name ANY + column ANY + outcome TEXT + severity ANY + violation_count ANY + measured_value ANY + evaluation_mode ANY + execution_error ANY + sql_digest TEXT join key + sample_digest ANY + +conflicts (view): record keys reused with different content; the first row is kept + dataset TEXT + record_key TEXT + existing_sha256 TEXT + rejected_sha256 TEXT + detected_at TEXT + invocation_id TEXT join key + invocation_started_at TEXT + schema_version INTEGER + producer_version TEXT + ingested_at TEXT + content_hash TEXT + +diagnostics (view): structured diagnostics at or above the configured severity + record_id TEXT + sequence INTEGER + occurred_at TEXT + severity TEXT + logger TEXT + source TEXT + message TEXT + fields TEXT + log_stream_id TEXT + run_id TEXT join key + resource_id TEXT join key + resource_attempt_id TEXT + operation_id TEXT + statement_id TEXT join key + invocation_id TEXT join key + invocation_started_at TEXT + schema_version INTEGER + producer_version TEXT + ingested_at TEXT + content_hash TEXT + +events (view): every lifecycle fact + event_id TEXT + invocation_sequence INTEGER + occurred_at TEXT + event_type TEXT + event_kind TEXT + severity TEXT + producer TEXT + event_schema_version INTEGER + run_id TEXT join key + resource_id TEXT join key + resource_attempt_id TEXT + operation_id TEXT + statement_id TEXT join key + command TEXT + resource_kind TEXT + resource_name TEXT + error_code TEXT + error_type TEXT + duration_ms REAL + exit_code INTEGER + query_id TEXT + sql_digest TEXT join key + intent TEXT + statement_kind TEXT + purpose TEXT + outcome TEXT + row_count INTEGER + affected_rows INTEGER + payload TEXT + invocation_id TEXT join key + invocation_started_at TEXT + schema_version INTEGER + producer_version TEXT + ingested_at TEXT + content_hash TEXT + +invocations (view): one row per invocation; status is completed, failed, running, abandoned or unknown + invocation_id TEXT join key + invocation_started_at ANY + started_at ANY + status ANY + ended_at ANY + duration_ms ANY + exit_code INTEGER + error_type TEXT + error_code TEXT + error_message ANY + command ANY + subcommand ANY + target TEXT + adapter TEXT + project_name TEXT + selection TEXT + exclusion TEXT + sqlbuild_version TEXT + git_revision TEXT + git_dirty INTEGER + host_token TEXT + pid INTEGER + external_context TEXT + recall_filter_active INTEGER + last_heartbeat_at ANY + +output_invocations (view): output completeness per invocation: gaps, drops, filtering, truncation, flush + invocation_id TEXT join key + invocation_started_at BLOB + chunks ANY + bytes ANY + stdout_bytes ANY + stderr_bytes ANY + truncated_bytes ANY + max_sequence ANY + missing_sequences ANY + reported_drops ANY + filtered ANY + failed ANY + filter_active ANY + flush_complete ANY + gap_reason ANY + complete ANY + +output_tail (view): the last 200 output chunks of each invocation as text + invocation_id TEXT join key + invocation_started_at TEXT + sequence INTEGER + stream TEXT + occurred_at TEXT + byte_count INTEGER + text TEXT + +output_transcripts (view): exact stdout, stderr and combined bytes per invocation + invocation_id TEXT join key + invocation_started_at ANY + stdout BLOB + stderr BLOB + combined BLOB + +statements (view): one row per statement; outcome no_terminal means no terminal fact arrived + invocation_id TEXT join key + invocation_started_at TEXT + statement_id TEXT join key + run_id TEXT join key + resource_id TEXT join key + resource_name ANY + resource_kind ANY + resource_attempt_id TEXT + operation_id TEXT + intent TEXT + statement_kind TEXT + purpose TEXT + query_id ANY + started_at ANY + submitted_at ANY + ended_at TEXT + outcome ANY + error_type TEXT + error_code TEXT + duration_ms REAL + row_count INTEGER + affected_rows INTEGER + sql_digest TEXT join key + sql_available ANY +``` + + diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py index 7a8e521a2b..449c188d93 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py @@ -12,11 +12,7 @@ from tests.integration.src.sqlbuild.cli.recall.helpers import recall_rows, run, write_project SHARED_STORE_TOML: str = ( - "\n[connections.history]\n" - 'adapter = "sqlite"\n' - 'path = "{path}"\n' - "\n[recall]\n" - "{keys}" + '\n[connections.history]\nadapter = "sqlite"\npath = "{path}"\n\n[recall]\n{keys}' ) @@ -138,7 +134,7 @@ def test_given_only_a_read_connection_when_running_then_reads_are_shared_and_wri 'ALTER TABLE "recall_heartbeats" ADD COLUMN "subcommand" TEXT;', 'DROP VIEW IF EXISTS "invocations";', 'CREATE VIEW IF NOT EXISTS "invocations" AS', - "INSERT OR IGNORE INTO \"recall_meta\"", + 'INSERT OR IGNORE INTO "recall_meta"', ), expected_up_to_date_note="is up to date with this SQLBuild version", ), diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py index e77ce07d8d..cf0964959e 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py @@ -55,8 +55,7 @@ def test_given_a_failed_command_when_recorded_then_its_cause_is_structured( failed: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) recorded: list[list[object]] = recall_rows( project_dir, - "SELECT command, status, exit_code, error_type, error_code, error_message " - "FROM invocations", + "SELECT command, status, exit_code, error_type, error_code, error_message FROM invocations", ) assert failed.returncode != 0 @@ -103,9 +102,7 @@ def test_given_a_deployment_or_checkout_when_recorded_then_the_git_revision_is_k _ = _git(repository, "worktree", "add", "-q", "-b", "orders", str(tmp_path / "linked")) project_dir = tmp_path / "linked" / "orders_shop" - built: subprocess.CompletedProcess[bytes] = run( - project_dir, "build", **test_case.environment - ) + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", **test_case.environment) assert built.returncode == 0, built.stderr.decode(errors="replace") assert recall_rows(project_dir, "SELECT git_revision, git_dirty FROM invocations") == [ diff --git a/tests/unit/scripts/docs_skill/_helpers/_test_types.py b/tests/unit/scripts/docs_skill/_helpers/_test_types.py index 3fd455a9ce..2c876ad37f 100644 --- a/tests/unit/scripts/docs_skill/_helpers/_test_types.py +++ b/tests/unit/scripts/docs_skill/_helpers/_test_types.py @@ -18,3 +18,11 @@ class ReferencePagesTestCase: stale_page: str expected_files: tuple[str, ...] expected_fragments: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class RecallViewsSectionTestCase: + description: str + guide: str + views: str + expected: str diff --git a/tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py b/tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py new file mode 100644 index 0000000000..b87e02831b --- /dev/null +++ b/tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py @@ -0,0 +1,41 @@ +import pytest + +from scripts.docs_skill._helpers.recall_schema import with_recall_views +from scripts.docs_skill.constants import SCHEMA_SECTION_END, SCHEMA_SECTION_START +from tests.unit.scripts.docs_skill._helpers._test_types import RecallViewsSectionTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + RecallViewsSectionTestCase( + description="an empty section is filled", + guide=f"# Guide\n\n{SCHEMA_SECTION_START}\n{SCHEMA_SECTION_END}\n", + views="orders (view)\n order_id TEXT\n", + expected=( + f"# Guide\n\n{SCHEMA_SECTION_START}\n\n```text\norders (view)\n order_id TEXT\n" + f"```\n\n{SCHEMA_SECTION_END}\n" + ), + ), + RecallViewsSectionTestCase( + description="a stale section is replaced and the text around it kept", + guide=( + f"intro\n{SCHEMA_SECTION_START}\n\n```text\nold (view)\n```\n\n" + f"{SCHEMA_SECTION_END}\noutro\n" + ), + views="customers (view)\n", + expected=( + f"intro\n{SCHEMA_SECTION_START}\n\n```text\ncustomers (view)\n```\n\n" + f"{SCHEMA_SECTION_END}\noutro\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_guide_when_the_views_section_is_rendered_then_only_that_section_changes( + test_case: RecallViewsSectionTestCase, +) -> None: + rendered: str = with_recall_views(test_case.guide, test_case.views) + + assert rendered == test_case.expected + assert with_recall_views(rendered, test_case.views) == rendered diff --git a/tests/unit/scripts/docs_skill/main/test_check.py b/tests/unit/scripts/docs_skill/main/test_check.py index e26bff16c7..6558cb6082 100644 --- a/tests/unit/scripts/docs_skill/main/test_check.py +++ b/tests/unit/scripts/docs_skill/main/test_check.py @@ -45,7 +45,14 @@ def test_given_reference_pages_when_checking_docs_skill_then_outdated_pages_fail (docs_root / "concepts").mkdir(parents=True) (tmp_path / "src/sidebar.json").write_text(_SIDEBAR_JSON, encoding="utf-8") (docs_root / "concepts/orders.mdx").write_text(_ORDERS_PAGE, encoding="utf-8") - arguments: list[str] = ["--docs-root", str(docs_root), "--output-dir", str(output_dir)] + arguments: list[str] = [ + "--docs-root", + str(docs_root), + "--output-dir", + str(output_dir), + "--investigation-guide", + "", + ] _ = generate_docs_skill(arguments) for relative_path, contents in test_case.output_edits.items(): (output_dir / relative_path).write_text(contents, encoding="utf-8") From 1b50b987a58720fd5081a658778bdf7616fbff3f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:24:16 +0100 Subject: [PATCH 11/18] fix: replace Recall views when a ClickHouse store's version row expired The schema version row expires with retention like any other row. A store reading as version 0 now has its views replaced by a writer allowed to, before the current version is recorded. A writer that may not replace them stays quiet as before and records no version, so sqb recall schema --ddl still lists the upgrade. --- .../_helpers/recall/clickhouse_store.rs | 14 +++-- .../commands/main/clickhouse/_test_types.py | 8 +++ .../main/clickhouse/test_clickhouse_recall.py | 54 +++++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 4b40395dea..43c36d561a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -273,7 +273,8 @@ impl ClickHouseStore { tables, }); } - let upgrade: bool = version > 0 && version < RECALL_SCHEMA_VERSION; + // A version of 0 means the version row expired with retention: the views may be from any earlier version, so they are replaced too. + let upgrade: bool = version < RECALL_SCHEMA_VERSION; for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { let statement: String = if upgrade { statement @@ -285,10 +286,13 @@ impl ClickHouseStore { match self.command(py, &statement, "create the Recall views") { Ok(()) => {} Err(error) if error.kind == StoreErrorKind::Permission => { - return Ok(ReadyStore { - behind: behind(self), - tables, - }); + // An expired version row says nothing about the views, so stay quiet, but never record a version the views may not have. + let behind: Vec = if version == 0 { + Vec::new() + } else { + behind(self) + }; + return Ok(ReadyStore { behind, tables }); } Err(error) => return Err(error), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 56081c6069..00505e1d98 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -215,3 +215,11 @@ class ClickHouseRecallPushdownTestCase: sql: str expected_rows: list[list[object]] max_read_fraction: float + + +@dataclass(frozen=True) +class ClickHouseRecallExpiredVersionTestCase: + description: str + stale_view: str + expected_statement_columns: tuple[tuple[object, ...], ...] + expected_version: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index 18350c48df..58ef6f4d2e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -13,6 +13,7 @@ ClickHouseRecallBehindStoreTestCase, ClickHouseRecallConcurrencyTestCase, ClickHouseRecallDuplicateTestCase, + ClickHouseRecallExpiredVersionTestCase, ClickHouseRecallExpiryTestCase, ClickHouseRecallFirstReleaseStoreTestCase, ClickHouseRecallLimitedWriterTestCase, @@ -889,3 +890,56 @@ def test_given_a_large_store_when_filtering_a_view_then_the_filter_reaches_the_t statistics: dict[str, int] = cast(dict[str, int], document["statistics"]) assert document["rows"] == test_case.expected_rows assert statistics["rows_read"] < stored_events * test_case.max_read_fraction + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallExpiredVersionTestCase( + description="views of an older version are replaced when its version row expired", + stale_view="SELECT invocation_id, statement_id FROM {database}.recall_events", + expected_statement_columns=((1,),), + expected_version=((2,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_store_whose_version_row_expired_when_a_writer_with_ddl_builds_then_views_upgrade( + test_case: ClickHouseRecallExpiredVersionTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + assert run(project_dir, "build").returncode == 0 + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"CREATE OR REPLACE VIEW {recall_database}.statements AS " + + test_case.stale_view.format(database=recall_database), + f"ALTER TABLE {recall_database}.recall_meta DELETE WHERE 1 SETTINGS mutations_sync = 2", + ), + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0, built.stderr + assert b"Recall" not in built.stderr + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count() FROM system.columns WHERE database = '{recall_database}' " + "AND table = 'statements' AND name = 'invocation_started_at'", + ) + == test_case.expected_statement_columns + ) + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT max(toInt64OrZero(value)) FROM {recall_database}.recall_meta " + "WHERE component = 'schema'", + ) + == test_case.expected_version + ) From 9d9310ddd40fd93be28a5eaa1a727a9ba8f863ca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:23:22 +0100 Subject: [PATCH 12/18] chore: satisfy static checks for the Recall batch Typed test cases and shared test helpers for the new Recall tests, the version row and DDL helpers split into single-entry modules, the shared Recall connection registry as a class attribute, and no literal comparisons or lossy result probes in the new native code. --- .../debug_report/tests/test_debug_report.rs | 71 +++++----- .../src/debug_report/tests/test_types.rs | 8 +- .../src/bindings/_helpers/cli/entry.rs | 29 +++-- .../bindings/_helpers/cli/failure_cause.rs | 12 +- .../_helpers/recall/clickhouse_store.rs | 29 ++--- .../src/bindings/_helpers/recall/command.rs | 17 +-- .../src/bindings/_helpers/recall/session.rs | 25 ++-- .../src/schema/_helpers/mod.rs | 1 + .../src/schema/_helpers/version_row.rs | 44 +++++++ .../src/schema/main/ddl_script.rs | 68 +--------- .../src/schema/main/ddl_statements.rs | 19 +++ crates/sqlbuild-recall/src/schema/main/mod.rs | 2 + .../src/schema/main/render_script.rs | 9 ++ .../src/schema/main/upgrade_statements.rs | 3 +- .../src/schema/tests/helpers.rs | 35 ++++- .../src/schema/tests/test_ddl.rs | 121 +++++++++--------- .../src/schema/tests/test_types.rs | 8 ++ crates/sqlbuild-recall/src/store/constants.rs | 2 + .../src/store/main/observe_sqlite_schema.rs | 3 +- scripts/docs_skill/_helpers/recall_schema.py | 8 +- scripts/docs_skill/main/check.py | 2 +- scripts/docs_skill/main/generate.py | 2 +- .../clickhouse/_helpers/recall_store.py | 16 ++- .../clickhouse/classes/recall_connections.py | 3 - .../classes/shared_recall_connections.py | 13 ++ .../cli/commands/main/clickhouse/helpers.py | 4 +- .../src/sqlbuild/cli/recall/_test_types.py | 19 ++- .../src/sqlbuild/cli/recall/helpers.py | 99 ++++++++++++++ .../sqlbuild/cli/recall/test_recall_cli.py | 71 ++++++---- .../cli/recall/test_recall_connections.py | 62 +++------ .../recall/test_recall_invocation_facts.py | 41 ++---- .../docs_skill/_helpers/_test_types.py | 2 +- .../docs_skill/_helpers/test_recall_schema.py | 10 +- 33 files changed, 518 insertions(+), 340 deletions(-) create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/version_row.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/ddl_statements.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/render_script.rs create mode 100644 src/sqlbuild/adapters/clickhouse/classes/shared_recall_connections.py diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index ba6e9103e3..7a2122f2d7 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -7,7 +7,9 @@ use crate::debug_report::models::{ ConnectionCheck, ConnectionValue, DebugFailure, DebugLine, DebugReport, DebugStatus, }; use crate::debug_report::tests::helpers::{line_summary, sample_report, text}; -use crate::debug_report::tests::test_types::{ConnectionLinesTestCase, FormatTestCase}; +use crate::debug_report::tests::test_types::{ + ConnectionLinesTestCase, DebugFailureTestCase, FormatTestCase, +}; /// Expectations recorded from the retired Python formatter. #[test] @@ -105,45 +107,52 @@ fn given_connection_settings_when_listing_then_values_are_sorted_masked_and_shor #[test] fn given_a_debug_report_when_its_failure_is_read_then_the_first_failed_check_is_coded() { - let passed = DebugReport { - connection: vec![DebugLine::checked( - "connection test", - "", - (DebugStatus::Ok, Some("connected")), - )], - ..DebugReport::default() - }; - let invalid_config = DebugReport { - configuration: vec![DebugLine::checked( - "project file", - "/repo/sqlbuild_project.toml", - (DebugStatus::Error, Some("invalid TOML")), - )], - ..DebugReport::default() - }; - let test_cases: [(&str, DebugReport, Option); 3] = [ - ( - "a failed connection test is a connection error", - sample_report(), - Some(DebugFailure { + let test_cases = [ + DebugFailureTestCase { + description: "a failed connection test is a connection error", + report: sample_report(), + expected_failure: Some(DebugFailure { error_type: "ConnectionError".to_owned(), error_code: "C980".to_owned(), message: "connection test: authentication failed".to_owned(), }), - ), - ( - "another failed check is a configuration error", - invalid_config, - Some(DebugFailure { + }, + DebugFailureTestCase { + description: "another failed check is a configuration error", + report: DebugReport { + configuration: vec![DebugLine::checked( + "project file", + "/repo/sqlbuild_project.toml", + (DebugStatus::Error, Some("invalid TOML")), + )], + ..DebugReport::default() + }, + expected_failure: Some(DebugFailure { error_type: "ConfigurationError".to_owned(), error_code: "C982".to_owned(), message: "project file: invalid TOML".to_owned(), }), - ), - ("a passing report has no failure", passed, None), + }, + DebugFailureTestCase { + description: "a passing report has no failure", + report: DebugReport { + connection: vec![DebugLine::checked( + "connection test", + "", + (DebugStatus::Ok, Some("connected")), + )], + ..DebugReport::default() + }, + expected_failure: None, + }, ]; - for (description, report, expected) in test_cases { - assert_eq!(debug_failure(&report), expected, "{description}"); + for test_case in test_cases { + assert_eq!( + debug_failure(&test_case.report), + test_case.expected_failure, + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_types.rs b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs index 4687aee553..685417e0c9 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_types.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugReport}; +use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugFailure, DebugReport}; pub(super) struct FormatTestCase { pub(super) description: &'static str, @@ -14,3 +14,9 @@ pub(super) struct ConnectionLinesTestCase { pub(super) check: ConnectionCheck, pub(super) expected_lines: Vec<(&'static str, &'static str, Option<&'static str>)>, } + +pub(super) struct DebugFailureTestCase { + pub(super) description: &'static str, + pub(super) report: DebugReport, + pub(super) expected_failure: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index ea80449cf9..469a5db3ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -7,7 +7,9 @@ use pyo3::exceptions::PySystemExit; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyTuple}; +use pyo3::types::{ + PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, +}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::dispatch::main::failure_report::failure_report; use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; @@ -29,6 +31,7 @@ use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::recall::constants::JSON_FORMAT; use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::recall::session::{ InvocationRecall, finish_invocation_recall, start_invocation_recall, @@ -208,11 +211,12 @@ fn write_machine_error( fallback_code: &str, ) -> PyResult<()> { let json_output: bool = if invocation.command_is("recall") { - invocation + let format: Option = invocation .args .getattr_opt("recall_format")? - .and_then(|format| format.extract::().ok()) - .is_some_and(|format| format == "json") + .map(|format| format.extract()) + .transpose()?; + format.as_deref() == Some(JSON_FORMAT) } else { invocation.command_is("plan") && invocation.flag("json")? }; @@ -556,16 +560,15 @@ fn observed_dispatch<'py>( payload.set_item("duration_ms", elapsed(started)?)?; payload.set_item("error_type", error.get_type(py).getattr("__name__")?)?; let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); - if let Some(code) = value.getattr_opt("code")? - && let Ok(code) = code.extract::() - { + let (code, message, _) = error_parts(&value, "")?; + let code: String = code.str()?.to_string_lossy().into_owned(); + if !code.is_empty() && !error.is_instance_of::(py) { payload.set_item("error_code", code)?; } - let message: String = match value.getattr_opt("message")? { - Some(message) if !message.is_none() => message.str()?.to_string(), - _ => value.str()?.to_string(), - }; - payload.set_item("error_message", short_message(&message))?; + payload.set_item( + "error_message", + short_message(&message.str()?.to_string_lossy()), + )?; publish(lifecycle_event(py, "invocation_failed", payload)?)?; Err(error) } @@ -684,7 +687,7 @@ fn parsed_invocation<'py>( ], ) } else { - operation.call0() + call_with(&operation.into_any(), &[]) }; let exit_status: Option = match &dispatched { Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs index 1deae61649..8c6af87304 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs @@ -1,6 +1,6 @@ //! The cause a command reports when it fails through its exit code instead of raising, for the invocation's terminal event. -use std::sync::Mutex; +use std::sync::{Mutex, MutexGuard, PoisonError}; /// A failure's error type, code and short message. #[derive(Clone, Debug, PartialEq, Eq)] @@ -12,14 +12,16 @@ pub(crate) struct FailureCause { static FAILURE_CAUSE: Mutex> = Mutex::new(None); +fn locked(slot: &Mutex) -> MutexGuard<'_, T> { + slot.lock().unwrap_or_else(PoisonError::into_inner) +} + /// Record why the running command is about to return a failing exit code. pub(crate) fn report_failure_cause(cause: FailureCause) { - if let Ok(mut slot) = FAILURE_CAUSE.lock() { - *slot = Some(cause); - } + *locked(&FAILURE_CAUSE) = Some(cause); } /// The cause the command reported, cleared so the next invocation starts without one. pub(crate) fn take_failure_cause() -> Option { - FAILURE_CAUSE.lock().ok().and_then(|mut slot| slot.take()) + locked(&FAILURE_CAUSE).take() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 43c36d561a..711f9dc8d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -25,7 +25,7 @@ use sqlbuild_recall::schema::constants::{ }; use sqlbuild_recall::schema::main::add_column_statement::add_column_statement; use sqlbuild_recall::schema::main::clickhouse_relation::clickhouse_relation; -use sqlbuild_recall::schema::main::ddl_script::render_script; +use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::main::retention_fix_statements::retention_fix_statements; use sqlbuild_recall::schema::main::table_columns::table_columns; use sqlbuild_recall::schema::main::table_spec::table_spec; @@ -228,23 +228,21 @@ impl ClickHouseStore { /// What the store holds, from its read columns, `views` and recorded `version`. fn observed(&self, views: &HashSet, version: i64) -> ObservedSchema { - ObservedSchema { - tables: self - .columns - .iter() - .map(|(table, columns)| { - ( - *table, - columns.iter().map(|column| column.name.clone()).collect(), - ) - }) - .collect(), + let mut observed = ObservedSchema { views: views.iter().cloned().collect(), version: (version > 0).then_some(version), + ..ObservedSchema::default() + }; + for (table, columns) in &self.columns { + let names = observed.tables.entry(*table).or_default(); + for column in columns { + names.insert(column.name.clone()); + } } + observed } - /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade; a user who may not run that DDL writes the columns that exist and gets the statements an administrator must run. + /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade or when the version row expired with retention (version 0), since the views may then be from any earlier version. A user who may not run that DDL writes the columns that exist and gets the statements an administrator must run; after an expired version row it stays quiet. A version is recorded only once the views are current. fn ensure_schema( &mut self, py: Python<'_>, @@ -273,7 +271,6 @@ impl ClickHouseStore { tables, }); } - // A version of 0 means the version row expired with retention: the views may be from any earlier version, so they are replaced too. let upgrade: bool = version < RECALL_SCHEMA_VERSION; for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { let statement: String = if upgrade { @@ -286,7 +283,6 @@ impl ClickHouseStore { match self.command(py, &statement, "create the Recall views") { Ok(()) => {} Err(error) if error.kind == StoreErrorKind::Permission => { - // An expired version row says nothing about the views, so stay quiet, but never record a version the views may not have. let behind: Vec = if version == 0 { Vec::new() } else { @@ -645,7 +641,7 @@ impl ClickHouseReader { self.store.location.database() } - /// At most `limit` rows of a read-only query, with the server's statistics. + /// At most `limit` rows of a read-only query, with the server's statistics. A failure on a store that is not created yet is a store failure (C971), not an invalid query. pub(crate) fn query( &self, py: Python<'_>, @@ -663,7 +659,6 @@ impl ClickHouseReader { "run the Recall query", ) .map_err(|failure| { - // A store not created yet is a store failure (C971), not a missing invocation. match self.require_store(py) { Err(missing) if missing.kind == StoreErrorKind::NotFound => { return StoreError { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index e51ef28f48..5e98ffa267 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -7,7 +7,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; use sqlbuild_recall::records::models::OutputStream; use sqlbuild_recall::schema::constants::VIEW_NAMES; -use sqlbuild_recall::schema::main::ddl_script::{ddl_statements, render_script}; +use sqlbuild_recall::schema::main::ddl_statements::ddl_statements; +use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::main::upgrade_statements::upgrade_statements; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; @@ -204,11 +205,9 @@ fn open_store(py: Python<'_>, settings: &RecallSettings, checked: bool) -> PyRes &ClickHouseLocation::new(config, database, settings.retention_days), ) .and_then(|reader| { - if checked { - if let Err(error) = reader.require_store(py) { - reader.close(); - return Err(error); - } + if checked && let Err(error) = reader.require_store(py) { + reader.close(); + return Err(error); } Ok(reader) }) @@ -240,8 +239,10 @@ fn open_store(py: Python<'_>, settings: &RecallSettings, checked: bool) -> PyRes /// The "no history yet" failure when a query failed because a writer is still creating the store's views, so it reads like the store not existing yet rather than an invalid query. fn not_created(store: &SqliteStore, path: &str) -> Option { - let observed = observe_sqlite_schema(store).ok()?; - let complete: bool = VIEW_NAMES.iter().all(|view| observed.views.contains(*view)); + let complete: bool = match observe_sqlite_schema(store) { + Ok(observed) => VIEW_NAMES.iter().all(|view| observed.views.contains(*view)), + Err(_unreadable) => return None, + }; (!complete).then(|| StoreError { kind: StoreErrorKind::Other, message: format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 7a1c43e8d9..3b8213e9cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -131,18 +131,19 @@ fn git_revision(project_dir: &Path) -> Option { return Some(revision.trim().to_owned()); } } - [&git, &common] - .into_iter() - .filter_map(|directory| file_text(&directory.join("packed-refs"))) - .flat_map(|packed| { - packed - .lines() - .filter_map(|line| line.split_once(' ')) - .filter(|(_, name)| *name == reference) - .map(|(revision, _)| revision.to_owned()) - .collect::>() - }) - .next() + for directory in [&git, &common] { + let Some(packed) = file_text(&directory.join("packed-refs")) else { + continue; + }; + let found = packed + .lines() + .filter_map(|line| line.split_once(' ')) + .find(|(_, name)| *name == reference); + if let Some((revision, _)) = found { + return Some(revision.to_owned()); + } + } + None } /// The revision a deployment sets in [`GIT_REVISION_ENVIRONMENT`], else the one the repository's files name. diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs index 8e71b76aae..275383fa1c 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod clickhouse_views; pub(crate) mod rendering; pub(crate) mod sqlite_views; +pub(crate) mod version_row; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/version_row.rs b/crates/sqlbuild-recall/src/schema/_helpers/version_row.rs new file mode 100644 index 0000000000..90b00ed303 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/version_row.rs @@ -0,0 +1,44 @@ +//! The insert recording this build's Recall schema version. + +use crate::schema::_helpers::rendering::clickhouse_table; +use crate::schema::constants::{ + DDL_SCRIPT_INVOCATION, META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, RECALL_SCHEMA_VERSION, +}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Dialect, Table}; + +/// The `recall_meta` row recording the schema version, so writers allowed only `SELECT` and `INSERT` find the store complete. +fn clickhouse_version_row(options: &DdlOptions) -> String { + format!( + "INSERT INTO {meta} (component, key, value, recorded_at, invocation_id, \ + invocation_started_at, schema_version, producer_version, ingested_at, content_hash)\n\ + SELECT '{META_SCHEMA_COMPONENT}', '{META_SCHEMA_VERSION_KEY}', \ + '{RECALL_SCHEMA_VERSION}', now64(6), '{DDL_SCRIPT_INVOCATION}', now64(6), \ + {RECALL_SCHEMA_VERSION}, '{version}', now64(6), ''", + meta = clickhouse_table(table_spec(Table::Meta).name, options), + version = env!("CARGO_PKG_VERSION"), + ) +} + +/// The SQLite `recall_meta` row recording the schema version. +fn sqlite_version_row() -> String { + let now = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + format!( + "INSERT OR IGNORE INTO \"{meta}\" (component, key, value, recorded_at, invocation_id, \ + invocation_started_at, schema_version, producer_version, ingested_at, content_hash)\n\ + VALUES ('{META_SCHEMA_COMPONENT}', '{META_SCHEMA_VERSION_KEY}', \ + '{RECALL_SCHEMA_VERSION}', {now}, '{DDL_SCRIPT_INVOCATION}', {now}, \ + {RECALL_SCHEMA_VERSION}, '{version}', {now}, '')", + meta = table_spec(Table::Meta).name, + version = env!("CARGO_PKG_VERSION"), + ) +} + +/// The insert recording this build's schema version, which a writer allowed only `SELECT` and `INSERT` may also run. +#[must_use] +pub(crate) fn version_row_statement(dialect: Dialect, options: &DdlOptions) -> String { + match dialect { + Dialect::Sqlite => sqlite_version_row(), + Dialect::ClickHouse => clickhouse_version_row(options), + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs index 51ba25f6f6..7dd0ee9662 100644 --- a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs +++ b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs @@ -1,70 +1,8 @@ //! `ddl_script`: the complete ordered script an administrator runs to create a Recall store. -use crate::schema::_helpers::rendering::clickhouse_table; -use crate::schema::constants::{ - DDL_SCRIPT_INVOCATION, META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, RECALL_SCHEMA_VERSION, -}; -use crate::schema::main::table_spec::table_spec; -use crate::schema::main::table_statements::table_statements; -use crate::schema::main::view_statements::view_statements; -use crate::schema::models::{DdlOptions, Dialect, Table}; - -/// The `recall_meta` row recording the schema version, so writers allowed only `SELECT` and `INSERT` find the store complete. -fn clickhouse_version_row(options: &DdlOptions) -> String { - format!( - "INSERT INTO {meta} (component, key, value, recorded_at, invocation_id, \ - invocation_started_at, schema_version, producer_version, ingested_at, content_hash)\n\ - SELECT '{META_SCHEMA_COMPONENT}', '{META_SCHEMA_VERSION_KEY}', \ - '{RECALL_SCHEMA_VERSION}', now64(6), '{DDL_SCRIPT_INVOCATION}', now64(6), \ - {RECALL_SCHEMA_VERSION}, '{version}', now64(6), ''", - meta = clickhouse_table(table_spec(Table::Meta).name, options), - version = env!("CARGO_PKG_VERSION"), - ) -} - -/// The SQLite `recall_meta` row recording the schema version. -fn sqlite_version_row() -> String { - let now = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; - format!( - "INSERT OR IGNORE INTO \"{meta}\" (component, key, value, recorded_at, invocation_id, \ - invocation_started_at, schema_version, producer_version, ingested_at, content_hash)\n\ - VALUES ('{META_SCHEMA_COMPONENT}', '{META_SCHEMA_VERSION_KEY}', \ - '{RECALL_SCHEMA_VERSION}', {now}, '{DDL_SCRIPT_INVOCATION}', {now}, \ - {RECALL_SCHEMA_VERSION}, '{version}', {now}, '')", - meta = table_spec(Table::Meta).name, - version = env!("CARGO_PKG_VERSION"), - ) -} - -/// The insert recording this build's schema version, which a writer allowed only `SELECT` and `INSERT` may also run. -#[must_use] -pub fn version_row_statement(dialect: Dialect, options: &DdlOptions) -> String { - match dialect { - Dialect::Sqlite => sqlite_version_row(), - Dialect::ClickHouse => clickhouse_version_row(options), - } -} - -/// Every table, index and view statement, then the schema version row. -#[must_use] -pub fn ddl_statements(dialect: Dialect, options: &DdlOptions) -> Vec { - let mut statements: Vec = table_statements(dialect, options); - statements.extend( - view_statements(dialect, options) - .into_iter() - .map(|(_, statement)| statement), - ); - statements.push(version_row_statement(dialect, options)); - statements -} - -/// `statements` as one script, each terminated by `;`. -#[must_use] -pub fn render_script(statements: &[String]) -> String { - let mut script: String = statements.join(";\n\n"); - script.push_str(";\n"); - script -} +use crate::schema::main::ddl_statements::ddl_statements; +use crate::schema::main::render_script::render_script; +use crate::schema::models::{DdlOptions, Dialect}; /// The complete creation script of [`ddl_statements`]. #[must_use] diff --git a/crates/sqlbuild-recall/src/schema/main/ddl_statements.rs b/crates/sqlbuild-recall/src/schema/main/ddl_statements.rs new file mode 100644 index 0000000000..9bf1f3908a --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/ddl_statements.rs @@ -0,0 +1,19 @@ +//! `ddl_statements`: every statement that creates a Recall store, in order. + +use crate::schema::_helpers::version_row::version_row_statement; +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Every table, index and view statement, then the schema version row. +#[must_use] +pub fn ddl_statements(dialect: Dialect, options: &DdlOptions) -> Vec { + let mut statements: Vec = table_statements(dialect, options); + statements.extend( + view_statements(dialect, options) + .into_iter() + .map(|(_, statement)| statement), + ); + statements.push(version_row_statement(dialect, options)); + statements +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index 3f6cca0f1d..7d875cb239 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -1,6 +1,8 @@ pub mod add_column_statement; pub mod clickhouse_relation; pub mod ddl_script; +pub mod ddl_statements; +pub mod render_script; pub mod retention_fix_statements; pub mod sqlite_retention_sql; pub mod table_columns; diff --git a/crates/sqlbuild-recall/src/schema/main/render_script.rs b/crates/sqlbuild-recall/src/schema/main/render_script.rs new file mode 100644 index 0000000000..89d9c58d43 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/render_script.rs @@ -0,0 +1,9 @@ +//! `render_script`: statements as one script an administrator runs. + +/// `statements` as one script, each terminated by `;`. +#[must_use] +pub fn render_script(statements: &[String]) -> String { + let mut script: String = statements.join(";\n\n"); + script.push_str(";\n"); + script +} diff --git a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs index 10b8806d4d..0b8c7430e2 100644 --- a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs @@ -1,9 +1,10 @@ //! `upgrade_statements`: what an administrator runs to bring an observed store up to this build's schema. use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::_helpers::version_row::version_row_statement; use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION}; use crate::schema::main::add_column_statement::add_column_statement; -use crate::schema::main::ddl_script::{ddl_statements, version_row_statement}; +use crate::schema::main::ddl_statements::ddl_statements; use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; use crate::schema::main::table_statements::table_statements; diff --git a/crates/sqlbuild-recall/src/schema/tests/helpers.rs b/crates/sqlbuild-recall/src/schema/tests/helpers.rs index 2c8fd299d6..f54d395f96 100644 --- a/crates/sqlbuild-recall/src/schema/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/schema/tests/helpers.rs @@ -1,8 +1,11 @@ -use crate::schema::constants::VIEW_NAMES; +use std::collections::BTreeSet; + +use crate::schema::constants::{ALL_TABLES, VIEW_NAMES}; use crate::schema::main::ddl_script::ddl_script; use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; +use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_statements::table_statements; -use crate::schema::models::{DdlOptions, Dialect}; +use crate::schema::models::{DdlOptions, Dialect, ObservedSchema, Table}; /// Options rendering into `observability` with 14-day retention, optionally on `cluster`. pub(crate) fn options(cluster: Option<&str>) -> DdlOptions { @@ -66,3 +69,31 @@ pub(crate) fn sqlite_schema_after(runs: usize) -> (usize, Vec) { .expect("the count query runs"); (usize::try_from(count).unwrap_or_default(), failures) } + +/// A store at `version` holding every table, column and view of this build, less `without_column` and `without_view`. +pub(crate) fn observed_store( + version: Option, + without_column: (Table, &str), + without_view: &str, +) -> ObservedSchema { + ObservedSchema { + tables: ALL_TABLES + .iter() + .map(|table| (*table, present_columns(*table, without_column))) + .collect(), + views: VIEW_NAMES + .iter() + .filter(|name| **name != without_view) + .map(|name| (*name).to_owned()) + .collect(), + version, + } +} + +fn present_columns(table: Table, (without_table, without_name): (Table, &str)) -> BTreeSet { + table_columns(table) + .into_iter() + .filter(|column| (table, column.name) != (without_table, without_name)) + .map(|column| column.name.to_owned()) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index 3a25189ea9..a8365c6e04 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -1,15 +1,17 @@ use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION, VIEW_NAMES}; use crate::schema::main::add_column_statement::add_column_statement; use crate::schema::main::ddl_script::ddl_script; -use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; use crate::schema::main::table_statements::table_statements; use crate::schema::main::upgrade_statements::upgrade_statements; use crate::schema::main::view_statements::view_statements; use crate::schema::models::{ColumnSpec, ColumnType, Dialect, ObservedSchema, Table}; -use crate::schema::tests::helpers::{clickhouse_table, contained, options, sqlite_schema_after}; +use crate::schema::tests::helpers::{ + clickhouse_table, contained, observed_store, options, sqlite_schema_after, +}; use crate::schema::tests::test_types::{ AddColumnTestCase, FragmentTestCase, SqliteScriptTestCase, StatementSetTestCase, + UpgradeStatementsTestCase, }; const NEW_COLUMN: ColumnSpec = ColumnSpec { @@ -205,80 +207,73 @@ fn given_a_new_column_when_upgraded_then_the_statement_is_additive_and_nullable( #[test] fn given_an_observed_store_when_upgraded_then_only_the_missing_additive_statements_are_listed() { - let current = || ObservedSchema { - tables: ALL_TABLES - .iter() - .map(|table| { - ( - *table, - table_columns(*table) - .iter() - .map(|column| column.name.to_owned()) - .collect(), - ) - }) - .collect(), - views: VIEW_NAMES.iter().map(|name| (*name).to_owned()).collect(), - version: Some(RECALL_SCHEMA_VERSION), - }; - let mut behind = current(); - behind.version = Some(RECALL_SCHEMA_VERSION - 1); - if let Some(columns) = behind.tables.get_mut(&Table::Heartbeats) { - columns.remove("subcommand"); - } - let mut missing_view = current(); - missing_view.views.remove("statements"); - let test_cases: [(&str, ObservedSchema, Dialect, Vec<&str>); 4] = [ - ( - "a current store needs nothing", - current(), - Dialect::Sqlite, - Vec::new(), - ), - ( - "an empty store gets the whole creation script", - ObservedSchema::default(), - Dialect::ClickHouse, - vec!["CREATE TABLE IF NOT EXISTS `observability`.`recall_meta`"], - ), - ( - "a store a version behind adds its column, replaces views and records the version", - behind, - Dialect::ClickHouse, - vec![ + let test_cases = [ + UpgradeStatementsTestCase { + description: "a current store needs nothing", + observed: observed_store(Some(RECALL_SCHEMA_VERSION), (Table::Meta, ""), ""), + dialect: Dialect::Sqlite, + expected_count: 0, + expected_prefixes: &[], + }, + UpgradeStatementsTestCase { + description: "an empty store gets the whole creation script", + observed: ObservedSchema::default(), + dialect: Dialect::ClickHouse, + expected_count: ALL_TABLES.len() + VIEW_NAMES.len() + 1, + expected_prefixes: &["CREATE TABLE IF NOT EXISTS `observability`.`recall_meta`"], + }, + UpgradeStatementsTestCase { + description: "a store a version behind adds its column, replaces views and records the version", + observed: observed_store( + Some(RECALL_SCHEMA_VERSION - 1), + (Table::Heartbeats, "subcommand"), + "", + ), + dialect: Dialect::ClickHouse, + expected_count: 1 + VIEW_NAMES.len() + 1, + expected_prefixes: &[ "ALTER TABLE `observability`.`recall_heartbeats` ADD COLUMN IF NOT EXISTS `subcommand` Nullable(String)", "CREATE OR REPLACE VIEW `observability`.`invocations`", "INSERT INTO `observability`.`recall_meta`", ], - ), - ( - "a current store missing a view only creates it", - missing_view, - Dialect::ClickHouse, - vec!["CREATE VIEW IF NOT EXISTS `observability`.`statements`"], - ), + }, + UpgradeStatementsTestCase { + description: "a SQLite store without a recorded version replaces its views", + observed: observed_store(None, (Table::Meta, ""), ""), + dialect: Dialect::Sqlite, + expected_count: 2 * VIEW_NAMES.len() + 1, + expected_prefixes: &[ + "DROP VIEW IF EXISTS \"invocations\"", + "INSERT OR IGNORE INTO", + ], + }, + UpgradeStatementsTestCase { + description: "a current store missing a view only creates it", + observed: observed_store(Some(RECALL_SCHEMA_VERSION), (Table::Meta, ""), "statements"), + dialect: Dialect::ClickHouse, + expected_count: 1, + expected_prefixes: &["CREATE VIEW IF NOT EXISTS `observability`.`statements`"], + }, ]; - for (description, observed, dialect, expected_prefixes) in test_cases { - let statements: Vec = upgrade_statements(dialect, &options(None), &observed); - let firsts: Vec = expected_prefixes + for test_case in test_cases { + let statements: Vec = + upgrade_statements(test_case.dialect, &options(None), &test_case.observed); + let found: Vec<&str> = test_case + .expected_prefixes .iter() - .map(|prefix| { + .copied() + .filter(|prefix| { statements .iter() .any(|statement| statement.starts_with(prefix)) }) .collect(); assert_eq!( - firsts, - vec![true; expected_prefixes.len()], - "{description}: {statements:#?}" + (statements.len(), found.as_slice()), + (test_case.expected_count, test_case.expected_prefixes), + "{}", + test_case.description ); - if expected_prefixes.is_empty() { - assert!(statements.is_empty(), "{description}: {statements:#?}"); - } - if description.starts_with("a current store missing") { - assert_eq!(statements.len(), 1, "{description}"); - } } } diff --git a/crates/sqlbuild-recall/src/schema/tests/test_types.rs b/crates/sqlbuild-recall/src/schema/tests/test_types.rs index e064d5472a..6f90fa620e 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_types.rs @@ -23,3 +23,11 @@ pub(crate) struct AddColumnTestCase { pub(crate) statement: String, pub(crate) expected_statement: &'static str, } + +pub(crate) struct UpgradeStatementsTestCase { + pub(crate) description: &'static str, + pub(crate) observed: crate::schema::models::ObservedSchema, + pub(crate) dialect: crate::schema::models::Dialect, + pub(crate) expected_count: usize, + pub(crate) expected_prefixes: &'static [&'static str], +} diff --git a/crates/sqlbuild-recall/src/store/constants.rs b/crates/sqlbuild-recall/src/store/constants.rs index f7ee4fb4ae..83d14eb638 100644 --- a/crates/sqlbuild-recall/src/store/constants.rs +++ b/crates/sqlbuild-recall/src/store/constants.rs @@ -30,3 +30,5 @@ pub const BINARY_UNIT_STEP: u64 = 1024; pub const DECIMAL_UNITS: [&str; 4] = ["kB", "MB", "GB", "TB"]; /// Binary size units above bytes, for kept output. pub const BINARY_UNITS: [&str; 4] = ["KiB", "MiB", "GiB", "TiB"]; +/// The `sqlite_schema.type` of a view. +pub(crate) const SQLITE_VIEW_TYPE: &str = "view"; diff --git a/crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs b/crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs index 94efbfe767..3dfaf34b37 100644 --- a/crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs +++ b/crates/sqlbuild-recall/src/store/main/observe_sqlite_schema.rs @@ -7,6 +7,7 @@ use crate::schema::main::table_spec::table_spec; use crate::schema::models::ObservedSchema; use crate::store::_helpers::schema::stored_columns; use crate::store::_helpers::sqlite::{checked, stored_schema_version}; +use crate::store::constants::SQLITE_VIEW_TYPE; use crate::store::errors::StoreError; use crate::store::models::SqliteStore; @@ -25,7 +26,7 @@ pub fn observe_sqlite_schema(store: &SqliteStore) -> Result str: +def _sqb(*, project_dir: Path, args: tuple[str, ...]) -> str: home: Path = project_dir / "home" completed: subprocess.CompletedProcess[str] = subprocess.run( [str(Path(sys.executable).with_name("sqlbuild-python")), *args], @@ -38,12 +38,12 @@ def render_recall_views() -> str: with tempfile.TemporaryDirectory() as directory: project_dir: Path = Path(directory) (project_dir / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") - script: str = _sqb(project_dir, "recall", "schema", "--ddl") + script: str = _sqb(project_dir=project_dir, args=("recall", "schema", "--ddl")) (project_dir / "logs").mkdir() with sqlite3.connect(project_dir / "logs" / "recall.sqlite3") as connection: connection.executescript(script) connection.close() - described: list[str] = _sqb(project_dir, "recall", "schema").splitlines() + described: list[str] = _sqb(project_dir=project_dir, args=("recall", "schema")).splitlines() views: list[str] = [] for line in described[1:]: if _TABLE_MARKER in line and not line.startswith(" "): @@ -52,7 +52,7 @@ def render_recall_views() -> str: return "\n".join(views).strip() + "\n" -def with_recall_views(guide: str, views: str) -> str: +def with_recall_views(*, guide: str, views: str) -> str: """`guide` with its generated schema section replaced by `views`.""" start: int = guide.index(SCHEMA_SECTION_START) + len(SCHEMA_SECTION_START) diff --git a/scripts/docs_skill/main/check.py b/scripts/docs_skill/main/check.py index 115739bdcf..410559cb32 100644 --- a/scripts/docs_skill/main/check.py +++ b/scripts/docs_skill/main/check.py @@ -22,7 +22,7 @@ def check_docs_skill(argv: list[str] | None = None) -> int: ) if args.investigation_guide is not None: guide: str = args.investigation_guide.read_text(encoding="utf-8") - if with_recall_views(guide, render_recall_views()) != guide: + if with_recall_views(guide=guide, views=render_recall_views()) != guide: outdated = (*outdated, args.investigation_guide.as_posix()) if not outdated: print(f"Docs skill reference pages in {args.output_dir} are up to date") diff --git a/scripts/docs_skill/main/generate.py b/scripts/docs_skill/main/generate.py index 5ba6fd0a12..45d4dfc26f 100644 --- a/scripts/docs_skill/main/generate.py +++ b/scripts/docs_skill/main/generate.py @@ -22,7 +22,7 @@ def generate_docs_skill(argv: list[str] | None = None) -> int: if args.investigation_guide is not None: guide: str = args.investigation_guide.read_text(encoding="utf-8") _ = args.investigation_guide.write_text( - with_recall_views(guide, render_recall_views()), encoding="utf-8" + with_recall_views(guide=guide, views=render_recall_views()), encoding="utf-8" ) return 0 diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py index 25c686a128..75e9bf19f1 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py @@ -9,7 +9,9 @@ from sqlbuild.adapters.clickhouse._helpers.connect_clickhouse import connect_clickhouse from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection -from sqlbuild.adapters.clickhouse.classes.recall_connections import RECALL_CONNECTIONS +from sqlbuild.adapters.clickhouse.classes.shared_recall_connections import ( + SharedRecallConnections, +) from sqlbuild.adapters.clickhouse.types import RecallCell, RecallFailure, RecallStatistics _CLIENT_MODULE: str = "clickhouse_connect" @@ -71,7 +73,7 @@ def connect_recall_store( connection: _ClickHouseConnection = connect_clickhouse(options) except Exception as error: return _failure(error), 0 - return None, RECALL_CONNECTIONS.add(connection=connection) + return None, SharedRecallConnections.connections.add(connection=connection) def run_recall_query( @@ -80,7 +82,7 @@ def run_recall_query( """Columns, rows of plain values and server statistics; strings not in UTF-8 stay bytes.""" try: - result: Any = RECALL_CONNECTIONS.get(handle=handle).client.query( + result: Any = SharedRecallConnections.connections.get(handle=handle).client.query( sql, settings=settings, query_formats=_BYTES_FORMATS ) except Exception as error: @@ -99,7 +101,7 @@ def run_recall_command(*, handle: int, sql: str) -> tuple[RecallFailure, None]: """Run one DDL statement.""" try: - RECALL_CONNECTIONS.get(handle=handle).client.command(sql) + SharedRecallConnections.connections.get(handle=handle).client.command(sql) except Exception as error: return _failure(error), None return None, None @@ -111,7 +113,7 @@ def insert_recall_rows( """Insert one `RowBinary` block; a retried block with the same token is deduplicated.""" try: - RECALL_CONNECTIONS.get(handle=handle).client.raw_insert( + SharedRecallConnections.connections.get(handle=handle).client.raw_insert( table, column_names=column_names, insert_block=payload, @@ -126,7 +128,9 @@ def insert_recall_rows( def close_recall_store(*, handle: int) -> None: """Close and forget the connection; a failure to close is ignored.""" - connection: _ClickHouseConnection | None = RECALL_CONNECTIONS.pop(handle=handle) + connection: _ClickHouseConnection | None = SharedRecallConnections.connections.pop( + handle=handle + ) if connection is None: return try: diff --git a/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py b/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py index e64abf6751..0eeea1f566 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py +++ b/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py @@ -35,6 +35,3 @@ def pop(self, *, handle: int) -> _ClickHouseConnection | None: with self._guard: return self._connections.pop(handle, None) - - -RECALL_CONNECTIONS: RecallConnections = RecallConnections() diff --git a/src/sqlbuild/adapters/clickhouse/classes/shared_recall_connections.py b/src/sqlbuild/adapters/clickhouse/classes/shared_recall_connections.py new file mode 100644 index 0000000000..a590e85dc1 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/shared_recall_connections.py @@ -0,0 +1,13 @@ +"""The process's open Recall store connections, shared by every handle native code holds.""" + +from __future__ import annotations + +from typing import ClassVar + +from sqlbuild.adapters.clickhouse.classes.recall_connections import RecallConnections + + +class SharedRecallConnections: + """The one registry of open Recall store connections in this process.""" + + connections: ClassVar[RecallConnections] = RecallConnections() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 777a44cd28..73f089dd06 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -484,9 +484,7 @@ def apply_script(*, config: dict[str, object], script: str) -> None: execute_clickhouse_sql( config=config, statements=tuple( - statement.strip().removesuffix(";") - for statement in script.split(";\n\n") - if statement.strip() + statement.strip().removesuffix(";") for statement in script.split(";\n\n") ), ) diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 879183d0de..1c82b43a37 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -1,4 +1,6 @@ +from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path @dataclass(frozen=True) @@ -248,7 +250,7 @@ class FailureCauseTestCase: @dataclass(frozen=True) class GitRevisionTestCase: description: str - layout: str + prepare: Callable[[Path], tuple[Path, str]] environment: dict[str, str] expected_dirty: object @@ -257,3 +259,18 @@ class GitRevisionTestCase: class LeanRecallTestCase: description: str args: tuple[str, ...] + expected_sink_log: bool + + +@dataclass(frozen=True) +class StatementStartTestCase: + description: str + sql: str + expected_rows: list[list[object]] + + +@dataclass(frozen=True) +class StoreBeingCreatedTestCase: + description: str + args: tuple[str, ...] + expected_error: str diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 0c76e4d683..fe2961b735 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -7,6 +7,7 @@ import os import re import signal +import sqlite3 import stat import subprocess import sys @@ -466,3 +467,101 @@ def logged_event_types(project_dir: Path, dropped_prefix: str) -> tuple[list[str ) dropped: operator.methodcaller = operator.methodcaller("startswith", dropped_prefix) return logged, list(itertools.filterfalse(dropped, logged)) + + +DEPLOYED_REVISION: str = "4f1c2e9" +SHARED_STORE_TOML: str = ( + '\n[connections.history]\nadapter = "sqlite"\npath = "{path}"\n\n[recall]\n{keys}' +) + + +def shared_store_project(*, parent: Path, store: Path, keys: str) -> Path: + """A project whose Recall settings set `keys` for the SQLite connection `history` at `store`.""" + + return write_project( + parent / "orders_shop", + extra_toml=SHARED_STORE_TOML.format(path=store.as_posix(), keys=keys), + ) + + +def store_schema(store: Path) -> list[tuple[object, ...]]: + """Every table and view of a SQLite store with its columns, to show reads changed nothing.""" + + with sqlite3.connect(store) as connection: + return connection.execute( + "SELECT m.type, m.name, p.name FROM sqlite_schema AS m " + "LEFT JOIN pragma_table_xinfo(m.name) AS p ORDER BY m.type, m.name, p.cid" + ).fetchall() + + +def make_store_behind(store: Path) -> None: + """Make a store look written by an older version: no newer column, views or version row.""" + + with sqlite3.connect(store) as connection: + views: list[str] = [ + row[0] + for row in connection.execute("SELECT name FROM sqlite_schema WHERE type = 'view'") + ] + connection.executescript( + "".join(f'DROP VIEW "{view}";' for view in views) + + 'ALTER TABLE "recall_heartbeats" DROP COLUMN "subcommand";' + + "DELETE FROM recall_meta WHERE component = 'schema';" + ) + connection.close() + + +def drop_store_views(store: Path) -> None: + """Drop every view of a SQLite store, as a store whose creation is still under way has none.""" + + with sqlite3.connect(store) as connection: + views: list[str] = [ + row[0] + for row in connection.execute("SELECT name FROM sqlite_schema WHERE type = 'view'") + ] + connection.executescript("".join(f'DROP VIEW "{view}";' for view in views)) + connection.close() + + +def git(directory: Path, *args: str) -> str: + """Run git with a fixed synthetic identity and return its stdout.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + ["git", "-c", "user.name=Orders", "-c", "user.email=orders@example.com", *args], + cwd=directory, + capture_output=True, + check=True, + text=True, + ) + return completed.stdout.strip() + + +def committed_repository(parent: Path) -> tuple[Path, str]: + """A git repository at `parent / "repository"` holding the project, and its commit.""" + + repository: Path = parent / "repository" + _ = write_project(repository / "orders_shop") + _ = git(repository, "init", "-q", "-b", "main") + _ = git(repository, "add", ".") + _ = git(repository, "commit", "-q", "-m", "orders") + return repository, git(repository, "rev-parse", "HEAD") + + +def deployed_project(parent: Path) -> tuple[Path, str]: + """A project outside any repository, and the revision its deployment sets in the environment.""" + + return write_project(parent / "orders_shop"), DEPLOYED_REVISION + + +def checkout_project(parent: Path) -> tuple[Path, str]: + """A project in a plain checkout, and the commit HEAD names.""" + + repository, revision = committed_repository(parent) + return repository / "orders_shop", revision + + +def worktree_project(parent: Path) -> tuple[Path, str]: + """A project in a linked git worktree, whose `.git` is a file, and the commit HEAD names.""" + + repository, revision = committed_repository(parent) + _ = git(repository, "worktree", "add", "-q", "-b", "orders", str(parent / "linked")) + return parent / "linked" / "orders_shop", revision diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index a4483176e8..6db796e1be 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -1,5 +1,4 @@ import json -import sqlite3 import subprocess from pathlib import Path @@ -9,9 +8,12 @@ DeprecatedFormatTestCase, JsonErrorTestCase, LeanRecallTestCase, + StatementStartTestCase, + StoreBeingCreatedTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( OUTPUT_LOG_SINK, + drop_store_views, recall_rows, run, write_project, @@ -114,9 +116,14 @@ def test_given_the_table_format_when_querying_then_it_works_with_a_deprecation_w LeanRecallTestCase( description="query", args=("recall", "query", "SELECT count(*) FROM invocations"), + expected_sink_log=False, + ), + LeanRecallTestCase( + description="schema", args=("recall", "schema"), expected_sink_log=False + ), + LeanRecallTestCase( + description="output", args=("recall", "output", "last"), expected_sink_log=False ), - LeanRecallTestCase(description="schema", args=("recall", "schema")), - LeanRecallTestCase(description="output", args=("recall", "output", "last")), ], ids=lambda case: case.description, ) @@ -133,42 +140,56 @@ def test_given_project_sinks_when_reading_recall_then_no_sink_runs_and_no_comput assert read.returncode == 0, read.stderr.decode(errors="replace") assert read.stdout - assert not (project_dir / "output.log").exists() + assert (project_dir / "output.log").exists() is test_case.expected_sink_log assert set((project_dir / "logs").glob("*/*")) == logs_before +@pytest.mark.parametrize( + "test_case", + [ + StatementStartTestCase( + description="statements carry their invocation's start time", + sql=( + "SELECT COUNT(*) > 0, COUNT(DISTINCT s.invocation_started_at = " + "i.invocation_started_at) FROM statements s JOIN invocations i " + "USING (invocation_id) WHERE s.invocation_started_at >= " + "(SELECT MIN(invocation_started_at) FROM invocations)" + ), + expected_rows=[[1, 1]], + ), + ], + ids=lambda case: case.description, +) def test_given_recorded_statements_when_filtered_by_start_time_then_the_view_exposes_it( - tmp_path: Path, + test_case: StatementStartTestCase, tmp_path: Path ) -> None: project_dir: Path = write_project(tmp_path / "orders_shop") assert run(project_dir, "build").returncode == 0 - rows: list[list[object]] = recall_rows( - project_dir, - "SELECT COUNT(*) > 0, COUNT(DISTINCT s.invocation_started_at = i.invocation_started_at) " - "FROM statements s JOIN invocations i USING (invocation_id) " - "WHERE s.invocation_started_at >= (SELECT MIN(invocation_started_at) FROM invocations)", - ) + rows: list[list[object]] = recall_rows(project_dir, test_case.sql) - assert rows == [[1, 1]] + assert rows == test_case.expected_rows +@pytest.mark.parametrize( + "test_case", + [ + StoreBeingCreatedTestCase( + description="a query before the views exist", + args=("recall", "query", "SELECT COUNT(*) FROM invocations"), + expected_error="error[C971]: no Recall history is recorded yet", + ), + ], + ids=lambda case: case.description, +) def test_given_a_store_whose_views_are_still_being_created_when_queried_then_it_is_not_recorded_yet( - tmp_path: Path, + test_case: StoreBeingCreatedTestCase, tmp_path: Path ) -> None: project_dir: Path = write_project(tmp_path / "orders_shop") assert run(project_dir, "build").returncode == 0 - with sqlite3.connect(project_dir / "logs" / "recall.sqlite3") as connection: - views: list[str] = [ - row[0] - for row in connection.execute("SELECT name FROM sqlite_schema WHERE type = 'view'") - ] - for view in views: - _ = connection.execute(f'DROP VIEW "{view}"') - - queried: subprocess.CompletedProcess[bytes] = run( - project_dir, "recall", "query", "SELECT COUNT(*) FROM invocations" - ) + drop_store_views(project_dir / "logs" / "recall.sqlite3") + + queried: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) assert queried.returncode == 1 - assert "error[C971]: no Recall history is recorded yet" in queried.stderr.decode() + assert test_case.expected_error in queried.stderr.decode() diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py index 449c188d93..bfd2c22d62 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py @@ -9,42 +9,16 @@ ReadOnlyReaderTestCase, RecallConnectionErrorTestCase, ) -from tests.integration.src.sqlbuild.cli.recall.helpers import recall_rows, run, write_project - -SHARED_STORE_TOML: str = ( - '\n[connections.history]\nadapter = "sqlite"\npath = "{path}"\n\n[recall]\n{keys}' +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + make_store_behind, + recall_rows, + run, + shared_store_project, + store_schema, + write_project, ) -def _shared_project(parent: Path, *, store: Path, keys: str) -> Path: - return write_project( - parent / "orders_shop", - extra_toml=SHARED_STORE_TOML.format(path=store.as_posix(), keys=keys), - ) - - -def _schema(store: Path) -> list[tuple[object, ...]]: - with sqlite3.connect(store) as connection: - return connection.execute( - "SELECT m.type, m.name, p.name FROM sqlite_schema AS m " - "LEFT JOIN pragma_table_xinfo(m.name) AS p ORDER BY m.type, m.name, p.cid" - ).fetchall() - - -def _make_store_behind(store: Path) -> None: - """Turn a current store into one a previous SQLBuild version wrote: no newer column, views or version.""" - - with sqlite3.connect(store) as connection: - views: list[str] = [ - row[0] - for row in connection.execute("SELECT name FROM sqlite_schema WHERE type = 'view'") - ] - for view in views: - _ = connection.execute(f'DROP VIEW "{view}"') - _ = connection.execute('ALTER TABLE "recall_heartbeats" DROP COLUMN "subcommand"') - _ = connection.execute("DELETE FROM recall_meta WHERE component = 'schema'") - - @pytest.mark.parametrize( "test_case", [ @@ -104,16 +78,16 @@ def test_given_only_a_read_connection_when_running_then_reads_are_shared_and_wri test_case: ReadOnlyReaderTestCase, tmp_path: Path ) -> None: store: Path = tmp_path / "shared" / "history.sqlite3" - writer: Path = _shared_project( - tmp_path / "writer", + writer: Path = shared_store_project( + parent=tmp_path / "writer", store=store, keys='write_connection = "history"\nread_connection = "history"\n', ) assert run(writer, "build").returncode == 0 - reader: Path = _shared_project( - tmp_path / "reader", store=store, keys='read_connection = "history"\n' + reader: Path = shared_store_project( + parent=tmp_path / "reader", store=store, keys='read_connection = "history"\n' ) - schema_before: list[tuple[object, ...]] = _schema(store) + schema_before: list[tuple[object, ...]] = store_schema(store) built: subprocess.CompletedProcess[bytes] = run(reader, "build") shared: list[list[object]] = recall_rows(reader, "SELECT command, status FROM invocations") @@ -122,7 +96,7 @@ def test_given_only_a_read_connection_when_running_then_reads_are_shared_and_wri assert b"Recall" not in built.stderr assert shared == test_case.expected_shared_invocations assert (reader / "logs" / "recall.sqlite3").exists() - assert _schema(store) == schema_before + assert store_schema(store) == schema_before @pytest.mark.parametrize( @@ -145,20 +119,20 @@ def test_given_a_store_one_version_behind_when_read_then_no_ddl_runs_and_the_pri test_case: BehindStoreTestCase, tmp_path: Path ) -> None: store: Path = tmp_path / "shared" / "history.sqlite3" - project_dir: Path = _shared_project( - tmp_path, + project_dir: Path = shared_store_project( + parent=tmp_path, store=store, keys='write_connection = "history"\nread_connection = "history"\n', ) assert run(project_dir, "build").returncode == 0 - _make_store_behind(store) - schema_before: list[tuple[object, ...]] = _schema(store) + make_store_behind(store) + schema_before: list[tuple[object, ...]] = store_schema(store) queried: subprocess.CompletedProcess[bytes] = run( project_dir, "recall", "query", "SELECT count(*) FROM recall_invocations" ) printed: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "schema", "--ddl") - schema_after_reads: list[tuple[object, ...]] = _schema(store) + schema_after_reads: list[tuple[object, ...]] = store_schema(store) with sqlite3.connect(store) as connection: connection.executescript(printed.stdout.decode()) repeated: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "schema", "--ddl") diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py index cf0964959e..f7b689c63f 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_invocation_facts.py @@ -7,7 +7,15 @@ FailureCauseTestCase, GitRevisionTestCase, ) -from tests.integration.src.sqlbuild.cli.recall.helpers import recall_rows, run, write_project +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + DEPLOYED_REVISION, + checkout_project, + deployed_project, + recall_rows, + run, + worktree_project, + write_project, +) BROKEN_TARGET: str = ( "\n[connections.broken]\n" @@ -18,17 +26,6 @@ ) -def _git(directory: Path, *args: str) -> str: - completed: subprocess.CompletedProcess[str] = subprocess.run( - ["git", "-c", "user.name=Orders", "-c", "user.email=orders@example.com", *args], - cwd=directory, - capture_output=True, - check=True, - text=True, - ) - return completed.stdout.strip() - - @pytest.mark.parametrize( "test_case", [ @@ -68,19 +65,19 @@ def test_given_a_failed_command_when_recorded_then_its_cause_is_structured( [ GitRevisionTestCase( description="the deployment's override wins", - layout="no_repository", - environment={"SQLBUILD_GIT_REVISION": "4f1c2e9", "SQLBUILD_GIT_DIRTY": "true"}, + prepare=deployed_project, + environment={"SQLBUILD_GIT_REVISION": DEPLOYED_REVISION, "SQLBUILD_GIT_DIRTY": "true"}, expected_dirty=1, ), GitRevisionTestCase( description="a git worktree reads HEAD through its .git file", - layout="worktree", + prepare=worktree_project, environment={}, expected_dirty=None, ), GitRevisionTestCase( description="a plain checkout reads HEAD from .git", - layout="checkout", + prepare=checkout_project, environment={}, expected_dirty=None, ), @@ -90,17 +87,7 @@ def test_given_a_failed_command_when_recorded_then_its_cause_is_structured( def test_given_a_deployment_or_checkout_when_recorded_then_the_git_revision_is_known( test_case: GitRevisionTestCase, tmp_path: Path ) -> None: - repository: Path = tmp_path / "repository" - project_dir: Path = write_project(repository / "orders_shop") - expected_revision: str = test_case.environment.get("SQLBUILD_GIT_REVISION", "") - if test_case.layout != "no_repository": - _ = _git(repository, "init", "-q", "-b", "main") - _ = _git(repository, "add", ".") - _ = _git(repository, "commit", "-q", "-m", "orders") - expected_revision = _git(repository, "rev-parse", "HEAD") - if test_case.layout == "worktree": - _ = _git(repository, "worktree", "add", "-q", "-b", "orders", str(tmp_path / "linked")) - project_dir = tmp_path / "linked" / "orders_shop" + project_dir, expected_revision = test_case.prepare(tmp_path) built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", **test_case.environment) diff --git a/tests/unit/scripts/docs_skill/_helpers/_test_types.py b/tests/unit/scripts/docs_skill/_helpers/_test_types.py index 2c876ad37f..f5230b6f12 100644 --- a/tests/unit/scripts/docs_skill/_helpers/_test_types.py +++ b/tests/unit/scripts/docs_skill/_helpers/_test_types.py @@ -25,4 +25,4 @@ class RecallViewsSectionTestCase: description: str guide: str views: str - expected: str + expected_guide: str diff --git a/tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py b/tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py index b87e02831b..b3f318ee9a 100644 --- a/tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py +++ b/tests/unit/scripts/docs_skill/_helpers/test_recall_schema.py @@ -12,7 +12,7 @@ description="an empty section is filled", guide=f"# Guide\n\n{SCHEMA_SECTION_START}\n{SCHEMA_SECTION_END}\n", views="orders (view)\n order_id TEXT\n", - expected=( + expected_guide=( f"# Guide\n\n{SCHEMA_SECTION_START}\n\n```text\norders (view)\n order_id TEXT\n" f"```\n\n{SCHEMA_SECTION_END}\n" ), @@ -24,7 +24,7 @@ f"{SCHEMA_SECTION_END}\noutro\n" ), views="customers (view)\n", - expected=( + expected_guide=( f"intro\n{SCHEMA_SECTION_START}\n\n```text\ncustomers (view)\n```\n\n" f"{SCHEMA_SECTION_END}\noutro\n" ), @@ -35,7 +35,7 @@ def test_given_a_guide_when_the_views_section_is_rendered_then_only_that_section_changes( test_case: RecallViewsSectionTestCase, ) -> None: - rendered: str = with_recall_views(test_case.guide, test_case.views) + rendered: str = with_recall_views(guide=test_case.guide, views=test_case.views) - assert rendered == test_case.expected - assert with_recall_views(rendered, test_case.views) == rendered + assert rendered == test_case.expected_guide + assert with_recall_views(guide=rendered, views=test_case.views) == test_case.expected_guide From e899469967b02779895b4646dc486e0c2c29652c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:13:38 +0100 Subject: [PATCH 13/18] fix: resolve only the Recall connection a command uses Recording resolves and expands only write_connection, and sqb recall only read_connection, so a machine holding only the writer's or only the reader's credentials works. The rule that write_connection needs read_connection now applies to the merged settings, so sqlbuild_local.toml or a target may supply either side; breaking it still fails every command with C975. Other unresolvable recording settings record nothing, as before. --- .../src/recall/main/recall_section.rs | 6 - crates/sqlbuild-config/src/recall/models.rs | 2 +- .../src/recall/tests/test_recall_section.rs | 6 - .../src/bindings/_helpers/cli/entry.rs | 14 +- .../src/bindings/_helpers/recall/command.rs | 10 +- .../src/bindings/_helpers/recall/session.rs | 21 ++- .../src/settings/_helpers/layers.rs | 23 ++- crates/sqlbuild-recall/src/settings/errors.rs | 10 + .../settings/main/resolve_recall_settings.rs | 22 +-- crates/sqlbuild-recall/src/settings/models.rs | 16 +- .../src/settings/tests/helpers.rs | 10 +- .../src/settings/tests/test_resolve.rs | 178 ++++++++++++++++-- .../src/settings/tests/test_types.rs | 4 +- .../src/sqlbuild/cli/recall/_test_types.py | 30 +++ .../src/sqlbuild/cli/recall/helpers.py | 5 + .../cli/recall/test_recall_connections.py | 85 +++++++++ 16 files changed, 377 insertions(+), 65 deletions(-) diff --git a/crates/sqlbuild-config/src/recall/main/recall_section.rs b/crates/sqlbuild-config/src/recall/main/recall_section.rs index 6901fbd9fb..fce31d1f59 100644 --- a/crates/sqlbuild-config/src/recall/main/recall_section.rs +++ b/crates/sqlbuild-config/src/recall/main/recall_section.rs @@ -73,12 +73,6 @@ pub fn recall_section(value: &ConfigValue, label: &str) -> Result, - /// The connection every command writes its history to; requires `read_connection`. + /// The connection every command writes its history to; the merged settings must also set `read_connection`. pub write_connection: Option, /// The connection `sqb recall query`, `schema` and `output` read. pub read_connection: Option, diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs index 9ae44329bc..61ca10eb0e 100644 --- a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -84,12 +84,6 @@ fn given_invalid_sections_when_validated_then_the_message_names_the_key() { and read_connection: set write_connection = \"history\" and \ read_connection = \"history\"", }, - InvalidSectionTestCase { - description: "a write connection needs a read connection", - value: table(&[("write_connection", text("history"))]), - label: "recall", - expected_message_prefix: "recall.write_connection requires recall.read_connection", - }, InvalidSectionTestCase { description: "unknown keys are listed", value: table(&[("mode", text("x"))]), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 469a5db3ef..851b770e52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -31,14 +31,14 @@ use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; -use crate::bindings::_helpers::recall::constants::JSON_FORMAT; +use crate::bindings::_helpers::recall::constants::{JSON_FORMAT, RECALL_SETTINGS_CODE}; use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::recall::session::{ InvocationRecall, finish_invocation_recall, start_invocation_recall, }; use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; -use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::record_class; +use crate::bindings::_helpers::records::construct::{error_class, new_error}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::workspace_commands::commands::maintain_sqlbuild_skills; @@ -581,7 +581,15 @@ fn handled_dispatch<'py>(invocation: &Invocation<'py>) -> PyResult = invocation.py(); let policy: InvocationPolicy = invocation_policy(&invocation.facts); report_skill_freshness(invocation, policy)?; - let dispatched: PyResult> = observed_dispatch(invocation, policy); + let dispatched: PyResult> = match &invocation.recall.configuration_error { + Some(message) => Err(new_error( + py, + ErrorId::CliUser, + message.clone(), + RECALL_SETTINGS_CODE, + )), + None => observed_dispatch(invocation, policy), + }; shared_transient_line_coordinator(py) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? .call_method0("clear_active")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index 5e98ffa267..5f9ca45f15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -12,7 +12,9 @@ use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::main::upgrade_statements::upgrade_statements; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; -use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use sqlbuild_recall::settings::models::{ + RecallSettings, SettingsRequest, StoreLocation, StorePurpose, +}; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; use sqlbuild_recall::store::main::describe_store::describe_store; use sqlbuild_recall::store::main::observe_sqlite_schema::observe_sqlite_schema; @@ -116,6 +118,7 @@ fn settings(py: Python<'_>, recall: &RecallRequest) -> PyResult<(PathBuf, Recall let request = SettingsRequest { project_dir: project_dir.clone(), selected_target: recall.selected_target.clone(), + purpose: StorePurpose::Read, }; let settings = resolve_recall_settings(&request) .map_err(|error| user_error(py, &error.message, RECALL_SETTINGS_CODE))?; @@ -177,7 +180,7 @@ impl ReadStore { /// The read store; `checked` first confirms a ClickHouse store exists, which a query skips and confirms only if it fails. fn open_store(py: Python<'_>, settings: &RecallSettings, checked: bool) -> PyResult { - match &settings.read_store { + match &settings.store { StoreLocation::Sqlite { path } => { let store = open_sqlite_store(path, OpenMode::ReadOnly).map_err(|error| { if error.kind == StoreErrorKind::NotFound { @@ -337,8 +340,7 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { /// Print what creates the read store, or brings it up to this build's schema, without changing it. fn run_schema_ddl(py: Python<'_>, settings: &RecallSettings) -> PyResult { - let (statements, place): (Result, StoreError>, String) = match &settings.read_store - { + let (statements, place): (Result, StoreError>, String) = match &settings.store { StoreLocation::Sqlite { path } => { let options = DdlOptions { database: None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 3b8213e9cf..9d1d0277bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -15,7 +15,9 @@ use sqlbuild_recall::records::models::{InvocationContext, OutputStream, RowEnvel use sqlbuild_recall::schema::main::ddl_script::ddl_script; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; -use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use sqlbuild_recall::settings::models::{ + RecallSettings, SettingsRequest, StoreLocation, StorePurpose, +}; use sqlbuild_recall::store::main::observe_store_file::observe_store_file; use sqlbuild_recall::store::main::quarantine_store::quarantine_store; use sqlbuild_recall::store::main::store_location_problem::store_location_problem; @@ -54,6 +56,8 @@ pub(crate) struct InvocationRecall { pub(crate) settings: Option<(RecallSettings, PathBuf)>, /// The Recall replacement sinks, configured once runtime extensions are discovered. pub(crate) replacements: Arc, + /// A Recall configuration mistake the command fails with before it runs. + pub(crate) configuration_error: Option, } /// The text of an optional attribute value. @@ -402,7 +406,7 @@ fn started_session( Ok(Some(Arc::new(session))) } -/// Begin recording when Recall is enabled, without waiting on the store; invalid settings record nothing and the command reports them. +/// Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. pub(crate) fn start_invocation_recall( py: Python<'_>, args: &Bound<'_, PyAny>, @@ -413,9 +417,17 @@ pub(crate) fn start_invocation_recall( let request = SettingsRequest { project_dir: project_dir.clone(), selected_target: text_value(&args.getattr("target")?)?, + purpose: StorePurpose::Record, }; - let Ok(settings) = resolve_recall_settings(&request) else { - return Ok(InvocationRecall::default()); + let settings: RecallSettings = match resolve_recall_settings(&request) { + Ok(settings) => settings, + Err(error) if error.invalid_configuration => { + return Ok(InvocationRecall { + configuration_error: Some(error.message), + ..InvocationRecall::default() + }); + } + Err(_unreachable_store) => return Ok(InvocationRecall::default()), }; let session: Option> = if settings.enabled && settings.project_name.is_some() { @@ -430,6 +442,7 @@ pub(crate) fn start_invocation_recall( command: facts.command.clone().unwrap_or_default(), settings: Some((settings, project_dir)), replacements: Arc::default(), + configuration_error: None, }) } diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs index d931d397e2..e716980373 100644 --- a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -13,7 +13,7 @@ use crate::settings::constants::{ SQLITE_PATH_KEY, }; use crate::settings::errors::SettingsError; -use crate::settings::models::StoreLocation; +use crate::settings::models::{StoreLocation, StorePurpose}; pub(crate) fn section( document: Option<&ConfigValue>, @@ -40,6 +40,27 @@ pub(crate) fn overlay(base: RecallSection, layer: RecallSection) -> RecallSectio } } +/// The key and connection the purpose uses from the merged settings; a write connection with no read connection on any layer is a configuration mistake. +pub(crate) fn purpose_connection<'a>( + effective: &'a RecallSection, + target: Option<&str>, + purpose: StorePurpose, +) -> Result<(&'static str, Option<&'a str>), SettingsError> { + if effective.write_connection.is_some() && effective.read_connection.is_none() { + let place: String = + target.map_or_else(String::new, |target| format!(" for target `{target}`")); + return Err(SettingsError::invalid_configuration(format!( + "recall.write_connection requires recall.read_connection{place}, the connection \ + `sqb recall` reads history with; set it in [recall], a target's recall table or \ + sqlbuild_local.toml (it may name the same connection)" + ))); + } + Ok(match purpose { + StorePurpose::Record => ("write_connection", effective.write_connection.as_deref()), + StorePurpose::Read => ("read_connection", effective.read_connection.as_deref()), + }) +} + /// The named connection's entries, local keys overriding project keys. fn connection_entries( documents: [Option<&ConfigValue>; 2], diff --git a/crates/sqlbuild-recall/src/settings/errors.rs b/crates/sqlbuild-recall/src/settings/errors.rs index fe4baa26dd..582b074c86 100644 --- a/crates/sqlbuild-recall/src/settings/errors.rs +++ b/crates/sqlbuild-recall/src/settings/errors.rs @@ -4,12 +4,22 @@ #[derive(Clone, Debug, PartialEq, Eq)] pub struct SettingsError { pub message: String, + /// A configuration mistake every command reports, not only a store this invocation cannot reach. + pub invalid_configuration: bool, } impl SettingsError { pub(crate) fn new(message: impl Into) -> Self { Self { message: message.into(), + invalid_configuration: false, + } + } + + pub(crate) fn invalid_configuration(message: impl Into) -> Self { + Self { + message: message.into(), + invalid_configuration: true, } } } diff --git a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs index 3346b36741..41852edf74 100644 --- a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs +++ b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs @@ -4,7 +4,7 @@ use sqlbuild_config::models::ConfigValue; use sqlbuild_config::recall::constants::RECALL_SECTION; use crate::settings::_helpers::documents::{read_document, string_at}; -use crate::settings::_helpers::layers::{overlay, section, store_location}; +use crate::settings::_helpers::layers::{overlay, purpose_connection, section, store_location}; use crate::settings::constants::{ DEFAULT_DIAGNOSTICS_MIN_SEVERITY, DEFAULT_RETENTION_DAYS, DEFAULT_SHUTDOWN_TIMEOUT_MS, LOCAL_FILE_NAME, PROJECT_FILE_NAME, @@ -12,7 +12,7 @@ use crate::settings::constants::{ use crate::settings::errors::SettingsError; use crate::settings::models::{RecallSettings, SettingsRequest}; -/// The effective settings: `targets..recall` over `[recall]` over the built-in local store, local configuration over project configuration at each level. +/// The effective settings: `targets..recall` over `[recall]` over the built-in local store, local configuration over project configuration at each level. Only the connection of the request's purpose is resolved, so a machine holding only the writer's or only the reader's credentials resolves its own side. pub fn resolve_recall_settings(request: &SettingsRequest) -> Result { let project = read_document(&request.project_dir.join(PROJECT_FILE_NAME))?; let local = read_document(&request.project_dir.join(LOCAL_FILE_NAME))?; @@ -43,23 +43,12 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result) = + purpose_connection(&effective, target.as_deref(), request.purpose)?; let store = store_location( &request.project_dir, documents, - ( - "write_connection", - effective.write_connection.as_deref(), - project_adapter.as_deref(), - ), - )?; - let read_store = store_location( - &request.project_dir, - documents, - ( - "read_connection", - effective.read_connection.as_deref(), - project_adapter.as_deref(), - ), + (key, connection, project_adapter.as_deref()), )?; let max_output_bytes: Option = match effective.max_output_bytes { Some(0) => None, @@ -70,7 +59,6 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result, - /// Where this invocation's own history is written. + /// The store of the request's purpose: where this invocation's history is written, or where `sqb recall` reads it. pub store: StoreLocation, - /// Where `sqb recall query`, `schema` and `output` read history. - pub read_store: StoreLocation, /// The configured write connection, `None` for the built-in local store. pub write_connection: Option, /// The configured read connection, `None` for the built-in local store. @@ -58,9 +56,19 @@ pub struct RecallSettings { pub destination_schema: Option, } -/// What selects the settings: the project directory and the `--target` option. +/// Which connection an invocation uses, so only that one is resolved and its environment references expanded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StorePurpose { + /// Recording the invocation's own history through `write_connection`. + Record, + /// `sqb recall query`, `schema` and `output` through `read_connection`. + Read, +} + +/// What selects the settings: the project directory, the `--target` option and the connection used. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SettingsRequest { pub project_dir: PathBuf, pub selected_target: Option, + pub purpose: StorePurpose, } diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs index 6330cf615e..12f1b04c14 100644 --- a/crates/sqlbuild-recall/src/settings/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -3,13 +3,17 @@ use std::path::PathBuf; use sqlbuild_config::models::ConfigValue; use crate::settings::main::resolve_recall_settings::resolve_recall_settings; -use crate::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use crate::settings::models::{RecallSettings, SettingsRequest, StoreLocation, StorePurpose}; /// The placeholder project directory resolved paths are reported under. pub(crate) const PROJECT: &str = ""; /// Settings resolved for a project holding `files`, debug-printed with its directory as `PROJECT`. -pub(crate) fn resolved(files: &[(&str, &str)], target: Option<&str>) -> Result { +pub(crate) fn resolved( + files: &[(&str, &str)], + target: Option<&str>, + purpose: StorePurpose, +) -> Result { let directory = tempfile::tempdir().expect("a temporary directory can be created"); for (name, text) in files { std::fs::write(directory.path().join(name), text).expect("test files can be written"); @@ -17,6 +21,7 @@ pub(crate) fn resolved(files: &[(&str, &str)], target: Option<&str>) -> Result RecallSettings { enabled: true, target: None, store: local_store(), - read_store: local_store(), write_connection: None, read_connection: None, retention_days: 30, diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index 62ab1d9d24..cb797cc9b6 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -1,4 +1,4 @@ -use crate::settings::models::{RecallSettings, StoreLocation}; +use crate::settings::models::{RecallSettings, StoreLocation, StorePurpose}; use crate::settings::tests::helpers::{central_store, defaults, local_store, resolved}; use crate::settings::tests::test_types::{ResolveErrorTestCase, ResolveTestCase}; @@ -42,6 +42,47 @@ const OUTPUT_LIMIT_PROJECT: &[(&str, &str)] = &[( [targets.dev]\n[targets.dev.recall]\nmax_output_bytes = 8388608\n[targets.prod]\n", )]; +const WRITER_CREDENTIALS_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n\ + [recall]\nwrite_connection = \"central\"\nread_connection = \"reader\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"secret\"\n\ + [connections.reader]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"${env:SQLBUILD_RECALL_READER_UNSET_FOR_TEST}\"\n", +)]; + +const READER_CREDENTIALS_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n\ + [recall]\nwrite_connection = \"writer\"\nread_connection = \"central\"\n\ + [connections.writer]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"${env:SQLBUILD_RECALL_WRITER_UNSET_FOR_TEST}\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"secret\"\n", +)]; + +const LOCAL_READ_PROJECT: &[(&str, &str)] = &[ + ( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n[recall]\nwrite_connection = \"central\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"secret\"\n", + ), + ( + "sqlbuild_local.toml", + "[recall]\nread_connection = \"central\"\n", + ), +]; + +const TARGET_WRITE_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n[recall]\nread_connection = \"central\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"secret\"\n\ + [targets.ci]\n[targets.ci.recall]\nwrite_connection = \"central\"\n", +)]; + #[test] fn given_project_layers_when_resolved_then_target_over_local_over_project_over_defaults() { let test_cases = [ @@ -49,6 +90,7 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "without a recall section local SQLite is on by default", files: DEFAULT_PROJECT, target: None, + purpose: StorePurpose::Record, expected_settings: RecallSettings { target: Some("dev".to_owned()), destination_schema: Some("analytics".to_owned()), @@ -59,15 +101,13 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "the target disables recall and the local file moves the store", files: LAYERED_PROJECT, target: Some("dev"), + purpose: StorePurpose::Record, expected_settings: RecallSettings { enabled: false, target: Some("dev".to_owned()), store: StoreLocation::Sqlite { path: "/tmp/opencode/elsewhere.sqlite3".into(), }, - read_store: StoreLocation::Sqlite { - path: "/tmp/opencode/elsewhere.sqlite3".into(), - }, write_connection: Some("history".to_owned()), read_connection: Some("history".to_owned()), retention_days: 10, @@ -79,10 +119,10 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "a ClickHouse connection keeps its options and records all output by default", files: LAYERED_PROJECT, target: Some("prod"), + purpose: StorePurpose::Record, expected_settings: RecallSettings { target: Some("prod".to_owned()), store: central_store(), - read_store: central_store(), write_connection: Some("central".to_owned()), read_connection: Some("central".to_owned()), retention_days: 10, @@ -95,10 +135,10 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "a configured output limit applies to a ClickHouse store", files: LAYERED_PROJECT, target: Some("capped"), + purpose: StorePurpose::Record, expected_settings: RecallSettings { target: Some("capped".to_owned()), store: central_store(), - read_store: central_store(), write_connection: Some("central".to_owned()), read_connection: Some("central".to_owned()), retention_days: 10, @@ -111,6 +151,7 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "a target raises the output limit over the project's unlimited one", files: OUTPUT_LIMIT_PROJECT, target: Some("dev"), + purpose: StorePurpose::Record, expected_settings: RecallSettings { target: Some("dev".to_owned()), max_output_bytes: Some(8_388_608), @@ -121,6 +162,7 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "zero records all output", files: OUTPUT_LIMIT_PROJECT, target: Some("prod"), + purpose: StorePurpose::Record, expected_settings: RecallSettings { target: Some("prod".to_owned()), max_output_bytes: None, @@ -131,16 +173,13 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "a connection on another adapter is reported as unsupported", files: LAYERED_PROJECT, target: Some("qa"), + purpose: StorePurpose::Record, expected_settings: RecallSettings { target: Some("qa".to_owned()), store: StoreLocation::Unsupported { connection: "other".to_owned(), adapter: "postgres".to_owned(), }, - read_store: StoreLocation::Unsupported { - connection: "other".to_owned(), - adapter: "postgres".to_owned(), - }, write_connection: Some("other".to_owned()), read_connection: Some("other".to_owned()), retention_days: 10, @@ -153,12 +192,12 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "a target read connection replaces only the read side", files: LAYERED_PROJECT, target: Some("laptop"), + purpose: StorePurpose::Record, expected_settings: RecallSettings { target: Some("laptop".to_owned()), store: StoreLocation::Sqlite { path: "/tmp/opencode/elsewhere.sqlite3".into(), }, - read_store: central_store(), write_connection: Some("history".to_owned()), read_connection: Some("central".to_owned()), retention_days: 10, @@ -171,18 +210,99 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d description: "a read connection alone reads shared history and writes the local store", files: READER_PROJECT, target: None, + purpose: StorePurpose::Record, expected_settings: RecallSettings { store: local_store(), - read_store: central_store(), read_connection: Some("central".to_owned()), ..defaults() }, }, + ResolveTestCase { + description: "a reader resolves the read connection a target sets", + files: LAYERED_PROJECT, + target: Some("laptop"), + purpose: StorePurpose::Read, + expected_settings: RecallSettings { + target: Some("laptop".to_owned()), + store: central_store(), + write_connection: Some("history".to_owned()), + read_connection: Some("central".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + max_output_bytes: None, + ..defaults() + }, + }, + ResolveTestCase { + description: "a reader with a read connection alone reads shared history", + files: READER_PROJECT, + target: None, + purpose: StorePurpose::Read, + expected_settings: RecallSettings { + store: central_store(), + read_connection: Some("central".to_owned()), + max_output_bytes: None, + ..defaults() + }, + }, + ResolveTestCase { + description: "a recording machine needs only the writer's environment", + files: WRITER_CREDENTIALS_PROJECT, + target: None, + purpose: StorePurpose::Record, + expected_settings: RecallSettings { + store: central_store(), + write_connection: Some("central".to_owned()), + read_connection: Some("reader".to_owned()), + max_output_bytes: None, + ..defaults() + }, + }, + ResolveTestCase { + description: "a reader needs only the reader's environment", + files: READER_CREDENTIALS_PROJECT, + target: None, + purpose: StorePurpose::Read, + expected_settings: RecallSettings { + store: central_store(), + write_connection: Some("writer".to_owned()), + read_connection: Some("central".to_owned()), + max_output_bytes: None, + ..defaults() + }, + }, + ResolveTestCase { + description: "a local read connection completes a project write connection", + files: LOCAL_READ_PROJECT, + target: None, + purpose: StorePurpose::Record, + expected_settings: RecallSettings { + store: central_store(), + write_connection: Some("central".to_owned()), + read_connection: Some("central".to_owned()), + max_output_bytes: None, + ..defaults() + }, + }, + ResolveTestCase { + description: "a target's write connection is completed by the project's read connection", + files: TARGET_WRITE_PROJECT, + target: Some("ci"), + purpose: StorePurpose::Record, + expected_settings: RecallSettings { + target: Some("ci".to_owned()), + store: central_store(), + write_connection: Some("central".to_owned()), + read_connection: Some("central".to_owned()), + max_output_bytes: None, + ..defaults() + }, + }, ]; for test_case in test_cases { assert_eq!( - resolved(test_case.files, test_case.target), + resolved(test_case.files, test_case.target, test_case.purpose), Ok(format!("{:?}", test_case.expected_settings)), "{}", test_case.description @@ -200,6 +320,7 @@ fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nwrite_connection = \"local\"\nread_connection = \"missing\"\n[connections.local]\npath = \"a.sqlite3\"\nadapter = \"sqlite\"\n", )], target: None, + purpose: StorePurpose::Read, expected_message_fragment: "recall.read_connection names unknown connection 'missing'", }, ResolveErrorTestCase { @@ -209,6 +330,7 @@ fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { "name = \"x\"\nadapter = \"duckdb\"\n[targets.dev.recall]\nretention = \"soon\"\n", )], target: Some("dev"), + purpose: StorePurpose::Record, expected_message_fragment: "targets.dev.recall.retention", }, ResolveErrorTestCase { @@ -220,13 +342,39 @@ fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { password = \"${env:SQLBUILD_RECALL_UNSET_FOR_TEST}\"\n", )], target: None, + purpose: StorePurpose::Read, expected_message_fragment: "connections.central.password references unset", }, + ResolveErrorTestCase { + description: "a reader without the reader's environment names it", + files: WRITER_CREDENTIALS_PROJECT, + target: None, + purpose: StorePurpose::Read, + expected_message_fragment: "connections.reader.password references unset", + }, + ResolveErrorTestCase { + description: "a recording machine without the writer's environment names it", + files: READER_CREDENTIALS_PROJECT, + target: None, + purpose: StorePurpose::Record, + expected_message_fragment: "connections.writer.password references unset", + }, + ResolveErrorTestCase { + description: "a write connection without a read connection anywhere is rejected", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[targets.ci.recall]\nwrite_connection = \"local\"\n\ + [connections.local]\npath = \"a.sqlite3\"\nadapter = \"sqlite\"\n", + )], + target: Some("ci"), + purpose: StorePurpose::Read, + expected_message_fragment: "recall.write_connection requires recall.read_connection for target `ci`", + }, ]; for test_case in test_cases { - let message: String = - resolved(test_case.files, test_case.target).expect_err("the settings are invalid"); + let message: String = resolved(test_case.files, test_case.target, test_case.purpose) + .expect_err("the settings are invalid"); assert!( message.contains(test_case.expected_message_fragment), "{}: {message}", diff --git a/crates/sqlbuild-recall/src/settings/tests/test_types.rs b/crates/sqlbuild-recall/src/settings/tests/test_types.rs index d038a10486..53d0f4f418 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_types.rs @@ -1,9 +1,10 @@ -use crate::settings::models::RecallSettings; +use crate::settings::models::{RecallSettings, StorePurpose}; pub(crate) struct ResolveTestCase { pub(crate) description: &'static str, pub(crate) files: &'static [(&'static str, &'static str)], pub(crate) target: Option<&'static str>, + pub(crate) purpose: StorePurpose, pub(crate) expected_settings: RecallSettings, } @@ -11,5 +12,6 @@ pub(crate) struct ResolveErrorTestCase { pub(crate) description: &'static str, pub(crate) files: &'static [(&'static str, &'static str)], pub(crate) target: Option<&'static str>, + pub(crate) purpose: StorePurpose, pub(crate) expected_message_fragment: &'static str, } diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 1c82b43a37..acf1f65bcf 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -274,3 +274,33 @@ class StoreBeingCreatedTestCase: description: str args: tuple[str, ...] expected_error: str + + +@dataclass(frozen=True) +class SplitCredentialsTestCase: + description: str + writer_environment: dict[str, str] + reader_environment: dict[str, str] + expected_invocations: list[list[object]] + + +@dataclass(frozen=True) +class LayeredConnectionsTestCase: + description: str + project_toml: str + local_toml: str + expected_invocations: list[list[object]] + + +@dataclass(frozen=True) +class RecallUsageErrorTestCase: + description: str + args: tuple[str, ...] + expected_message: str + + +@dataclass(frozen=True) +class RecallDiagnosticsTestCase: + description: str + global_args: tuple[str, ...] + expected_debug_log: bool diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index fe2961b735..7624e25b30 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -470,6 +470,11 @@ def logged_event_types(project_dir: Path, dropped_prefix: str) -> tuple[list[str DEPLOYED_REVISION: str = "4f1c2e9" +SPLIT_CREDENTIALS_TOML: str = ( + '\n[connections.writer]\nadapter = "sqlite"\npath = "${env:ORDERS_RECALL_WRITER_PATH}"\n' + '\n[connections.reader]\nadapter = "sqlite"\npath = "${env:ORDERS_RECALL_READER_PATH}"\n' + '\n[recall]\nwrite_connection = "writer"\nread_connection = "reader"\n' +) SHARED_STORE_TOML: str = ( '\n[connections.history]\nadapter = "sqlite"\npath = "{path}"\n\n[recall]\n{keys}' ) diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py index bfd2c22d62..6d52f48cec 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py @@ -1,3 +1,4 @@ +import json import sqlite3 import subprocess from pathlib import Path @@ -6,10 +7,13 @@ from tests.integration.src.sqlbuild.cli.recall._test_types import ( BehindStoreTestCase, + LayeredConnectionsTestCase, ReadOnlyReaderTestCase, RecallConnectionErrorTestCase, + SplitCredentialsTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( + SPLIT_CREDENTIALS_TOML, make_store_behind, recall_rows, run, @@ -150,3 +154,84 @@ def test_given_a_store_one_version_behind_when_read_then_no_ddl_runs_and_the_pri assert rebuilt.returncode == 0 assert b"Recall" not in rebuilt.stderr assert recall_rows(project_dir, "SELECT count(*) FROM invocations") == [[2]] + + +@pytest.mark.parametrize( + "test_case", + [ + SplitCredentialsTestCase( + description="the writer needs only its variable and the reader only its own", + writer_environment={"ORDERS_RECALL_WRITER_PATH": "history.sqlite3"}, + reader_environment={"ORDERS_RECALL_READER_PATH": "history.sqlite3"}, + expected_invocations=[["build", "completed"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_split_credentials_when_each_side_runs_then_it_resolves_only_its_connection( + test_case: SplitCredentialsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", + extra_toml=SPLIT_CREDENTIALS_TOML, + ) + + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", **test_case.writer_environment + ) + read: subprocess.CompletedProcess[bytes] = run( + project_dir, + "recall", + "query", + "--json", + "SELECT command, status FROM invocations", + **test_case.reader_environment, + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert read.returncode == 0, read.stderr.decode(errors="replace") + assert json.loads(read.stdout)["rows"] == test_case.expected_invocations + assert not (project_dir / "logs" / "recall.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + LayeredConnectionsTestCase( + description="a local read connection completes the project's write connection", + project_toml='[recall]\nwrite_connection = "history"\n', + local_toml='[recall]\nread_connection = "history"\n', + expected_invocations=[["build", "completed"]], + ), + LayeredConnectionsTestCase( + description="a target's write connection is completed by the project's read connection", + project_toml=( + '[recall]\nread_connection = "history"\n' + '[targets.dev.recall]\nwrite_connection = "history"\n' + ), + local_toml="", + expected_invocations=[["build", "completed"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_connections_split_across_layers_when_building_then_history_is_recorded( + test_case: LayeredConnectionsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", + extra_toml=( + '\n[connections.history]\nadapter = "sqlite"\npath = "history.sqlite3"\n\n' + + test_case.project_toml + ), + ) + _ = (project_dir / "sqlbuild_local.toml").write_text(test_case.local_toml) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert ( + recall_rows(project_dir, "SELECT command, status FROM invocations") + == test_case.expected_invocations + ) + assert (project_dir / "history.sqlite3").exists() From ef9f8164b83987dbacf684a1a53710c1a8cec6fd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:13:55 +0100 Subject: [PATCH 14/18] fix: keep the server's error for an unknown relation in a ClickHouse Recall query A query naming a relation the complete store lacks, such as a misspelled view, reported 'not recorded yet' (C971). It is now retried once, in case the views were created since it ran, and otherwise fails as an invalid query (C974) with the server's message. A store that is not created yet still reports C971. --- .../_helpers/recall/clickhouse_store.rs | 52 +++++++++---------- .../commands/main/clickhouse/_test_types.py | 7 +++ .../main/clickhouse/test_clickhouse_recall.py | 33 ++++++++++++ 3 files changed, 66 insertions(+), 26 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 711f9dc8d4..d20d97a678 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -641,7 +641,7 @@ impl ClickHouseReader { self.store.location.database() } - /// At most `limit` rows of a read-only query, with the server's statistics. A failure on a store that is not created yet is a store failure (C971), not an invalid query. + /// At most `limit` rows of a read-only query, with the server's statistics. A failure on a store that is not created yet is a store failure (C971), not an invalid query. An unknown relation on a complete store is retried once, since its views may have been created since the query ran; a relation still unknown is the query's own error (C974). pub(crate) fn query( &self, py: Python<'_>, @@ -652,37 +652,37 @@ impl ClickHouseReader { if let Some(limit) = limit { settings.push(("limit", i64::try_from(limit).unwrap_or(i64::MAX - 1) + 1)); } - let (mut result, statistics) = calls::query( - py, - self.store.connection()?, - (sql, &settings), - "run the Recall query", - ) - .map_err(|failure| { - match self.require_store(py) { + let run = || { + calls::query( + py, + self.store.connection()?, + (sql, &settings), + "run the Recall query", + ) + }; + let rejected = |failure: StoreError| StoreError { + kind: if failure.kind == StoreErrorKind::Busy { + StoreErrorKind::Other + } else { + StoreErrorKind::Query + }, + message: failure.message, + }; + let (mut result, statistics) = match run() { + Ok(found) => found, + Err(failure) => match self.require_store(py) { Err(missing) if missing.kind == StoreErrorKind::NotFound => { - return StoreError { + return Err(StoreError { kind: StoreErrorKind::Other, message: missing.message, - }; + }); } Ok(()) if failure.message.contains(UNKNOWN_TABLE_ERROR) => { - return StoreError { - kind: StoreErrorKind::Other, - message: self.not_created_message(), - }; + run().map_err(rejected)? } - Ok(()) | Err(_) => {} - } - StoreError { - kind: if failure.kind == StoreErrorKind::Busy { - StoreErrorKind::Other - } else { - StoreErrorKind::Query - }, - message: failure.message, - } - })?; + Ok(()) | Err(_) => return Err(rejected(failure)), + }, + }; if let Some(limit) = limit && result.rows.len() > limit { diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 00505e1d98..fbcdefd133 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -223,3 +223,10 @@ class ClickHouseRecallExpiredVersionTestCase: stale_view: str expected_statement_columns: tuple[tuple[object, ...], ...] expected_version: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRecallUnknownRelationTestCase: + description: str + sql: str + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index 58ef6f4d2e..9e873b01d7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -24,6 +24,7 @@ ClickHouseRecallQueryOutputSinkTestCase, ClickHouseRecallRetentionConflictTestCase, ClickHouseRecallRetentionTestCase, + ClickHouseRecallUnknownRelationTestCase, ClickHouseRecallUnlimitedOutputTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( @@ -943,3 +944,35 @@ def test_given_a_store_whose_version_row_expired_when_a_writer_with_ddl_builds_t ) == test_case.expected_version ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallUnknownRelationTestCase( + description="a misspelled view on a complete store is the query's own error", + sql="SELECT * FROM invocation", + expected_fragments=("error[C974]", "invocation"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_complete_store_when_a_query_names_an_unknown_relation_then_the_server_error_shows( + test_case: ClickHouseRecallUnknownRelationTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + assert run(project_dir, "build").returncode == 0 + + queried: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "query", test_case.sql) + + stderr: str = queried.stderr.decode(errors="replace") + assert queried.returncode == 1 + assert [fragment in stderr for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ] + assert "C971" not in stderr From 0aa969d8b82b397352f0e3bd6e66c0d03a233b73 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:14:00 +0100 Subject: [PATCH 15/18] fix: never downgrade ClickHouse Recall views after the version row expires The schema version row expires with retention. A writer that finds none now compares the stored tables' and views' columns with its own, both additive: it replaces the views only on an older store, records its version on an older or current one, and leaves a newer store untouched. Writers append the highest version they see again once the newest row is older than half the retention, so it does not expire while the store is in use. A writer that may not upgrade an older store without a version row prints the daily upgrade notice instead of failing silently. --- .../_helpers/recall/clickhouse_store.rs | 126 +++++++++++--- .../src/clickhouse/main/store_state_sql.rs | 12 +- .../sqlbuild-recall/src/clickhouse/models.rs | 4 +- .../sqlbuild-recall/src/schema/constants.rs | 3 + crates/sqlbuild-recall/src/schema/main/mod.rs | 3 + .../src/schema/main/schema_age.rs | 29 ++++ .../src/schema/main/version_plan.rs | 35 ++++ .../src/schema/main/view_columns.rs | 33 ++++ crates/sqlbuild-recall/src/schema/models.rs | 29 ++++ crates/sqlbuild-recall/src/schema/tests.rs | 2 + .../src/schema/tests/helpers.rs | 24 ++- .../src/schema/tests/test_types.rs | 12 ++ .../src/schema/tests/test_version.rs | 120 +++++++++++++ .../commands/main/clickhouse/_test_types.py | 23 +++ .../main/clickhouse/test_clickhouse_recall.py | 159 ++++++++++++++++++ 15 files changed, 583 insertions(+), 31 deletions(-) create mode 100644 crates/sqlbuild-recall/src/schema/main/schema_age.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/version_plan.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/view_columns.rs create mode 100644 crates/sqlbuild-recall/src/schema/tests/test_version.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index d20d97a678..a30b915092 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -1,6 +1,6 @@ //! The ClickHouse Recall store: written and read through the ClickHouse adapter's connection. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant, SystemTime}; @@ -27,12 +27,17 @@ use sqlbuild_recall::schema::main::add_column_statement::add_column_statement; use sqlbuild_recall::schema::main::clickhouse_relation::clickhouse_relation; use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::main::retention_fix_statements::retention_fix_statements; +use sqlbuild_recall::schema::main::schema_age::schema_age; use sqlbuild_recall::schema::main::table_columns::table_columns; use sqlbuild_recall::schema::main::table_spec::table_spec; use sqlbuild_recall::schema::main::table_statements::table_statements; use sqlbuild_recall::schema::main::upgrade_statements::upgrade_statements; +use sqlbuild_recall::schema::main::version_plan::version_plan; +use sqlbuild_recall::schema::main::view_columns::view_columns; use sqlbuild_recall::schema::main::view_statements::view_statements; -use sqlbuild_recall::schema::models::{ColumnSpec, DdlOptions, Dialect, ObservedSchema, Table}; +use sqlbuild_recall::schema::models::{ + ColumnSpec, DdlOptions, Dialect, ObservedSchema, SchemaAge, StoreVersion, Table, VersionPlan, +}; use sqlbuild_recall::store::constants::LAST_INVOCATION_SELECTOR; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; use sqlbuild_recall::store::models::{ @@ -90,6 +95,8 @@ struct StoreState { tables: Vec<(String, String)>, views: HashSet, version: i64, + /// Seconds since the newest version row was recorded. + version_age_seconds: i64, } /// A store ready to write: its tables with their `CREATE` queries, and the upgrade an administrator must run when this user could not. @@ -157,22 +164,70 @@ impl ClickHouseStore { tables.push((name, text(row.get(2)))); } } - let version: i64 = if tables + let (version, version_age_seconds): (i64, i64) = if tables .iter() .any(|(name, _)| name == table_spec(Table::Meta).name) { let found = self.query(py, &sql.schema_version, "read the Recall schema version")?; - integer(found.rows.first().and_then(|row| row.first())) + let row = found.rows.first(); + ( + integer(row.and_then(|row| row.first())), + integer(row.and_then(|row| row.get(1))), + ) } else { - 0 + (0, 0) }; Ok(StoreState { tables, views, version, + version_age_seconds, }) } + /// The recorded version, or without one how the tables, read by `read_columns`, and the views compare with this build's. + fn store_version( + &self, + py: Python<'_>, + state: &StoreState, + ) -> Result { + if state.version > 0 { + return Ok(StoreVersion::Recorded { + version: state.version, + age_seconds: state.version_age_seconds, + }); + } + let mut expected = view_columns().map_err(|message| StoreError { + kind: StoreErrorKind::Other, + message: format!("could not list this build's Recall view columns: {message}"), + })?; + let mut observed: BTreeMap> = BTreeMap::new(); + for table in ALL_TABLES { + let name: &'static str = table_spec(table).name; + expected.insert( + name, + table_columns(table) + .iter() + .map(|column| column.name.to_owned()) + .collect(), + ); + let stored = self.columns.get(&table).map_or(&[][..], Vec::as_slice); + observed.insert( + name.to_owned(), + stored.iter().map(|column| column.name.clone()).collect(), + ); + } + let sql = store_state_sql(self.location.database()); + let found = self.query(py, &sql.view_columns, "inspect the Recall views")?; + for row in &found.rows { + observed + .entry(text(row.first())) + .or_default() + .insert(text(row.get(1))); + } + Ok(StoreVersion::Unrecorded(schema_age(&expected, &observed))) + } + fn require_database(&self, py: Python<'_>) -> Result<(), StoreError> { let found = self.query( py, @@ -242,26 +297,25 @@ impl ClickHouseStore { observed } - /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade or when the version row expired with retention (version 0), since the views may then be from any earlier version. A user who may not run that DDL writes the columns that exist and gets the statements an administrator must run; after an expired version row it stays quiet. A version is recorded only once the views are current. + /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade; when retention removed the version row, the views' columns tell whether the store is older, current or newer, and a newer store's views are never replaced. A user who may not run that DDL writes the columns that exist and gets the statements an administrator must run. A version is recorded only once the views are settled, never lower than the store holds, and again before retention removes the newest row. fn ensure_schema( &mut self, py: Python<'_>, envelope: &RowEnvelope, ) -> Result { - let StoreState { - tables, - views, - version, - } = self.state(py)?; - if tables.len() < ALL_TABLES.len() { - self.create(py, envelope, &views)?; + let state: StoreState = self.state(py)?; + if state.tables.len() < ALL_TABLES.len() { + self.create(py, envelope, &state.views)?; return Ok(ReadyStore { tables: self.state(py)?.tables, behind: Vec::new(), }); } self.read_columns(py)?; - let before: ObservedSchema = self.observed(&views, version); + let store_version: StoreVersion = self.store_version(py, &state)?; + let plan: VersionPlan = version_plan(store_version, self.location.options.retention_days); + let StoreState { tables, views, .. } = state; + let before: ObservedSchema = self.observed(&views, observed_version(store_version)); let behind = |store: &Self| { upgrade_statements(Dialect::ClickHouse, &store.location.options, &before) }; @@ -271,9 +325,8 @@ impl ClickHouseStore { tables, }); } - let upgrade: bool = version < RECALL_SCHEMA_VERSION; for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { - let statement: String = if upgrade { + let statement: String = if plan.replace_views { statement } else if !views.contains(name) { statement.replacen("CREATE OR REPLACE VIEW", "CREATE VIEW IF NOT EXISTS", 1) @@ -283,18 +336,16 @@ impl ClickHouseStore { match self.command(py, &statement, "create the Recall views") { Ok(()) => {} Err(error) if error.kind == StoreErrorKind::Permission => { - let behind: Vec = if version == 0 { - Vec::new() - } else { - behind(self) - }; - return Ok(ReadyStore { behind, tables }); + return Ok(ReadyStore { + behind: behind(self), + tables, + }); } Err(error) => return Err(error), } } - if version < RECALL_SCHEMA_VERSION { - self.record_version(py, envelope)?; + if let Some(version) = plan.record { + self.record_version(py, envelope, version)?; } Ok(ReadyStore { tables, @@ -325,7 +376,7 @@ impl ClickHouseStore { self.command(py, &statement, "create the Recall views")?; } } - self.record_version(py, envelope) + self.record_version(py, envelope, RECALL_SCHEMA_VERSION) } /// Add the additive columns the stored tables lack; `true` when the user may not. @@ -372,14 +423,19 @@ impl ClickHouseStore { } /// Record this build's schema version with an insert, which a writer allowed only `SELECT` and `INSERT` may run. - fn record_version(&self, py: Python<'_>, envelope: &RowEnvelope) -> Result<(), StoreError> { + fn record_version( + &self, + py: Python<'_>, + envelope: &RowEnvelope, + version: i64, + ) -> Result<(), StoreError> { let now = format_timestamp(SystemTime::now()); let row = meta_row( envelope, ( META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, - &RECALL_SCHEMA_VERSION.to_string(), + &version.to_string(), ), &now, ); @@ -626,7 +682,12 @@ impl ClickHouseReader { let observed = store.require_database(py).and_then(|()| { let state: StoreState = store.state(py)?; store.read_columns(py)?; - Ok(store.observed(&state.views, state.version)) + let version: i64 = if state.tables.len() < ALL_TABLES.len() { + state.version + } else { + observed_version(store.store_version(py, &state)?) + }; + Ok(store.observed(&state.views, version)) }); store.close(); Ok(upgrade_statements( @@ -853,3 +914,12 @@ impl ClickHouseReader { self.store.close(); } } + +/// The version `upgrade_statements` compares with: a store without a version row whose views are current or newer needs no view replacement. +fn observed_version(store: StoreVersion) -> i64 { + match store { + StoreVersion::Recorded { version, .. } => version, + StoreVersion::Unrecorded(SchemaAge::Older) => 0, + StoreVersion::Unrecorded(SchemaAge::Current | SchemaAge::Newer) => RECALL_SCHEMA_VERSION, + } +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs b/crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs index d8c021948b..4078f26baf 100644 --- a/crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs +++ b/crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs @@ -20,6 +20,11 @@ pub fn store_state_sql(database: &str) -> StoreStateSql { for view in VIEW_NAMES { names.push(string_literal(view)); } + let views: String = VIEW_NAMES + .iter() + .map(|view| string_literal(view)) + .collect::>() + .join(", "); let database_literal: String = string_literal(database); let options = DdlOptions { database: Some(database.to_owned()), @@ -37,11 +42,16 @@ pub fn store_state_sql(database: &str) -> StoreStateSql { WHERE database = {database_literal} AND table IN ({tables}) ORDER BY table, position" ), schema_version: format!( - "SELECT toInt64(max(toInt64OrZero(value))) AS schema_version FROM {} \ + "SELECT toInt64(max(toInt64OrZero(value))) AS schema_version, \ + toInt64(dateDiff('second', max(recorded_at), now64(6))) AS age_seconds FROM {} \ WHERE component = {} AND key = {}", clickhouse_relation(table_spec(Table::Meta).name, &options), string_literal(META_SCHEMA_COMPONENT), string_literal(META_SCHEMA_VERSION_KEY) ), + view_columns: format!( + "SELECT table, name FROM system.columns \ + WHERE database = {database_literal} AND table IN ({views})" + ), } } diff --git a/crates/sqlbuild-recall/src/clickhouse/models.rs b/crates/sqlbuild-recall/src/clickhouse/models.rs index c4e322c556..5d1d79f133 100644 --- a/crates/sqlbuild-recall/src/clickhouse/models.rs +++ b/crates/sqlbuild-recall/src/clickhouse/models.rs @@ -61,6 +61,8 @@ pub struct StoreStateSql { pub relations: String, /// `table`, `name`, `type` of every column of the Recall tables, in table order. pub columns: String, - /// The highest schema version recorded, 0 when none. + /// The highest schema version recorded, 0 when none, and the age in seconds of the newest version row. pub schema_version: String, + /// `table`, `name` of every column of the Recall views present. + pub view_columns: String, } diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 942caabe09..b48373d847 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -348,3 +348,6 @@ pub(crate) const RETENTION_CANDIDATE_TABLES: [Table; 5] = [ Table::Output, Table::Diagnostics, ]; + +/// Seconds in a day, for comparing a version row's age with the retention. +pub(crate) const SECONDS_PER_DAY: i64 = 86_400; diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index 7d875cb239..eb5220048f 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -4,9 +4,12 @@ pub mod ddl_script; pub mod ddl_statements; pub mod render_script; pub mod retention_fix_statements; +pub mod schema_age; pub mod sqlite_retention_sql; pub mod table_columns; pub mod table_spec; pub mod table_statements; pub mod upgrade_statements; +pub mod version_plan; +pub mod view_columns; pub mod view_statements; diff --git a/crates/sqlbuild-recall/src/schema/main/schema_age.rs b/crates/sqlbuild-recall/src/schema/main/schema_age.rs new file mode 100644 index 0000000000..6b038a6cf1 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/schema_age.rs @@ -0,0 +1,29 @@ +//! `schema_age`: how a store's tables and views compare with this build's. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::schema::models::SchemaAge; + +/// Compare the `observed` columns of each table and view with this build's `expected` ones; both evolve additively, so a missing relation or column means an older store and an unknown column a newer one. A relation this build does not define is ignored. +#[must_use] +pub fn schema_age( + expected: &BTreeMap<&'static str, BTreeSet>, + observed: &BTreeMap>, +) -> SchemaAge { + let mut older: bool = false; + for (relation, columns) in expected { + let Some(present) = observed.get(*relation) else { + older = true; + continue; + }; + if present.difference(columns).next().is_some() { + return SchemaAge::Newer; + } + older |= columns.difference(present).next().is_some(); + } + if older { + SchemaAge::Older + } else { + SchemaAge::Current + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/version_plan.rs b/crates/sqlbuild-recall/src/schema/main/version_plan.rs new file mode 100644 index 0000000000..d5344b88db --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/version_plan.rs @@ -0,0 +1,35 @@ +//! `version_plan`: whether a writer replaces the views and which version it records. + +use crate::schema::constants::{RECALL_SCHEMA_VERSION, SECONDS_PER_DAY}; +use crate::schema::models::{SchemaAge, StoreVersion, VersionPlan}; + +/// Upgrade an older store and record this build's version; never replace a newer store's views or record a lower version than it holds. A version row older than half the retention is appended again with the highest version seen, so retention never removes the last one. +#[must_use] +pub fn version_plan(store: StoreVersion, retention_days: u32) -> VersionPlan { + match store { + StoreVersion::Recorded { version, .. } if version < RECALL_SCHEMA_VERSION => VersionPlan { + replace_views: true, + record: Some(RECALL_SCHEMA_VERSION), + }, + StoreVersion::Recorded { + version, + age_seconds, + } => VersionPlan { + replace_views: false, + record: (age_seconds.saturating_mul(2) > i64::from(retention_days) * SECONDS_PER_DAY) + .then_some(version), + }, + StoreVersion::Unrecorded(SchemaAge::Older) => VersionPlan { + replace_views: true, + record: Some(RECALL_SCHEMA_VERSION), + }, + StoreVersion::Unrecorded(SchemaAge::Current) => VersionPlan { + replace_views: false, + record: Some(RECALL_SCHEMA_VERSION), + }, + StoreVersion::Unrecorded(SchemaAge::Newer) => VersionPlan { + replace_views: false, + record: None, + }, + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/view_columns.rs b/crates/sqlbuild-recall/src/schema/main/view_columns.rs new file mode 100644 index 0000000000..ac8be7893f --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/view_columns.rs @@ -0,0 +1,33 @@ +//! `view_columns`: the columns each view of this build exposes. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::schema::constants::VIEW_NAMES; +use crate::schema::main::ddl_script::ddl_script; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Each view's column names, from a scratch in-memory store built with this build's DDL, or the SQLite error text. +pub fn view_columns() -> Result>, String> { + let connection = rusqlite::Connection::open_in_memory().map_err(|error| error.to_string())?; + let options = DdlOptions { + database: None, + retention_days: 1, + cluster: None, + }; + connection + .execute_batch(&ddl_script(Dialect::Sqlite, &options)) + .map_err(|error| error.to_string())?; + let mut columns: BTreeMap<&'static str, BTreeSet> = BTreeMap::new(); + let mut statement = connection + .prepare("SELECT name FROM pragma_table_info(?1)") + .map_err(|error| error.to_string())?; + for view in VIEW_NAMES { + let names = statement + .query_map([view], |row| row.get::<_, String>(0)) + .map_err(|error| error.to_string())? + .collect::, _>>() + .map_err(|error| error.to_string())?; + columns.insert(view, names); + } + Ok(columns) +} diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index dbc4d55943..198ae24fdb 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -118,3 +118,32 @@ pub struct ObservedSchema { /// The highest recorded schema version; `None` when none is recorded. pub version: Option, } + +/// How a store's tables and views compare with this build's, from their columns. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SchemaAge { + /// A relation is missing or lacks a column of this build, and none has a column this build does not know. + Older, + /// Every table and view has exactly this build's columns. + Current, + /// A relation has a column this build does not know: a newer build created it. + Newer, +} + +/// What a writer knows about the store's schema version. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoreVersion { + /// The highest recorded version, and the age in seconds of the newest version row. + Recorded { version: i64, age_seconds: i64 }, + /// No version row is left, as after retention expired it; the columns tell the store's age. + Unrecorded(SchemaAge), +} + +/// What a writer does to the views and the version row. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VersionPlan { + /// Replace every view with this build's definition. + pub replace_views: bool, + /// The version to append once the views are settled, if any. + pub record: Option, +} diff --git a/crates/sqlbuild-recall/src/schema/tests.rs b/crates/sqlbuild-recall/src/schema/tests.rs index e448cf632c..710a546c44 100644 --- a/crates/sqlbuild-recall/src/schema/tests.rs +++ b/crates/sqlbuild-recall/src/schema/tests.rs @@ -4,3 +4,5 @@ mod ddl; mod helpers; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_version.rs"] +mod version; diff --git a/crates/sqlbuild-recall/src/schema/tests/helpers.rs b/crates/sqlbuild-recall/src/schema/tests/helpers.rs index f54d395f96..ae37f631e1 100644 --- a/crates/sqlbuild-recall/src/schema/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/schema/tests/helpers.rs @@ -1,10 +1,11 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use crate::schema::constants::{ALL_TABLES, VIEW_NAMES}; use crate::schema::main::ddl_script::ddl_script; use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_columns::view_columns; use crate::schema::models::{DdlOptions, Dialect, ObservedSchema, Table}; /// Options rendering into `observability` with 14-day retention, optionally on `cluster`. @@ -97,3 +98,24 @@ fn present_columns(table: Table, (without_table, without_name): (Table, &str)) - .map(|column| column.name.to_owned()) .collect() } + +/// This build's view columns, keyed by owned names as a store reports them. +pub(crate) fn current_views() -> BTreeMap> { + view_columns() + .expect("this build's views can be listed") + .into_iter() + .map(|(view, columns)| (view.to_owned(), columns)) + .collect() +} + +/// This build's view columns with `added` added to and `removed` taken from `statements`. +pub(crate) fn changed_statements_view( + added: &[&str], + removed: &[&str], +) -> BTreeMap> { + let mut views = current_views(); + let statements = views.get_mut("statements").expect("statements is a view"); + statements.extend(added.iter().map(|column| (*column).to_owned())); + statements.retain(|column| !removed.contains(&column.as_str())); + views +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_types.rs b/crates/sqlbuild-recall/src/schema/tests/test_types.rs index 6f90fa620e..287b54032f 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_types.rs @@ -31,3 +31,15 @@ pub(crate) struct UpgradeStatementsTestCase { pub(crate) expected_count: usize, pub(crate) expected_prefixes: &'static [&'static str], } + +pub(crate) struct SchemaAgeTestCase { + pub(crate) description: &'static str, + pub(crate) observed: std::collections::BTreeMap>, + pub(crate) expected_age: crate::schema::models::SchemaAge, +} + +pub(crate) struct VersionPlanTestCase { + pub(crate) description: &'static str, + pub(crate) store: crate::schema::models::StoreVersion, + pub(crate) expected_plan: crate::schema::models::VersionPlan, +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_version.rs b/crates/sqlbuild-recall/src/schema/tests/test_version.rs new file mode 100644 index 0000000000..a4130c497f --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests/test_version.rs @@ -0,0 +1,120 @@ +use crate::schema::constants::RECALL_SCHEMA_VERSION; +use crate::schema::main::schema_age::schema_age; +use crate::schema::main::version_plan::version_plan; +use crate::schema::main::view_columns::view_columns; +use crate::schema::models::{SchemaAge, StoreVersion, VersionPlan}; +use crate::schema::tests::helpers::{changed_statements_view, current_views}; +use crate::schema::tests::test_types::{SchemaAgeTestCase, VersionPlanTestCase}; + +const RETENTION_DAYS: u32 = 30; +const DAY: i64 = 86_400; + +#[test] +fn given_stored_view_columns_when_compared_then_the_store_is_older_current_or_newer() { + let mut missing_view = current_views(); + missing_view.remove("invocations"); + let test_cases = [ + SchemaAgeTestCase { + description: "this build's columns are current", + observed: current_views(), + expected_age: SchemaAge::Current, + }, + SchemaAgeTestCase { + description: "a view lacking a column is older", + observed: changed_statements_view(&[], &["invocation_started_at"]), + expected_age: SchemaAge::Older, + }, + SchemaAgeTestCase { + description: "a missing view is older", + observed: missing_view, + expected_age: SchemaAge::Older, + }, + SchemaAgeTestCase { + description: "an unknown column is newer, even beside a missing one", + observed: changed_statements_view(&["rows_scanned"], &["invocation_started_at"]), + expected_age: SchemaAge::Newer, + }, + ]; + + let expected = view_columns().expect("this build's views can be listed"); + for test_case in test_cases { + assert_eq!( + schema_age(&expected, &test_case.observed), + test_case.expected_age, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_stays_alive() { + let test_cases = [ + VersionPlanTestCase { + description: "an older recorded version upgrades", + store: StoreVersion::Recorded { + version: RECALL_SCHEMA_VERSION - 1, + age_seconds: 0, + }, + expected_plan: VersionPlan { + replace_views: true, + record: Some(RECALL_SCHEMA_VERSION), + }, + }, + VersionPlanTestCase { + description: "a fresh current row needs nothing", + store: StoreVersion::Recorded { + version: RECALL_SCHEMA_VERSION, + age_seconds: DAY, + }, + expected_plan: VersionPlan { + replace_views: false, + record: None, + }, + }, + VersionPlanTestCase { + description: "a newer row past half the retention is re-recorded at its version", + store: StoreVersion::Recorded { + version: RECALL_SCHEMA_VERSION + 1, + age_seconds: 16 * DAY, + }, + expected_plan: VersionPlan { + replace_views: false, + record: Some(RECALL_SCHEMA_VERSION + 1), + }, + }, + VersionPlanTestCase { + description: "an expired row over older views upgrades", + store: StoreVersion::Unrecorded(SchemaAge::Older), + expected_plan: VersionPlan { + replace_views: true, + record: Some(RECALL_SCHEMA_VERSION), + }, + }, + VersionPlanTestCase { + description: "an expired row over current views only records the version", + store: StoreVersion::Unrecorded(SchemaAge::Current), + expected_plan: VersionPlan { + replace_views: false, + record: Some(RECALL_SCHEMA_VERSION), + }, + }, + VersionPlanTestCase { + description: "an expired row over newer views changes nothing", + store: StoreVersion::Unrecorded(SchemaAge::Newer), + expected_plan: VersionPlan { + replace_views: false, + record: None, + }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + version_plan(test_case.store, RETENTION_DAYS), + test_case.expected_plan, + "{}", + test_case.description + ); + } +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index fbcdefd133..fe3dfe2402 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -230,3 +230,26 @@ class ClickHouseRecallUnknownRelationTestCase: description: str sql: str expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseRecallNewerStoreTestCase: + description: str + newer_view: str + expected_future_columns: tuple[tuple[object, ...], ...] + expected_version_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRecallVersionRefreshTestCase: + description: str + row_age_days: int + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRecallUnrecordedBehindTestCase: + description: str + stale_view: str + expected_notice: str + expected_version_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index 9e873b01d7..f851ad2c6b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -17,6 +17,7 @@ ClickHouseRecallExpiryTestCase, ClickHouseRecallFirstReleaseStoreTestCase, ClickHouseRecallLimitedWriterTestCase, + ClickHouseRecallNewerStoreTestCase, ClickHouseRecallOutputTestCase, ClickHouseRecallPartialInsertTestCase, ClickHouseRecallPermissionTestCase, @@ -26,6 +27,8 @@ ClickHouseRecallRetentionTestCase, ClickHouseRecallUnknownRelationTestCase, ClickHouseRecallUnlimitedOutputTestCase, + ClickHouseRecallUnrecordedBehindTestCase, + ClickHouseRecallVersionRefreshTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( FIRST_RELEASE_EDITS, @@ -976,3 +979,159 @@ def test_given_a_complete_store_when_a_query_names_an_unknown_relation_then_the_ True for _ in test_case.expected_fragments ] assert "C971" not in stderr + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallNewerStoreTestCase( + description="views with a column this build does not know are left alone", + newer_view="SELECT *, 1 AS future_column FROM {database}.recall_events", + expected_future_columns=((1,),), + expected_version_rows=((0,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_newer_store_whose_version_row_expired_when_an_older_writer_builds_then_views_stay( + test_case: ClickHouseRecallNewerStoreTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + assert run(project_dir, "build").returncode == 0 + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"CREATE OR REPLACE VIEW {recall_database}.statements AS " + + test_case.newer_view.format(database=recall_database), + f"ALTER TABLE {recall_database}.recall_meta DELETE WHERE 1 SETTINGS mutations_sync = 2", + ), + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0, built.stderr + assert b"Recall" not in built.stderr + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count() FROM system.columns WHERE database = '{recall_database}' " + "AND table = 'statements' AND name = 'future_column'", + ) + == test_case.expected_future_columns + ) + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count() FROM {recall_database}.recall_meta WHERE component = 'schema'", + ) + == test_case.expected_version_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallVersionRefreshTestCase( + description="a version row past half the retention is recorded again", + row_age_days=20, + expected_rows=((2, 2, 1),), + ), + ClickHouseRecallVersionRefreshTestCase( + description="a recent version row is left as it is", + row_age_days=1, + expected_rows=((1, 2, 0),), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_aging_version_row_when_building_then_it_is_refreshed_before_retention_drops_it( + test_case: ClickHouseRecallVersionRefreshTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + assert run(project_dir, "build").returncode == 0 + meta: str = f"{recall_database}.recall_meta" + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"CREATE TABLE {recall_database}.aged_meta AS {meta}", + f"INSERT INTO {recall_database}.aged_meta SELECT * REPLACE (recorded_at - " + f"toIntervalDay({test_case.row_age_days}) AS recorded_at) FROM {meta} " + "WHERE component = 'schema'", + f"ALTER TABLE {meta} DELETE WHERE component = 'schema' SETTINGS mutations_sync = 2", + f"INSERT INTO {meta} SELECT * FROM {recall_database}.aged_meta", + f"DROP TABLE {recall_database}.aged_meta", + ), + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0, built.stderr + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count(), max(toInt64OrZero(value)), " + f"countIf(recorded_at > now64(6) - toIntervalHour(1)) FROM {meta} " + "WHERE component = 'schema'", + ) + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallUnrecordedBehindTestCase( + description="a SELECT, INSERT writer warns when an expired row hides older views", + stale_view="SELECT invocation_id, statement_id FROM {database}.recall_events", + expected_notice="is behind this SQLBuild version, and this user may not upgrade it", + expected_version_rows=((0,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_older_views_and_an_expired_version_row_when_a_limited_writer_builds_then_it_warns( + test_case: ClickHouseRecallUnrecordedBehindTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + admin: Path = write_recall_project( + parent=tmp_path / "admin", config=clickhouse_e2e_config, database=recall_database + ) + assert run(admin, "build").returncode == 0 + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"CREATE OR REPLACE VIEW {recall_database}.statements AS " + + test_case.stale_view.format(database=recall_database), + f"ALTER TABLE {recall_database}.recall_meta DELETE WHERE 1 SETTINGS mutations_sync = 2", + ), + ) + + with limited_user(config=clickhouse_e2e_config, database=recall_database) as writer: + project_dir: Path = write_recall_project( + parent=tmp_path / "writer", config=writer, database=recall_database + ) + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + stderr: str = built.stderr.decode(errors="replace") + assert built.returncode == 0, stderr + assert stderr.count(test_case.expected_notice) == 1 + assert "sqb recall schema --ddl" in stderr + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count() FROM {recall_database}.recall_meta WHERE component = 'schema'", + ) + == test_case.expected_version_rows + ) From f84bcd51960143a79c3688d7862e60d603c9a7cc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:35:49 +0100 Subject: [PATCH 16/18] fix: harden Recall failure records, --ddl and recall diagnostics A command's exit-code failure cause is now kept per thread, so concurrent invocations never take each other's. A raised error whose type, code or message cannot be read still publishes invocation_failed and reaches its caller. sqb recall schema --ddl with --json or --format is a usage error. sqb recall routes diagnostics again, so --debug shows them and a failure prints only its coded error, while still keeping no compute log. Documents the merged connection rule, per-side resolution and the schema version row. --- .../sqlbuild-cli/src/argv/command_table.json | 1 + .../src/bindings/_helpers/cli/entry.rs | 51 +++++++++++----- .../bindings/_helpers/cli/failure_cause.rs | 13 ++--- crates/sqlbuild-python/src/bindings/tests.rs | 2 + .../src/bindings/tests/helpers.rs | 22 +++++++ .../src/bindings/tests/test_failure_cause.rs | 33 +++++++++++ .../src/bindings/tests/test_types.rs | 6 ++ fensu.toml | 2 +- .../docs/concepts/observability/recall.md | 14 ++++- .../entrypoint/_dispatch_with_compute_logs.py | 11 +++- .../sqlbuild/cli/recall/test_recall_cli.py | 58 +++++++++++++++++++ .../cli/commands/main/entry/_test_types.py | 7 +++ .../cli/commands/main/entry/helpers.py | 8 +++ .../cli/commands/main/entry/test_main.py | 37 ++++++++++++ .../docs/concepts/observability/recall.mdx | 14 ++++- 15 files changed, 251 insertions(+), 28 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 8fe116fcff..0377f47be8 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -7028,6 +7028,7 @@ "const": true, "default": false, "help": "print the SQL that creates the Recall store or brings it up to this version", + "mutex_group": 0, "group": 1 }, { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 851b770e52..61132a215e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -8,7 +8,8 @@ use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{ - PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, + PyBool, PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, + PyTypeMethods, }; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::dispatch::main::failure_report::failure_report; @@ -53,6 +54,8 @@ const DIFF_PUBLICATION_CODE: &str = "C257"; const DIFF_EXECUTION_STATUS: &str = "execution_failed"; const DIFF_OUTCOME_PREFIX: &str = "Query diff outcome "; const ERROR_MESSAGE_CHARACTERS: usize = 500; +/// The error type recorded when a raised error cannot be read. +const FALLBACK_ERROR_TYPE: &str = "Exception"; /// Expected staying-Python command failures, loaded only once a command raised. const EXPECTED_COMMAND_FAILURES: [ExecutorHost; 4] = [ ExecutorHost::AuditExecutionError, @@ -497,6 +500,28 @@ fn short_message(message: &str) -> String { line.chars().take(ERROR_MESSAGE_CHARACTERS).collect() } +/// A raised error's type name, code (empty for `SystemExit` or none) and short message; a part that cannot be read is left empty, so the original error and its terminal event survive. +fn raised_cause(py: Python<'_>, error: &PyErr) -> (String, String, String) { + let error_type: String = error.get_type(py).name().map_or_else( + |_unreadable| FALLBACK_ERROR_TYPE.to_owned(), + |name| name.to_string_lossy().into_owned(), + ); + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let readable = |part: &Bound<'_, PyAny>| -> String { + part.str() + .map(|text| text.to_string_lossy().into_owned()) + .unwrap_or_default() + }; + let (code, message): (String, String) = match error_parts(&value, "") { + Ok((code, message, _)) => (readable(&code), short_message(&readable(&message))), + Err(_unreadable) => (String::new(), String::new()), + }; + if error.is_instance_of::(py) { + return (error_type, String::new(), message); + } + (error_type, code, message) +} + /// Dispatch once while publishing canonical invocation lifecycle facts. fn observed_dispatch<'py>( invocation: &Invocation<'py>, @@ -558,17 +583,13 @@ fn observed_dispatch<'py>( let payload: Bound<'py, PyDict> = PyDict::new(py); payload.set_item("command", &command)?; payload.set_item("duration_ms", elapsed(started)?)?; - payload.set_item("error_type", error.get_type(py).getattr("__name__")?)?; - let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); - let (code, message, _) = error_parts(&value, "")?; - let code: String = code.str()?.to_string_lossy().into_owned(); - if !code.is_empty() && !error.is_instance_of::(py) { + let (error_type, code, message): (String, String, String) = + raised_cause(py, &error); + payload.set_item("error_type", error_type)?; + if !code.is_empty() { payload.set_item("error_code", code)?; } - payload.set_item( - "error_message", - short_message(&message.str()?.to_string_lossy()), - )?; + payload.set_item("error_message", message)?; publish(lifecycle_event(py, "invocation_failed", payload)?)?; Err(error) } @@ -684,7 +705,9 @@ fn parsed_invocation<'py>( Ok(handled_dispatch(&invocation)?.unbind()) }, )?; - let dispatched: PyResult> = if policy.captured { + let dispatched: PyResult> = if policy.creates_project { + call_with(&operation.into_any(), &[]) + } else { call_with( &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, &[ @@ -692,10 +715,12 @@ fn parsed_invocation<'py>( ("identity", identity.clone()), ("operation", operation.into_any()), ("recall", recall_sink), + ( + "capture", + PyBool::new(py, policy.captured).to_owned().into_any(), + ), ], ) - } else { - call_with(&operation.into_any(), &[]) }; let exit_status: Option = match &dispatched { Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs index 8c6af87304..65428569b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/failure_cause.rs @@ -1,6 +1,6 @@ //! The cause a command reports when it fails through its exit code instead of raising, for the invocation's terminal event. -use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::cell::RefCell; /// A failure's error type, code and short message. #[derive(Clone, Debug, PartialEq, Eq)] @@ -10,18 +10,17 @@ pub(crate) struct FailureCause { pub(crate) message: String, } -static FAILURE_CAUSE: Mutex> = Mutex::new(None); - -fn locked(slot: &Mutex) -> MutexGuard<'_, T> { - slot.lock().unwrap_or_else(PoisonError::into_inner) +thread_local! { + /// Per thread, since an invocation dispatches and reports on the thread that runs it, so concurrent invocations never read each other's cause. + static FAILURE_CAUSE: RefCell> = const { RefCell::new(None) }; } /// Record why the running command is about to return a failing exit code. pub(crate) fn report_failure_cause(cause: FailureCause) { - *locked(&FAILURE_CAUSE) = Some(cause); + FAILURE_CAUSE.with(|slot| *slot.borrow_mut() = Some(cause)); } /// The cause the command reported, cleared so the next invocation starts without one. pub(crate) fn take_failure_cause() -> Option { - locked(&FAILURE_CAUSE).take() + FAILURE_CAUSE.with(|slot| slot.borrow_mut().take()) } diff --git a/crates/sqlbuild-python/src/bindings/tests.rs b/crates/sqlbuild-python/src/bindings/tests.rs index 18e57b5f55..76ff959782 100644 --- a/crates/sqlbuild-python/src/bindings/tests.rs +++ b/crates/sqlbuild-python/src/bindings/tests.rs @@ -2,6 +2,8 @@ mod coverage; #[path = "tests/test_delta_facts.rs"] mod delta_facts; +#[path = "tests/test_failure_cause.rs"] +mod failure_cause; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_image.rs"] diff --git a/crates/sqlbuild-python/src/bindings/tests/helpers.rs b/crates/sqlbuild-python/src/bindings/tests/helpers.rs index c51ca5e950..24aa5c45cc 100644 --- a/crates/sqlbuild-python/src/bindings/tests/helpers.rs +++ b/crates/sqlbuild-python/src/bindings/tests/helpers.rs @@ -5,6 +5,7 @@ use sqlbuild_cache::project_reuse::types::ImageSections; use sqlbuild_config::models::ConfigValue; use sqlbuild_rules::models::{EvaluateRequest, RetainedRows}; +use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; use crate::bindings::_helpers::compile_context::delta::BaseRules; use crate::bindings::_helpers::compile_context::delta::ModelCoverage; use crate::bindings::_helpers::compile_context::delta::PendingRules; @@ -263,3 +264,24 @@ pub(crate) fn order(models: &[(&str, &[&str])]) -> Vec<(String, Vec)> { .map(|(name, upstream)| ((*name).to_owned(), strings(upstream))) .collect() } + +/// A connection failure's cause, as `sqb debug` reports it. +pub(crate) fn connection_cause() -> FailureCause { + FailureCause { + error_type: "ConnectionError".to_owned(), + error_code: "C980".to_owned(), + message: "connection test: refused".to_owned(), + } +} + +/// Report the cause on the calling thread, as the running invocation does. +pub(crate) fn report_here() { + report_failure_cause(connection_cause()); +} + +/// Report the cause on another thread, as a concurrent invocation does. +pub(crate) fn report_elsewhere() { + std::thread::spawn(report_here) + .join() + .expect("the reporting thread finishes"); +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs b/crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs new file mode 100644 index 0000000000..6133d4820d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/tests/test_failure_cause.rs @@ -0,0 +1,33 @@ +use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; +use crate::bindings::tests::helpers::{report_elsewhere, report_here}; +use crate::bindings::tests::test_types::FailureCauseTestCase; + +#[test] +fn given_a_reported_cause_when_taken_then_only_the_reporting_thread_sees_it() { + let test_cases = [ + FailureCauseTestCase { + description: "a cause reported by this invocation is taken once", + report: report_here, + expected_code: Some("C980"), + }, + FailureCauseTestCase { + description: "a concurrent invocation's cause is never taken", + report: report_elsewhere, + expected_code: None, + }, + ]; + + for test_case in test_cases { + drop(take_failure_cause()); + (test_case.report)(); + + let taken: Option = take_failure_cause().map(|cause| cause.error_code); + + assert_eq!( + (taken.as_deref(), take_failure_cause()), + (test_case.expected_code, None), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/tests/test_types.rs b/crates/sqlbuild-python/src/bindings/tests/test_types.rs index c5785986b5..e3f1a5535b 100644 --- a/crates/sqlbuild-python/src/bindings/tests/test_types.rs +++ b/crates/sqlbuild-python/src/bindings/tests/test_types.rs @@ -65,3 +65,9 @@ pub(crate) struct DeltaFactTestCase { pub(crate) all_sources: bool, pub(crate) expected_safe: bool, } + +pub(crate) struct FailureCauseTestCase { + pub(crate) description: &'static str, + pub(crate) report: fn(), + pub(crate) expected_code: Option<&'static str>, +} diff --git a/fensu.toml b/fensu.toml index 8f39033b71..ff2c76ed43 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1672,7 +1672,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 7f875b4941..3f9fdc3d71 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -165,7 +165,7 @@ enabled = false # do not record runs against the ci target | Key | Default | Description | |-----|---------|-------------| | `enabled` | `true` | Set to `false` to stop recording | -| `write_connection` | (none) | The connection every command writes its history to: a `sqlite` connection whose `path` names the store, or a `clickhouse` connection. Requires `read_connection`. Defaults to `logs/recall.sqlite3` | +| `write_connection` | (none) | The connection every command writes its history to: a `sqlite` connection whose `path` names the store, or a `clickhouse` connection. Requires `read_connection` in the merged settings: `[recall]`, the target's table or `sqlbuild_local.toml` may set either. Defaults to `logs/recall.sqlite3` | | `read_connection` | (none) | The connection `sqb recall query`, `schema` and `output` read; may name the same connection as `write_connection`. Set alone to read shared history while this machine's own commands record to `logs/recall.sqlite3` | | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | @@ -261,7 +261,9 @@ retention = "30d" anything in the store, so someone investigating production runs needs no credentials that could rewrite or delete its history. Both keys may name one connection when separate users are not wanted. A machine that only reads shared history, such as a laptop investigating production runs, -sets only `read_connection`; its own commands keep recording to the local store. +sets only `read_connection`; its own commands keep recording to the local store. Each side +resolves only its own connection, so environment references in the other connection, such as the +writer's password on a reader's laptop, need not be set. ### Users and grants @@ -285,7 +287,8 @@ sqb recall schema --ddl --target prod > recall.sql # review, then run it with Repeat the same step after upgrading SQLBuild: `sqb recall schema --ddl` reads the store and prints only what this version adds (`ALTER TABLE … ADD COLUMN`, replaced views and the schema version row), or says that the store is up to date. Changes are always additive, and older SQLBuild versions keep -writing to an upgraded store. +writing to an upgraded store. `--ddl` prints SQL, so it cannot be combined with `--json` or +`--format`. - **Setup.** A writer allowed to run DDL creates the `recall_*` MergeTree tables and the views above in that database on first use, and adds the columns and views a newer SQLBuild introduces. If the @@ -295,6 +298,11 @@ writing to an upgraded store. command still runs and exits as it otherwise would. History is still written, to the columns that exist, and Recall prints one warning (at most once a day) with the exact statements an administrator should run; `sqb recall schema --ddl` prints them again. +- **Schema version.** The store's version is a `recall_meta` row, which retention expires like any + other. Writers append it again once it is older than half the retention, with the highest version + recorded. When it has expired anyway, a writer compares the stored tables' and views' columns + with its own: it upgrades an older store, records its version on a current one, and never + replaces the views of a store a newer SQLBuild version created. - **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index 49ce8eab0a..544b6fb887 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -33,8 +33,9 @@ def dispatch_with_compute_logs( identity: ExecutionIdentity, operation: Callable[[], int], recall: RecallLifecycleSink | None = None, + capture: bool = True, ) -> int: - """Run one composed dispatch while retaining exact process streams best-effort.""" + """Run one dispatch retaining process streams best-effort, or only routing diagnostics.""" project_dir: Path = Path(args.project_dir) if args.project_dir is not None else Path.cwd() project_roots: tuple[str, ...] = diagnostic_project_roots(project_dir=project_dir) @@ -43,6 +44,14 @@ def dispatch_with_compute_logs( use_color=(not args.no_color) and supports_color(), project_roots=project_roots, ) + if not capture: + with configure_diagnostics( + invocation_id=identity.invocation_id, + debug=args.debug, + use_color=routing_options.use_color, + project_roots=project_roots, + ): + return operation() started_at: datetime = datetime.now(UTC) command: str = "unknown" if args.command is None else str(args.command) metadata: CaptureMetadata = CaptureMetadata( diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index 6db796e1be..4834953e71 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -8,6 +8,8 @@ DeprecatedFormatTestCase, JsonErrorTestCase, LeanRecallTestCase, + RecallDiagnosticsTestCase, + RecallUsageErrorTestCase, StatementStartTestCase, StoreBeingCreatedTestCase, ) @@ -193,3 +195,59 @@ def test_given_a_store_whose_views_are_still_being_created_when_queried_then_it_ assert queried.returncode == 1 assert test_case.expected_error in queried.stderr.decode() + + +@pytest.mark.parametrize( + "test_case", + [ + RecallUsageErrorTestCase( + description="--ddl prints SQL, so it cannot be combined with --json", + args=("recall", "schema", "--ddl", "--json"), + expected_message="argument --json: not allowed with argument --ddl", + ), + ], + ids=lambda case: case.description, +) +def test_given_conflicting_recall_options_when_parsed_then_it_is_a_usage_error( + test_case: RecallUsageErrorTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + rejected: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert rejected.returncode == 2 + assert rejected.stdout == b"" + assert test_case.expected_message in rejected.stderr.decode() + + +@pytest.mark.parametrize( + "test_case", + [ + RecallDiagnosticsTestCase( + description="--debug routes the failure's diagnostics to the console", + global_args=("--debug",), + expected_debug_log=True, + ), + RecallDiagnosticsTestCase( + description="without --debug only the coded error is printed", + global_args=(), + expected_debug_log=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failing_recall_query_when_run_then_diagnostics_follow_the_debug_option( + test_case: RecallDiagnosticsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + assert run(project_dir, "build").returncode == 0 + + failed: subprocess.CompletedProcess[bytes] = run( + project_dir, *test_case.global_args, "recall", "query", "SELECT * FROM missing_orders" + ) + + stderr: str = failed.stderr.decode(errors="replace") + assert failed.returncode == 1 + assert "error[C974]" in stderr + assert ("[error] cli" in stderr) is test_case.expected_debug_log + assert ("CliUserError" in stderr) is test_case.expected_debug_log diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 334d3df385..3d5ba43db7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -160,3 +160,10 @@ class MachineProgressRoutingCase: command: str | None = None dbt_args: tuple[str, ...] = () expected_stdout_excludes: str = "Project compile FAIL" + + +@dataclass(frozen=True) +class UnreadableErrorTestCase: + description: str + expected_error_type: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index c8e6d04851..47022aea48 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -21,6 +21,14 @@ class JsonOutputRequest(Protocol): json_output_path: Path | None +class UnprintableError(Exception): + """A handler error whose message raises when read.""" + + def __str__(self) -> str: + """Fail as a broken `__str__` does.""" + raise RuntimeError("unreadable") + + def invalid_loader_destination_error(_: Path) -> SpecConfigError: """Return the contract error produced by an invalid loader destination.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index b3fbacdb99..2db7418051 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -46,8 +46,10 @@ MainErrorRenderingTestCase, MainTestCase, SkillFreshnessNoticeTestCase, + UnreadableErrorTestCase, ) from tests.unit.src.sqlbuild.cli.commands.main.entry.helpers import ( + UnprintableError, build_handlers, build_json_recording_handler, invalid_loader_destination_error, @@ -2647,6 +2649,41 @@ def run_compile(_request: CompileCommandRequest) -> int: assert events[-1].payload["error_type"] == "ValueError" +@pytest.mark.parametrize( + "test_case", + [ + UnreadableErrorTestCase( + description="an error whose message cannot be read still reaches its caller", + expected_error_type="UnprintableError", + expected_message="", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_unprintable_handler_error_when_dispatching_then_it_propagates_after_its_terminal( + tmp_path: Path, test_case: UnreadableErrorTestCase +) -> None: + def run_compile(_request: CompileCommandRequest) -> int: + raise UnprintableError + + with ( + patch.object(EventDispatcher, "publish_lifecycle") as publish, + pytest.raises(UnprintableError), + ): + _ = _main_with_dependencies( + argv=["--project-dir", str(tmp_path), "compile"], + handlers=build_handlers(run_compile=run_compile), + ) + events: tuple[LifecycleEvent, ...] = tuple(call.args[0] for call in publish.call_args_list) + + assert tuple(event.event_type for event in events) == ( + "invocation_started", + "invocation_failed", + ) + assert events[-1].payload["error_type"] == test_case.expected_error_type + assert events[-1].payload["error_message"] == test_case.expected_message + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 43c9c0ba68..b3b6e9c8b3 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -148,7 +148,7 @@ enabled = false # do not record runs against the ci target | Key | Default | Description | |-----|---------|-------------| | `enabled` | `true` | Set to `false` to stop recording | -| `write_connection` | (none) | The connection every command writes its history to: a `sqlite` connection whose `path` names the store, or a `clickhouse` connection. Requires `read_connection`. Defaults to `logs/recall.sqlite3` | +| `write_connection` | (none) | The connection every command writes its history to: a `sqlite` connection whose `path` names the store, or a `clickhouse` connection. Requires `read_connection` in the merged settings: `[recall]`, the target's table or `sqlbuild_local.toml` may set either. Defaults to `logs/recall.sqlite3` | | `read_connection` | (none) | The connection `sqb recall query`, `schema` and `output` read; may name the same connection as `write_connection`. Set alone to read shared history while this machine's own commands record to `logs/recall.sqlite3` | | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | @@ -244,7 +244,9 @@ retention = "30d" anything in the store, so someone investigating production runs needs no credentials that could rewrite or delete its history. Both keys may name one connection when separate users are not wanted. A machine that only reads shared history, such as a laptop investigating production runs, -sets only `read_connection`; its own commands keep recording to the local store. +sets only `read_connection`; its own commands keep recording to the local store. Each side +resolves only its own connection, so environment references in the other connection, such as the +writer's password on a reader's laptop, need not be set. ### Users and grants @@ -268,7 +270,8 @@ sqb recall schema --ddl --target prod > recall.sql # review, then run it with Repeat the same step after upgrading SQLBuild: `sqb recall schema --ddl` reads the store and prints only what this version adds (`ALTER TABLE … ADD COLUMN`, replaced views and the schema version row), or says that the store is up to date. Changes are always additive, and older SQLBuild versions keep -writing to an upgraded store. +writing to an upgraded store. `--ddl` prints SQL, so it cannot be combined with `--json` or +`--format`. - **Setup.** A writer allowed to run DDL creates the `recall_*` MergeTree tables and the views above in that database on first use, and adds the columns and views a newer SQLBuild introduces. If the @@ -278,6 +281,11 @@ writing to an upgraded store. command still runs and exits as it otherwise would. History is still written, to the columns that exist, and Recall prints one warning (at most once a day) with the exact statements an administrator should run; `sqb recall schema --ddl` prints them again. +- **Schema version.** The store's version is a `recall_meta` row, which retention expires like any + other. Writers append it again once it is older than half the retention, with the highest version + recorded. When it has expired anyway, a writer compares the stored tables' and views' columns + with its own: it upgrades an older store, records its version on a current one, and never + replaces the views of a store a newer SQLBuild version created. - **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. From 48e67e537f8b55161f2297bf92a6033ca69ea347 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:41:15 +0100 Subject: [PATCH 17/18] fix: report misconfigured Recall write connections and skip unseen views An unknown write connection, a non-string adapter, a SQLite connection without a path, an empty ClickHouse database or an unterminated environment reference now fails the command with C975, like a write connection without a read connection; an unset environment variable still only skips recording. A ClickHouse writer that sees none of the views, as when granted only the tables, issues no view DDL, warns of no view upgrade and records no version it cannot vouch for. A test checks that both dialects' views expose the same columns. --- .../_helpers/recall/clickhouse_store.rs | 18 ++- .../src/schema/main/schema_age.rs | 6 +- .../src/schema/main/version_plan.rs | 44 ++++-- crates/sqlbuild-recall/src/schema/models.rs | 12 +- .../src/schema/tests/helpers.rs | 55 +++++++ .../src/schema/tests/test_types.rs | 6 + .../src/schema/tests/test_version.rs | 137 +++++++++++++++++- .../src/settings/_helpers/documents.rs | 2 +- .../src/settings/_helpers/layers.rs | 8 +- crates/sqlbuild-recall/src/settings/errors.rs | 2 +- .../src/settings/tests/helpers.rs | 20 +++ .../src/settings/tests/test_resolve.rs | 51 ++++++- .../src/settings/tests/test_types.rs | 7 + .../src/sqlbuild/cli/recall/_test_types.py | 6 + .../cli/recall/test_recall_connections.py | 30 ++++ 15 files changed, 370 insertions(+), 34 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index a30b915092..31b43d292c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -195,6 +195,7 @@ impl ClickHouseStore { return Ok(StoreVersion::Recorded { version: state.version, age_seconds: state.version_age_seconds, + views_visible: !state.views.is_empty(), }); } let mut expected = view_columns().map_err(|message| StoreError { @@ -225,7 +226,11 @@ impl ClickHouseStore { .or_default() .insert(text(row.get(1))); } - Ok(StoreVersion::Unrecorded(schema_age(&expected, &observed))) + Ok(StoreVersion::Unrecorded(schema_age( + &expected, + &observed, + &VIEW_NAMES, + ))) } fn require_database(&self, py: Python<'_>) -> Result<(), StoreError> { @@ -325,7 +330,12 @@ impl ClickHouseStore { tables, }); } - for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { + let view_ddl: Vec<(&str, String)> = if plan.view_ddl { + view_statements(Dialect::ClickHouse, &self.location.options) + } else { + Vec::new() + }; + for (name, statement) in view_ddl { let statement: String = if plan.replace_views { statement } else if !views.contains(name) { @@ -920,6 +930,8 @@ fn observed_version(store: StoreVersion) -> i64 { match store { StoreVersion::Recorded { version, .. } => version, StoreVersion::Unrecorded(SchemaAge::Older) => 0, - StoreVersion::Unrecorded(SchemaAge::Current | SchemaAge::Newer) => RECALL_SCHEMA_VERSION, + StoreVersion::Unrecorded(SchemaAge::Current | SchemaAge::Newer | SchemaAge::Unseen) => { + RECALL_SCHEMA_VERSION + } } } diff --git a/crates/sqlbuild-recall/src/schema/main/schema_age.rs b/crates/sqlbuild-recall/src/schema/main/schema_age.rs index 6b038a6cf1..8f0ea2bc55 100644 --- a/crates/sqlbuild-recall/src/schema/main/schema_age.rs +++ b/crates/sqlbuild-recall/src/schema/main/schema_age.rs @@ -4,12 +4,16 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::schema::models::SchemaAge; -/// Compare the `observed` columns of each table and view with this build's `expected` ones; both evolve additively, so a missing relation or column means an older store and an unknown column a newer one. A relation this build does not define is ignored. +/// Compare the `observed` columns of each table and view with this build's `expected` ones; both evolve additively, so a missing relation or column means an older store and an unknown column a newer one. When none of `views` is observed, the user may simply not see them, so nothing is judged. A relation this build does not define is ignored. #[must_use] pub fn schema_age( expected: &BTreeMap<&'static str, BTreeSet>, observed: &BTreeMap>, + views: &[&str], ) -> SchemaAge { + if !views.iter().any(|view| observed.contains_key(*view)) { + return SchemaAge::Unseen; + } let mut older: bool = false; for (relation, columns) in expected { let Some(present) = observed.get(*relation) else { diff --git a/crates/sqlbuild-recall/src/schema/main/version_plan.rs b/crates/sqlbuild-recall/src/schema/main/version_plan.rs index d5344b88db..4a82fb6390 100644 --- a/crates/sqlbuild-recall/src/schema/main/version_plan.rs +++ b/crates/sqlbuild-recall/src/schema/main/version_plan.rs @@ -3,32 +3,44 @@ use crate::schema::constants::{RECALL_SCHEMA_VERSION, SECONDS_PER_DAY}; use crate::schema::models::{SchemaAge, StoreVersion, VersionPlan}; -/// Upgrade an older store and record this build's version; never replace a newer store's views or record a lower version than it holds. A version row older than half the retention is appended again with the highest version seen, so retention never removes the last one. +/// Upgrade an older store and record this build's version; never replace a newer store's views or record a lower version than it holds. A version row older than half the retention is appended again with the highest version seen, so retention never removes the last one. A user who sees no view runs no view statements and, without a version row, records none. #[must_use] pub fn version_plan(store: StoreVersion, retention_days: u32) -> VersionPlan { + let refresh = |version: i64, age_seconds: i64| { + (age_seconds.saturating_mul(2) > i64::from(retention_days) * SECONDS_PER_DAY) + .then_some(version) + }; + let plan = |replace_views: bool, record: Option| VersionPlan { + replace_views, + view_ddl: true, + record, + }; match store { - StoreVersion::Recorded { version, .. } if version < RECALL_SCHEMA_VERSION => VersionPlan { - replace_views: true, - record: Some(RECALL_SCHEMA_VERSION), - }, StoreVersion::Recorded { version, age_seconds, + views_visible: false, } => VersionPlan { replace_views: false, - record: (age_seconds.saturating_mul(2) > i64::from(retention_days) * SECONDS_PER_DAY) - .then_some(version), - }, - StoreVersion::Unrecorded(SchemaAge::Older) => VersionPlan { - replace_views: true, - record: Some(RECALL_SCHEMA_VERSION), - }, - StoreVersion::Unrecorded(SchemaAge::Current) => VersionPlan { - replace_views: false, - record: Some(RECALL_SCHEMA_VERSION), + view_ddl: false, + record: (version >= RECALL_SCHEMA_VERSION) + .then(|| refresh(version, age_seconds)) + .flatten(), }, - StoreVersion::Unrecorded(SchemaAge::Newer) => VersionPlan { + StoreVersion::Recorded { version, .. } if version < RECALL_SCHEMA_VERSION => { + plan(true, Some(RECALL_SCHEMA_VERSION)) + } + StoreVersion::Recorded { + version, + age_seconds, + .. + } => plan(false, refresh(version, age_seconds)), + StoreVersion::Unrecorded(SchemaAge::Older) => plan(true, Some(RECALL_SCHEMA_VERSION)), + StoreVersion::Unrecorded(SchemaAge::Current) => plan(false, Some(RECALL_SCHEMA_VERSION)), + StoreVersion::Unrecorded(SchemaAge::Newer) => plan(false, None), + StoreVersion::Unrecorded(SchemaAge::Unseen) => VersionPlan { replace_views: false, + view_ddl: false, record: None, }, } diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 198ae24fdb..022e4f0b84 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -128,13 +128,19 @@ pub enum SchemaAge { Current, /// A relation has a column this build does not know: a newer build created it. Newer, + /// No view is visible, as to a user granted only the tables, so the store's age is unknown. + Unseen, } /// What a writer knows about the store's schema version. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum StoreVersion { - /// The highest recorded version, and the age in seconds of the newest version row. - Recorded { version: i64, age_seconds: i64 }, + /// The highest recorded version, the age in seconds of the newest version row, and whether any view is visible to this user. + Recorded { + version: i64, + age_seconds: i64, + views_visible: bool, + }, /// No version row is left, as after retention expired it; the columns tell the store's age. Unrecorded(SchemaAge), } @@ -144,6 +150,8 @@ pub enum StoreVersion { pub struct VersionPlan { /// Replace every view with this build's definition. pub replace_views: bool, + /// Run view statements at all: replacements, or creating missing views. A user who sees no view issues none, since they would fail on every start. + pub view_ddl: bool, /// The version to append once the views are settled, if any. pub record: Option, } diff --git a/crates/sqlbuild-recall/src/schema/tests/helpers.rs b/crates/sqlbuild-recall/src/schema/tests/helpers.rs index ae37f631e1..d67b4d9c1d 100644 --- a/crates/sqlbuild-recall/src/schema/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/schema/tests/helpers.rs @@ -119,3 +119,58 @@ pub(crate) fn changed_statements_view( statements.retain(|column| !removed.contains(&column.as_str())); views } + +/// Views that pass a table through with `SELECT *`, and that table. +const PASSTHROUGH_VIEWS: [(&str, Table); 3] = [ + ("events", Table::Events), + ("diagnostics", Table::Diagnostics), + ("conflicts", Table::Conflicts), +]; + +/// The columns a ClickHouse view body exposes: its passed-through table's, or the aliases of its final top-level `SELECT` list. +pub(crate) fn clickhouse_view_columns(view: &str, body: &str) -> BTreeSet { + PASSTHROUGH_VIEWS + .iter() + .find(|(name, _)| *name == view) + .map(|(_, table)| { + table_columns(*table) + .into_iter() + .map(|column| column.name.to_owned()) + .collect() + }) + .unwrap_or_else(|| select_list_names(body)) +} + +fn select_list_names(body: &str) -> BTreeSet { + let start: usize = body.rfind("\nSELECT").map_or(0, |index| index + 1); + let list: &str = body[start + "SELECT".len()..] + .split("\nFROM ") + .next() + .unwrap_or_default(); + top_level_items(list) + .iter() + .map(|item| item_name(item)) + .collect() +} + +fn top_level_items(list: &str) -> Vec { + let marked: String = list + .chars() + .scan((0_i32, false), |(depth, quoted), character| { + let separator: bool = *depth == 0 && !*quoted && character == ','; + *quoted ^= character == '\''; + *depth += i32::from(!*quoted && character == '('); + *depth -= i32::from(!*quoted && character == ')'); + Some([character, '\u{0}'][usize::from(separator)]) + }) + .collect(); + marked.split('\u{0}').map(str::to_owned).collect() +} + +fn item_name(item: &str) -> String { + let normalized: String = item.split_whitespace().collect::>().join(" "); + normalized.rsplit_once(" AS ").map_or_else( + || normalized.rsplit('.').next().unwrap_or_default().to_owned(), + |(_, alias)| alias.to_owned(), + ) +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_types.rs b/crates/sqlbuild-recall/src/schema/tests/test_types.rs index 287b54032f..00dc8d54c6 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_types.rs @@ -43,3 +43,9 @@ pub(crate) struct VersionPlanTestCase { pub(crate) store: crate::schema::models::StoreVersion, pub(crate) expected_plan: crate::schema::models::VersionPlan, } + +pub(crate) struct ViewColumnsTestCase { + pub(crate) description: &'static str, + pub(crate) view: &'static str, + pub(crate) expected_same: bool, +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_version.rs b/crates/sqlbuild-recall/src/schema/tests/test_version.rs index a4130c497f..09f50ae617 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_version.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_version.rs @@ -1,10 +1,17 @@ -use crate::schema::constants::RECALL_SCHEMA_VERSION; +use std::collections::BTreeMap; + +use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; +use crate::schema::constants::{RECALL_SCHEMA_VERSION, VIEW_NAMES}; use crate::schema::main::schema_age::schema_age; use crate::schema::main::version_plan::version_plan; use crate::schema::main::view_columns::view_columns; use crate::schema::models::{SchemaAge, StoreVersion, VersionPlan}; -use crate::schema::tests::helpers::{changed_statements_view, current_views}; -use crate::schema::tests::test_types::{SchemaAgeTestCase, VersionPlanTestCase}; +use crate::schema::tests::helpers::{ + changed_statements_view, clickhouse_view_columns, current_views, options, +}; +use crate::schema::tests::test_types::{ + SchemaAgeTestCase, VersionPlanTestCase, ViewColumnsTestCase, +}; const RETENTION_DAYS: u32 = 30; const DAY: i64 = 86_400; @@ -29,6 +36,11 @@ fn given_stored_view_columns_when_compared_then_the_store_is_older_current_or_ne observed: missing_view, expected_age: SchemaAge::Older, }, + SchemaAgeTestCase { + description: "a user who sees no view cannot judge the store", + observed: BTreeMap::new(), + expected_age: SchemaAge::Unseen, + }, SchemaAgeTestCase { description: "an unknown column is newer, even beside a missing one", observed: changed_statements_view(&["rows_scanned"], &["invocation_started_at"]), @@ -39,7 +51,7 @@ fn given_stored_view_columns_when_compared_then_the_store_is_older_current_or_ne let expected = view_columns().expect("this build's views can be listed"); for test_case in test_cases { assert_eq!( - schema_age(&expected, &test_case.observed), + schema_age(&expected, &test_case.observed, &VIEW_NAMES), test_case.expected_age, "{}", test_case.description @@ -55,9 +67,11 @@ fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_sta store: StoreVersion::Recorded { version: RECALL_SCHEMA_VERSION - 1, age_seconds: 0, + views_visible: true, }, expected_plan: VersionPlan { replace_views: true, + view_ddl: true, record: Some(RECALL_SCHEMA_VERSION), }, }, @@ -66,9 +80,11 @@ fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_sta store: StoreVersion::Recorded { version: RECALL_SCHEMA_VERSION, age_seconds: DAY, + views_visible: true, }, expected_plan: VersionPlan { replace_views: false, + view_ddl: true, record: None, }, }, @@ -77,9 +93,11 @@ fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_sta store: StoreVersion::Recorded { version: RECALL_SCHEMA_VERSION + 1, age_seconds: 16 * DAY, + views_visible: true, }, expected_plan: VersionPlan { replace_views: false, + view_ddl: true, record: Some(RECALL_SCHEMA_VERSION + 1), }, }, @@ -88,6 +106,7 @@ fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_sta store: StoreVersion::Unrecorded(SchemaAge::Older), expected_plan: VersionPlan { replace_views: true, + view_ddl: true, record: Some(RECALL_SCHEMA_VERSION), }, }, @@ -96,6 +115,7 @@ fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_sta store: StoreVersion::Unrecorded(SchemaAge::Current), expected_plan: VersionPlan { replace_views: false, + view_ddl: true, record: Some(RECALL_SCHEMA_VERSION), }, }, @@ -104,6 +124,42 @@ fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_sta store: StoreVersion::Unrecorded(SchemaAge::Newer), expected_plan: VersionPlan { replace_views: false, + view_ddl: true, + record: None, + }, + }, + VersionPlanTestCase { + description: "a user who sees no view and finds no version row does nothing", + store: StoreVersion::Unrecorded(SchemaAge::Unseen), + expected_plan: VersionPlan { + replace_views: false, + view_ddl: false, + record: None, + }, + }, + VersionPlanTestCase { + description: "a user who sees no view only refreshes an aging current row", + store: StoreVersion::Recorded { + version: RECALL_SCHEMA_VERSION, + age_seconds: 16 * DAY, + views_visible: false, + }, + expected_plan: VersionPlan { + replace_views: false, + view_ddl: false, + record: Some(RECALL_SCHEMA_VERSION), + }, + }, + VersionPlanTestCase { + description: "a user who sees no view never records over an older row", + store: StoreVersion::Recorded { + version: RECALL_SCHEMA_VERSION - 1, + age_seconds: 16 * DAY, + views_visible: false, + }, + expected_plan: VersionPlan { + replace_views: false, + view_ddl: false, record: None, }, }, @@ -118,3 +174,76 @@ fn given_a_store_version_when_planned_then_views_never_downgrade_and_the_row_sta ); } } + +#[test] +fn given_each_view_when_both_dialects_are_built_then_they_expose_the_same_columns() { + let test_cases = [ + ViewColumnsTestCase { + description: "invocations exposes the same columns in both dialects", + view: "invocations", + expected_same: true, + }, + ViewColumnsTestCase { + description: "statements exposes the same columns in both dialects", + view: "statements", + expected_same: true, + }, + ViewColumnsTestCase { + description: "audit_results exposes the same columns in both dialects", + view: "audit_results", + expected_same: true, + }, + ViewColumnsTestCase { + description: "output_invocations exposes the same columns in both dialects", + view: "output_invocations", + expected_same: true, + }, + ViewColumnsTestCase { + description: "output_tail exposes the same columns in both dialects", + view: "output_tail", + expected_same: true, + }, + ViewColumnsTestCase { + description: "output_transcripts exposes the same columns in both dialects", + view: "output_transcripts", + expected_same: true, + }, + ViewColumnsTestCase { + description: "events exposes the same columns in both dialects", + view: "events", + expected_same: true, + }, + ViewColumnsTestCase { + description: "diagnostics exposes the same columns in both dialects", + view: "diagnostics", + expected_same: true, + }, + ViewColumnsTestCase { + description: "artifacts exposes the same columns in both dialects", + view: "artifacts", + expected_same: true, + }, + ViewColumnsTestCase { + description: "conflicts exposes the same columns in both dialects", + view: "conflicts", + expected_same: true, + }, + ]; + + let sqlite = view_columns().expect("this build's views can be listed"); + let bodies: BTreeMap<&str, String> = + clickhouse_view_bodies(&options(None)).into_iter().collect(); + for test_case in test_cases { + let clickhouse = clickhouse_view_columns( + test_case.view, + bodies.get(test_case.view).map_or("", String::as_str), + ); + let expected = sqlite.get(test_case.view).cloned().unwrap_or_default(); + assert_eq!( + clickhouse == expected && !expected.is_empty(), + test_case.expected_same, + "{}: ClickHouse {clickhouse:?}, SQLite {expected:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/documents.rs b/crates/sqlbuild-recall/src/settings/_helpers/documents.rs index 1fb6e4e52d..f5b3a55ec2 100644 --- a/crates/sqlbuild-recall/src/settings/_helpers/documents.rs +++ b/crates/sqlbuild-recall/src/settings/_helpers/documents.rs @@ -55,7 +55,7 @@ pub(crate) fn expand_environment(text: &str, label: &str) -> Result { - return Err(SettingsError::new(format!( + return Err(SettingsError::invalid_configuration(format!( "connections.{name}.database must name the existing ClickHouse database Recall \ writes to" ))); @@ -152,7 +152,7 @@ pub(crate) fn store_location( }); }; let entries = connection_entries(documents, name).ok_or_else(|| { - SettingsError::new(format!( + SettingsError::invalid_configuration(format!( "recall.{key} names unknown connection '{name}'; define [connections.{name}]" )) })?; @@ -165,7 +165,7 @@ pub(crate) fn store_location( let adapter: String = match value(CONNECTION_ADAPTER_KEY) { Some(ConfigValue::String(adapter)) => adapter.trim().to_owned(), Some(_) => { - return Err(SettingsError::new(format!( + return Err(SettingsError::invalid_configuration(format!( "connections.{name}.adapter must be a string" ))); } @@ -181,7 +181,7 @@ pub(crate) fn store_location( }); } let Some(ConfigValue::String(path)) = value(SQLITE_PATH_KEY) else { - return Err(SettingsError::new(format!( + return Err(SettingsError::invalid_configuration(format!( "connections.{name}.path must name the SQLite file Recall writes" ))); }; diff --git a/crates/sqlbuild-recall/src/settings/errors.rs b/crates/sqlbuild-recall/src/settings/errors.rs index 582b074c86..375050d80c 100644 --- a/crates/sqlbuild-recall/src/settings/errors.rs +++ b/crates/sqlbuild-recall/src/settings/errors.rs @@ -4,7 +4,7 @@ #[derive(Clone, Debug, PartialEq, Eq)] pub struct SettingsError { pub message: String, - /// A configuration mistake every command reports, not only a store this invocation cannot reach. + /// A configuration mistake every command reports, such as an unknown or incomplete connection; an unset environment variable or unreachable store is not one. pub invalid_configuration: bool, } diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs index 12f1b04c14..9c20bf9474 100644 --- a/crates/sqlbuild-recall/src/settings/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -76,3 +76,23 @@ pub(crate) fn defaults() -> RecallSettings { destination_schema: None, } } + +/// Why settings for a project holding `files` do not resolve: the message and whether it is a configuration mistake. +pub(crate) fn resolution_error( + files: &[(&str, &str)], + target: Option<&str>, + purpose: StorePurpose, +) -> Option<(String, bool)> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + for (name, text) in files { + std::fs::write(directory.path().join(name), text).expect("test files can be written"); + } + let request = SettingsRequest { + project_dir: directory.path().to_path_buf(), + selected_target: target.map(str::to_owned), + purpose, + }; + resolve_recall_settings(&request) + .err() + .map(|error| (error.message, error.invalid_configuration)) +} diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index cb797cc9b6..fea7a4fcf9 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -1,6 +1,10 @@ use crate::settings::models::{RecallSettings, StoreLocation, StorePurpose}; -use crate::settings::tests::helpers::{central_store, defaults, local_store, resolved}; -use crate::settings::tests::test_types::{ResolveErrorTestCase, ResolveTestCase}; +use crate::settings::tests::helpers::{ + central_store, defaults, local_store, resolution_error, resolved, +}; +use crate::settings::tests::test_types::{ + ErrorKindTestCase, ResolveErrorTestCase, ResolveTestCase, +}; const DEFAULT_PROJECT: &[(&str, &str)] = &[( "sqlbuild_project.toml", @@ -382,3 +386,46 @@ fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { ); } } + +#[test] +fn given_a_write_connection_problem_when_recording_then_only_mistakes_fail_the_command() { + let test_cases = [ + ErrorKindTestCase { + description: "a misspelled write connection is a configuration mistake", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nwrite_connection = \"histroy_writer\"\n\ + read_connection = \"history\"\n[connections.history]\nadapter = \"sqlite\"\npath = \"h.sqlite3\"\n", + )], + expected_fragment: "recall.write_connection names unknown connection 'histroy_writer'", + expected_invalid_configuration: true, + }, + ErrorKindTestCase { + description: "a SQLite write connection without a path is a configuration mistake", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nwrite_connection = \"history\"\n\ + read_connection = \"history\"\n[connections.history]\nadapter = \"sqlite\"\n", + )], + expected_fragment: "connections.history.path must name the SQLite file", + expected_invalid_configuration: true, + }, + ErrorKindTestCase { + description: "an unset writer variable only skips recording", + files: READER_CREDENTIALS_PROJECT, + expected_fragment: "connections.writer.password references unset", + expected_invalid_configuration: false, + }, + ]; + + for test_case in test_cases { + let (message, invalid) = resolution_error(test_case.files, None, StorePurpose::Record) + .expect("the settings do not resolve"); + assert_eq!( + (message.contains(test_case.expected_fragment), invalid), + (true, test_case.expected_invalid_configuration), + "{}: {message}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/settings/tests/test_types.rs b/crates/sqlbuild-recall/src/settings/tests/test_types.rs index 53d0f4f418..3496faf26f 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_types.rs @@ -15,3 +15,10 @@ pub(crate) struct ResolveErrorTestCase { pub(crate) purpose: StorePurpose, pub(crate) expected_message_fragment: &'static str, } + +pub(crate) struct ErrorKindTestCase { + pub(crate) description: &'static str, + pub(crate) files: &'static [(&'static str, &'static str)], + pub(crate) expected_fragment: &'static str, + pub(crate) expected_invalid_configuration: bool, +} diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index acf1f65bcf..4109cb2f7d 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -304,3 +304,9 @@ class RecallDiagnosticsTestCase: description: str global_args: tuple[str, ...] expected_debug_log: bool + + +@dataclass(frozen=True) +class UnsetWriterVariableTestCase: + description: str + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py index 6d52f48cec..04fdac798d 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_connections.py @@ -11,6 +11,7 @@ ReadOnlyReaderTestCase, RecallConnectionErrorTestCase, SplitCredentialsTestCase, + UnsetWriterVariableTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( SPLIT_CREDENTIALS_TOML, @@ -47,6 +48,14 @@ args=("build",), expected_message="recall.write_connection requires recall.read_connection", ), + RecallConnectionErrorTestCase( + description="a misspelled write connection fails the build", + recall_toml=( + '[recall]\nwrite_connection = "histroy_writer"\nread_connection = "history"\n' + ), + args=("build",), + expected_message="error[C975]: recall.write_connection names unknown connection", + ), ], ids=lambda case: case.description, ) @@ -235,3 +244,24 @@ def test_given_connections_split_across_layers_when_building_then_history_is_rec == test_case.expected_invocations ) assert (project_dir / "history.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + UnsetWriterVariableTestCase( + description="an unset writer variable skips recording without failing the build", + expected_exit_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unset_writer_variable_when_building_then_the_command_still_succeeds( + test_case: UnsetWriterVariableTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=SPLIT_CREDENTIALS_TOML) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == test_case.expected_exit_code, built.stderr.decode(errors="replace") + assert not (project_dir / "history.sqlite3").exists() From 6652e74bd5fb49d5e93b0277fc533693aed5ce4d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 03:14:07 +0100 Subject: [PATCH 18/18] test: keep coded stderr in compiler goldens when --json prints an error document plan --json now prints {"error": ...} on stdout when it fails. The golden builder read that document as a report with no diagnostics; it now records the coded stderr for it, so the goldens keep checking the error. --- .../_helpers/goldens/goldens.py | 3 ++- scripts/compiler_differential/constants.py | 1 + .../_helpers/goldens/helpers.py | 18 ++++++++++++++++++ .../_helpers/goldens/test_compiler_goldens.py | 19 ++++++++++++++++++- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/scripts/compiler_differential/_helpers/goldens/goldens.py b/scripts/compiler_differential/_helpers/goldens/goldens.py index 460a111fc9..4bf6581bad 100644 --- a/scripts/compiler_differential/_helpers/goldens/goldens.py +++ b/scripts/compiler_differential/_helpers/goldens/goldens.py @@ -32,6 +32,7 @@ GOLDEN_SUFFIX, GOLDEN_VERSION_MASK, GOLDEN_WORK_MASK, + MACHINE_ERROR_KEY, PROJECT_DIRECTORY, ) from scripts.compiler_differential.models import ( @@ -155,7 +156,7 @@ def _command( "exit_code": "timed out" if outcome.timed_out else outcome.exit_code, "diagnostics": ( report.get("diagnostics", []) - if report is not None + if report is not None and MACHINE_ERROR_KEY not in report else _lines( text=normalize_stderr(_masked(text=outcome.stderr, masked_paths=masked_paths)) ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 68c62c7c03..80357dea36 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -23,6 +23,7 @@ GOLDEN_LABEL: str = "golden" GOLDEN_FULL_REPORT_LABEL_PREFIX: str = "refactor-" GOLDEN_REPORT_KEY: str = "report" +MACHINE_ERROR_KEY: str = "error" GOLDEN_MANIFEST_DROPPED_KEYS: frozenset[str] = frozenset({"metadata", "child_map"}) GOLDEN_MANIFEST_RESOURCE_SECTIONS: tuple[str, ...] = ("nodes", "sources", "macros") GOLDEN_RESOURCE_DROPPED_FIELDS: frozenset[str] = frozenset( diff --git a/tests/unit/scripts/compiler_differential/_helpers/goldens/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/goldens/helpers.py index d6cd0be258..68579f0ebd 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/goldens/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/goldens/helpers.py @@ -42,3 +42,21 @@ def project_run(*, message: str, compiled_sql: str) -> ProjectRun: } ), ) + + +def machine_error_run() -> ProjectRun: + """A `plan --json` run that failed with the machine error document and a coded stderr.""" + + error: dict[str, object] = {"error": {"code": "C214", "message": "no manifest", "help": None}} + return ProjectRun( + outcomes=( + CommandOutcome( + label="plan", + exit_code=1, + stdout=json.dumps(error), + stderr="error[C214]: no manifest\n", + ), + ), + compiled={}, + manifest=None, + ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py b/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py index 05bc76784c..b87eefc4e5 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py +++ b/tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py @@ -20,6 +20,7 @@ ) from tests.unit.scripts.compiler_differential._helpers.goldens.helpers import ( PROJECT_PATH, + machine_error_run, project_run, ) @@ -98,7 +99,23 @@ def test_given_corpus_project_when_locating_its_golden_then_only_golden_corpora_ "parent_map": {"model.orders.customer_totals": []}, }, }, - ) + ), + GoldenPayloadTestCase( + description="machine_error_document_keeps_the_coded_stderr", + run=machine_error_run(), + masked_paths=(), + expected_payload={ + "commands": [ + { + "label": "plan", + "exit_code": 1, + "diagnostics": ["error[C214]: no manifest", ""], + } + ], + "compiled": {}, + "manifest": None, + }, + ), ], ids=lambda case: case.description, )