From 8685dcd47d34f1fe76eb3202a7f4e534fd9fcf0a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:38:51 +0100 Subject: [PATCH 01/34] fix(rename): rename contract-declared columns and audit arguments with the column A MODEL columns block without commas between entries hid every entry after the first from sqb rename, so the model's own declaration was left for a hand edit that the next run then refused because the project no longer compiled. Column entries now split where the compiler splits them. String arguments of attached audits are renamed only when the generic audit's SQL definition splices them in as SQL (`@where_condition`), and only where they reference the audited model: a subquery reading another model keeps that model's column, and one reading the audited model is renamed too. Arguments the audit quotes (`@'prefix'`), arguments of audits with no SQL definition, unparseable ones and ones reading relations of unknown columns are listed for a hand edit. Refs: CHI-640 --- .../_helpers/edits/header_edits.rs | 105 ++++++++++- .../_helpers/planning/audit_arguments.rs | 75 ++++++++ .../_helpers/planning/column_planning.rs | 163 +++++++++++++++--- .../_helpers/planning/column_references.rs | 57 ++++++ .../src/refactoring/_helpers/planning/mod.rs | 1 + .../src/refactoring/constants.rs | 12 ++ .../src/refactoring/tests/helpers.rs | 65 ++++++- .../refactoring/tests/test_header_edits.rs | 114 +++++++++++- .../src/refactoring/tests/test_types.rs | 7 + .../sqlbuild/references/docs/cli/rename.md | 3 +- .../cli/commands/main/refactor/helpers.py | 75 ++++++++ .../main/refactor/test_rename_columns.py | 59 +++++++ website/src/content/docs/docs/cli/rename.mdx | 3 +- 13 files changed, 706 insertions(+), 33 deletions(-) create mode 100644 crates/sqlbuild-refactor/src/refactoring/_helpers/planning/audit_arguments.rs diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/header_edits.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/header_edits.rs index 56f0c50fdc..ef6ec15a1f 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/header_edits.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/header_edits.rs @@ -1,6 +1,7 @@ //! Token-exact MODEL and SCHEMA header edits, as `header_edits.py` and `schema_edits.py` make. use sqlbuild_core::text::main::is_word_char::is_word_char; +use sqlbuild_model_config::header_metadata::constants::AUDIT_OPTION_KEYS; use sqlbuild_sqltext::compiler::main::model_header_matching::match_batch; use sqlbuild_sqltext::compiler::main::model_header_tokenizing::tokenize_one; use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; @@ -9,10 +10,11 @@ use crate::refactoring::_helpers::edits::text_edits::{text_edit, whole_word_offs use crate::refactoring::_helpers::scanning::chars::{find, rfind, slice, starts_with}; use crate::refactoring::_helpers::scanning::sql_sites::embedded_ref_spans; use crate::refactoring::constants::{ - COLUMN_VALUED_CONFIG_KEYS, COLUMNS_KEY, CURSOR_INPUTS_KEY, HEADER_CLOSERS, + AUDITS_KEY, COLUMN_VALUED_CONFIG_KEYS, COLUMNS_KEY, CURSOR_INPUTS_KEY, HEADER_CLOSERS, HEADER_DESCRIPTION_KEY, HEADER_INDENT, HEADER_KEY_AND_VALUE_TOKENS, HEADER_OPEN_PAREN, - HEADER_OPENERS, HEADER_SEPARATOR, MIGRATE_FROM_KEY, PARENTHESIZED_EMPTY_TOKENS, REF_FUNCTION, - RELATIONSHIPS_AUDIT, RELATIONSHIPS_FIELD_KEY, RELATIONSHIPS_TO_KEY, SCHEMA_KEYWORD, + HEADER_OPENERS, HEADER_SEPARATOR, LIST_OPENER, MIGRATE_FROM_KEY, PARENTHESIZED_EMPTY_TOKENS, + REF_FUNCTION, RELATIONSHIPS_AUDIT, RELATIONSHIPS_FIELD_KEY, RELATIONSHIPS_TO_KEY, + SCHEMA_KEYWORD, }; use crate::refactoring::errors::RefactorError; use crate::refactoring::models::{EditKind, TextEdit}; @@ -159,6 +161,40 @@ fn nested_entries<'a>(tokens: &[&'a HeaderToken], depth: i64) -> Vec(tokens: &[&'a HeaderToken]) -> Vec> { + let mut groups: Vec> = vec![Vec::new()]; + let mut previous: Option<&HeaderToken> = None; + for token in tokens { + if token.depth < 1 { + continue; + } + let closed_entry = token.depth == 1 + && is_value_kind(token.kind) + && previous.is_some_and(|item| { + item.depth == 1 + && item.kind == HeaderTokenKind::Symbol + && HEADER_CLOSERS.contains(&item.value.as_str()) + }); + previous = Some(token); + if token.depth == 1 && is_separator(token) { + groups.push(Vec::new()); + continue; + } + if closed_entry { + groups.push(Vec::new()); + } + if let Some(group) = groups.last_mut() { + group.push(token); + } + } + groups + .into_iter() + .filter(|group| group.first().is_some_and(|first| is_value_kind(first.kind))) + .map(|group| (group[0], group[1..].to_vec())) + .collect() +} + /// Top-level `key value` entries keyed by their first word. fn header_entries(tokens: &[HeaderToken]) -> Vec<(String, Vec<&HeaderToken>)> { let all: Vec<&HeaderToken> = tokens.iter().collect(); @@ -298,7 +334,7 @@ pub(crate) fn column_entry_edits( if key != COLUMNS_KEY { continue; } - for (name_token, values) in nested_entries(&entry[1..], 1) { + for (name_token, values) in column_entries(&entry[1..]) { if !is_value(name_token, old) { continue; } @@ -383,6 +419,67 @@ pub(crate) fn insert_header_entry_edit( ) } +/// One keyed string argument of an attached audit instance. +pub(crate) struct AuditArgument<'a> { + pub(crate) token: &'a HeaderToken, + /// The generic audit the instance attaches, such as `accepted_values`. + pub(crate) audit: String, + pub(crate) key: String, +} + +/// Keyed string arguments of `audits [...]` instances, with their audit and argument names. +pub(crate) fn audit_argument_strings(tokens: &[HeaderToken]) -> Vec> { + let mut found: Vec> = Vec::new(); + let mut audit_depths: Vec = Vec::new(); + let mut instance: String = String::new(); + for (index, token) in tokens.iter().enumerate() { + let list_opener = index > 0 && tokens[index - 1].value == AUDITS_KEY; + while !list_opener + && audit_depths + .last() + .is_some_and(|depth| token.depth < *depth) + { + audit_depths.pop(); + } + let opens_audits = token.kind == HeaderTokenKind::Word + && token.value == AUDITS_KEY + && tokens.get(index + 1).is_some_and(|next| { + next.kind == HeaderTokenKind::Symbol && next.value == LIST_OPENER + }); + if opens_audits { + audit_depths.push(token.depth + 1); + continue; + } + if token.kind == HeaderTokenKind::Word && audit_depths.last() == Some(&token.depth) { + instance.clone_from(&token.value); + } + let preceding = index + .checked_sub(HEADER_KEY_AND_VALUE_TOKENS) + .map(|before| (&tokens[before], &tokens[before + 1])); + let Some((before_key, key)) = preceding else { + continue; + }; + if audit_depths.is_empty() || token.kind != HeaderTokenKind::String { + continue; + } + let keyed = key.kind == HeaderTokenKind::Word + && key.depth == token.depth + && before_key.kind == HeaderTokenKind::Symbol + && (before_key.value == HEADER_OPEN_PAREN || before_key.value == HEADER_SEPARATOR); + let option = AUDIT_OPTION_KEYS.contains(&key.value.as_str()) + || key.value == RELATIONSHIPS_TO_KEY + || key.value == RELATIONSHIPS_FIELD_KEY; + if keyed && !option { + found.push(AuditArgument { + token, + audit: instance.clone(), + key: key.value.clone(), + }); + } + } + found +} + /// Header offsets that still mention a word outside descriptions. pub(crate) fn unhandled_word_offsets( text: &[char], diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/audit_arguments.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/audit_arguments.rs new file mode 100644 index 0000000000..cc6cd98a64 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/audit_arguments.rs @@ -0,0 +1,75 @@ +//! How a generic audit's SQL uses one argument: spliced in as SQL, quoted, or unknown. + +use std::path::Path; + +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::constants::{ + AUDIT_PARAMETER_SIGIL, GENERIC_AUDIT_DIRECTORY, SQL_STRING_QUOTE, +}; +use crate::refactoring::models::{AuthoredFile, SqlFileRole}; + +/// How every definition of a generic audit renders one argument. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ArgumentUsage { + /// Every definition splices the argument in as SQL with `@name`. + Sql, + /// A definition quotes the argument as a string literal with `@'name'`. + Literal, + /// No project SQL definition shows how the argument is used. + Unknown, +} + +/// How the project's SQL definitions of `audit` render argument `key`. +pub(crate) fn argument_usage(files: &[AuthoredFile], audit: &str, key: &str) -> ArgumentUsage { + let definitions: Vec<&AuthoredFile> = files + .iter() + .filter(|file| file.role == SqlFileRole::Audit && defines_generic_audit(&file.path, audit)) + .collect(); + let mut usage = ArgumentUsage::Unknown; + for definition in definitions { + let (spliced, quoted) = parameter_uses(&chars(&definition.contents), key); + if quoted { + return ArgumentUsage::Literal; + } + if !spliced { + return ArgumentUsage::Unknown; + } + usage = ArgumentUsage::Sql; + } + usage +} + +/// Whether `path` is a generic audit definition file named `audit`. +fn defines_generic_audit(path: &str, audit: &str) -> bool { + let path = Path::new(path); + path.file_stem().is_some_and(|stem| stem == audit) + && path + .parent() + .and_then(Path::file_name) + .is_some_and(|directory| directory == GENERIC_AUDIT_DIRECTORY) +} + +/// Whether the SQL uses `@key` unquoted and whether it uses `@'key'`. +fn parameter_uses(text: &[char], key: &str) -> (bool, bool) { + let key: Vec = chars(key); + let mut spliced = false; + let mut quoted = false; + for (index, character) in text.iter().enumerate() { + if *character != AUDIT_PARAMETER_SIGIL { + continue; + } + let rest = &text[index + 1..]; + if rest.starts_with(&key) && !rest.get(key.len()).is_some_and(|next| is_word(*next)) { + spliced = true; + } + let quoted_name = rest.first() == Some(&SQL_STRING_QUOTE) + && rest[1..].starts_with(&key) + && rest.get(key.len() + 1) == Some(&SQL_STRING_QUOTE); + quoted = quoted || quoted_name; + } + (spliced, quoted) +} + +fn is_word(character: char) -> bool { + character.is_alphanumeric() || character == '_' +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_planning.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_planning.rs index 0a15a1168a..c11fe3cd76 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_planning.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_planning.rs @@ -3,29 +3,31 @@ use std::collections::{BTreeSet, HashSet, VecDeque}; use crate::refactoring::_helpers::edits::header_edits::{ - add_column_entry_edit, column_config_edits, column_entry_edits, consumer_column_header_edits, - header_tokens, schema_column_edits, unhandled_word_offsets, + HeaderToken, add_column_entry_edit, audit_argument_strings, column_config_edits, + column_entry_edits, consumer_column_header_edits, header_tokens, schema_column_edits, + unhandled_word_offsets, }; use crate::refactoring::_helpers::edits::text_edits::{ - RefactorParts, manual_at, merge_parts, path_edits, + RefactorParts, manual_at, merge_parts, path_edits, whole_word_offsets, }; use crate::refactoring::_helpers::edits::yaml_edits::yaml_column_edits; use crate::refactoring::_helpers::files::project_files::{ AuthoredBody, ProjectSqlFile, authored_bodies, project_sql_files, yaml_files, }; +use crate::refactoring::_helpers::planning::audit_arguments::{ArgumentUsage, argument_usage}; use crate::refactoring::_helpers::planning::column_references::{ - BodyContext, BodyMapping, ColumnQuery, ResourceColumns, analyze_column, columns_of, - consumer_edits, output_edits, resource_columns, + BodyContext, BodyEdits, BodyMapping, ColumnFacts, ColumnQuery, ExpressionSite, ResourceColumns, + analyze_column, columns_of, consumer_edits, expression_edits, output_edits, resource_columns, }; use crate::refactoring::_helpers::planning::model_planning::{ find_model, needs_column_migration, validate_identifier, }; -use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::_helpers::scanning::chars::{chars, slice}; use crate::refactoring::_helpers::scanning::scan_context::ScanContext; -use crate::refactoring::_helpers::scanning::sql_sites::{analysis_sql, model_body}; +use crate::refactoring::_helpers::scanning::sql_sites::{AnalysisSql, analysis_sql, model_body}; use crate::refactoring::constants::{ - EXPECTED_FIXTURE_PREFIX, MIGRATE_FROM_KEY, MODEL_RESOURCE_TYPE, REF_FIXTURE_PREFIX, REF_KIND, - ROOT_SCOPE, + AUDIT_ARGUMENT_TABLE, ESCAPE_CHARACTER, EXPECTED_FIXTURE_PREFIX, MIGRATE_FROM_KEY, + MODEL_RESOURCE_TYPE, REF_FIXTURE_PREFIX, REF_KIND, ROOT_SCOPE, }; use crate::refactoring::errors::RefactorError; use crate::refactoring::models::{ @@ -272,7 +274,7 @@ fn declarations( ])) } -fn header_parts( +pub(crate) fn header_parts( context: &ColumnRenameContext<'_>, model: &ModelFacts, ) -> Result { @@ -293,18 +295,23 @@ fn header_parts( let mut edits = column_config_edits(&text, &tokens, old, new); edits.extend(entry_edits.unwrap_or_default()); edits.extend(added); - let handled: Vec = edits.iter().map(|edit| edit.start).collect(); - let manual: Vec = unhandled_word_offsets(&text, &tokens, old, &handled) - .into_iter() - .map(|offset| { - manual_at( - path, - &text, - Some(offset), - format!("the MODEL header mentions {old} here; update it by hand"), - ) - }) - .collect(); + let mut handled: Vec = edits.iter().map(|edit| edit.start).collect(); + let arguments = audit_argument_parts(context, model, (path, &text), &tokens)?; + handled.extend(arguments.handled); + edits.extend(arguments.edits.edits); + let mut manual: Vec = arguments.edits.manual; + manual.extend( + unhandled_word_offsets(&text, &tokens, old, &handled) + .into_iter() + .map(|offset| { + manual_at( + path, + &text, + Some(offset), + format!("the MODEL header mentions {old} here; update it by hand"), + ) + }), + ); let migrations = if migrate { vec![MigrationDeclaration { model_name: model.name.clone(), @@ -322,6 +329,118 @@ fn header_parts( }) } +/// Audit argument edits and the header string tokens they settle. +struct ArgumentParts { + edits: BodyEdits, + handled: Vec, +} + +/// Rename the column in audit string arguments the audit's SQL splices in as expressions. +fn audit_argument_parts( + context: &ColumnRenameContext<'_>, + model: &ModelFacts, + file: (&str, &[char]), + tokens: &[HeaderToken], +) -> Result { + let (path, text) = file; + let (old, _) = context.names(); + let mut parts = ArgumentParts { + edits: BodyEdits::default(), + handled: Vec::new(), + }; + for argument in audit_argument_strings(tokens) { + let token = argument.token; + let expression = slice(text, token.start + 1, token.end.saturating_sub(1)); + let mentions = whole_word_offsets(&chars(&expression), old); + if mentions.is_empty() { + continue; + } + parts.handled.push(token.start); + let usage = argument_usage( + &context.facts.authored_files, + &argument.audit, + &argument.key, + ); + if usage != ArgumentUsage::Sql { + let reason = non_sql_argument_reason(old, &argument.audit, &argument.key, usage); + parts + .edits + .manual + .extend(mentions.into_iter().map(|offset| { + manual_at(path, text, Some(token.start + 1 + offset), reason.clone()) + })); + continue; + } + let prefix = format!("SELECT * FROM {AUDIT_ARGUMENT_TABLE} WHERE "); + let site = ExpressionSite { + path, + contents: text, + prefix: prefix.chars().count(), + offset: token.start + 1, + }; + let rendered: AnalysisSql = analysis_sql(&chars(&expression), context.scan); + let unknown_relation = rendered + .tables + .iter() + .find(|((kind, name), _)| columns_of(&context.columns, kind, name).is_empty()) + .map(|((_, name), _)| name.clone()); + if let Some(relation) = unknown_relation { + let reason = format!( + "this audit argument reads {relation}, whose columns SQLBuild does not know, so {old} here is not renamed; update it by hand" + ); + parts + .edits + .manual + .extend(mentions.into_iter().map(|offset| { + manual_at(path, text, Some(token.start + 1 + offset), reason.clone()) + })); + continue; + } + let mut target_tables: BTreeSet = rendered.placeholders(REF_KIND, &model.name); + target_tables.insert(AUDIT_ARGUMENT_TABLE.to_owned()); + let mut tables = vec![( + (REF_KIND.to_owned(), model.name.clone()), + BTreeSet::from([AUDIT_ARGUMENT_TABLE.to_owned()]), + )]; + tables.extend(rendered.tables); + let analysis = AnalysisSql { + sql: format!("{prefix}{}", rendered.sql), + tables, + }; + let analysed = if expression.contains(ESCAPE_CHARACTER) { + ColumnFacts::default() + } else { + analyze_column( + &analysis, + &context.facts.dialect, + &context.columns, + &ColumnQuery { + column: old, + target_tables, + target_ctes: BTreeSet::new(), + output_scopes: BTreeSet::new(), + }, + )? + }; + let result = expression_edits(&analysed, &site, context.names()); + parts.edits.edits.extend(result.edits); + parts.edits.manual.extend(result.manual); + } + Ok(parts) +} + +/// Why an audit argument that is not known to be SQL is left for a hand edit. +fn non_sql_argument_reason(old: &str, audit: &str, key: &str, usage: ArgumentUsage) -> String { + match usage { + ArgumentUsage::Literal => format!( + "audit {audit} quotes argument {key} as a string, so {old} here is not renamed; update it by hand if it names the column" + ), + _ => format!( + "no SQL definition of audit {audit} shows whether argument {key} is SQL, so {old} here is not renamed; update it by hand if it names the column" + ), + } +} + fn consumers<'a>( context: &ColumnRenameContext<'a>, model: &ModelFacts, diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_references.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_references.rs index 38e6ad1a97..af00f40706 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_references.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_references.rs @@ -426,6 +426,63 @@ pub(crate) fn output_edits( result } +/// Where one SQL expression authored inside a header string sits in its analysis query and file. +pub(crate) struct ExpressionSite<'a> { + pub(crate) path: &'a str, + pub(crate) contents: &'a [char], + /// The length of the analysis query text before the expression. + pub(crate) prefix: usize, + /// The file offset of the expression's first character. + pub(crate) offset: usize, +} + +/// Rename every reference to the column in one analysed header-string expression. +pub(crate) fn expression_edits( + facts: &ColumnFacts, + site: &ExpressionSite<'_>, + names: (&str, &str), +) -> BodyEdits { + let (old, new) = names; + let locate = |start: usize| { + start + .checked_sub(site.prefix) + .map(|start| site.offset + start) + }; + if !facts.parsed { + return BodyEdits::manual(manual_at( + site.path, + site.contents, + Some(site.offset), + format!( + "this audit argument mentions {old} but is not a SQL expression SQLBuild can parse; update it by hand" + ), + )); + } + let mut result = BodyEdits::default(); + for reference in &facts.references { + let (Some(start), Some(end)) = (locate(reference.name_start), locate(reference.name_end)) + else { + continue; + }; + let authored = slice(site.contents, start, end); + result.edits.push(text_edit( + site.contents, + (start, end), + respell(&authored, new), + EditKind::Header, + )); + } + for unresolved in &facts.unresolved { + result.manual.push(manual_at( + site.path, + site.contents, + unresolved.start.and_then(locate).or(Some(site.offset)), + format!("cannot tell whether {old} in this audit argument is the renamed column; update it by hand"), + )); + } + result +} + fn scope_role(scope: &str) -> String { if scope == ROOT_SCOPE { scope.to_owned() diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs index 0554329300..3817d5ef15 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod audit_arguments; pub(crate) mod column_planning; pub(crate) mod column_references; pub(crate) mod declaration_moves; diff --git a/crates/sqlbuild-refactor/src/refactoring/constants.rs b/crates/sqlbuild-refactor/src/refactoring/constants.rs index e60ecdb0ad..84645fd5f6 100644 --- a/crates/sqlbuild-refactor/src/refactoring/constants.rs +++ b/crates/sqlbuild-refactor/src/refactoring/constants.rs @@ -13,6 +13,17 @@ pub const CTE_SCOPE_PREFIX: &str = "cte:"; pub const MIGRATE_FROM_KEY: &str = "migrate_from"; pub const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; pub const COLUMNS_KEY: &str = "columns"; +pub const AUDITS_KEY: &str = "audits"; +/// The relation an audit string argument is analysed against, standing for the audited model. +pub const AUDIT_ARGUMENT_TABLE: &str = "__sqb_audit_target__"; +/// The escape character of quoted MODEL header values. +pub const ESCAPE_CHARACTER: char = '\\'; +/// The quote of SQL string literals. +pub const SQL_STRING_QUOTE: char = '\''; +/// The character that starts a generic audit parameter such as `@expression`. +pub const AUDIT_PARAMETER_SIGIL: char = '@'; +/// The directory holding generic audit definitions, as in `audits/generic/`. +pub const GENERIC_AUDIT_DIRECTORY: &str = "generic"; pub const RELATIONSHIPS_AUDIT: &str = "relationships"; pub const RELATIONSHIPS_TO_KEY: &str = "to"; pub const RELATIONSHIPS_FIELD_KEY: &str = "field"; @@ -39,6 +50,7 @@ pub const HEADER_DESCRIPTION_KEY: &str = "description"; pub const HEADER_INDENT: &str = " "; pub const HEADER_SEPARATOR: &str = ","; pub const HEADER_OPEN_PAREN: &str = "("; +pub const LIST_OPENER: &str = "["; pub const HEADER_OPENERS: [&str; 3] = ["(", "[", "{"]; pub const HEADER_CLOSERS: [&str; 3] = [")", "]", "}"]; pub const PARENTHESIZED_EMPTY_TOKENS: usize = 2; diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/helpers.rs b/crates/sqlbuild-refactor/src/refactoring/tests/helpers.rs index dac3d1fcc5..7f01e62d17 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests/helpers.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests/helpers.rs @@ -12,6 +12,9 @@ use crate::refactoring::_helpers::edits::header_edits::{ use crate::refactoring::_helpers::edits::text_edits::{apply_text_edits, file_changes, text_edit}; use crate::refactoring::_helpers::edits::yaml_edits::{yaml_column_edits, yaml_model_edits}; use crate::refactoring::_helpers::files::workspace::commit_changes; +use crate::refactoring::_helpers::planning::column_planning::{ + column_rename_context, header_parts, +}; use crate::refactoring::_helpers::planning::column_references::{ BodyContext, BodyMapping, ColumnQuery, ResourceColumns, analyze_column, consumer_edits, }; @@ -22,9 +25,9 @@ use crate::refactoring::_helpers::scanning::scan_context::ScanContext; use crate::refactoring::_helpers::scanning::sql_sites::{ModelBody, analysis_sql, resource_sites}; use crate::refactoring::errors::{RefactorError, RefactorErrorKind}; use crate::refactoring::models::{ - DeclarationMoves, DeclarationPlacement, DestinationFacts, DiscoveredFile, EditKind, - ExpansionSpan, FileChange, ModelFacts, PlacedDeclaration, RefactorFacts, RefactorOperation, - RefactorRequest, RelocatedDeclaration, TextEdit, + AuthoredFile, DeclarationMoves, DeclarationPlacement, DestinationFacts, DiscoveredFile, + EditKind, ExpansionSpan, FileChange, ModelFacts, PlacedDeclaration, RefactorFacts, + RefactorOperation, RefactorRequest, RelocatedDeclaration, SqlFileRole, TextEdit, }; use crate::refactoring::tests::test_types::{ExpectedSpans, PlannedMoves, Spans}; @@ -587,3 +590,59 @@ pub(crate) fn physical_columns(columns: &[(&str, &str)]) -> Vec<(String, String) .map(|(name, column_type)| ((*name).to_owned(), (*column_type).to_owned())) .collect() } + +/// Header edits and manual locations of renaming `orders.is_flagged` to `is_marked`. +pub(super) fn owner_header_parts( + contents: &str, +) -> Result<(Vec, Vec), RefactorError> { + let mut orders = model("orders", Some("table"), "analytics"); + orders.inferred_columns = vec!["order_id".to_owned(), "is_flagged".to_owned()]; + let mut vip_flags = model("vip_flags", Some("table"), "analytics"); + vip_flags.inferred_columns = vec!["is_flagged".to_owned()]; + let unknown_flags = model("unknown_flags", Some("table"), "analytics"); + let mut project_facts = facts( + "/nonexistent/project", + vec![orders.clone(), vip_flags, unknown_flags], + ); + project_facts.model_files = vec![DiscoveredFile { + path: orders.path.clone(), + contents: contents.to_owned(), + }]; + project_facts.authored_files = [ + ( + "flag_check", + "SELECT * FROM @relation WHERE NOT (@where_condition)", + ), + ( + "prefix_check", + "SELECT * FROM @relation WHERE @'prefix' = ''", + ), + ] + .into_iter() + .map(|(name, body)| AuthoredFile { + role: SqlFileRole::Audit, + path: format!("audits/generic/{name}.sql"), + contents: format!("AUDIT ();\n{body}\n"), + texts: vec![body.to_owned()], + }) + .collect(); + let mut rename = request(RefactorOperation::RenameColumn, "is_marked", None); + rename.column_name = Some("is_flagged".to_owned()); + let scan = context("duckdb"); + let rename_context = column_rename_context(&project_facts, &rename, &scan)?; + let parts = header_parts(&rename_context, &orders)?; + let edits: Vec = parts.edits.into_iter().map(|(_, edit)| edit).collect(); + let manual = parts + .manual + .into_iter() + .map(|item| { + format!( + "{}:{} {}", + item.line.unwrap_or_default(), + item.column.unwrap_or_default(), + item.reason + ) + }) + .collect(); + Ok((described(&edits), manual)) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_header_edits.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_header_edits.rs index d87bb0cb0d..5c5f7923ce 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests/test_header_edits.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_header_edits.rs @@ -1,9 +1,11 @@ use crate::refactoring::errors::RefactorError; use crate::refactoring::tests::helpers::{ applied, consumer_model_edits, described, inserted_entry, owner_column_edits, - schema_field_edits, schema_model_edits, + owner_header_parts, schema_field_edits, schema_model_edits, +}; +use crate::refactoring::tests::test_types::{ + HeaderEditsTestCase, HeaderPartsTestCase, InsertEntryTestCase, }; -use crate::refactoring::tests::test_types::{HeaderEditsTestCase, InsertEntryTestCase}; const CONSUMER: &str = "MODEL (\n description 'Reads __ref(\"stg_orders\") rows.',\n cursor_inputs (stg_orders (order_date)),\n relationships (to stg_orders, field order_id),\n materialized incremental,\n);\n\nSELECT 1\n"; const OWNER: &str = "MODEL (\n materialized incremental,\n unique_key order_id,\n columns (\n order_id (description 'Order key.'),\n amount,\n ),\n);\n\nSELECT 1\n"; @@ -34,6 +36,16 @@ fn given_headers_when_renaming_then_token_exact_edits_are_planned() -> Result<() ], expected_contents: "MODEL (\n materialized incremental,\n unique_key order_key,\n columns (\n order_key (migrate_from order_id, description 'Order key.'),\n amount,\n ),\n);\n\nSELECT 1\n", }, + HeaderEditsTestCase { + description: "newline-separated columns entries without commas are found", + contents: "MODEL (\n materialized incremental,\n columns (\n amount (type INT)\n order_id (type VARCHAR)\n ),\n);\n\nSELECT 1\n", + plan: owner_column_edits, + expected_edits: &[ + "5:5 header \"order_id\" -> \"order_key\"", + "5:15 migration \"\" -> \"migrate_from order_id\"", + ], + expected_contents: "MODEL (\n materialized incremental,\n columns (\n amount (type INT)\n order_key (migrate_from order_id, type VARCHAR)\n ),\n);\n\nSELECT 1\n", + }, HeaderEditsTestCase { description: "a SCHEMA header target is renamed; a longer keyword is not a SCHEMA header", contents: SCHEMA_FILE, @@ -106,3 +118,101 @@ fn given_header_layouts_when_inserting_entry_then_python_spacing_is_kept() } Ok(()) } + +#[test] +fn given_owner_header_when_renaming_column_then_declarations_and_audit_arguments_follow() +-> Result<(), RefactorError> { + let test_cases = [ + HeaderPartsTestCase { + description: "a comma-less column declaration is renamed instead of left for a hand edit", + contents: "MODEL (\n columns (\n order_id (type VARCHAR)\n is_flagged (type BOOLEAN)\n ),\n);\nSELECT 1\n", + expected_edits: &["4:5 header \"is_flagged\" -> \"is_marked\""], + expected_manual: &[], + }, + HeaderPartsTestCase { + description: "column and model audit arguments rename column references, not string literals", + contents: "MODEL (\n columns (\n is_flagged (audits [flag_check (where_condition \"is_flagged AND 'is_flagged' <> ''\")]),\n ),\n audits [flag_check (name \"orders_flag\", where_condition \"NOT is_flagged\")],\n);\nSELECT 1\n", + expected_edits: &[ + "3:5 header \"is_flagged\" -> \"is_marked\"", + "3:54 header \"is_flagged\" -> \"is_marked\"", + "5:64 header \"is_flagged\" -> \"is_marked\"", + ], + expected_manual: &[], + }, + HeaderPartsTestCase { + description: "an audit argument that is not a parseable expression is listed for a hand edit", + contents: "MODEL (\n audits [flag_check (where_condition \"is_flagged ((\")],\n);\nSELECT 1\n", + expected_edits: &[], + expected_manual: &[ + "2:40 this audit argument mentions is_flagged but is not a SQL expression SQLBuild can parse; update it by hand", + ], + }, + HeaderPartsTestCase { + description: "a qualified reference the analysis cannot resolve is listed for a hand edit", + contents: "MODEL (\n audits [flag_check (where_condition \"NOT orders.is_flagged\")],\n);\nSELECT 1\n", + expected_edits: &[], + expected_manual: &[ + "2:44 cannot tell whether is_flagged in this audit argument is the renamed column; update it by hand", + ], + }, + HeaderPartsTestCase { + description: "a subquery reading another model keeps its own column", + contents: "MODEL (\n audits [flag_check (where_condition 'is_flagged IN (SELECT is_flagged FROM __ref(\"vip_flags\"))')],\n);\nSELECT 1\n", + expected_edits: &["2:40 header \"is_flagged\" -> \"is_marked\""], + expected_manual: &[], + }, + HeaderPartsTestCase { + description: "a subquery reading the audited model renames its column too", + contents: "MODEL (\n audits [flag_check (where_condition 'is_flagged IN (SELECT is_flagged FROM __ref(\"orders\"))')],\n);\nSELECT 1\n", + expected_edits: &[ + "2:40 header \"is_flagged\" -> \"is_marked\"", + "2:62 header \"is_flagged\" -> \"is_marked\"", + ], + expected_manual: &[], + }, + HeaderPartsTestCase { + description: "a subquery reading a model of unknown columns is not renamed", + contents: "MODEL (\n audits [flag_check (where_condition 'is_flagged IN (SELECT is_flagged FROM __ref(\"unknown_flags\"))')],\n);\nSELECT 1\n", + expected_edits: &[], + expected_manual: &[ + "2:40 this audit argument reads unknown_flags, whose columns SQLBuild does not know, so is_flagged here is not renamed; update it by hand", + "2:62 this audit argument reads unknown_flags, whose columns SQLBuild does not know, so is_flagged here is not renamed; update it by hand", + ], + }, + HeaderPartsTestCase { + description: "an argument the audit quotes as a string is listed, not renamed", + contents: "MODEL (\n audits [prefix_check (prefix \"is_flagged\")],\n);\nSELECT 1\n", + expected_edits: &[], + expected_manual: &[ + "2:33 audit prefix_check quotes argument prefix as a string, so is_flagged here is not renamed; update it by hand if it names the column", + ], + }, + HeaderPartsTestCase { + description: "an argument of an audit with no SQL definition is listed, not renamed", + contents: "MODEL (\n audits [python_check (where_condition \"NOT is_flagged\")],\n);\nSELECT 1\n", + expected_edits: &[], + expected_manual: &[ + "2:46 no SQL definition of audit python_check shows whether argument where_condition is SQL, so is_flagged here is not renamed; update it by hand if it names the column", + ], + }, + HeaderPartsTestCase { + description: "audit option values and list literals are not audit arguments", + contents: "MODEL (\n audits [accepted_values (name \"is_flagged\", values [\"is_flagged\"])],\n);\nSELECT 1\n", + expected_edits: &[], + expected_manual: &[ + "2:34 the MODEL header mentions is_flagged here; update it by hand", + "2:56 the MODEL header mentions is_flagged here; update it by hand", + ], + }, + ]; + for test_case in test_cases { + let (edits, manual) = owner_header_parts(test_case.contents)?; + assert_eq!(edits, test_case.expected_edits, "{}", test_case.description); + assert_eq!( + manual, test_case.expected_manual, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs index 87f8028685..869adc4ac1 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs @@ -210,3 +210,10 @@ pub(crate) struct SourceContractTestCase { pub(crate) overwrite: bool, pub(crate) expected_edit: ContractEdit, } + +pub(super) struct HeaderPartsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_edits: &'static [&'static str], + pub(super) expected_manual: &'static [&'static str], +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index 85b3ab7a29..042f572a9e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -76,7 +76,8 @@ Renaming a column rewrites the model that produces it and every query that reads - the producing projection, as `amount AS revenue`, or the existing alias; - every downstream reference, including filters, joins, grouping, and ordering; -- column declarations and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- column declarations, including the model's own `MODEL (columns (...))` entries with or without commas between them, and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- string arguments of audits attached in `MODEL` headers, such as `where_condition "is_flagged = TRUE"` or `expression "amount > 0"`, when the generic audit's SQL definition splices the argument in as SQL with `@where_condition`. These arguments are parsed as SQL expressions, so only references to the audited model's column change: string literals stay as written, and a subquery reading another model, such as `customer_id IN (SELECT customer_id FROM __ref("vip_customers"))`, keeps that model's column. You are asked to edit an argument by hand when it cannot be parsed, when it reads a relation whose columns SQLBuild does not know, when the audit quotes it as a string with `@'prefix'`, or when no SQL definition of the audit is found; - fixture columns of `__ref__` and `__expected__` CTEs. By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 968e55733b..a1b8b383a4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -184,6 +184,81 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: "materialized table,", "materialized table,\n model_schema order_shape," ), } +FLAGGED_ORDERS: str = ( + "MODEL (\n" + ' description "Orders with a flag",\n' + " materialized table,\n" + " columns (\n" + " order_id (type VARCHAR)\n" + " is_flagged (type BOOLEAN)\n" + " ),\n" + ");\n\n" + "SELECT\n" + " CAST('o1' AS VARCHAR) AS order_id,\n" + " CAST(TRUE AS BOOLEAN) AS is_flagged\n" +) +FLAGGED_ORDER_SUMMARY: str = ( + "MODEL (\n" + ' description "Orders summary",\n' + " materialized table,\n" + ");\n\n" + "WITH orders AS (\n" + " SELECT *\n" + ' FROM __ref("flagged_orders")\n' + ")\n\n" + "SELECT\n" + " order_id,\n" + " is_flagged\n" + "FROM orders\n" +) +FLAGGED_CONTRACT: dict[str, str] = { + "models/marts/flagged_orders.sql": FLAGGED_ORDERS, + "models/marts/flagged_order_summary.sql": FLAGGED_ORDER_SUMMARY, +} +FLAGGED_AUDITS: dict[str, str] = { + "models/marts/_sqlbuild/_audits/generic/flag_check.sql": ( + "AUDIT ();\n\nSELECT *\nFROM @relation\nWHERE @where_condition AND order_id IS NULL\n" + ), + "models/marts/flagged_orders.sql": FLAGGED_ORDERS.replace( + "is_flagged (type BOOLEAN)", + "is_flagged (type BOOLEAN, audits [flag_check " + "(where_condition \"is_flagged = TRUE AND 'is_flagged' <> ''\")])", + ).replace( + " ),\n);", + ' ),\n audits [flag_check (name "orders_flag", where_condition "NOT is_flagged")],\n);', + ), + "models/marts/flagged_order_summary.sql": FLAGGED_ORDER_SUMMARY.replace( + " materialized table,\n", + ' materialized table,\n audits [flag_check (where_condition "is_flagged IS NULL")],\n', + ), +} +_FLAG_CHECK: str = ( + "AUDIT ();\n\nSELECT *\nFROM @relation\nWHERE @where_condition AND order_id IS NULL\n" +) +FLAGGED_SUBQUERY_AUDITS: dict[str, str] = { + "models/marts/_sqlbuild/_audits/generic/flag_check.sql": _FLAG_CHECK, + "models/marts/vip_flags.sql": ( + 'MODEL (description "VIP flags", materialized table);\n' + "SELECT CAST(TRUE AS BOOLEAN) AS is_flagged\n" + ), + "models/marts/flagged_orders.sql": FLAGGED_ORDERS.replace( + " ),\n);", + " ),\n audits [\n" + ' flag_check (name "vip_flag", where_condition ' + "'is_flagged IN (SELECT is_flagged FROM __ref(\"vip_flags\"))'),\n" + ' flag_check (name "own_flag", where_condition ' + "'is_flagged IN (SELECT is_flagged FROM __ref(\"flagged_orders\"))'),\n" + " ],\n);", + ), +} +FLAGGED_LITERAL_AUDIT: dict[str, str] = { + "models/marts/_sqlbuild/_audits/generic/prefix_check.sql": ( + "AUDIT ();\n\nSELECT *\nFROM @relation\nWHERE @'prefix' = ''\n" + ), + "models/marts/flagged_orders.sql": FLAGGED_ORDERS.replace( + " ),\n);", ' ),\n audits [prefix_check (prefix "is_flagged")],\n);' + ), +} ORDERS_MACRO: dict[str, str] = { "sqlbuild_project.toml": _PROJECT_TOML + "\n[references]\nenforce_explicit = false\n", "models/staging/_sqlbuild/_macros/orders.py": ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py index a46b665363..f96ce95eec 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_columns.py @@ -18,6 +18,10 @@ from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( CENTS_MACRO, CUSTOMER_DECLARATIONS, + FLAGGED_AUDITS, + FLAGGED_CONTRACT, + FLAGGED_LITERAL_AUDIT, + FLAGGED_SUBQUERY_AUDITS, ORDER_AMOUNT_CHECK, ORDER_EXPORT, ORDER_SHAPE, @@ -127,6 +131,53 @@ "order_export": ("order_id", "customer_id", "revenue", "order_date"), }, ), + ColumnRenameE2ETestCase( + description="a column declared in the model's own contract renames in one step", + command=("rename", "column:flagged_orders.is_flagged", "is_marked", "--cascade"), + extra_files=FLAGGED_CONTRACT, + expected_fragments={ + "models/marts/flagged_orders.sql": ( + " is_marked (type BOOLEAN)\n", + "CAST(TRUE AS BOOLEAN) AS is_marked\n", + ), + "models/marts/flagged_order_summary.sql": (" is_marked\nFROM orders",), + }, + expected_columns={ + "flagged_orders": ("order_id", "is_marked"), + "flagged_order_summary": ("order_id", "is_marked"), + }, + ), + ColumnRenameE2ETestCase( + description="audit where_condition arguments rename the column as SQL", + command=("rename", "column:flagged_orders.is_flagged", "is_marked", "--cascade"), + extra_files=FLAGGED_AUDITS, + expected_fragments={ + "models/marts/flagged_orders.sql": ( + "where_condition \"is_marked = TRUE AND 'is_flagged' <> ''\"", + 'where_condition "NOT is_marked"', + ), + "models/marts/flagged_order_summary.sql": ('where_condition "is_marked IS NULL"',), + }, + expected_columns={ + "flagged_orders": ("order_id", "is_marked"), + "flagged_order_summary": ("order_id", "is_marked"), + }, + ), + ColumnRenameE2ETestCase( + description="audit subqueries rename only columns of the audited model", + command=("rename", "column:flagged_orders.is_flagged", "is_marked"), + extra_files=FLAGGED_SUBQUERY_AUDITS, + expected_fragments={ + "models/marts/flagged_orders.sql": ( + "'is_marked IN (SELECT is_flagged FROM __ref(\"vip_flags\"))'", + "'is_marked IN (SELECT is_marked FROM __ref(\"flagged_orders\"))'", + ), + }, + expected_columns={ + "flagged_orders": ("order_id", "is_marked"), + "vip_flags": ("is_flagged",), + }, + ), ], ids=lambda case: case.description, ) @@ -207,6 +258,14 @@ def test_given_column_rename_when_applied_then_project_tests_and_builds( expected_reason="reference to stg_orders produced by a macro", expected_paths=("models/staging/stg_order_count.sql",), ), + RefusedRefactorE2ETestCase( + description="audit argument the audit quotes as a string needs a manual check", + command=("rename", "column:flagged_orders.is_flagged", "is_marked"), + extra_files=FLAGGED_LITERAL_AUDIT, + expected_status="refused", + expected_reason="audit prefix_check quotes argument prefix as a string", + expected_paths=("models/marts/flagged_orders.sql",), + ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index 7431ffc6c2..4946eb90d2 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -61,7 +61,8 @@ Renaming a column rewrites the model that produces it and every query that reads - the producing projection, as `amount AS revenue`, or the existing alias; - every downstream reference, including filters, joins, grouping, and ordering; -- column declarations and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- column declarations, including the model's own `MODEL (columns (...))` entries with or without commas between them, and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; +- string arguments of audits attached in `MODEL` headers, such as `where_condition "is_flagged = TRUE"` or `expression "amount > 0"`, when the generic audit's SQL definition splices the argument in as SQL with `@where_condition`. These arguments are parsed as SQL expressions, so only references to the audited model's column change: string literals stay as written, and a subquery reading another model, such as `customer_id IN (SELECT customer_id FROM __ref("vip_customers"))`, keeps that model's column. You are asked to edit an argument by hand when it cannot be parsed, when it reads a relation whose columns SQLBuild does not know, when the audit quotes it as a string with `@'prefix'`, or when no SQL definition of the audit is found; - fixture columns of `__ref__` and `__expected__` CTEs. By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. From 0d12e362e703d40cec4c5c00264a3fdbdcf60cd3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:14:50 +0100 Subject: [PATCH 02/34] fix(cli): report an unreadable --select-file as a coded error Reading a missing or unreadable selector file raised a raw FileNotFoundError with a traceback from compile, plan and build. The read now fails with error[C114] naming the file, in text and --json output. Refs: CHI-501 --- .../src/bindings/_helpers/cli/dispatch.rs | 41 ++++++++-- .../cli/commands/main/compile/_test_types.py | 7 ++ .../cli/commands/main/compile/helpers.py | 7 ++ .../main/compile/test_compile_selection.py | 80 +++++++++++++++++++ 4 files changed, 130 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 32abc9ad06..da8cc80927 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -38,7 +38,7 @@ use crate::bindings::_helpers::workspace_commands::commands::{ run_playground_command, run_skills_command, }; use pyo3::IntoPyObjectExt; -use pyo3::exceptions::{PyAttributeError, PyTypeError}; +use pyo3::exceptions::{PyAttributeError, PyOSError, PyTypeError, PyUnicodeError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_cli::diff_report::errors::DiffUserError; @@ -60,6 +60,10 @@ use crate::bindings::_helpers::records::registry; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; const UTF8: &str = "utf-8"; +/// The code of a `--select-file` that cannot be read. +const SELECTOR_FILE_CODE: &str = "C114"; +const SELECTOR_FILE_HELP: &str = + "Pass the path of a readable UTF-8 file, relative to the current directory or absolute."; /// The `diff` arguments that compare raw queries instead of project models. const RAW_QUERY_ATTRIBUTES: [&str; 4] = [ "left_query", @@ -255,10 +259,11 @@ fn selector_inputs<'py>( let path: Bound<'py, PyAny> = path_class(py)?.call1((raw_path?,))?; let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item("encoding", UTF8)?; - let lines: Vec = path - .call_method("read_text", (), Some(&keywords))? - .call_method0("splitlines")? - .extract()?; + let text: Bound<'py, PyAny> = match path.call_method("read_text", (), Some(&keywords)) { + Ok(text) => text, + Err(error) => return Err(selector_file_error(py, &path, &error)?), + }; + let lines: Vec = text.call_method0("splitlines")?.extract()?; let mut count: usize = 0; for line in &lines { let stripped: &str = python_strip(line); @@ -280,6 +285,32 @@ fn selector_inputs<'py>( Ok((PyTuple::new(py, selectors)?, PyTuple::new(py, files)?)) } +/// The user error for a `--select-file` that cannot be read, or `error` when it is unexpected. +fn selector_file_error(py: Python<'_>, path: &Bound<'_, PyAny>, error: &PyErr) -> PyResult { + let readable = + error.is_instance_of::(py) || error.is_instance_of::(py); + if !readable { + return Ok(error.clone_ref(py)); + } + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let reason: Bound<'_, PyAny> = value + .getattr_opt("strerror")? + .unwrap_or_else(|| value.clone()); + let reason: String = if reason.is_none() { + value.str()?.to_string() + } else { + reason.str()?.to_string() + }; + Ok(user_error( + py, + DispatchFailure { + message: format!("cannot read selector file {}: {reason}", path.str()?), + code: SELECTOR_FILE_CODE, + help: Some(SELECTOR_FILE_HELP), + }, + )) +} + /// Call the route's command implementation, or the handler a test injected for its slot. fn call_route<'py>( invocation: &Invocation<'py>, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 429b1573c5..ad7eacb81d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -232,6 +232,13 @@ class CompileSelectionTestCase: unexpected_stdout_fragments: tuple[str, ...] +@dataclass(frozen=True) +class SelectorFileErrorTestCase: + description: str + command: tuple[str, ...] + expected_json_codes: tuple[str, ...] + + @dataclass(frozen=True) class DeepSqlAnalysisCompileTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 4d584bdfcf..b852152b73 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5088,3 +5088,10 @@ def restore_default_interrupt() -> None: """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" _ = signal.signal(signal.SIGINT, signal.SIG_DFL) + + +def json_diagnostic_codes(stdout: str) -> tuple[str, ...]: + """Return the diagnostic codes of a compile JSON document; empty stdout has none.""" + + document: dict[str, Any] = json.loads(stdout or "{}") + return tuple(str(diagnostic["code"]) for diagnostic in document.get("diagnostics", ())) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py index 18978d26f8..88940fd26e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py @@ -9,9 +9,36 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileSelectionTestCase, + SelectorFileErrorTestCase, ) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import json_diagnostic_codes from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +_PROJECT_TOML: str = ( + 'name = "compile_selection"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "selection.duckdb"\n\n' + "[targets.dev]\n" + 'schema = "dev"\n' +) +_SOURCE_FILES: dict[str, str] = { + "sources/raw.yml": ( + "sources:\n" + " - name: raw__orders\n" + " description: Raw orders.\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.');\n\nSELECT order_id FROM __source(\"raw__orders\")\n" + ), +} + @pytest.mark.parametrize( "test_case", @@ -97,5 +124,58 @@ def test_given_model_selection_when_compiling_then_limits_offline_analysis_and_r assert fragment not in result.stdout +@pytest.mark.parametrize( + "test_case", + [ + SelectorFileErrorTestCase( + description="compile names the missing selector file", + command=("compile",), + expected_json_codes=(), + ), + SelectorFileErrorTestCase( + description="compile json reports the missing selector file as a diagnostic", + command=("compile", "--json"), + expected_json_codes=("C114",), + ), + SelectorFileErrorTestCase( + description="plan names the missing selector file", + command=("plan",), + expected_json_codes=(), + ), + SelectorFileErrorTestCase( + description="build names the missing selector file", + command=("build",), + expected_json_codes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_missing_selector_file_when_running_then_reports_a_coded_error( + test_case: SelectorFileErrorTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="compile_selection", + repo_files={"sqlbuild_project.toml": _PROJECT_TOML, **_SOURCE_FILES}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command, "--select-file", "missing.txt"), + project_dir=project_dir, + working_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, test_case.description + output + assert "Traceback" not in output, test_case.description + output + assert "error[C114]: cannot read selector file missing.txt" in result.stderr, ( + test_case.description + output + ) + assert json_diagnostic_codes(result.stdout) == test_case.expected_json_codes, ( + test_case.description + output + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 39ec24f3cb2d9248980f1a5436c676ca26dea685 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:14:50 +0100 Subject: [PATCH 03/34] fix(rules): evaluate model rules on a selection without models With only declaration rules selected, model rule evaluation took the first model unconditionally, so a source-only selection sliced an empty model list and panicked. The model limit is now capped by the number of models. Refs: CHI-530 --- .../src/engine/_helpers/evaluation.rs | 6 ++- .../cli/commands/main/compile/_test_types.py | 7 ++++ .../main/compile/test_compile_selection.py | 42 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 1c8e9e2bb0..383296000a 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -112,7 +112,11 @@ fn evaluate_native_models( "SQBRDECLARATION201" | "SQBRDECLARATION301" | "SQBRTEST301" ) }); - let model_limit = if all_models_required { models.len() } else { 1 }; + let model_limit = if all_models_required { + models.len() + } else { + models.len().min(1) + }; let mut bucket = cache .map(|store| store.native_rules_bucket(model_rules_fingerprint)) .transpose()? diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ad7eacb81d..0a68348ecf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -239,6 +239,13 @@ class SelectorFileErrorTestCase: expected_json_codes: tuple[str, ...] +@dataclass(frozen=True) +class SourceSelectionRulesTestCase: + description: str + selection_args: tuple[str, ...] + expected_stdout_fragment: str + + @dataclass(frozen=True) class DeepSqlAnalysisCompileTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py index 88940fd26e..14a265e4f1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileSelectionTestCase, SelectorFileErrorTestCase, + SourceSelectionRulesTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import json_diagnostic_codes from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -177,5 +178,46 @@ def test_given_missing_selector_file_when_running_then_reports_a_coded_error( ) +@pytest.mark.parametrize( + "test_case", + [ + SourceSelectionRulesTestCase( + description="an exact source selection evaluates rules without models", + selection_args=("--select", "source:raw__orders"), + expected_stdout_fragment="Project compiled 0 models", + ), + SourceSelectionRulesTestCase( + description="a source glob selection evaluates rules without models", + selection_args=("--select", "source:raw_*"), + expected_stdout_fragment="Project compiled 0 models", + ), + ], + ids=lambda case: case.description, +) +def test_given_rules_and_source_only_selection_when_compiling_then_compiles( + test_case: SourceSelectionRulesTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="compile_selection", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML + '\n[rules]\nselect = ["SQBRDECLARATION201"]\n', + **_SOURCE_FILES, + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", *test_case.selection_args), + project_dir=project_dir, + working_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 0, test_case.description + output + assert "panicked" not in output, test_case.description + output + assert test_case.expected_stdout_fragment in result.stdout, test_case.description + output + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 0d95ece8b03a23ece11578c3f24c57af379cb0de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:14:51 +0100 Subject: [PATCH 04/34] fix(janitor): refuse cleanly when confirmation cannot be read Without --auto-approve, janitor prompted on a non-interactive stdin and ended with an EOFError traceback. Like build confirmations, it now refuses before prompting with error[C503] and tells the user to pass --auto-approve or run it in a terminal. Refs: CHI-534 --- .../_helpers/janitor/janitor_command.rs | 28 ++++++++- .../cli/commands/main/janitor/_test_types.py | 11 ++++ .../cli/commands/main/janitor/test_janitor.py | 57 +++++++++++++++++++ 3 files changed, 93 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 0b2d70a16e..48e4e1ded7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -8,6 +8,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::janitor_planning::main::resolve_retention_days::resolve_retention_days; +use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::janitor_text::errors::JanitorSettingsError; use sqlbuild_core::janitor_text::main::janitor_completion_text::janitor_completion_text; @@ -24,6 +25,7 @@ use sqlbuild_core::janitor_text::models::JanitorPlanText; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; +use crate::bindings::_helpers::diff_report::support::cli_error; use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, @@ -35,8 +37,15 @@ use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; +use crate::bindings::_helpers::scenario_command::setup::stream_isatty; const COMMAND_NAME: &str = "janitor"; +/// The code of a cleanup that needs a confirmation stdin cannot give. +const CONFIRMATION_CODE: &str = "C503"; +const CONFIRMATION_REFUSAL: &str = + "janitor cleanup requires confirmation, but stdin is not a terminal"; +const CONFIRMATION_HELP: &str = "Run sqb janitor in a terminal to confirm, or pass --auto-approve."; +const LINE_ENDINGS: [char; 2] = ['\n', '\r']; fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { py.import(module)?.getattr(name) @@ -157,12 +166,25 @@ fn settings<'py>( new_record(py, RecordId::JanitorSettings, (), Some(&options)) } -/// Ask for the exact confirmation phrase; an interrupt cancels. +/// Ask for the exact confirmation phrase; an interrupt cancels, and a non-interactive stdin refuses. fn confirmed(py: Python<'_>, plan: &JanitorPlanText) -> PyResult { + let stdin: Bound<'_, PyAny> = imported(py, "sys", "stdin")?; + if stdin.is_none() || !stream_isatty(&stdin)? { + return Err(cli_error( + py, + ErrorId::CliUser, + &DiffUserError { + message: CONFIRMATION_REFUSAL.to_owned(), + code: CONFIRMATION_CODE.to_owned(), + help: Some(CONFIRMATION_HELP.to_owned()), + }, + )); + } write(py, &janitor_confirmation_prompt(plan))?; flush(py)?; - match imported(py, "builtins", "input")?.call0() { - Ok(response) => Ok(response.extract::()? == janitor_confirmation_phrase(plan)), + match stdin.call_method0("readline") { + Ok(response) => Ok(response.extract::()?.trim_end_matches(LINE_ENDINGS) + == janitor_confirmation_phrase(plan)), Err(error) if error.is_instance_of::(py) => { write(py, "\n")?; Ok(false) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index b96a3de3be..dfe682a71f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -175,3 +175,14 @@ class JanitorTargetErrorE2ETestCase: janitor_command: tuple[str, ...] expected_exit_code: int expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JanitorNonInteractiveE2ETestCase: + """Test case for a janitor confirmation that stdin cannot answer.""" + + description: str + janitor_command: tuple[str, ...] + expected_exit_code: int + expected_stderr_fragments: tuple[str, ...] + expected_existing_tables: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 810db559b5..35f6ba4e68 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -13,6 +13,7 @@ JanitorDirectStatePruningE2ETestCase, JanitorDisabledE2ETestCase, JanitorInvalidConfigE2ETestCase, + JanitorNonInteractiveE2ETestCase, JanitorSourceOverlapE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( @@ -153,6 +154,62 @@ def test_given_stale_relations_when_running_direct_janitor_then_it_archives_trac assert "Eligible expired locks" not in janitor_result.stdout +@pytest.mark.parametrize( + "test_case", + [ + JanitorNonInteractiveE2ETestCase( + description="janitor without auto-approve refuses when stdin is not a terminal", + janitor_command=("--no-color", "janitor"), + expected_exit_code=1, + expected_stderr_fragments=( + "error[C503]: janitor cleanup requires confirmation, but stdin is not a terminal", + "pass --auto-approve", + ), + expected_existing_tables=("janitor_tracked_extra", "janitor_untracked_extra"), + ) + ], + ids=lambda case: case.description, +) +def test_given_piped_stdin_when_janitor_needs_confirmation_then_it_refuses_cleanly( + test_case: JanitorNonInteractiveE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_janitor_project( + tmp_path=tmp_path, + project_name="janitor_non_interactive_project", + janitor_config=dedent( + """ + enabled = true + retention_days = 0 + """ + ), + ) + db_path: Path = project_dir / "janitor.duckdb" + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--full-refresh"), + project_dir=project_dir, + ) + create_janitor_demo_relations(db_path=db_path) + + janitor_result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.janitor_command, + project_dir=project_dir, + input_text="", + ) + + output: str = janitor_result.stdout + janitor_result.stderr + assert build_result.returncode == 0, build_result.stdout + build_result.stderr + assert janitor_result.returncode == test_case.expected_exit_code, test_case.description + output + assert "Traceback" not in output, test_case.description + output + assert all( + fragment in janitor_result.stderr for fragment in test_case.expected_stderr_fragments + ), test_case.description + output + assert all( + table_exists(db_path=db_path, table_name=table_name) + for table_name in test_case.expected_existing_tables + ), test_case.description + + @pytest.mark.parametrize( "test_case", [ From ad47c7ce04638044908b733c526c7a31ebac78c1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:14:51 +0100 Subject: [PATCH 05/34] fix(macros): reject list and dict keys in macro argument dicts A dict key such as [1] or (1, [2]) parsed, then failed with a raw TypeError when the Python dict was built. The argument parser now rejects list and dict keys, also inside tuple keys, with a located P001 error and help. Refs: CHI-571 --- .../macro_arguments/_helpers/evaluation.rs | 18 +++++++++++-- .../src/macro_arguments/_helpers/failures.rs | 11 ++++++++ .../tests/test_parse_macro_arguments.rs | 18 +++++++++++++ .../_helpers/corpus/failure_cases.py | 18 +++++++++++++ ...e-error-macro-argument-unhashable-key.json | 27 +++++++++++++++++++ 5 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-unhashable-key.json diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs index bb2ecdd966..cbd6346417 100644 --- a/crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs @@ -4,7 +4,7 @@ use std::collections::VecDeque; use crate::macro_arguments::_helpers::failures::{ Failure, bytes_literal, complex_literal, dict_unpacking, ellipsis, formatted_string, - keyword_expansion, typed_reference, unary, unsupported, + keyword_expansion, typed_reference, unary, unhashable_key, unsupported, }; use crate::macro_arguments::_helpers::lexer::NumberToken; use crate::macro_arguments::_helpers::syntax_tree::{CallArguments, Expr, Keyword, Node}; @@ -68,7 +68,11 @@ fn evaluate(node: &Node) -> Result { let Some(key) = key else { return Err(dict_unpacking(value.position)); }; - values.push((evaluate(key)?, evaluate(value)?)); + let key_value: ArgumentValue = evaluate(key)?; + if !is_hashable(&key_value) { + return Err(unhashable_key(key.position)); + } + values.push((key_value, evaluate(value)?)); } Ok(ArgumentValue::Dict(values)) } @@ -87,6 +91,16 @@ fn evaluate(node: &Node) -> Result { } } +/// Whether Python can hash the value; a nested call's value is left for Python to check. +fn is_hashable(value: &ArgumentValue) -> bool { + match value { + ArgumentValue::List(_) | ArgumentValue::Dict(_) => false, + ArgumentValue::Tuple(items) => items.iter().all(is_hashable), + ArgumentValue::Negative(inner) | ArgumentValue::Positive(inner) => is_hashable(inner), + _ => true, + } +} + fn evaluate_all(items: &[Node]) -> Result, Failure> { items.iter().map(evaluate).collect() } diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs index e1b52f9b76..c623fd2acc 100644 --- a/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs @@ -11,6 +11,8 @@ const ELLIPSIS_HELP: &str = "Pass None, or a string the macro understands, inste const FORMATTED_HELP: &str = "Pass the parts as separate arguments and format them in the macro"; const REFERENCE_HELP: &str = "Write one quoted resource name, for example __ref(\"orders\")"; const UNPACKING_HELP: &str = "Write each argument or key out explicitly"; +const HASHABLE_KEY_HELP: &str = "Use a string, number, True, False, None, or a tuple of them as \ + a dict key"; const KEYWORD_HELP: &str = "Pass each keyword argument once, after every positional argument"; const SURROGATE_HELP: &str = "Lone surrogates are not valid Unicode text and cannot be sent to a \ warehouse; write the character itself or the escape of a valid code point"; @@ -108,6 +110,15 @@ pub(crate) fn dict_unpacking(position: usize) -> Failure { failure("must not use dict unpacking", UNPACKING_HELP, position) } +/// A dict key Python cannot hash: a list or dict, or a tuple containing one. +pub(crate) fn unhashable_key(position: usize) -> Failure { + failure( + "use a list or dict as a dict key", + HASHABLE_KEY_HELP, + position, + ) +} + pub(crate) fn keyword_order(detail: &str, position: usize) -> Failure { failure(detail, KEYWORD_HELP, position) } diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs index eaf557532c..27dcafc30c 100644 --- a/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs +++ b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs @@ -70,6 +70,24 @@ fn given_argument_text_when_parsing_then_values_match_python_literals() { nested: &[], expected_plan: "Err(ArgumentError { detail: \"use \'...\' (Ellipsis)\", help: \"Pass None, or a string the macro understands, instead of \'...\'\", line: 1, column: 1 })", }, + ParseMacroArgumentsTestCase { + description: "a list dict key is rejected at the key", + text: "{'a': 1, [1]: 2}", + nested: &[], + expected_plan: "Err(ArgumentError { detail: \"use a list or dict as a dict key\", help: \"Use a string, number, True, False, None, or a tuple of them as a dict key\", line: 1, column: 10 })", + }, + ParseMacroArgumentsTestCase { + description: "a tuple dict key holding a list is rejected", + text: "{(1, [2]): 2}", + nested: &[], + expected_plan: "Err(ArgumentError { detail: \"use a list or dict as a dict key\", help: \"Use a string, number, True, False, None, or a tuple of them as a dict key\", line: 1, column: 2 })", + }, + ParseMacroArgumentsTestCase { + description: "tuple, number and reference dict keys are accepted", + text: "{(1, 'a'): 2, -1: 3, __ref('a'): 4}", + nested: &[], + expected_plan: "Ok(MacroArguments { positional: [Dict([(Tuple([Int { radix: 10, digits: \"1\" }, Str(\"a\")]), Int { radix: 10, digits: \"2\" }), (Negative(Int { radix: 10, digits: \"1\" }), Int { radix: 10, digits: \"3\" }), (TypedReference { function: \"__ref\", name: \"a\" }, Int { radix: 10, digits: \"4\" })])], keywords: [], typed_references: [(\"__ref\", \"a\")] })", + }, ParseMacroArgumentsTestCase { description: "an f-string is rejected", text: "f'{x}'", diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 8fa79fe75b..d58dab3217 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -818,6 +818,24 @@ def engine_error_cases() -> tuple[FailureCase, ...]: "b'orders'" ), ), + failure_case( + name="engine-error-macro-argument-unhashable-key", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + 'SELECT customer_id, @cents({[1]: 2}) AS cents\nFROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' " + "use a list or dict as a dict key at line 5, column 29" + ), + expected_location=(5, 29), + expected_help=( + "Use a string, number, True, False, None, or a tuple of them as a dict key" + ), + ), failure_case( name="engine-error-macro-argument-complex", expected_code="P001", diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-unhashable-key.json b/tests/goldens/compiler/failure/engine-error-macro-argument-unhashable-key.json new file mode 100644 index 0000000000..9477f06115 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-unhashable-key.json @@ -0,0 +1,27 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' use a list or dict as a dict key at line 5, column 29", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29 + }, + "help": "Use a string, number, True, False, None, or a tuple of them as a dict key" + } + ] + } + ], + "compiled": {}, + "manifest": null +} From cdfebe5c9f9ff5502cd106b066ca7257b3fc4049 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:15:02 +0100 Subject: [PATCH 06/34] fix(cli): report a warehouse connection failure as a coded error When the warehouse connection could not be opened, for example a DuckDB file in a missing directory, build, seed, audit, test, plan, load, query and diff printed the failed connection line and then the adapter's raw exception with a traceback. Connection failures now raise error[C115] naming the adapter and the reason, with help pointing at sqb debug. SQLBuild's own coded errors pass through unchanged; a client library error that carries its own `code`, such as a Google API error, is still reported as C115. Refs: CHI-596 --- .../bindings/_helpers/check/check_command.rs | 6 +- .../src/bindings/_helpers/cli/entry.rs | 3 +- .../bindings/_helpers/clone/clone_command.rs | 4 +- .../_helpers/compile_driver/plan_result.rs | 3 +- .../_helpers/contract_command/command.rs | 3 +- .../bindings/_helpers/diff_report/command.rs | 5 +- .../_helpers/freshness/freshness_command.rs | 5 +- .../_helpers/janitor/janitor_phases.rs | 3 +- .../bindings/_helpers/load/load_command.rs | 5 +- .../_helpers/native_host/connections.rs | 22 +++ .../bindings/_helpers/native_host/executor.rs | 12 ++ .../src/bindings/_helpers/native_host/mod.rs | 1 + .../bindings/_helpers/query_command/query.rs | 4 +- src/sqlbuild/_native_host.py | 8 + src/sqlbuild/_native_host.pyi | 6 + .../executor/load/_helpers/scheduling.py | 7 +- .../executor/pipeline/_helpers/auditing.py | 7 +- .../executor/pipeline/_helpers/scenario.py | 7 +- .../executor/pipeline/_helpers/seeding.py | 7 +- .../executor/pipeline/_helpers/testing.py | 7 +- src/sqlbuild/executor/pipeline/constants.py | 3 + src/sqlbuild/executor/pipeline/exceptions.py | 13 ++ .../raise_warehouse_connection_failure.py | 25 ++++ .../cli/commands/main/build/_test_types.py | 9 ++ .../main/build/test_lifecycle_commands.py | 137 ++++++++++++++++++ .../pipeline/_helpers/test_auditing.py | 11 +- .../sqlbuild/runtime/contracts/_test_types.py | 1 + .../runtime/contracts/test_open_connection.py | 13 +- 28 files changed, 310 insertions(+), 27 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/connections.rs create mode 100644 src/sqlbuild/executor/pipeline/constants.py create mode 100644 src/sqlbuild/executor/pipeline/main/raise_warehouse_connection_failure.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 98d7fd7328..385086be84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_selection::selectors::parse_project_selector; @@ -564,8 +565,9 @@ fn execute<'py>( ) -> PyResult>> { let py: Python<'py> = invocation.py(); let adapter: Bound<'py, PyAny> = native_field(invocation, "adapter")?; - let connection: Bound<'py, PyAny> = - adapter.call_method1("connect", (native_field(invocation, "connection_config")?,))?; + let connection: Bound<'py, PyAny> = adapter + .call_method1("connect", (native_field(invocation, "connection_config")?,)) + .map_err(|error| connection_failure(&adapter, &error))?; let event_writer: Bound<'py, PyAny> = Bound::new(py, ExecutionEventWriter::new(py, None)?)?.into_any(); let executed: PyResult>> = (|| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 6323521345..676a05d255 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -43,9 +43,10 @@ const DIFF_PUBLICATION_CODE: &str = "C257"; const DIFF_EXECUTION_STATUS: &str = "execution_failed"; const DIFF_OUTCOME_PREFIX: &str = "Query diff outcome "; /// Expected staying-Python command failures, loaded only once a command raised. -const EXPECTED_COMMAND_FAILURES: [ExecutorHost; 2] = [ +const EXPECTED_COMMAND_FAILURES: [ExecutorHost; 3] = [ ExecutorHost::AuditExecutionError, ExecutorHost::ExecutionDurationLimitError, + ExecutorHost::WarehouseConnectionError, ]; /// Expected native command failures. const EXPECTED_NATIVE_FAILURES: [ErrorId; 2] = [ErrorId::Discovery, ErrorId::SpecConfig]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index de1ab3c393..098bca4a05 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -33,6 +33,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; @@ -214,7 +215,8 @@ fn connect<'py>( let connection: Bound<'py, PyAny> = within( &lifecycle(py, ("clone", "clone_target_connection"))?, |_| invocation.adapter.call_method1("connect", (&config,)), - )?; + ) + .map_err(|error| connection_failure(&invocation.adapter, &error))?; let summary: String = format!( "{} ({:.2}s)", invocation.adapter_name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index c013edc2c1..407cf70237 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -19,6 +19,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::fields::field as native_field; @@ -280,7 +281,7 @@ pub(crate) fn connection_with_hooks<'py>( if error.is_instance_of::(hooks.py()) && !on_error.is_none() { on_error.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; } - return Err(error); + return Err(connection_failure(adapter, &error)); } }; let on_complete: Bound<'py, PyAny> = native_field(hooks, "on_connection_complete")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index ff774e700f..56b69f6503 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -199,7 +200,7 @@ fn compare<'py>( if exception(py, &error) { connection_timing(&reporter, "on_connection_error", connection_started)?; } - return Err(error); + return Err(connection_failure(&adapter, &error)); } }; let compared: PyResult> = (|| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 1a830d2dcf..4b0ff2b799 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -37,6 +37,7 @@ use crate::bindings::_helpers::diff_report::support::{ use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; use crate::bindings::_helpers::frontier::encoder::emit_error; use crate::bindings::_helpers::native_host::adapter::AdapterHost; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; @@ -441,7 +442,9 @@ fn run_model_diff( ], )? }; - let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (&connection_config,))?; + let connection: Bound<'_, PyAny> = adapter + .call_method1("connect", (&connection_config,)) + .map_err(|error| connection_failure(&adapter, &error))?; let extras: [(&str, Bound<'_, PyAny>); 5] = [ ("bounded", native_field(request, "bounded")?), ("max_column_examples", object(py, column_examples)?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index c4841c9d00..a26b38c856 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -22,6 +22,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::planner_selection::selectors::{ expand_project_upstream_keys, select_project_graph, }; @@ -221,7 +222,9 @@ fn observe( ], )?), )?; - let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; + let connection: Bound<'_, PyAny> = adapter + .call_method1("connect", (connection_config,)) + .map_err(|error| connection_failure(adapter, &error))?; let observed = (|| { let project: Bound<'_, PyAny> = native_field(graph, "project")?; let by_identity: Option> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 639e89515b..5fc363b8da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::native_host::connections::connection_failure; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -172,7 +173,7 @@ pub(crate) fn connect_janitor_warehouse<'py>( } Err(error) => { connection_timing(&progress, "on_connection_error", started)?; - Err(error) + Err(connection_failure(&compiled.adapter, &error)) } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 40531e77e1..e7b86460f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -35,6 +35,7 @@ use crate::bindings::_helpers::load::load_selection::{ LoadProject, LoaderDefaults, loader_defaults, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; @@ -168,7 +169,9 @@ fn check_reference_targets( .iter() .map(|source| native_field(source, "name")?.extract()) .collect::>()?; - let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; + let connection: Bound<'_, PyAny> = adapter + .call_method1("connect", (connection_config,)) + .map_err(|error| connection_failure(adapter, &error))?; let checked: PyResult<()> = (|| { let locations: Vec> = invocation .references diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/connections.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/connections.rs new file mode 100644 index 0000000000..447ddf442d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/connections.rs @@ -0,0 +1,22 @@ +//! The coded error commands raise when the warehouse connection cannot be opened. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; + +/// The coded error to raise for a failed `adapter.connect`, or the failure itself when coded. +pub(crate) fn connection_failure(adapter: &Bound<'_, PyAny>, error: &PyErr) -> PyErr { + let py: Python<'_> = adapter.py(); + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("adapter", adapter)?; + options.set_item("error", error.value(py))?; + host::entry(py, ExecutorHost::RaiseWarehouseConnectionFailure)?.call((), Some(&options)) + })(); + match raised { + Ok(_) => error.clone_ref(py), + Err(failure) => failure, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs index fe6bf9c1e5..c7528eda68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs @@ -68,6 +68,8 @@ pub(crate) enum ExecutorHost { MigrationRelationForLocation, IsCompatibilityView, AuditExecutionError, + WarehouseConnectionError, + RaiseWarehouseConnectionFailure, AuditPipelineCallbacks, RunAuditPipelineWithCallbacks, RunBuildPipeline, @@ -486,6 +488,16 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Exception, edge: HostEdge::Executor, }, + HostEntry { + name: "WarehouseConnectionError", + kind: HostKind::Exception, + edge: HostEdge::Executor, + }, + HostEntry { + name: "raise_warehouse_connection_failure", + kind: HostKind::Function, + edge: HostEdge::Executor, + }, HostEntry { name: "AuditPipelineCallbacks", kind: HostKind::Class, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index c9c575a440..3efbda9aea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,6 +1,7 @@ //! The one Python module native code imports by path, and the staying-Python entries it names. pub(crate) mod adapter; +pub(crate) mod connections; pub(crate) mod constants; pub(crate) mod executor; pub(crate) mod facade; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index 9b72b40611..0adfe98ad0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -17,6 +17,7 @@ use crate::bindings::_helpers::command_runtime::connection::resolve_project_conn use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; +use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; use crate::bindings::_helpers::query_command::query_values::{ @@ -164,7 +165,8 @@ fn run_sql<'py>( )?; let connection: Bound<'py, PyAny> = project .adapter - .call_method1("connect", (connection_config,))?; + .call_method1("connect", (connection_config,)) + .map_err(|error| connection_failure(&project.adapter, &error))?; let limit_value: Bound<'py, PyAny> = match limit { Some(limit) => PyInt::new(py, limit).into_any(), None => py.None().into_bound(py), diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 3ee235e68e..4adc8800e5 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -381,6 +381,14 @@ "is_compatibility_view", ), "AuditExecutionError": ("sqlbuild.executor.pipeline.exceptions", "AuditExecutionError"), + "WarehouseConnectionError": ( + "sqlbuild.executor.pipeline.exceptions", + "WarehouseConnectionError", + ), + "raise_warehouse_connection_failure": ( + "sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure", + "raise_warehouse_connection_failure", + ), "AuditPipelineCallbacks": ("sqlbuild.executor.pipeline.main.run", "AuditPipelineCallbacks"), "run_audit_pipeline_with_callbacks": ( "sqlbuild.executor.pipeline.main.run", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 1dde9af711..7e222cc35e 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -270,6 +270,12 @@ from sqlbuild.executor.old_name_views._helpers.janitor import ( is_compatibility_view as is_compatibility_view, ) from sqlbuild.executor.pipeline.exceptions import AuditExecutionError as AuditExecutionError +from sqlbuild.executor.pipeline.exceptions import ( + WarehouseConnectionError as WarehouseConnectionError, +) +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure as raise_warehouse_connection_failure, +) from sqlbuild.executor.pipeline.main.run import AuditPipelineCallbacks as AuditPipelineCallbacks from sqlbuild.executor.pipeline.main.run import ( run_audit_pipeline_with_callbacks as run_audit_pipeline_with_callbacks, diff --git a/src/sqlbuild/executor/load/_helpers/scheduling.py b/src/sqlbuild/executor/load/_helpers/scheduling.py index bedf7263e1..e7737c7d6e 100644 --- a/src/sqlbuild/executor/load/_helpers/scheduling.py +++ b/src/sqlbuild/executor/load/_helpers/scheduling.py @@ -35,6 +35,9 @@ LoadRuntimeParams, ) from sqlbuild.executor.load.types import LoadProgressCallback +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure, +) from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback @@ -150,13 +153,13 @@ def open_load_connections( try: for _ in range(effective_concurrency): connections.append(adapter.connect(connection_config)) - except Exception: + except Exception as error: if on_connection_error is not None: on_connection_error(effective_concurrency, elapsed_seconds=time.monotonic() - start) connection: Any for connection in connections: adapter.close(connection) - raise + raise_warehouse_connection_failure(adapter=adapter, error=error) if on_connection_complete is not None: on_connection_complete(effective_concurrency, elapsed_seconds=time.monotonic() - start) return tuple(connections) diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index 91696eb327..1065a65c81 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -39,6 +39,9 @@ AuditExecutionError, AuditOutcomeError, ) +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure, +) from sqlbuild.executor.pipeline.models import AuditExecutionFailure, AuditPipelineCallbacks from sqlbuild.executor.scheduling.main._run_worker import run_worker_with_completion from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback @@ -289,7 +292,7 @@ def _run_audits( connection_config=connection_config, connection_count=worker_count, ) - except BaseException: + except BaseException as error: if callbacks.on_connection_error is not None: try: callbacks.on_connection_error( @@ -297,7 +300,7 @@ def _run_audits( ) except BaseException: pass - raise + raise_warehouse_connection_failure(adapter=adapter, error=error) active_error: BaseException | None = None try: if callbacks.on_connection_complete is not None: diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 918f030374..decc58ee66 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -42,6 +42,9 @@ timed_out_scenario_result, ) from sqlbuild.executor.pipeline._helpers.settings import resolve_promotion_mode +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure, +) from sqlbuild.executor.scenario.classes.prepared_scenario_schemas import PreparedScenarioSchemas from sqlbuild.executor.scenario.classes.running_scenarios import RunningScenarios from sqlbuild.executor.scenario.classes.scenario_interrupts import ScenarioInterrupts @@ -421,10 +424,10 @@ def _scenario_target_connections( connection_config=connection_config, connection_count=connection_count, ) - except Exception: + except Exception as error: if hooks.on_connection_error is not None: hooks.on_connection_error(connection_count, elapsed_seconds=time.monotonic() - start) - raise + raise_warehouse_connection_failure(adapter=adapter, error=error) if hooks.on_connection_complete is not None: hooks.on_connection_complete(connection_count, elapsed_seconds=time.monotonic() - start) recovery: ConnectionRecovery = ConnectionRecovery( diff --git a/src/sqlbuild/executor/pipeline/_helpers/seeding.py b/src/sqlbuild/executor/pipeline/_helpers/seeding.py index e608c77529..71bfa9083f 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/seeding.py +++ b/src/sqlbuild/executor/pipeline/_helpers/seeding.py @@ -12,6 +12,9 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.build.models import SeedExecutionResult +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure, +) from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.main._execute import execute_seed from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback @@ -46,13 +49,13 @@ def run_seed_pipeline( try: for _ in range(effective_concurrency): connections.append(adapter.connect(connection_config)) - except Exception: + except Exception as error: if on_connection_error is not None: on_connection_error(effective_concurrency, elapsed_seconds=time.monotonic() - start) connection: Any for connection in connections: adapter.close(connection) - raise + raise_warehouse_connection_failure(adapter=adapter, error=error) if on_connection_complete is not None: on_connection_complete(effective_concurrency, elapsed_seconds=time.monotonic() - start) diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index a1eb9145ff..da9881cdde 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -30,6 +30,9 @@ close_connections, open_worker_connections, ) +from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( + raise_warehouse_connection_failure, +) from sqlbuild.executor.pipeline.models import TestPipelineCallbacks from sqlbuild.executor.scheduling.classes.recovering_connection_pool import RecoveringConnectionPool from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -85,12 +88,12 @@ def run_test_pipeline( connection_config=connection_config, connection_count=worker_count, ) - except BaseException: + except BaseException as error: if resolved_callbacks.on_connection_error is not None: resolved_callbacks.on_connection_error( worker_count, elapsed_seconds=time.monotonic() - start ) - raise + raise_warehouse_connection_failure(adapter=adapter, error=error) if resolved_callbacks.on_connection_complete is not None: resolved_callbacks.on_connection_complete( worker_count, elapsed_seconds=time.monotonic() - start diff --git a/src/sqlbuild/executor/pipeline/constants.py b/src/sqlbuild/executor/pipeline/constants.py new file mode 100644 index 0000000000..fd137bcba7 --- /dev/null +++ b/src/sqlbuild/executor/pipeline/constants.py @@ -0,0 +1,3 @@ +"""Execution pipeline constants.""" + +SQLBUILD_PACKAGE: str = "sqlbuild" diff --git a/src/sqlbuild/executor/pipeline/exceptions.py b/src/sqlbuild/executor/pipeline/exceptions.py index 1e0500db42..889f24321c 100644 --- a/src/sqlbuild/executor/pipeline/exceptions.py +++ b/src/sqlbuild/executor/pipeline/exceptions.py @@ -32,3 +32,16 @@ class AuditConcurrencyError(RuntimeError): class AuditOutcomeError(RuntimeError): """Raised when audit execution returns an unknown quality outcome.""" + + +class WarehouseConnectionError(Exception): + """Raised when a command cannot open its warehouse connection.""" + + code: str = "C115" + + def __init__(self, *, adapter_name: str, reason: str) -> None: + self.message: str = f"could not connect to the {adapter_name} warehouse: {reason}" + self.help: str = ( + "Check the connection settings of the target; `sqb debug` tests the connection." + ) + super().__init__(self.message) diff --git a/src/sqlbuild/executor/pipeline/main/raise_warehouse_connection_failure.py b/src/sqlbuild/executor/pipeline/main/raise_warehouse_connection_failure.py new file mode 100644 index 0000000000..d1b2f3951b --- /dev/null +++ b/src/sqlbuild/executor/pipeline/main/raise_warehouse_connection_failure.py @@ -0,0 +1,25 @@ +"""Public entry for the coded failure of a warehouse connection a command cannot open.""" + +from __future__ import annotations + +from typing import NoReturn + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.pipeline.constants import SQLBUILD_PACKAGE +from sqlbuild.executor.pipeline.exceptions import WarehouseConnectionError + + +def raise_warehouse_connection_failure(*, adapter: BaseAdapter, error: BaseException) -> NoReturn: + """Raise the coded connection error for `error`, or `error` itself when SQLBuild coded it.""" + + if not isinstance(error, Exception) or _is_sqlbuild_coded(error): + raise error + raise WarehouseConnectionError( + adapter_name=getattr(adapter, "adapter_name", None) or type(adapter).__name__, + reason=str(error) or type(error).__name__, + ) from error + + +def _is_sqlbuild_coded(error: Exception) -> bool: + package: str = type(error).__module__.partition(".")[0] + return package == SQLBUILD_PACKAGE and isinstance(getattr(error, "code", None), str) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 692244d217..118451fa56 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1336,3 +1336,12 @@ class ModelConfigEnginesBuildE2ETestCase: expected_exit_code: int expected_output_fragments: tuple[str, ...] expected_relations: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class WarehouseConnectionFailureE2ETestCase: + description: str + command: tuple[str, ...] + expected_exit_code: int + expected_stderr_fragments: tuple[str, ...] + extra_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py index b6d5e77e67..d71e77b767 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py @@ -10,13 +10,68 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( LifecycleCommandsBuildE2ETestCase, + WarehouseConnectionFailureE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( assert_fragments_in_order, + prepare_inline_project, prepare_waffle_shop, run_sqb, ) +_UNREACHABLE_WAREHOUSE_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "unreachable_warehouse"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "missing_directory/nested/orders.duckdb"\n\n' + "[targets.dev]\n" + 'schema = "dev"\n' + ), + "seeds/order_statuses.csv": "status_id,status\n1,placed\n", + "seeds/order_statuses.yml": ( + "seeds:\n" + " - name: order_statuses\n" + " description: Order statuses.\n" + " columns:\n" + " - name: status_id\n" + " type: INTEGER\n" + " audits: [not_null]\n" + " - name: status\n" + " type: VARCHAR\n" + ), + "models/statuses.sql": ( + "MODEL (description 'Statuses.', materialized table, " + "columns (status_id (audits [not_null])));\n\n" + 'SELECT status_id FROM __seed("order_statuses")\n' + ), + "tests/unit/test_statuses.sql": ( + "TEST();\n\n" + "WITH __seed__order_statuses AS (SELECT 1 AS status_id, 'placed' AS status),\n" + "__expected__statuses AS (SELECT 1 AS status_id)\n" + "SELECT 1\n" + ), +} +_FORBIDDEN_ADAPTER_FILES: dict[str, str] = { + "sqlbuild_project.toml": _UNREACHABLE_WAREHOUSE_FILES["sqlbuild_project.toml"] + .replace('adapter = "duckdb"', 'adapter = "duckdb_forbidden"') + .replace("missing_directory/nested/orders.duckdb", "orders.duckdb"), + "adapters/duckdb_forbidden.py": ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class Forbidden(Exception):\n" + " code = 403\n\n\n" + "class DuckDbForbiddenAdapter(DuckDbAdapter):\n" + ' adapter_name = "duckdb_forbidden"\n\n' + " def connect(self, config):\n" + ' raise Forbidden("403 access denied for the orders project")\n' + ), +} +_CONNECTION_FAILURE: tuple[str, ...] = ( + "error[C115]: could not connect to the duckdb warehouse", + "`sqb debug` tests the connection", +) + @pytest.mark.parametrize( "test_case", @@ -165,3 +220,85 @@ def test_given_waffle_shop_when_running_core_lifecycle_commands_then_outputs_are full_refresh_build.stdout + full_refresh_build.stderr ) assert test_case.expected_full_refresh_fragment in full_refresh_build.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseConnectionFailureE2ETestCase( + description=f"{' '.join(command)} reports an unreachable warehouse as a coded error", + command=command, + expected_exit_code=1, + expected_stderr_fragments=_CONNECTION_FAILURE, + ) + for command in ( + ("build",), + ("seed",), + ("audit",), + ("plan",), + ("test",), + ("query", "SELECT 1"), + ) + ], + ids=lambda case: case.description, +) +def test_given_unreachable_warehouse_when_running_command_then_reports_coded_error( + test_case: WarehouseConnectionFailureE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="unreachable_warehouse", + repo_files=_UNREACHABLE_WAREHOUSE_FILES, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, test_case.description + output + assert "Traceback" not in output, test_case.description + output + assert all(fragment in result.stderr for fragment in test_case.expected_stderr_fragments), ( + test_case.description + output + ) + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseConnectionFailureE2ETestCase( + description="a client error carrying its own code is still a coded connection error", + command=("build",), + expected_exit_code=1, + expected_stderr_fragments=( + "error[C115]: could not connect to the duckdb_forbidden warehouse: " + "403 access denied for the orders project", + ), + extra_files=_FORBIDDEN_ADAPTER_FILES, + ) + ], + ids=lambda case: case.description, +) +def test_given_client_error_with_own_code_when_connecting_then_reports_coded_error( + test_case: WarehouseConnectionFailureE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="unreachable_warehouse", + repo_files={**_UNREACHABLE_WAREHOUSE_FILES, **test_case.extra_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, test_case.description + output + assert "Traceback" not in output, test_case.description + output + assert all(fragment in result.stderr for fragment in test_case.expected_stderr_fragments), ( + test_case.description + output + ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 5c02f6ec76..a248f7b307 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -42,7 +42,11 @@ from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing -from sqlbuild.executor.pipeline.exceptions import AuditExecutionError, AuditOutcomeError +from sqlbuild.executor.pipeline.exceptions import ( + AuditExecutionError, + AuditOutcomeError, + WarehouseConnectionError, +) from sqlbuild.executor.pipeline.main.run import ( AuditPipelineCallbacks, run_audit_pipeline, @@ -907,10 +911,13 @@ def test_given_partial_connection_open_when_later_open_fails_then_opened_connect test_case: AuditConcurrencyTestCase, ) -> None: adapter: Mock = Mock(spec=BaseAdapter) + adapter.adapter_name = "duckdb" opened: object = object() adapter.connect.side_effect = (opened, LookupError("open failure")) - with pytest.raises(LookupError, match="open failure"): + with pytest.raises( + WarehouseConnectionError, match="could not connect to the duckdb warehouse: open failure" + ): run_audit_pipeline( plan=PlanOutput(audit_entries=(audit_entry("first"), audit_entry("second"))), connection_config={}, diff --git a/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py b/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py index 7b431a5856..8c0b00e543 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py @@ -17,6 +17,7 @@ class OpenConnectionFailureTestCase: connection_config: dict[str, object] monotonic_times: tuple[float, float] expected_error: BaseException + expected_raised_type: type[BaseException] expected_event_order: tuple[str, ...] expected_elapsed_seconds: tuple[float, ...] expected_connection_counts: tuple[int, ...] diff --git a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py index 5e8083d0e7..77f2ef8aa6 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import open_connection_with_hooks +from sqlbuild.executor.pipeline.exceptions import WarehouseConnectionError from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.runtime.contracts._test_types import ( OpenConnectionFailureTestCase, @@ -75,10 +76,11 @@ def on_complete(connection_count: int, *, elapsed_seconds: int | float) -> None: "test_case", [ OpenConnectionFailureTestCase( - description="failed connection reports the original exception", + description="failed connection reports a coded error caused by the original", connection_config={"database": "unavailable.duckdb"}, monotonic_times=(20.0, 21.25), expected_error=RuntimeError("connection unavailable"), + expected_raised_type=WarehouseConnectionError, expected_event_order=("start", "connect", "error"), expected_elapsed_seconds=(1.25,), expected_connection_counts=(1, 1), @@ -88,6 +90,7 @@ def on_complete(connection_count: int, *, elapsed_seconds: int | float) -> None: connection_config={"database": "unavailable.duckdb"}, monotonic_times=(20.0, 21.25), expected_error=KeyboardInterrupt(), + expected_raised_type=KeyboardInterrupt, expected_event_order=("start", "connect"), expected_elapsed_seconds=(), expected_connection_counts=(1,), @@ -95,7 +98,7 @@ def on_complete(connection_count: int, *, elapsed_seconds: int | float) -> None: ], ids=lambda case: case.description, ) -def test_given_connection_hooks_when_opening_fails_then_reports_error_and_reraises_identity( +def test_given_connection_hooks_when_opening_fails_then_reports_error_and_raises_original_cause( test_case: OpenConnectionFailureTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -118,7 +121,7 @@ def on_error(connection_count: int, *, elapsed_seconds: int | float) -> None: connection_counts.append(connection_count) elapsed_seconds_values.append(float(elapsed_seconds)) - with pytest.raises(type(test_case.expected_error)) as exc_info: + with pytest.raises(test_case.expected_raised_type) as exc_info: open_connection_with_hooks( adapter=adapter, connection_config=test_case.connection_config, @@ -128,7 +131,9 @@ def on_error(connection_count: int, *, elapsed_seconds: int | float) -> None: ), ) - assert exc_info.value is test_case.expected_error + assert test_case.expected_error in (exc_info.value, exc_info.value.__cause__), ( + test_case.description + ) assert adapter.connected_config is test_case.connection_config assert tuple(events) == test_case.expected_event_order assert tuple(elapsed_seconds_values) == test_case.expected_elapsed_seconds From 14678062fd3a87d7a5d37331eda1ed771164ec87 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:15:03 +0100 Subject: [PATCH 07/34] fix(diff): report unknown key columns and missing relations as coded errors sqb diff with a --key column that does not exist, or with the relation missing on either side, failed with a raw warehouse error and a traceback. Key columns are now checked against both relations (X311), and a failed schema read whose relation does not exist reports which side lacks it (X310). Refs: CHI-600 --- .../executor/diff/_helpers/execution.py | 109 +++++++++++++++++- .../cli/commands/main/diff/test_diff.py | 38 ++++++ 2 files changed, 142 insertions(+), 5 deletions(-) diff --git a/src/sqlbuild/executor/diff/_helpers/execution.py b/src/sqlbuild/executor/diff/_helpers/execution.py index 72d630ed94..fea5d32897 100644 --- a/src/sqlbuild/executor/diff/_helpers/execution.py +++ b/src/sqlbuild/executor/diff/_helpers/execution.py @@ -50,11 +50,21 @@ def execute_model_diff( unique_key = options.unique_key_override elif _model_has_unique_key(right_model): unique_key = get_unique_key(right_model) - schema_result: SchemaDiffResult = adapter.diff_schema( - connection=connection, - left=left_relation, - right=right_relation, - ) + try: + schema_result: SchemaDiffResult = adapter.diff_schema( + connection=connection, + left=left_relation, + right=right_relation, + ) + except Exception as error: + _raise_missing_relation( + adapter=adapter, + connection=connection, + name=name, + sides=(("FROM", left_model, left_relation), ("TO", right_model, right_relation)), + error=error, + ) + raise if options.max_columns is not None: observed_columns: int = max( schema_result.left_column_count, @@ -367,6 +377,12 @@ def _execute_model_row_diff( (*get_row_diff_exclude_columns(right_model), *options.excluded_columns_override) ) ) + _require_key_columns( + adapter=adapter, + connection=connection, + relations=(left_relation, right_relation), + keys=resolved_unique_key, + ) intersection: tuple[str, ...] = tuple( key for key in resolved_unique_key if key in excluded_columns ) @@ -516,3 +532,86 @@ def _model_has_unique_key(model: Any) -> bool: if isinstance(raw, list | tuple): return any(isinstance(item, str) and item for item in raw) return False + + +def _raise_missing_relation( + *, + adapter: BaseAdapter, + connection: Any, + name: str, + sides: tuple[tuple[str, Any, str], tuple[str, Any, str]], + error: Exception, +) -> None: + """Raise a coded error when a schema diff failed because one side's relation is missing.""" + + left, right = sides + left_missing: bool = _relation_missing( + adapter=adapter, connection=connection, model=left[1], error=error + ) + right_missing: bool = not left_missing and _relation_missing( + adapter=adapter, connection=connection, model=right[1], error=error + ) + missing: tuple[tuple[str, Any, str], ...] = tuple( + side for side, absent in ((left, left_missing), (right, right_missing)) if absent + ) + if not missing: + return + label, _, relation = missing[0] + raise ExecutorInputError( + f"model '{name}' has no relation {relation} in the {label} target", + code="X310", + help="Build the model in that target first, or leave it out of --select.", + ) from error + + +def _relation_missing( + *, adapter: BaseAdapter, connection: Any, model: Any, error: Exception +) -> bool: + """Whether the model's relation is absent; a failed check re-raises the diff's own error.""" + + try: + exists: bool = adapter.relation_exists( + connection=connection, + database=model.destination.database, + schema=model.destination.schema, + name=model.destination.name, + ) + except Exception: + raise error # noqa: B904 + return not exists + + +def _require_key_columns( + *, + adapter: BaseAdapter, + connection: Any, + relations: tuple[str, str], + keys: tuple[str, ...], +) -> None: + """Reject row diff key columns that one of the relations does not have.""" + + if not keys: + return + left, right = relations + _require_relation_keys(adapter=adapter, connection=connection, relation=left, keys=keys) + _require_relation_keys(adapter=adapter, connection=connection, relation=right, keys=keys) + + +def _require_relation_keys( + *, adapter: BaseAdapter, connection: Any, relation: str, keys: tuple[str, ...] +) -> None: + columns: set[str] = { + column.name.lower() + for column in adapter.describe_relation(connection=connection, relation=relation) + } + missing: tuple[str, ...] = tuple(key for key in keys if key.lower() not in columns) + if not missing: + return + noun: str = "column" if len(missing) == 1 else "columns" + verb: str = "does" if len(missing) == 1 else "do" + raise ExecutorInputError( + f"row diff key {noun} {', '.join(repr(key) for key in missing)} " + f"{verb} not exist in {relation}", + code="X311", + help="Pass --key with columns both relations have, or fix the model's unique_key.", + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py b/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py index 18bbf2c089..3e0e6b856f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py @@ -432,6 +432,44 @@ expected_exit_code=1, expected_stderr_fragments=("model 'daily_revenue' requires unique_key for row diff",), ), + DiffCommandE2ETestCase( + description="an unknown cli key column fails with a coded error", + command=( + "--no-color", + "diff", + "prod:dev", + "--full", + "--key", + "nope", + "--select", + "daily_revenue", + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[X311]", + "row diff key column 'nope' does not exist in prod.daily_revenue", + ), + ), + DiffCommandE2ETestCase( + description="a relation missing on the FROM side fails with a coded error", + mutation_sql=("DROP TABLE prod.daily_revenue",), + command=("--no-color", "diff", "prod:dev", "--full", "--select", "daily_revenue"), + expected_exit_code=1, + expected_stderr_fragments=( + "error[X310]", + "model 'daily_revenue' has no relation prod.daily_revenue in the FROM target", + ), + ), + DiffCommandE2ETestCase( + description="a relation missing on the TO side fails with a coded error", + mutation_sql=("DROP TABLE dev.daily_revenue",), + command=("--no-color", "diff", "prod:dev", "--full", "--select", "daily_revenue"), + expected_exit_code=1, + expected_stderr_fragments=( + "error[X310]", + "model 'daily_revenue' has no relation dev.daily_revenue in the TO target", + ), + ), DiffCommandE2ETestCase( description="cli key enables model without declared unique key", command=( From cd669ba72460ccc7d5bbed9ea85d0f0d2831477d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:15:16 +0100 Subject: [PATCH 08/34] fix(plan): draw the tree bar under detail rows of non-last entries Detail rows under every plan entry used a blank prefix, so the vertical bar of a non-last entry broke under its cursor, bounds, reason and diff rows. Detail and nested rows now carry the bar when their entry has a later sibling. Refs: CHI-533 --- .../src/cli_text/_helpers/tree_rows.rs | 49 ++++++++++++++++--- .../sqlbuild-core/src/cli_text/constants.rs | 5 ++ .../src/cli_text/tests/test_cli_text.rs | 12 +++++ .../sqlbuild/references/docs/cli/plan.md | 10 ++-- .../test_identity_handover.py | 10 ++-- .../pre_build/test_pre_build_work_display.py | 4 +- website/src/content/docs/docs/cli/plan.mdx | 10 ++-- 7 files changed, 76 insertions(+), 24 deletions(-) diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs b/crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs index db02ee17d0..ff47cb2eb3 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs @@ -3,7 +3,8 @@ use crate::cli_text::_helpers::bounded_sections::strip_ansi; use crate::cli_text::_helpers::layout::tree_connector; use crate::cli_text::constants::{ - ENTRY_ROW_PREFIX, LEAF_KEY_SEPARATOR, LEAF_ROW_PREFIX, NESTED_ROW_PREFIX, TREE_EXEMPT_CHARS, + ENTRY_ROW_PREFIX, LEAF_KEY_SEPARATOR, LEAF_ROW_PREFIX, NESTED_ROW_PREFIX, STYLE_ESCAPE_END, + STYLE_ESCAPE_START, TREE_CONTINUATION_GLYPH, TREE_EXEMPT_CHARS, }; use crate::cli_text::models::{CliStyle, RowKind}; @@ -42,19 +43,49 @@ fn is_last(kinds: &[RowKind], index: usize, row_kind: RowKind) -> bool { true } +/// A nested row whose leaf indentation becomes a non-last entry's bar, keeping style escapes. +fn continued(line: String, continuation: Option<&str>) -> String { + let Some(continuation) = continuation else { + return line; + }; + let mut kept: String = String::with_capacity(line.len()); + let mut removed: usize = 0; + let mut characters = line.chars().peekable(); + while removed < LEAF_ROW_PREFIX.len() { + match characters.next() { + Some(STYLE_ESCAPE_START) => { + kept.push(STYLE_ESCAPE_START); + for character in characters.by_ref() { + kept.push(character); + if character == STYLE_ESCAPE_END { + break; + } + } + } + Some(' ') => removed += 1, + _ => return line, + } + } + kept.extend(characters); + format!("{continuation}{kept}") +} + /// `_treeify_plan_lines`: entry rows and their detail rows drawn with tree connectors. pub(crate) fn treeify(lines: Vec) -> Vec { let style: CliStyle = CliStyle { use_color: true }; let kinds: Vec = lines.iter().map(|line| classify(line)).collect(); let mut out: Vec = Vec::with_capacity(lines.len()); + let mut bar: Option = None; for (index, line) in lines.into_iter().enumerate() { match kinds[index] { RowKind::Entry => { - let connector: String = - tree_connector(style, is_last(&kinds, index, RowKind::Entry)); + let last: bool = is_last(&kinds, index, RowKind::Entry); + bar = (!last).then(|| style.muted(TREE_CONTINUATION_GLYPH)); + let connector: String = tree_connector(style, last); out.push(format!("{connector} {}", &line[ENTRY_ROW_PREFIX.len()..])); } RowKind::Leaf => { + let continuation: &str = bar.as_deref().unwrap_or(LEAF_ROW_PREFIX); let connector: String = tree_connector(style, is_last(&kinds, index, RowKind::Leaf)); let content: String = match line.strip_prefix(LEAF_ROW_PREFIX) { @@ -63,19 +94,23 @@ pub(crate) fn treeify(lines: Vec) -> Vec { }; if let Some((key, value)) = content.split_once(LEAF_KEY_SEPARATOR) { out.push(format!( - "{LEAF_ROW_PREFIX}{connector} {} {value}", + "{continuation}{connector} {} {value}", style.muted(key) )); } else if content.ends_with(':') { out.push(format!( - "{LEAF_ROW_PREFIX}{connector} {}", + "{continuation}{connector} {}", style.muted(&content) )); } else { - out.push(format!("{LEAF_ROW_PREFIX}{connector} {content}")); + out.push(format!("{continuation}{connector} {content}")); } } - RowKind::Nested | RowKind::Other => out.push(line), + RowKind::Nested => out.push(continued(line, bar.as_deref())), + RowKind::Other => { + bar = None; + out.push(line); + } } } out diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index c356a7ab7b..483b68b65d 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -16,6 +16,11 @@ pub(crate) const PHASE_OK_GLYPH: &str = "\u{2713}"; pub(crate) const PHASE_FAIL_GLYPH: &str = "\u{2717}"; pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}"; pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}"; +/// The bar a non-last entry's detail rows carry under its connector. +pub(crate) const TREE_CONTINUATION_GLYPH: &str = "\u{2502} "; +/// The start of a terminal style escape and the character that ends it. +pub(crate) const STYLE_ESCAPE_START: char = '\u{1b}'; +pub(crate) const STYLE_ESCAPE_END: char = 'm'; /// The human report lists at most this many models. pub(crate) const HUMAN_MODEL_LIMIT: usize = 100; pub(crate) const MIN_MODEL_NAME_WIDTH: usize = 24; diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 3e2632fd59..5531a03b4e 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -158,6 +158,18 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_text: "Plan ready 3 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{251c}\u{2500}\u{2500} policy replay_on_change=full\n \u{2514}\u{2500}\u{2500} query diff:\n -SELECT 1\n +SELECT 2\n\nModels (1)\n\u{2514}\u{2500}\u{2500} customers table\n\nWarnings (1)\n\u{2514}\u{2500}\u{2500} orders\n \u{251c}\u{2500}\u{2500} orders lag\n \u{2514}\u{2500}\u{2500} second line", expected_completion: "\u{2713} Plan complete 3 selected", }, + PlanTextTestCase { + description: "detail rows under a non-last entry keep the tree's vertical bar", + report: plan_report(vec![ + ModelPlanText { + query_diff: Some(vec![" -SELECT 1".to_owned()]), + ..table_model("orders", "query_changed") + }, + table_model("customers", "query_changed"), + ]), + expected_text: "Plan ready 2 selected\n\nQuery changed (2)\n\u{251c}\u{2500}\u{2500} orders full rebuild\n\u{2502} \u{251c}\u{2500}\u{2500} policy replay_on_change=full\n\u{2502} \u{2514}\u{2500}\u{2500} query diff:\n\u{2502} -SELECT 1\n\u{2514}\u{2500}\u{2500} customers full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full", + expected_completion: "\u{2713} Plan complete 2 selected", + }, PlanTextTestCase { description: "every active model is a full refresh", report: plan_report(vec![ diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 808e85b2f5..d094dda1e6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -60,7 +60,7 @@ Sources to load (2) Changed functions (2) ├── table_fn__customer_orders table function - └── reason first run +│ └── reason first run └── udf__is_completed_order sql udf └── reason first run @@ -73,11 +73,11 @@ First run (12) ├── dim_customers table ├── fact_orders table ├── customer_status_snapshot merge (timestamp) - ├── cursor last_ordered_at (timestamp) - └── bounds computed at run time, once model fact_orders is up to date +│ ├── cursor last_ordered_at (timestamp) +│ └── bounds computed at run time, once model fact_orders is up to date ├── hourly_order_activity delete_insert (timestamp, microbatch) - ├── cursor activity_hour (timestamp) - ├── mode microbatch +│ ├── cursor activity_hour (timestamp) +│ ├── mode microbatch ... └── order_status_index delete_insert (integer) ├── cursor order_id (integer) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py index 7802a14a9d..3e3b1b0c4b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_identity_handover.py @@ -148,12 +148,12 @@ def test_given_renamed_chain_with_full_replay_when_building_then_migrated_histor description="plan lists a renamed view as a migration that recreates it", expected_fragment=( "├── customer_orders_enriched migrate main.orders_enriched -> " - "main.customer_orders_enriched\n ├── transfer recreate (view)\n" - " ├── discovery automatic\n" - " ├── hint matched by unchanged definition; sqb rename " + "main.customer_orders_enriched\n│ ├── transfer recreate (view)\n" + "│ ├── discovery automatic\n" + "│ ├── hint matched by unchanged definition; sqb rename " "rewrites references and declares migrate_from so the move does not depend on discovery\n" - " └── old name main.orders_enriched\n" - " └── left for janitor old_name_view false" + "│ └── old name main.orders_enriched\n" + "│ └── left for janitor old_name_view false" ), ) ], diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index f3c1786018..83936e3c0c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -207,8 +207,8 @@ def test_given_colour_off_when_formatting_plan_then_migration_text_is_plain( expected_text=( "Column migrations (3)\n" "├── fact_orders migrate columns\n" - " ├── amount -> revenue rename in place (automatic)\n" - " └── tax -> tax_amount already renamed, record\n" + "│ ├── amount -> revenue rename in place (automatic)\n" + "│ └── tax -> tax_amount already renamed, record\n" "└── order_history migrate columns\n" " └── status -> order_status rename in place" ), diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 4cb337be29..582c72b52c 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -51,7 +51,7 @@ Sources to load (2) Changed functions (2) ├── table_fn__customer_orders table function - └── reason first run +│ └── reason first run └── udf__is_completed_order sql udf └── reason first run @@ -64,11 +64,11 @@ First run (12) ├── dim_customers table ├── fact_orders table ├── customer_status_snapshot merge (timestamp) - ├── cursor last_ordered_at (timestamp) - └── bounds computed at run time, once model fact_orders is up to date +│ ├── cursor last_ordered_at (timestamp) +│ └── bounds computed at run time, once model fact_orders is up to date ├── hourly_order_activity delete_insert (timestamp, microbatch) - ├── cursor activity_hour (timestamp) - ├── mode microbatch +│ ├── cursor activity_hour (timestamp) +│ ├── mode microbatch ... └── order_status_index delete_insert (integer) ├── cursor order_id (integer) From e4033373be174763cc2dba5a1b423b46ce16f035 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:15:16 +0100 Subject: [PATCH 09/34] fix(build): show target and outcome on standalone audit rows Seed, source and end audits printed as 'audit not_null OK', without their target, and a failing warn-severity audit still showed OK. Their rows now name the target and column, as in 'not_null (customers.customer_id) PASS', and show PASS, WARN, FAIL or INSUFFICIENT from the audit outcome. The partial semantic checks notice also says '1 model' instead of '1 models'. Refs: CHI-519 --- .../cli_text/main/semantic_coverage_notice.rs | 7 ++- .../src/cli_text/tests/test_cli_text.rs | 7 +++ .../src/progress/_helpers/resources.rs | 44 +++++++++++++- .../sqlbuild-core/src/progress/constants.rs | 10 ++++ crates/sqlbuild-core/src/progress/models.rs | 8 +++ .../src/progress/tests/helpers.rs | 33 ++++++++++- .../src/progress/tests/test_progress.rs | 58 ++++++++++++++++++- .../src/progress/tests/test_types.rs | 8 +++ .../src/bindings/_helpers/cli/progress.rs | 8 +++ src/sqlbuild/_native.pyi | 1 + .../main/_record_audit_progress_outcome.py | 18 ++++++ .../executor/build/_helpers/end_audits.py | 6 ++ .../executor/build/_helpers/source_audits.py | 6 ++ .../executor/pipeline/_helpers/auditing.py | 6 ++ .../cli/commands/main/build/_test_types.py | 8 +++ .../cli/commands/main/build/helpers.py | 27 +++++++++ .../main/build/test_audit_failures.py | 48 +++++++++++++++ .../classes/test_native_progress_projector.py | 12 ++-- 18 files changed, 301 insertions(+), 14 deletions(-) create mode 100644 src/sqlbuild/executor/auditing/main/_record_audit_progress_outcome.py diff --git a/crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs b/crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs index e8cfe029d2..e3b5712f1d 100644 --- a/crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs +++ b/crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs @@ -2,6 +2,8 @@ use std::collections::BTreeMap; +use crate::cli_text::_helpers::summary_text::count_noun; + use crate::cli_text::constants::{ DEFAULT_ACTION, INVALID_UPSTREAM, NOTICE_DISPLAY_LIMIT, OPEN_INPUT, OPEN_SOURCE, }; @@ -53,9 +55,10 @@ pub fn semantic_coverage_notice(reasons: &[(String, Vec)]) -> Option = vec![format!( - "Semantic checks were partial for {} models:", - reasons.len() + "Semantic checks were partial for {}:", + count_noun(count, "model", None) )]; for (reason, models) in groups.iter().take(NOTICE_DISPLAY_LIMIT) { let suffix: String = if models.len() > NOTICE_DISPLAY_LIMIT { diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 5531a03b4e..4bc1ad7077 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -78,6 +78,13 @@ fn given_partial_reasons_when_building_notice_then_groups_by_reason() { "Semantic checks were partial for 2 models:\n zeta: SQL analysis disabled; enable sql_analysis, then run `sqb compile --no-cache`.\n alpha, zeta: reads open source raw_orders; run `sqb contract generate --from --select source:raw_orders --write` and enforce the complete source contract.\n alpha: weird; declare the input shape, then run `sqb compile --no-cache`.", ), }, + NoticeTestCase { + description: "one partial model is counted in the singular", + reasons: vec![("alpha".to_owned(), vec!["SQL analysis disabled".to_owned()])], + expected_notice: Some( + "Semantic checks were partial for 1 model:\n alpha: SQL analysis disabled; enable sql_analysis, then run `sqb compile --no-cache`.", + ), + }, NoticeTestCase { description: "complete checks print no notice", reasons: Vec::new(), diff --git a/crates/sqlbuild-core/src/progress/_helpers/resources.rs b/crates/sqlbuild-core/src/progress/_helpers/resources.rs index f1ca1aeaca..32c55fa84a 100644 --- a/crates/sqlbuild-core/src/progress/_helpers/resources.rs +++ b/crates/sqlbuild-core/src/progress/_helpers/resources.rs @@ -4,7 +4,8 @@ use std::collections::VecDeque; use crate::cli_text::models::CliStyle; use crate::progress::constants::{ - FAILED_SUFFIX, KIND_WIDTH, SCENARIO_RESOURCE_PREFIX, SKIPPED_SUFFIX, + AUDIT_OUTCOME_STATUSES, AUDIT_RESOURCE_KIND, AUDIT_RESOURCE_PREFIX, FAILED_SUFFIX, KIND_WIDTH, + SCENARIO_RESOURCE_PREFIX, SKIPPED_SUFFIX, }; use crate::progress::models::{ProgressEvent, ProgressProjection}; use crate::terminal_events::models::TerminalRequest; @@ -19,6 +20,27 @@ fn display_resource_name(resource_id: Option<&str>, resource_name: &str) -> Stri format!("{scenario_name}/{resource_name}") } +/// `name (target.column)` from an audit identifier `audit::[:[:]]`. +fn audit_display_name(resource_id: Option<&str>, resource_name: &str) -> String { + let Some(identity) = resource_id.and_then(|id| id.strip_prefix(AUDIT_RESOURCE_PREFIX)) else { + return resource_name.to_owned(); + }; + let attached: Vec<&str> = identity.split(':').skip(2).collect(); + if attached.is_empty() { + return resource_name.to_owned(); + } + format!("{resource_name} ({})", attached.join(".")) +} + +/// The row name of one resource: audits name their target, scenario resources their scenario. +fn resource_display_name(kind: &str, resource_id: Option<&str>, resource_name: &str) -> String { + if kind == AUDIT_RESOURCE_KIND { + audit_display_name(resource_id, resource_name) + } else { + display_resource_name(resource_id, resource_name) + } +} + /// Remember an attempt, keeping the position of its first sighting. fn store_attempt( mut projection: ProgressProjection, @@ -84,7 +106,8 @@ pub(crate) fn consume_resource_start( use_color: projection.settings.use_color, } .status("START"); - let display_name: String = display_resource_name(Some(resource_id), resource_name); + let display_name: String = + resource_display_name(resource_kind, Some(resource_id), resource_name); lines.push(format!( " {counter}{resource_kind: = projection + .audit_outcomes + .get(attempt_id) + .and_then(|outcome| { + AUDIT_OUTCOME_STATUSES + .iter() + .find(|(value, _)| value == outcome) + .map(|(_, status)| *status) + }); let status_text: &str = if event.event_type.ends_with(FAILED_SUFFIX) { "FAIL" } else if event.event_type.ends_with(SKIPPED_SUFFIX) { "SKIP" + } else if let Some(status) = recorded { + status + } else if audit { + "PASS" } else { "OK" }; @@ -148,7 +185,8 @@ pub(crate) fn render_resource_terminal( .duration_ms .map(|duration| format!(" {:.2}s", duration / 1000.0)) .unwrap_or_default(); - let display_name: String = display_resource_name(event.resource_id.as_deref(), resource_name); + let display_name: String = + resource_display_name(resource_kind, event.resource_id.as_deref(), resource_name); let line: String = format!(" {resource_kind:, pub(crate) quiet_operation_ids: HashSet, pub(crate) enriched_resource_names: HashSet, + /// Audit outcomes by attempt, which the audit's own terminal row shows. + pub(crate) audit_outcomes: HashMap, pub(crate) resource_names: HashMap, pub(crate) statement_starts: HashMap, pub(crate) statement_query_ids: HashMap, @@ -108,6 +110,12 @@ impl ProgressProjection { .insert(resource_name.to_owned()); } + /// Record how one audit attempt ended, before its terminal row renders. + pub fn record_audit_outcome(&mut self, resource_attempt_id: &str, outcome: &str) { + self.audit_outcomes + .insert(resource_attempt_id.to_owned(), outcome.to_owned()); + } + /// Whether a visible operation with this name currently owns progress. #[must_use] pub fn is_operation_active(&self, operation_name: &str) -> bool { diff --git a/crates/sqlbuild-core/src/progress/tests/helpers.rs b/crates/sqlbuild-core/src/progress/tests/helpers.rs index a654c48407..bdbd5d6d28 100644 --- a/crates/sqlbuild-core/src/progress/tests/helpers.rs +++ b/crates/sqlbuild-core/src/progress/tests/helpers.rs @@ -4,7 +4,7 @@ use crate::progress::main::consume_progress_event::consume_progress_event; use crate::progress::models::{ ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, }; -use crate::progress::tests::test_types::ProjectionTestCase; +use crate::progress::tests::test_types::{AuditRowTestCase, ProjectionTestCase}; use crate::terminal_events::models::TerminalRequest; /// One `orders` resource attempt event. @@ -84,3 +84,34 @@ pub(super) fn orders_claim() -> TerminalRequest { pub(super) fn plain() -> ProgressSettings { ProgressSettings::default() } + +/// The start and terminal rows of one standalone `not_null` audit attempt. +pub(super) fn audit_rows(test_case: &AuditRowTestCase) -> Vec { + let mut projection: ProgressProjection = ProgressProjection::new(plain()); + for (attempt_id, outcome) in &test_case.recorded_outcomes { + projection.record_audit_outcome(attempt_id, outcome); + } + let mut lines: Vec = Vec::new(); + for (event_id, event_type) in [ + ("e1", "resource_attempt_started"), + ("e2", test_case.terminal_event), + ] { + let event: ProgressEvent = ProgressEvent { + event_id: event_id.to_owned(), + event_type: event_type.to_owned(), + resource_id: Some(test_case.resource_id.to_owned()), + resource_attempt_id: Some("a1".to_owned()), + payload: ProgressPayload { + resource_name: Some("not_null".to_owned()), + resource_kind: Some("audit".to_owned()), + duration_ms: Some(10.0), + ..ProgressPayload::default() + }, + ..ProgressEvent::default() + }; + let (consumed, event_lines) = consume_progress_event(projection, &event); + projection = consumed; + lines.extend(event_lines); + } + lines +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress.rs b/crates/sqlbuild-core/src/progress/tests/test_progress.rs index a3ff9d8de3..9ffb9e3b11 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress.rs @@ -1,8 +1,8 @@ use crate::progress::models::{ProgressEvent, ProgressPayload, ProgressSettings}; use crate::progress::tests::helpers::{ - attempt_event, operation_event, orders_claim, plain, project_case, + attempt_event, audit_rows, operation_event, orders_claim, plain, project_case, }; -use crate::progress::tests::test_types::ProjectionTestCase; +use crate::progress::tests::test_types::{AuditRowTestCase, ProjectionTestCase}; #[test] fn given_lifecycle_facts_when_projecting_then_progress_lines_match() { @@ -211,3 +211,57 @@ fn given_lifecycle_facts_when_projecting_then_progress_lines_match() { ); } } + +#[test] +fn given_standalone_audit_when_projecting_then_row_names_target_and_outcome() { + let test_cases = [ + AuditRowTestCase { + description: "a seed column audit names its target and passes", + resource_id: "audit:not_null:seed:customers:customer_id", + terminal_event: "resource_attempt_completed", + recorded_outcomes: vec![("a1", "pass")], + expected_lines: vec![ + " audit not_null (customers.customer_id) START", + " audit not_null (customers.customer_id) PASS 0.01s", + ], + }, + AuditRowTestCase { + description: "a completed audit with a warn outcome shows WARN", + resource_id: "audit:not_null:source:raw_orders:order_id", + terminal_event: "resource_attempt_completed", + recorded_outcomes: vec![("a1", "warn")], + expected_lines: vec![ + " audit not_null (raw_orders.order_id) START", + " audit not_null (raw_orders.order_id) WARN 0.01s", + ], + }, + AuditRowTestCase { + description: "a failed singular audit has no target and shows FAIL", + resource_id: "audit:not_null:singular", + terminal_event: "resource_attempt_failed", + recorded_outcomes: vec![("a1", "error")], + expected_lines: vec![ + " audit not_null START", + " audit not_null FAIL 0.01s", + ], + }, + AuditRowTestCase { + description: "a completed audit without its own recorded outcome passes", + resource_id: "audit:not_null:model:orders", + terminal_event: "resource_attempt_completed", + recorded_outcomes: vec![("a2", "warn")], + expected_lines: vec![ + " audit not_null (orders) START", + " audit not_null (orders) PASS 0.01s", + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + audit_rows(&test_case), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs index 0a9e9d687d..1606615f18 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_types.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -49,3 +49,11 @@ pub(super) struct StatusSpinnerLineTestCase { pub(super) style: CliStyle, pub(super) expected_line: &'static str, } + +pub(super) struct AuditRowTestCase { + pub(super) description: &'static str, + pub(super) resource_id: &'static str, + pub(super) terminal_event: &'static str, + pub(super) recorded_outcomes: Vec<(&'static str, &'static str)>, + pub(super) expected_lines: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index e7d1df94fb..5878950bff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -268,6 +268,14 @@ impl NativeProgressProjector { Ok(()) } + /// Record how one audit attempt ended, so its terminal row shows the outcome. + #[pyo3(signature = (*, resource_attempt_id, outcome))] + fn record_audit_outcome(&self, resource_attempt_id: &str, outcome: &str) -> PyResult<()> { + self.locked()? + .record_audit_outcome(resource_attempt_id, outcome); + Ok(()) + } + /// Return whether a visible canonical operation currently owns progress. #[pyo3(name = "is_operation_active", signature = (*, operation_name))] pub(crate) fn operation_active(&self, operation_name: &str) -> PyResult { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index dca9789df7..67501a09dd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3045,6 +3045,7 @@ class NativeProgressProjector: def restore(self, token: Token[NativeProgressProjector | None]) -> None: ... def configure_resources(self, *, ordinals: Mapping[str, int], total: int) -> None: ... def expect_resource_enrichment(self, *, resource_name: str) -> None: ... + def record_audit_outcome(self, *, resource_attempt_id: str, outcome: str) -> None: ... def is_operation_active(self, *, operation_name: str) -> bool: ... def consume(self, event: LifecycleEvent) -> None: ... def consume_resource_terminal( diff --git a/src/sqlbuild/executor/auditing/main/_record_audit_progress_outcome.py b/src/sqlbuild/executor/auditing/main/_record_audit_progress_outcome.py new file mode 100644 index 0000000000..a8b728d177 --- /dev/null +++ b/src/sqlbuild/executor/auditing/main/_record_audit_progress_outcome.py @@ -0,0 +1,18 @@ +"""Audit outcomes for the progress rows of standalone audits.""" + +from __future__ import annotations + +from sqlbuild._native import NativeProgressProjector, current_native_progress_projector +from sqlbuild.executor.auditing.models import AuditExecutionResult + + +def record_audit_progress_outcome( + *, resource_attempt_id: str, result: AuditExecutionResult +) -> None: + """Show the audit's outcome, such as WARN, on its terminal row in the active progress.""" + + projector: NativeProgressProjector | None = current_native_progress_projector() + if projector is not None: + projector.record_audit_outcome( + resource_attempt_id=resource_attempt_id, outcome=result.outcome.value + ) diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index b49ea7f544..db077aac03 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -15,6 +15,9 @@ from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) +from sqlbuild.executor.auditing.main._record_audit_progress_outcome import ( + record_audit_progress_outcome, +) from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) @@ -62,6 +65,9 @@ def run_end_audits( run_scope_phase=AuditRunScope.FINAL, quality_scope=quality_scope, ) + record_audit_progress_outcome( + resource_attempt_id=lifecycle.resource_attempt_id, result=result + ) if result.outcome == AuditOutcome.ERROR: lifecycle.failed() results.append(result) diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index b43fa5aa2d..0942ce4e50 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -16,6 +16,9 @@ from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) +from sqlbuild.executor.auditing.main._record_audit_progress_outcome import ( + record_audit_progress_outcome, +) from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( publish_completed_audit_results, ) @@ -83,6 +86,9 @@ def run_pending_source_audits( run_scope_phase=AuditRunScope.FINAL, quality_scope="source", ) + record_audit_progress_outcome( + resource_attempt_id=lifecycle.resource_attempt_id, result=result + ) if result.outcome == AuditOutcome.ERROR: lifecycle.failed() audit_results.append(result) diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index 1065a65c81..ceae487286 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -22,6 +22,9 @@ from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.main._project_results import project_audit_result_batch +from sqlbuild.executor.auditing.main._record_audit_progress_outcome import ( + record_audit_progress_outcome, +) from sqlbuild.executor.auditing.main.audit_result_publication_scope import ( audit_result_publication_scope, ) @@ -580,6 +583,9 @@ def _execute_entry( run_scope_phase=AuditRunScope.FINAL, quality_scope="standalone", ) + record_audit_progress_outcome( + resource_attempt_id=lifecycle.resource_attempt_id, result=result + ) if result.outcome == AuditOutcome.ERROR: lifecycle.failed() return result diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 118451fa56..7b30f393d6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1345,3 +1345,11 @@ class WarehouseConnectionFailureE2ETestCase: expected_exit_code: int expected_stderr_fragments: tuple[str, ...] extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class StandaloneAuditRowE2ETestCase: + description: str + status_values: tuple[str, ...] + expected_exit_code: int + expected_rows: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index c3856bf578..98556b8bf9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1115,3 +1115,30 @@ def dynamic_category_project(*, tmp_path: Path, name_pattern_entry: str) -> Path ), }, ) + + +def seed_audit_files(status_values: tuple[str, ...]) -> dict[str, str]: + """A seed with a not_null audit and a warn-severity accepted_values audit.""" + + return { + "sqlbuild_project.toml": ( + 'name = "seed_audit_rows"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "seed_audits.duckdb"\n\n[targets.dev]\nschema = "dev"\n' + ), + "seeds/order_statuses.csv": "status_id,status\n1,placed\n2,shipped\n", + "seeds/order_statuses.yml": ( + "seeds:\n" + " - name: order_statuses\n" + " description: Order statuses.\n" + " columns:\n" + " - name: status_id\n" + " type: INTEGER\n" + " audits: [not_null]\n" + " - name: status\n" + " type: VARCHAR\n" + " audits:\n" + " - accepted_values:\n" + f" values: [{', '.join(status_values)}]\n" + " severity: warn\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py index 6a8674fac6..30e4ba2ce0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py @@ -11,7 +11,9 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( AuditFailureBuildE2ETestCase, MeasurementAuditCliE2ETestCase, + StandaloneAuditRowE2ETestCase, ) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import seed_audit_files from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( prepare_inline_project, run_sqb, @@ -265,3 +267,49 @@ def test_given_measurement_audit_when_running_cli_then_outcome_controls_exit_and assert result.returncode == test_case.expected_exit_code assert test_case.expected_output in result.stdout assert "FAIL" in result.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + StandaloneAuditRowE2ETestCase( + description="passing seed audits name their column and show PASS", + status_values=("placed", "shipped"), + expected_exit_code=0, + expected_rows=( + " audit not_null (order_statuses.status_id) PASS", + " audit accepted_values (order_statuses.status) PASS", + ), + ), + StandaloneAuditRowE2ETestCase( + description="a failing warn seed audit shows WARN", + status_values=("placed",), + expected_exit_code=0, + expected_rows=( + " audit not_null (order_statuses.status_id) PASS", + " audit accepted_values (order_statuses.status) WARN", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_seed_audits_when_building_then_rows_show_target_and_outcome( + test_case: StandaloneAuditRowE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="seed_audit_rows", + repo_files=seed_audit_files(test_case.status_values), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, test_case.description + output + assert all(row in result.stdout for row in test_case.expected_rows), ( + test_case.description + output + ) diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py index 856756ee1c..1cf91dea05 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py @@ -365,8 +365,8 @@ def test_given_retried_resource_when_final_result_claims_then_prior_attempt_rend RetryProjectionCase( description="same generic audit column on distinct targets", expected_lines=( - " 1/1 audit not_null (id) START", - " 1/1 audit not_null (id) START", + " 1/1 audit not_null (orders.id) START", + " 1/1 audit not_null (customers.id) START", " orders audit OK 0.10s rich", " customers audit OK 0.20s rich", ), @@ -381,7 +381,7 @@ def test_given_same_name_distinct_resources_when_claimed_by_id_then_terminals_do ) -> None: stream: _FlushRecordingStream = _FlushRecordingStream() projector: NativeProgressProjector = NativeProgressProjector(stream=stream, use_color=False) - projector.configure_resources(ordinals={"not_null (id)": 1}, total=1) + projector.configure_resources(ordinals={"not_null": 1}, total=1) first_start: LifecycleEvent = replace( lifecycle_event( "resource_attempt_started", @@ -390,7 +390,7 @@ def test_given_same_name_distinct_resources_when_claimed_by_id_then_terminals_do resource_attempt_id="first-attempt", payload={ "resource_kind": "audit", - "resource_name": "not_null (id)", + "resource_name": "not_null", "attempt_number": 1, }, ), @@ -418,11 +418,11 @@ def test_given_same_name_distinct_resources_when_claimed_by_id_then_terminals_do for event in (first_start, second_start, second_terminal, first_terminal): projector.consume(event) first_duration: float | None = projector.consume_resource_terminal( - resource_name="not_null (id)", resource_id="audit:not_null:model:orders:id" + resource_name="not_null", resource_id="audit:not_null:model:orders:id" ) stream.write(f" orders audit OK {cast(float, first_duration) / 1000.0:.2f}s rich\n") second_duration: float | None = projector.consume_resource_terminal( - resource_name="not_null (id)", resource_id="audit:not_null:model:customers:id" + resource_name="not_null", resource_id="audit:not_null:model:customers:id" ) stream.write(f" customers audit OK {cast(float, second_duration) / 1000.0:.2f}s rich\n") projector.close() From 72a945dab2d7d21c35bca4bb600adb9026559e69 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:15:17 +0100 Subject: [PATCH 10/34] fix(plan): exit 1 when the plan has error findings sqb plan always exited 0 and printed 'Plan complete with errors' when the plan had error-severity findings that sqb build refuses, such as M114 name claims, so CI could not gate on it. Text and --json plans now exit 1 and the final line reads 'Plan failed with errors'; plans with only warnings still exit 0. The plan docs record the change as a release note. Refs: CHI-429 --- .../cli_text/main/format_plan_completion.rs | 8 +-- .../src/cli_text/tests/test_cli_text.rs | 13 ++++ .../bindings/_helpers/plan_command/plan.rs | 19 ++++- .../sqlbuild/references/docs/cli/plan.md | 10 +++ .../cli/commands/main/plan/_test_types.py | 9 +++ .../cli/commands/main/plan/helpers.py | 43 +++++++++++ .../main/plan/test_plan_exit_codes.py | 72 +++++++++++++++++++ .../main/column_migrations/helpers.py | 13 +++- .../commands/main/model_migrations/helpers.py | 7 +- website/src/content/docs/docs/cli/plan.mdx | 9 +++ 10 files changed, 193 insertions(+), 10 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_exit_codes.py diff --git a/crates/sqlbuild-core/src/cli_text/main/format_plan_completion.rs b/crates/sqlbuild-core/src/cli_text/main/format_plan_completion.rs index b386b0bb0c..322bde9466 100644 --- a/crates/sqlbuild-core/src/cli_text/main/format_plan_completion.rs +++ b/crates/sqlbuild-core/src/cli_text/main/format_plan_completion.rs @@ -1,10 +1,10 @@ //! The completion line that closes text plan output. use crate::cli_text::_helpers::plan_sections::{full_refresh_layout, ready_summary}; -use crate::cli_text::constants::{ERROR_SEVERITY_VALUE, PHASE_OK_GLYPH}; +use crate::cli_text::constants::{ERROR_SEVERITY_VALUE, PHASE_FAIL_GLYPH, PHASE_OK_GLYPH}; use crate::cli_text::models::{CliStyle, PlanTextReport}; -/// `✓ Plan complete `, warning-styled when the plan has errors. +/// `✓ Plan complete `, or `✗ Plan failed` when the plan has errors a build refuses. #[must_use] pub fn format_plan_completion(report: &PlanTextReport) -> String { let style: CliStyle = CliStyle { @@ -16,8 +16,8 @@ pub fn format_plan_completion(report: &PlanTextReport) -> String { .any(|warning| warning.severity == ERROR_SEVERITY_VALUE); let (glyph, label) = if has_errors { ( - style.warning(PHASE_OK_GLYPH), - style.warning_strong("Plan complete with errors"), + style.error(PHASE_FAIL_GLYPH), + style.error_strong("Plan failed with errors"), ) } else { (style.success(PHASE_OK_GLYPH), "Plan complete".to_owned()) diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 4bc1ad7077..60b9f89cc3 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -177,6 +177,19 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_text: "Plan ready 2 selected\n\nQuery changed (2)\n\u{251c}\u{2500}\u{2500} orders full rebuild\n\u{2502} \u{251c}\u{2500}\u{2500} policy replay_on_change=full\n\u{2502} \u{2514}\u{2500}\u{2500} query diff:\n\u{2502} -SELECT 1\n\u{2514}\u{2500}\u{2500} customers full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full", expected_completion: "\u{2713} Plan complete 2 selected", }, + PlanTextTestCase { + description: "an error finding makes the final line state failure", + report: PlanTextReport { + warnings: vec![PlanWarningText { + severity: "error".to_owned(), + message: "would replace analytics.orders".to_owned(), + model_name: None, + }], + ..plan_report(vec![table_model("orders", "no_change")]) + }, + expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} orders table\n\nErrors (1)\n\u{2514}\u{2500}\u{2500} would replace analytics.orders", + expected_completion: "\u{2717} Plan failed with errors 1 selected", + }, PlanTextTestCase { description: "every active model is a full refresh", report: plan_report(vec![ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 7653cd894b..ddf569199b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -30,6 +30,8 @@ use crate::bindings::_helpers::records::ids::RecordId; const COMMAND_NAME: &str = "plan"; const DEFAULT_SECTION_ENTRIES: i64 = 50; +/// The `severity` value of a plan finding that blocks a build. +const ERROR_SEVERITY: &str = "error"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Request fields passed to the compile pipeline under the same name. const PIPELINE_FIELDS: [&str; 11] = [ @@ -300,7 +302,22 @@ pub(crate) fn run_plan_command(request: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = with_verbose_inspection(py, verbose, || compile_pipeline(request, &invocation))?; write_output(request, &invocation, &pipeline)?; - Ok(0) + Ok(i32::from(plan_has_errors(&native_field( + &pipeline, + "plan_output", + )?)?)) +} + +/// Whether any plan finding has error severity, which `sqb build` refuses. +fn plan_has_errors(plan: &Bound<'_, PyAny>) -> PyResult { + for warning in native_field(plan, "warnings")?.try_iter()? { + let severity: Bound<'_, PyAny> = + native_field(&native_field(&warning?, "severity")?, "value")?; + if severity.extract::()? == ERROR_SEVERITY { + return Ok(true); + } + } + Ok(false) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index d094dda1e6..fa6806eb22 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/cli/plan/ - Usage - Flags - Output +- Exit codes - Missing upstream dependencies - Missing source tables @@ -132,6 +133,15 @@ Column migrations (1) When a removed column and an added column look related but are not an exact rename, the schema diff shows a hint such as `similar to amount; if this is a rename, add revenue (migrate_from amount)` until the change is built. +## Exit codes + +`sqb plan` exits `0` when the plan is ready, including plans that only have warnings. It exits `1` +when the plan has an error-severity finding that `sqb build` would refuse, such as an `M114` name +claim; the final line then reads `✗ Plan failed with errors`. `sqb plan --json` uses the same exit +codes, so CI can gate on either form. + +Release note: earlier versions printed `Plan complete with errors` and exited `0` for such plans. + ## Missing upstream dependencies Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index 810b51661a..067e857ff1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -63,3 +63,12 @@ class MissingSourceTableE2ETestCase: expected_output_fragments: tuple[str, ...] = () expected_recorded_source_names: tuple[str, ...] = () expected_statuses: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class PlanExitCodeE2ETestCase: + description: str + added_files: dict[str, str] + plan_args: tuple[str, ...] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 6dd49631fe..bce8148381 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -173,3 +173,46 @@ def prepare_missing_source_tables_project(*, tmp_path: Path) -> Path: ): execute_duckdb(db_path=db_path, sql=statement) return project_dir + + +_EXIT_CODE_PROJECT_TOML: str = ( + 'name = "plan_exit_codes"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "plan.duckdb"\n\n' + "[targets.dev]\n" + 'schema = "analytics"\n' +) +_REVENUE_MODEL: str = "MODEL (description 'Revenue.', materialized table);\n\nSELECT 1 AS amount\n" +_DAILY_REVENUE_MODEL: str = ( + "MODEL (description 'Daily revenue.', materialized table, migrate_from revenue);\n\n" + "SELECT 1 AS amount\n" +) +RECLAIMED_NAME: dict[str, str] = {"models/revenue.sql": _REVENUE_MODEL} +REMOVABLE_MIGRATION_WARNING: str = "migrate_from can be removed from 'daily_revenue'" +NAME_CLAIM_ERROR: str = "model 'revenue' would replace analytics.revenue" + + +def migrated_project(tmp_path: Path) -> Path: + """A project whose `revenue` table was migrated to `daily_revenue` and built.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="plan_exit_codes", + repo_files={ + "sqlbuild_project.toml": _EXIT_CODE_PROJECT_TOML, + "models/revenue.sql": _REVENUE_MODEL, + }, + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + (project_dir / "models/revenue.sql").unlink() + (project_dir / "models/daily_revenue.sql").write_text(_DAILY_REVENUE_MODEL, encoding="utf-8") + migrated: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert migrated.returncode == 0, migrated.stdout + migrated.stderr + return project_dir diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_exit_codes.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_exit_codes.py new file mode 100644 index 0000000000..d49308145d --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_plan_exit_codes.py @@ -0,0 +1,72 @@ +"""E2E coverage for the exit code of `sqb plan`.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import PlanExitCodeE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( + NAME_CLAIM_ERROR, + RECLAIMED_NAME, + REMOVABLE_MIGRATION_WARNING, + migrated_project, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + PlanExitCodeE2ETestCase( + description="a plan with only warnings exits 0", + added_files={}, + plan_args=(), + expected_exit_code=0, + expected_output_fragments=( + REMOVABLE_MIGRATION_WARNING, + "\u2713 Plan complete 1 selected", + ), + ), + PlanExitCodeE2ETestCase( + description="a plan with an error finding exits 1 and states failure", + added_files=RECLAIMED_NAME, + plan_args=(), + expected_exit_code=1, + expected_output_fragments=( + NAME_CLAIM_ERROR, + "\u2717 Plan failed with errors 2 selected", + ), + ), + PlanExitCodeE2ETestCase( + description="a json plan with an error finding exits 1", + added_files=RECLAIMED_NAME, + plan_args=("--json",), + expected_exit_code=1, + expected_output_fragments=(NAME_CLAIM_ERROR,), + ), + ], + ids=lambda case: case.description, +) +def test_given_plan_findings_when_planning_then_exit_code_reflects_errors( + test_case: PlanExitCodeE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = migrated_project(tmp_path) + relative_path: str + contents: str + for relative_path, contents in test_case.added_files.items(): + (project_dir / relative_path).write_text(contents, encoding="utf-8") + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", *test_case.plan_args), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, test_case.description + output + assert all(fragment in output for fragment in test_case.expected_output_fragments), ( + test_case.description + output + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index 7500593196..64177a1fcf 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -246,11 +246,18 @@ def plan_text(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> str: def plan_json(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> dict[str, Any]: - """Run sqb plan --json and parse stdout.""" + """Run sqb plan --json, check its exit code against its findings, and parse stdout.""" result: CliRun = run_sqb(project_dir=project_dir, args=("plan", "--json"), capsys=capsys) - assert result.exit_code == 0, result.output - return json.loads(result.stdout) + payload: dict[str, Any] = json.loads(result.stdout) + assert result.exit_code == plan_exit_code(payload), result.output + return payload + + +def plan_exit_code(plan: dict[str, Any]) -> int: + """Return the exit code of a plan: 1 when any finding has error severity, else 0.""" + + return int(any(warning["severity"] == "error" for warning in plan["warnings"])) def planned_column_migrations(plan: dict[str, Any]) -> tuple[tuple[str, str, str, str], ...]: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 6828de306d..98ac4033b8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -302,8 +302,11 @@ def plan_json( ["--project-dir", str(project_dir), "--no-color", "plan", "--json", *args] ) captured: CaptureResult[str] = capsys.readouterr() - assert exit_code == 0, captured.out + captured.err - return json.loads(captured.out) + payload: dict[str, Any] = json.loads(captured.out) + assert exit_code == int( + any(warning["severity"] == "error" for warning in payload["warnings"]) + ), captured.out + captured.err + return payload def model_entry(*, plan: dict[str, Any], name: str) -> dict[str, Any]: diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 582c72b52c..9340cfa2ef 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -123,6 +123,15 @@ Column migrations (1) When a removed column and an added column look related but are not an exact rename, the schema diff shows a hint such as `similar to amount; if this is a rename, add revenue (migrate_from amount)` until the change is built. +## Exit codes + +`sqb plan` exits `0` when the plan is ready, including plans that only have warnings. It exits `1` +when the plan has an error-severity finding that `sqb build` would refuse, such as an `M114` name +claim; the final line then reads `✗ Plan failed with errors`. `sqb plan --json` uses the same exit +codes, so CI can gate on either form. + +Release note: earlier versions printed `Plan complete with errors` and exited `0` for such plans. + ## Missing upstream dependencies Planning a scoped selection whose upstream inputs were never built in the target fails with `S301` rather than producing a plan that would break at build time: From 9db6b32291d3120a403f125b1879eb9797404e36 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:36:53 +0100 Subject: [PATCH 11/34] fix(test): run unit tests against temporary copies of project functions `sqb test` called the built warehouse function, so an edited UDF or table function was tested at its last-built definition, and a never-built one failed with "Run `sqb build` first". Before the tests run, SQLBuild now creates a session-scoped copy of every function the selected tests call, and of every function those call (the test plan now carries the transitive closure), then points calls in the tests' SQL code at the copies; calls in string literals and comments keep their text. No permanent DDL is issued. The copy set is decided on the first test connection and recreated on every other connection and on recovery reconnects; a connection that cannot create it stops the run with error[T006] naming the function and connection. Adapters opt in through the optional StrictAdapter method `render_create_temporary_function`: DuckDB and MotherDuck use TEMP MACRO, PostgreSQL uses pg_temp, Snowflake and Databricks use prefixed TEMPORARY FUNCTIONs. BigQuery and SQL Server keep calling the built function. When no copy can be created, tests fall back to the built function (a caller whose callee has no copy too), and a missing one reports the reason with the build hint. Refs: CHI-555 --- .../src/bindings/_helpers/cli/entry.rs | 3 +- .../bindings/_helpers/native_host/executor.rs | 6 + .../_helpers/planner_entries/command_plans.rs | 38 +- .../references/docs/concepts/adapters.md | 2 +- .../docs/concepts/adapters/bigquery.md | 5 + .../docs/concepts/adapters/databricks.md | 6 + .../docs/concepts/adapters/duckdb.md | 6 + .../docs/concepts/adapters/motherduck.md | 6 + .../docs/concepts/adapters/postgres.md | 7 + .../docs/concepts/adapters/snowflake.md | 8 + .../docs/concepts/adapters/sqlserver.md | 5 + .../references/docs/concepts/testing.md | 34 ++ src/sqlbuild/_native_host.py | 4 + src/sqlbuild/_native_host.pyi | 3 + .../contract/classes/duckdb_backed_adapter.py | 20 + .../contract/classes/strict_adapter.py | 10 + src/sqlbuild/adapter/contract/models.py | 8 + .../databricks/classes/databricks_adapter.py | 27 ++ src/sqlbuild/adapters/databricks/constants.py | 1 + .../postgres/classes/postgres_adapter.py | 20 + .../snowflake/classes/snowflake_adapter.py | 29 ++ src/sqlbuild/adapters/snowflake/constants.py | 1 + .../pipeline/_helpers/temporary_functions.py | 362 ++++++++++++++++++ .../executor/pipeline/_helpers/testing.py | 105 ++++- src/sqlbuild/executor/pipeline/exceptions.py | 17 + src/sqlbuild/executor/pipeline/models.py | 15 +- .../cli/commands/main/test/_test_types.py | 32 ++ .../main/test/test_temporary_functions.py | 312 +++++++++++++++ .../classes/strict_adapter/_test_types.py | 14 + .../strict_adapter/test_strict_adapter.py | 17 +- .../test_temporary_function_rendering.py | 146 +++++++ .../executor/pipeline/_helpers/_test_types.py | 17 + .../executor/pipeline/_helpers/helpers.py | 89 +++++ .../_helpers/test_temporary_functions.py | 75 ++++ .../pipeline/_helpers/test_testing.py | 126 ++++++ .../content/docs/docs/concepts/adapters.mdx | 2 +- .../docs/docs/concepts/adapters/bigquery.mdx | 5 + .../docs/concepts/adapters/databricks.mdx | 6 + .../docs/docs/concepts/adapters/duckdb.mdx | 6 + .../docs/concepts/adapters/motherduck.mdx | 6 + .../docs/docs/concepts/adapters/postgres.mdx | 7 + .../docs/docs/concepts/adapters/snowflake.mdx | 7 + .../docs/docs/concepts/adapters/sqlserver.mdx | 5 + .../content/docs/docs/concepts/testing.mdx | 34 ++ 44 files changed, 1637 insertions(+), 17 deletions(-) create mode 100644 src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_temporary_functions.py create mode 100644 tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_temporary_function_rendering.py create mode 100644 tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_temporary_functions.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 676a05d255..d4500c5832 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -43,10 +43,11 @@ const DIFF_PUBLICATION_CODE: &str = "C257"; const DIFF_EXECUTION_STATUS: &str = "execution_failed"; const DIFF_OUTCOME_PREFIX: &str = "Query diff outcome "; /// Expected staying-Python command failures, loaded only once a command raised. -const EXPECTED_COMMAND_FAILURES: [ExecutorHost; 3] = [ +const EXPECTED_COMMAND_FAILURES: [ExecutorHost; 4] = [ ExecutorHost::AuditExecutionError, ExecutorHost::ExecutionDurationLimitError, ExecutorHost::WarehouseConnectionError, + ExecutorHost::TemporaryTestFunctionError, ]; /// Expected native command failures. const EXPECTED_NATIVE_FAILURES: [ErrorId; 2] = [ErrorId::Discovery, ErrorId::SpecConfig]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs index c7528eda68..2f31d6fa79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs @@ -69,6 +69,7 @@ pub(crate) enum ExecutorHost { IsCompatibilityView, AuditExecutionError, WarehouseConnectionError, + TemporaryTestFunctionError, RaiseWarehouseConnectionFailure, AuditPipelineCallbacks, RunAuditPipelineWithCallbacks, @@ -493,6 +494,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Exception, edge: HostEdge::Executor, }, + HostEntry { + name: "TemporaryTestFunctionError", + kind: HostKind::Exception, + edge: HostEdge::Executor, + }, HostEntry { name: "raise_warehouse_connection_failure", kind: HostKind::Function, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 5ac80284d5..284d458d3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -23,7 +23,10 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, + PyTuple, +}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; @@ -190,6 +193,37 @@ pub(crate) fn build_audit_command_plan<'py>( ) } +/// The tests' direct function dependencies and every project function those call. +fn called_function_keys<'py>( + project: &Bound<'py, PyAny>, + direct: Vec>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let functions: Bound<'py, PyDict> = PyDict::new(py); + for function in native_field(project, "functions")?.try_iter()? { + let function = function?; + functions.set_item(native_field(&function, "key")?, function)?; + } + let called: Bound<'py, PySet> = PySet::empty(py)?; + let mut pending: Vec> = direct; + while let Some(key) = pending.pop() { + if called.contains(&key)? { + continue; + } + called.add(&key)?; + let Some(function) = functions.get_item(&key)? else { + continue; + }; + for dependency in native_field(&function, "deps")?.try_iter()? { + let dependency = dependency?; + if functions.contains(&dependency)? { + pending.push(dependency); + } + } + } + Ok(called) +} + /// Project selected SQL tests from canonical static state. #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -235,7 +269,7 @@ pub(crate) fn build_test_command_plan<'py>( function_keys.push(key?); } } - let function_keys = PyFrozenSet::new(py, function_keys)?; + let function_keys = called_function_keys(&project, function_keys)?; let mut functions: Vec> = Vec::new(); for function in native_field(&project, "functions")?.try_iter()? { let function = function?; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md index e56fc97d91..32a57fc8e7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md @@ -102,7 +102,7 @@ Set `sql_analysis_dialect_name` to the SQL dialect name that matches your engine For full control with no inherited defaults, subclass `StrictAdapter` instead. Every method is abstract and must be implemented explicitly. SQLBuild raises a clear error listing any unimplemented methods. -A few `StrictAdapter` methods are optional and have a documented default. `create_sql_test_output` and `drop_sql_test_output` let an adapter store a unit test's actual output once so every comparison reads it (see [Single-read comparison](testing.md#single-read-comparison)). By default `create_sql_test_output` returns `None`, and SQLBuild runs the test's inline comparison query instead, so adapters that do not implement them keep working unchanged. +A few `StrictAdapter` methods are optional and have a documented default. `create_sql_test_output` and `drop_sql_test_output` let an adapter store a unit test's actual output once so every comparison reads it (see [Single-read comparison](testing.md#single-read-comparison)). By default `create_sql_test_output` returns `None`, and SQLBuild runs the test's inline comparison query instead, so adapters that do not implement them keep working unchanged. `render_create_temporary_function` returns the call target and DDL for a session-scoped copy of a project function, which `sqb test` calls instead of the built function (see [Functions in `sqb test`](testing.md#functions-in-sqb-test)). By default it returns `None`, and tests call the built function. ### Discovery rules diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md index 8d0141e523..b450317391 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/bigquery.md @@ -65,3 +65,8 @@ NUMERIC '2.4700' BigQuery does not support arrays of arrays. A nested list or set requested with `render_as array` fails at compile time rather than producing invalid BigQuery SQL. Parenthesized `value_list` rendering remains available for ordinary scalar collections used with `IN`. See [Collections and Rendering](../constants/collections-and-rendering.md#native-array-rendering) for declarations, rendering precedence, and the cross-adapter matrix. + +## Functions in unit tests + +BigQuery temporary functions exist only inside one script, so `sqb test` calls the built function. +Run `sqb build` before testing a new or changed function. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md index 61748c1a68..0053f2e4fc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/databricks.md @@ -45,3 +45,9 @@ The target's `database` selects the Unity Catalog catalog, and its `schema` sele On connect, SQLBuild runs `USE CATALOG` and `USE SCHEMA` statements from the active target to set the authoritative session namespace. + +## Functions in unit tests + +`sqb test` creates each SQL UDF and table function the selected tests call as a session +`TEMPORARY FUNCTION __sqb_test__`. Python UDFs cannot be temporary, so tests call the built +Python function. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index 51a57dccb7..35506f687c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -60,3 +60,9 @@ DuckDB defaults to `staged` promotion: tables are materialized into a staging ta [settings] table_promotion_mode = "staged" ``` + +## Functions in unit tests + +`sqb test` copies each SQL UDF and table function the selected tests call into a `CREATE TEMP +MACRO` on every test connection, called as `temp.main.`, so tests use the current project +definition without a build. Python UDFs are registered on each test connection. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md index d54f46af87..cf92a5859c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md @@ -77,3 +77,9 @@ schema = "dev" ``` This lets you develop and test locally with zero MotherDuck compute cost, then deploy to MotherDuck in production. SQLBuild's [scenario replay](../scenarios.md) also runs locally in DuckDB regardless of the production adapter. + +## Functions in unit tests + +`sqb test` copies each SQL UDF and table function the selected tests call into a session +`TEMP MACRO`, the same as [DuckDB](duckdb.md#functions-in-unit-tests), so the +functions in your MotherDuck database are not replaced. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md index 41ff1723bf..97337b9aa1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md @@ -64,3 +64,10 @@ connection = "postgres" database = "analytics" schema = "dev" ``` + +## Functions in unit tests + +`sqb test` creates each SQL function the selected tests call in the session's `pg_temp` schema and +calls it as `pg_temp.`, so tests use the current project definition without a build. Temporary +functions belong to one server session: behind a transaction-mode pooler, run `sqb test` through a +direct connection or a session-mode pool. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index a48a472d01..ea93ec6ce1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/snowflake/ - State tables - Shared connections across targets - Cost estimates +- Functions in unit tests Snowflake requires the optional `snowflake-connector-python` dependency: @@ -137,3 +138,10 @@ not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60 cloud-services credits, contract adjustments, or multi-cluster billing. Run metadata and query IDs are stored under `target/executions//`; that statement ledger stores only an SQL digest, not SQL text. Executed SQL artifacts are stored separately under the sensitive `target/run/` tree. + +## Functions in unit tests + +`sqb test` creates each UDF and table function the selected tests call, SQL or Python, as a +`TEMPORARY FUNCTION __SQB_TEST__` in the session's current schema. The session needs a current +schema and permission to create functions there; otherwise tests call the built function. See +[Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md index 3bba118653..3939785771 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlserver.md @@ -80,3 +80,8 @@ connection = "sqlserver" database = "analytics" schema = "dev" ``` + +## Functions in unit tests + +SQL Server has no session-scoped functions, so `sqb test` calls the built function. Run `sqb build` +before testing a new or changed function. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 48dc842484..6b6a0b0fa1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -590,6 +590,40 @@ __udf_expected__ AS ( UDFs are warehouse objects, so the function is created before the test runs. During `sqb build`, UDF tests run after the function is created but before any model that uses it. +### Functions in `sqb test` + +`sqb test` does not need a build to test the current version of a function. Before the tests run, +SQLBuild creates a session-scoped temporary copy of every project UDF and table function the +selected tests call, including functions those functions call, from the definitions in your +project. The tests call the copies, so an edited function is tested without `sqb build`, and the +built function in the target schema is never created, replaced or dropped. The copies disappear +when the test connections close. + +| Adapter | Temporary copy | +|---|---| +| DuckDB, MotherDuck | `CREATE TEMP MACRO`, called as `temp.main.`. Python UDFs are registered on each test connection. | +| PostgreSQL | SQL function in the session's `pg_temp` schema, called as `pg_temp.` | +| Snowflake | `CREATE TEMPORARY FUNCTION __sqb_test__` in the session's current schema, for SQL and Python UDFs and table functions | +| Databricks | `CREATE TEMPORARY FUNCTION __sqb_test__` for SQL functions | +| BigQuery, SQL Server | Not supported; tests call the built function | + +When an adapter cannot create a copy (a Databricks Python UDF, for example), or the warehouse +refuses to create it (the role lacks permission, or the Snowflake session has no current schema), +the test calls the built function instead. If that function has not been built either, the test +fails with an error that names the function and the reason no copy was created, and asks you to run +`sqb build` first. +A function whose callee has no copy calls the built functions too, so one test never mixes copies +and built functions. + +SQLBuild decides which copies exist on the first test connection and then creates the same copies +on every other test connection, including a connection that replaces one a timeout left unusable. +If one of those connections cannot create a copy, `sqb test` stops with error `T006`, which names +the function and the connection. + +PostgreSQL temporary functions belong to one server session. Behind a transaction-mode connection +pooler, a test's statements can reach a different server session that does not hold the copies, so +run `sqb test` for projects with functions through a direct connection or a session-mode pool. + ### Table function tests Test table functions by calling them in `__table_fn_actual__` and comparing against `__table_fn_expected__`: diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 4adc8800e5..634ab0f4fe 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -385,6 +385,10 @@ "sqlbuild.executor.pipeline.exceptions", "WarehouseConnectionError", ), + "TemporaryTestFunctionError": ( + "sqlbuild.executor.pipeline.exceptions", + "TemporaryTestFunctionError", + ), "raise_warehouse_connection_failure": ( "sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure", "raise_warehouse_connection_failure", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 7e222cc35e..08c7f6dee1 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -270,6 +270,9 @@ from sqlbuild.executor.old_name_views._helpers.janitor import ( is_compatibility_view as is_compatibility_view, ) from sqlbuild.executor.pipeline.exceptions import AuditExecutionError as AuditExecutionError +from sqlbuild.executor.pipeline.exceptions import ( + TemporaryTestFunctionError as TemporaryTestFunctionError, +) from sqlbuild.executor.pipeline.exceptions import ( WarehouseConnectionError as WarehouseConnectionError, ) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index e57bbd8024..bb1ad30829 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -72,6 +72,7 @@ SnapshotChangeTarget, TableFreshnessMetadata, TableFreshnessRequest, + TemporaryFunction, ) from sqlbuild.adapter.contract.types import ( CursorKind, @@ -1330,6 +1331,25 @@ def render_create_function( return (f"CREATE OR REPLACE MACRO {destination}({argument_sql}) AS TABLE\n{body_sql}",) return (f"CREATE OR REPLACE MACRO {destination}({argument_sql}) AS (\n{body_sql}\n)",) + def render_create_temporary_function( + self, *, name: str, definition: FunctionDefinition + ) -> TemporaryFunction | None: + """Copy a SQL function into a temporary macro; Python functions register per connection.""" + + if definition.language != FunctionLanguage.SQL: + return None + identifier: str = self.render_identifier(name) + argument_sql: str = ", ".join(str(arg.name) for arg in definition.arguments) + body: str = ( + f"AS TABLE\n{definition.body_sql}" + if definition.return_columns + else f"AS (\n{definition.body_sql}\n)" + ) + return TemporaryFunction( + target=f"temp.main.{identifier}", + statements=(f"CREATE OR REPLACE TEMP MACRO {identifier}({argument_sql}) {body}",), + ) + def supports_python_functions(self) -> bool: return True diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index c1164d99c3..b9733a11f8 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -16,6 +16,7 @@ ColumnInfo, CursorValue, ExpressionInferenceProfile, + FunctionDefinition, MigrationStagePlan, RelationGrant, RelationReadProbe, @@ -24,6 +25,7 @@ SnapshotChangeTarget, TableFreshnessMetadata, TableFreshnessRequest, + TemporaryFunction, ) from sqlbuild.adapter.contract.types import ( FrameworkType, @@ -122,6 +124,14 @@ def render_cursor_operand(self, *, expression: str, cursor_type: str | None) -> del cursor_type return expression + def render_create_temporary_function( + self, *, name: str, definition: FunctionDefinition + ) -> TemporaryFunction | None: + """Optional: a session-scoped function copy for unit tests, or `None` if unsupported.""" + + del name, definition + return None + @abstractmethod def lists_relations_authoritatively(self, *, database: str | None, schema: str) -> bool: """Return whether ``list_relations`` reports every relation in this scope.""" diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index c6e237e612..3678afe53a 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -284,6 +284,14 @@ class FunctionDefinition: source_file_path: Path | None = None +@dataclass(frozen=True) +class TemporaryFunction: + """A session-scoped copy of a project function: its call target and creating DDL.""" + + target: str + statements: tuple[str, ...] + + @dataclass(frozen=True) class QueryResult: """Rows returned by an ad hoc warehouse query.""" diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 91ff016fd2..9006a13ce2 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -80,6 +80,7 @@ SnapshotSqlDialect, TableFreshnessMetadata, TableFreshnessRequest, + TemporaryFunction, ) from sqlbuild.adapter.contract.types import ( BuiltinAdapter, @@ -128,6 +129,7 @@ NON_ROW_RESULT_COLUMN_NAMES, RELATION_NOT_FOUND_ERROR_CLASSES, TABLE_RELATION_METADATA_TYPES, + TEMPORARY_FUNCTION_PREFIX, UNDEFINED_TABLE_SQLSTATE, ) from sqlbuild.compiler.compile.types import FunctionLanguage @@ -1679,6 +1681,31 @@ def render_create_function( f"RETURN {body_sql}", ) + def render_create_temporary_function( + self, *, name: str, definition: FunctionDefinition + ) -> TemporaryFunction | None: + """Copy a SQL function into a prefixed session TEMPORARY function, called unqualified.""" + + if definition.language != FunctionLanguage.SQL: + return None + target: str = self.render_identifier(f"{TEMPORARY_FUNCTION_PREFIX}{name}") + statements: tuple[str, ...] = self.render_create_function( + destination=target, + arguments=definition.arguments, + returns=definition.returns, + body_sql=definition.body_sql, + return_columns=definition.return_columns, + ) + return TemporaryFunction( + target=target, + statements=tuple( + statement.replace( + "CREATE OR REPLACE FUNCTION ", "CREATE OR REPLACE TEMPORARY FUNCTION ", 1 + ) + for statement in statements + ), + ) + def _render_create_python_function( self, *, diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index 5401a62f46..3ab8be030e 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -16,3 +16,4 @@ UNDEFINED_TABLE_SQLSTATE: str = "42P01" DETAILED_TABLE_INFORMATION_MARKER: str = "# detailed table information" DESCRIBE_DETAIL_ROW_WIDTH: int = 2 +TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 2ccd006066..8a8e9af2d6 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -77,6 +77,7 @@ SnapshotSqlDialect, TableFreshnessMetadata, TableFreshnessRequest, + TemporaryFunction, ) from sqlbuild.adapter.contract.types import ( BuiltinAdapter, @@ -537,6 +538,25 @@ def render_create_function( f"LANGUAGE SQL AS $$\n{body_sql}\n$$", ) + def render_create_temporary_function( + self, *, name: str, definition: FunctionDefinition + ) -> TemporaryFunction | None: + """Copy a SQL function into the session's `pg_temp` schema, called schema-qualified.""" + + if definition.language != FunctionLanguage.SQL: + return None + target: str = f"pg_temp.{self.render_identifier(name)}" + return TemporaryFunction( + target=target, + statements=self.render_create_function( + destination=target, + arguments=definition.arguments, + returns=definition.returns, + body_sql=definition.body_sql, + return_columns=definition.return_columns, + ), + ) + def render_append( self, *, destination: str, sql: str, columns: tuple[str, ...] | None = None ) -> tuple[str, ...]: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 05a67ee557..0cb29d1848 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -79,6 +79,7 @@ SnapshotSqlDialect, TableFreshnessMetadata, TableFreshnessRequest, + TemporaryFunction, ) from sqlbuild.adapter.contract.types import ( BuiltinAdapter, @@ -152,6 +153,7 @@ SHOW_LIKE_ESCAPE_CHARACTER, STATUS_COLUMN_NAME, SUCCESS_STATUS_TOKENS, + TEMPORARY_FUNCTION_PREFIX, TRUE_METADATA_VALUE, VIEW_RELATION_TYPE_TOKEN, ) @@ -2829,6 +2831,33 @@ def render_create_function( f"AS $$\n{body_sql}\n$$", ) + def render_create_temporary_function( + self, *, name: str, definition: FunctionDefinition + ) -> TemporaryFunction | None: + """Copy a function into a session TEMPORARY function in the session's current schema.""" + + target: str = self.render_identifier(f"{TEMPORARY_FUNCTION_PREFIX}{name}") + statements: tuple[str, ...] = self.render_create_function( + destination=target, + arguments=definition.arguments, + returns=definition.returns, + body_sql=definition.body_sql, + return_columns=definition.return_columns, + language=definition.language, + runtime_version=definition.runtime_version, + entry_point=definition.entry_point, + packages=definition.packages, + ) + return TemporaryFunction( + target=target, + statements=tuple( + statement.replace( + "CREATE OR REPLACE FUNCTION ", "CREATE OR REPLACE TEMPORARY FUNCTION ", 1 + ) + for statement in statements + ), + ) + def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> str: if cursor_type == CursorKind.INTEGER: return value diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 26a745d85c..3816e406b6 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -28,3 +28,4 @@ STATE_TABLE_FALLBACK_RETENTION_DAYS: int = 1 STATE_TABLE_RETENTION_PARAMETER: str = "DATA_RETENTION_TIME_IN_DAYS" INVALID_PARAMETER_VALUE_ERRNO: int = 1008 +TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" diff --git a/src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py b/src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py new file mode 100644 index 0000000000..734d328678 --- /dev/null +++ b/src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py @@ -0,0 +1,362 @@ +"""Session-scoped copies of project functions so unit tests call the current definitions.""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Iterable +from dataclasses import replace +from typing import Any + +from sqlbuild._native import ( + ChainStep, + CompiledObjectKey, + FunctionPlanEntry, + SqlCodeScanner, + SqlTestAssertionStep, + SqlTestPlanEntry, +) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import FunctionDefinition, TemporaryFunction +from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor +from sqlbuild.diagnostics.main.log_debug_event import log_debug_event +from sqlbuild.executor.pipeline.exceptions import TemporaryTestFunctionError +from sqlbuild.executor.pipeline.models import TemporaryTestFunctions +from sqlbuild.runtime.execution_limits.exceptions import ( + ExecutionDurationLimitError, + ItemTimeoutError, +) + +_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") +_IDENTIFIER_QUOTES: frozenset[str] = frozenset({'"', "`", "["}) +_SCANNER_CONTEXT: str = "SQL test function call rewriting" + + +def create_temporary_test_functions( + *, + function_entries: tuple[FunctionPlanEntry, ...], + required_keys: Iterable[CompiledObjectKey], + adapter: BaseAdapter, + connection: Any, +) -> TemporaryTestFunctions: + """Create a temporary copy of each required function and the functions its body calls.""" + + syntax: dict[str, object] = adapter.sql_lexical_syntax.native_mapping + entries_by_key: dict[CompiledObjectKey, FunctionPlanEntry] = _entries_by_key(function_entries) + keys_by_target: dict[str, CompiledObjectKey] = _keys_by_target(entries_by_key) + replacements: dict[str, str] = {} + created_keys: set[CompiledObjectKey] = set() + statements: list[tuple[str, str]] = [] + failures: dict[CompiledObjectKey, str] = {} + key: CompiledObjectKey + for key in _dependency_order( + required_keys=required_keys, entries_by_key=entries_by_key, syntax=syntax + ): + entry: FunctionPlanEntry = entries_by_key[key] + uncopied: tuple[CompiledObjectKey, ...] = tuple( + callee + for callee in _called_function_keys( + sql=entry.body_sql, keys_by_target=keys_by_target, syntax=syntax + ) + if callee in failures + ) + if uncopied: + failures[key] = ( + f"function {uncopied[0].name}, which it calls, has no temporary copy: " + f"{failures[uncopied[0]]}" + ) + continue + deployed_target: str = str(entry.destination.qualified_name) + temporary: TemporaryFunction | None = adapter.render_create_temporary_function( + name=entry.name, + definition=FunctionDefinition( + destination=deployed_target, + arguments=entry.arguments, + returns=entry.returns, + body_sql=rewrite_function_calls( + sql=entry.body_sql, replacements=replacements, syntax=syntax + ), + return_columns=entry.return_columns, + language=entry.language, + runtime_version=entry.runtime_version, + entry_point=entry.entry_point, + packages=entry.packages, + source_file_path=entry.source_file_path, + ), + ) + if temporary is None: + failures[key] = ( + f"the {adapter.adapter_name} adapter cannot create a temporary copy of " + f"{entry.language} function {entry.name} for the test run" + ) + continue + failure: str | None = _execute_statements( + adapter=adapter, connection=connection, statements=temporary.statements + ) + if failure is not None: + failures[key] = f"its temporary copy for the test run could not be created: {failure}" + continue + replacements[deployed_target] = temporary.target + created_keys.add(key) + statements.extend((entry.name, statement) for statement in temporary.statements) + return TemporaryTestFunctions( + statements=tuple(statements), + created_keys=frozenset(created_keys), + replacements=replacements, + failures=failures, + ) + + +def install_temporary_test_functions( + *, + adapter: BaseAdapter, + connection: Any, + functions: TemporaryTestFunctions, + connection_label: str, +) -> None: + """Create the copies the first connection created on another one, or fail with T006.""" + + function_name: str + statement: str + for function_name, statement in functions.statements: + try: + _ = adapter.execute(connection=connection, sql=statement) + except (ItemTimeoutError, ExecutionDurationLimitError): + raise + except Exception as error: + raise TemporaryTestFunctionError( + function_name=function_name, + connection=connection_label, + reason=DiagnosticRecordRedactor.text(str(error)) or type(error).__name__, + ) from error + + +def called_function_keys( + *, + function_entries: tuple[FunctionPlanEntry, ...], + keys: Iterable[CompiledObjectKey], + adapter: BaseAdapter, +) -> tuple[CompiledObjectKey, ...]: + """The planned functions among `keys` and every planned function their bodies call.""" + + return _dependency_order( + required_keys=keys, + entries_by_key=_entries_by_key(function_entries), + syntax=adapter.sql_lexical_syntax.native_mapping, + ) + + +def rewrite_test_entry( + *, entry: SqlTestPlanEntry, replacements: dict[str, str], syntax: dict[str, object] +) -> SqlTestPlanEntry: + """Point a test's function calls at their temporary copies.""" + + if not replacements: + return entry + return replace( + entry, + chain=tuple( + _rewrite_chain_step(step=step, replacements=replacements, syntax=syntax) + for step in entry.chain + ), + assertions=tuple( + _rewrite_assertion_step(step=step, replacements=replacements, syntax=syntax) + for step in entry.assertions + ), + ) + + +def rewrite_function_calls( + *, sql: str, replacements: dict[str, str], syntax: dict[str, object] +) -> str: + """Replace each deployed function target called in SQL code with its temporary target.""" + + if not replacements: + return sql + pieces: list[str] = [] + copied_to: int = 0 + match: re.Match[str] + for match in _code_matches(pattern=_call_pattern(targets=replacements), sql=sql, syntax=syntax): + pieces.append(sql[copied_to : match.start()]) + pieces.append(replacements[match.group(1)]) + copied_to = match.end() + pieces.append(sql[copied_to:]) + return "".join(pieces) + + +def _execute_statements( + *, adapter: BaseAdapter, connection: Any, statements: tuple[str, ...] +) -> str | None: + try: + statement: str + for statement in statements: + _ = adapter.execute(connection=connection, sql=statement) + except (ItemTimeoutError, ExecutionDurationLimitError): + raise + except Exception as error: + reason: str = DiagnosticRecordRedactor.text(str(error)) + log_debug_event( + logger=_LOGGER, + message="Temporary test function could not be created; using the built function", + sqlbuild_error=reason, + ) + return reason + return None + + +def _entries_by_key( + function_entries: tuple[FunctionPlanEntry, ...], +) -> dict[CompiledObjectKey, FunctionPlanEntry]: + return { + entry.key: entry + for entry in function_entries + if entry.destination.qualified_name is not None + } + + +def _keys_by_target( + entries_by_key: dict[CompiledObjectKey, FunctionPlanEntry], +) -> dict[str, CompiledObjectKey]: + return {str(entry.destination.qualified_name): key for key, entry in entries_by_key.items()} + + +def _dependency_order( + *, + required_keys: Iterable[CompiledObjectKey], + entries_by_key: dict[CompiledObjectKey, FunctionPlanEntry], + syntax: dict[str, object], +) -> tuple[CompiledObjectKey, ...]: + keys_by_target: dict[str, CompiledObjectKey] = _keys_by_target(entries_by_key) + ordered: list[CompiledObjectKey] = [] + visiting: set[CompiledObjectKey] = set() + pending: list[tuple[CompiledObjectKey, bool]] = [ + (key, False) for key in reversed(tuple(required_keys)) if key in entries_by_key + ] + while pending: + key, expanded = pending.pop() + if key in ordered: + continue + if expanded: + ordered.append(key) + continue + if key in visiting: + continue + visiting.add(key) + pending.append((key, True)) + pending.extend( + (dependency, False) + for dependency in _called_function_keys( + sql=entries_by_key[key].body_sql, keys_by_target=keys_by_target, syntax=syntax + ) + if dependency != key and dependency not in ordered + ) + return tuple(ordered) + + +def _called_function_keys( + *, sql: str, keys_by_target: dict[str, CompiledObjectKey], syntax: dict[str, object] +) -> tuple[CompiledObjectKey, ...]: + if not keys_by_target: + return () + return tuple( + keys_by_target[match.group(1)] + for match in _code_matches( + pattern=_call_pattern(targets=keys_by_target), sql=sql, syntax=syntax + ) + ) + + +def _call_pattern(*, targets: Iterable[str]) -> re.Pattern[str]: + alternatives: str = "|".join( + re.escape(target) for target in sorted(targets, key=lambda value: len(value), reverse=True) + ) + return re.compile(rf"(? tuple[re.Match[str], ...]: + matches: tuple[re.Match[str], ...] = tuple(pattern.finditer(sql)) + if not matches: + return () + hidden: tuple[tuple[int, int], ...] = _comment_and_string_ranges(sql=sql, syntax=syntax) + return tuple(match for match in matches if not _inside(offset=match.start(), ranges=hidden)) + + +def _inside(*, offset: int, ranges: tuple[tuple[int, int], ...]) -> bool: + start: int + end: int + for start, end in ranges: + if start <= offset < end: + return True + return False + + +def _comment_and_string_ranges( + *, sql: str, syntax: dict[str, object] +) -> tuple[tuple[int, int], ...]: + scanner: SqlCodeScanner = SqlCodeScanner(sql, syntax, _SCANNER_CONTEXT) + ranges: list[tuple[int, int]] = [] + index: int = 0 + while index < len(sql): + end: int | None = scanner.non_code_end(index) + if end is None: + index += 1 + continue + if sql[index] not in _IDENTIFIER_QUOTES: + ranges.append((index, end)) + index = max(end, index + 1) + return tuple(ranges) + + +def _rewrite_chain_step( + *, step: ChainStep, replacements: dict[str, str], syntax: dict[str, object] +) -> ChainStep: + return replace( + step, + resolved_sql=rewrite_function_calls( + sql=step.resolved_sql, replacements=replacements, syntax=syntax + ), + expected_cte_sql=_rewrite_optional( + sql=step.expected_cte_sql, replacements=replacements, syntax=syntax + ), + lifted_ctes=_rewrite_ctes(ctes=step.lifted_ctes, replacements=replacements, syntax=syntax), + comparison_body_sql=_rewrite_optional( + sql=step.comparison_body_sql, replacements=replacements, syntax=syntax + ), + expected_lifted_ctes=_rewrite_ctes( + ctes=step.expected_lifted_ctes, replacements=replacements, syntax=syntax + ), + ) + + +def _rewrite_assertion_step( + *, step: SqlTestAssertionStep, replacements: dict[str, str], syntax: dict[str, object] +) -> SqlTestAssertionStep: + return replace( + step, + resolved_sql=rewrite_function_calls( + sql=step.resolved_sql, replacements=replacements, syntax=syntax + ), + lifted_ctes=_rewrite_ctes(ctes=step.lifted_ctes, replacements=replacements, syntax=syntax), + comparison_body_sql=_rewrite_optional( + sql=step.comparison_body_sql, replacements=replacements, syntax=syntax + ), + ) + + +def _rewrite_optional( + *, sql: str | None, replacements: dict[str, str], syntax: dict[str, object] +) -> str | None: + if sql is None: + return None + return rewrite_function_calls(sql=sql, replacements=replacements, syntax=syntax) + + +def _rewrite_ctes( + *, ctes: tuple[tuple[str, str], ...], replacements: dict[str, str], syntax: dict[str, object] +) -> tuple[tuple[str, str], ...]: + return tuple( + (name, rewrite_function_calls(sql=sql, replacements=replacements, syntax=syntax)) + for name, sql in ctes + ) diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index da9881cdde..7ff175af98 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -30,10 +30,16 @@ close_connections, open_worker_connections, ) +from sqlbuild.executor.pipeline._helpers.temporary_functions import ( + called_function_keys, + create_temporary_test_functions, + install_temporary_test_functions, + rewrite_test_entry, +) from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( raise_warehouse_connection_failure, ) -from sqlbuild.executor.pipeline.models import TestPipelineCallbacks +from sqlbuild.executor.pipeline.models import TemporaryTestFunctions, TestPipelineCallbacks from sqlbuild.executor.scheduling.classes.recovering_connection_pool import RecoveringConnectionPool from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.constants import SQL_TEST_EXECUTION_ERROR_CODE @@ -99,8 +105,14 @@ def run_test_pipeline( worker_count, elapsed_seconds=time.monotonic() - start ) active_error: BaseException | None = None + installed_functions: list[TemporaryTestFunctions] = [] recovery: ConnectionRecovery = ConnectionRecovery( - adapter=adapter, connect=lambda: adapter.connect(dict(connection_config)) + adapter=adapter, + connect=lambda: _connect_test_connection( + adapter=adapter, + connection_config=connection_config, + installed_functions=installed_functions, + ), ) try: preflight_start: float = time.monotonic() @@ -114,11 +126,27 @@ def run_test_pipeline( metadata={"item_count": len(plan.test_entries)}, attributes=OperationAttributes(phase="setup", target_kind="sql_test"), ): - missing_functions_by_test = _prepare_test_functions( + missing_functions_by_test, temporary_functions = _prepare_test_functions( plan=plan, adapter=adapter, connection=connections[0], ) + for ordinal, connection in enumerate(connections[1:], start=2): + install_temporary_test_functions( + adapter=adapter, + connection=connection, + functions=temporary_functions, + connection_label=f"test connection {ordinal} of {len(connections)}", + ) + installed_functions.append(temporary_functions) + entries = tuple( + rewrite_test_entry( + entry=entry, + replacements=temporary_functions.replacements, + syntax=adapter.sql_lexical_syntax.native_mapping, + ) + for entry in entries + ) if resolved_callbacks.on_progress is not None: resolved_callbacks.on_progress( f"Prepared test functions. ({time.monotonic() - preflight_start:.2f}s)" @@ -286,25 +314,67 @@ def _test_resource_name(entry: SqlTestPlanEntry) -> str: return entry.name +def _connect_test_connection( + *, + adapter: BaseAdapter, + connection_config: dict[str, object], + installed_functions: list[TemporaryTestFunctions], +) -> Any: + connection: Any = adapter.connect(dict(connection_config)) + try: + functions: TemporaryTestFunctions + for functions in installed_functions: + install_temporary_test_functions( + adapter=adapter, + connection=connection, + functions=functions, + connection_label="a replacement test connection", + ) + except BaseException: + adapter.close(connection) + raise + return connection + + def _prepare_test_functions( *, plan: PlanOutput, adapter: BaseAdapter, connection: Any, -) -> dict[CompiledObjectKey, tuple[str, ...]]: +) -> tuple[dict[CompiledObjectKey, tuple[str, ...]], TemporaryTestFunctions]: function_entries: dict[CompiledObjectKey, FunctionPlanEntry] = { entry.key: entry for entry in plan.function_entries } + distinct_deps: set[tuple[CompiledObjectKey, ...]] = { + test_entry.function_deps for test_entry in plan.test_entries + } + closures: dict[tuple[CompiledObjectKey, ...], tuple[CompiledObjectKey, ...]] = { + deps: _function_closure(plan=plan, keys=deps, adapter=adapter) for deps in distinct_deps + } + deps_by_test: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { + test_entry.key: closures[test_entry.function_deps] for test_entry in plan.test_entries + } required_targets_by_key: dict[CompiledObjectKey, CompiledRelationLocation] = {} test_entry: SqlTestPlanEntry for test_entry in plan.test_entries: dep: CompiledObjectKey - for dep in test_entry.function_deps: + for dep in deps_by_test[test_entry.key]: function_target: CompiledRelationLocation | None = plan.function_locations.get(dep.name) if function_target is not None: required_targets_by_key[dep] = function_target if not required_targets_by_key: - return {} + return {}, TemporaryTestFunctions() + temporary_functions: TemporaryTestFunctions = create_temporary_test_functions( + function_entries=plan.function_entries, + required_keys=tuple(required_targets_by_key), + adapter=adapter, + connection=connection, + ) + created_key: CompiledObjectKey + for created_key in temporary_functions.created_keys: + _ = required_targets_by_key.pop(created_key, None) + if not required_targets_by_key: + return {}, temporary_functions existing_function_names: set[tuple[str | None, str | None, str]] = set() for database, schema, names in _group_function_names(required_targets_by_key.values()): @@ -333,7 +403,9 @@ def _prepare_test_functions( _normalize_required_name(target.name), ) if function_key not in existing_function_names: - missing_by_key[key] = key.name + missing_by_key[key] = _missing_function_label( + name=key.name, reason=temporary_functions.failures.get(key) + ) _register_connection_scoped_python_functions( missing_by_key=missing_by_key, @@ -345,11 +417,26 @@ def _prepare_test_functions( missing_by_test: dict[CompiledObjectKey, tuple[str, ...]] = {} for test_entry in plan.test_entries: missing_names: tuple[str, ...] = tuple( - missing_by_key[dep] for dep in test_entry.function_deps if dep in missing_by_key + missing_by_key[dep] for dep in deps_by_test[test_entry.key] if dep in missing_by_key ) if missing_names: missing_by_test[test_entry.key] = missing_names - return missing_by_test + return missing_by_test, temporary_functions + + +def _function_closure( + *, plan: PlanOutput, keys: tuple[CompiledObjectKey, ...], adapter: BaseAdapter +) -> tuple[CompiledObjectKey, ...]: + called: tuple[CompiledObjectKey, ...] = called_function_keys( + function_entries=plan.function_entries, keys=keys, adapter=adapter + ) + return tuple(dict.fromkeys((*keys, *called))) + + +def _missing_function_label(*, name: str, reason: str | None) -> str: + if reason is None: + return name + return f"{name} ({reason})" def _register_connection_scoped_python_functions( diff --git a/src/sqlbuild/executor/pipeline/exceptions.py b/src/sqlbuild/executor/pipeline/exceptions.py index 889f24321c..e56371e14d 100644 --- a/src/sqlbuild/executor/pipeline/exceptions.py +++ b/src/sqlbuild/executor/pipeline/exceptions.py @@ -45,3 +45,20 @@ def __init__(self, *, adapter_name: str, reason: str) -> None: "Check the connection settings of the target; `sqb debug` tests the connection." ) super().__init__(self.message) + + +class TemporaryTestFunctionError(Exception): + """Raised when a test connection cannot create the temporary function copies tests call.""" + + code: str = "T006" + + def __init__(self, *, function_name: str, connection: str, reason: str) -> None: + self.message: str = ( + f"could not create the temporary copy of function {function_name} on {connection}: " + f"{reason}" + ) + self.help: str = ( + "The tests call the temporary copies, which the first test connection created; " + "check that every connection of the target may create temporary functions." + ) + super().__init__(self.message) diff --git a/src/sqlbuild/executor/pipeline/models.py b/src/sqlbuild/executor/pipeline/models.py index 9190a261b0..c821813fe3 100644 --- a/src/sqlbuild/executor/pipeline/models.py +++ b/src/sqlbuild/executor/pipeline/models.py @@ -3,10 +3,10 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any -from sqlbuild._native import AuditPlanEntry, SqlTestPlanEntry +from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, SqlTestPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildCallbacks, @@ -43,6 +43,17 @@ class TestPipelineCallbacks: on_test_complete: Callable[[SqlTestExecutionResult], None] | None = None +@dataclass(frozen=True) +class TemporaryTestFunctions: + """Temporary function copies created for one test run and the functions left without one.""" + + statements: tuple[tuple[str, str], ...] = () + """Each `(function name, statement)` that created a copy, in creation order.""" + created_keys: frozenset[CompiledObjectKey] = frozenset() + replacements: dict[str, str] = field(default_factory=dict) + failures: dict[CompiledObjectKey, str] = field(default_factory=dict) + + @dataclass(frozen=True) class AuditExecutionFailure: """Safe public description of one standalone audit runtime failure.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 4a49c1b251..a8353abd4d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -438,3 +438,35 @@ class TransactionTimeoutE2ETestCase: expected_output_fragments: tuple[str, ...] expected_published_rows: tuple[tuple[int, int], ...] expected_later_model_rows: tuple[tuple[int], ...] + + +@dataclass(frozen=True) +class TemporaryFunctionE2ETestCase: + """Test case for unit tests that call an edited project function.""" + + description: str + built_body: str + edited_body: str + expected_exit_code: int + expected_stdout_fragment: str + expected_built_result: bool + + +@dataclass(frozen=True) +class TemporaryCalleeE2ETestCase: + """Test case for unit tests whose function calls another project function.""" + + description: str + build_first: bool + callee_body: str + expected_exit_code: int + expected_stdout_fragment: str + + +@dataclass(frozen=True) +class TemporaryFunctionConnectionE2ETestCase: + """Test case for a test connection that cannot create the temporary copies.""" + + description: str + expected_exit_code: int + expected_stderr_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_temporary_functions.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_temporary_functions.py new file mode 100644 index 0000000000..7ca584b869 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_temporary_functions.py @@ -0,0 +1,312 @@ +"""Unit tests call the project's current function definitions without a build.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + TemporaryCalleeE2ETestCase, + TemporaryFunctionConnectionE2ETestCase, + TemporaryFunctionE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_FUNCTION_PATH: str = "functions/sql/is_completed_order.sql" +_FUNCTION_HEADER: str = ( + "FUNCTION (description 'Whether an order status counts as completed.', " + "arguments (order_status VARCHAR), returns BOOLEAN);\n" +) +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders_temporary_functions"\nadapter = "duckdb"\n' + '[connection]\ndatabase = "orders.duckdb"\n' + ), + "models/order_statuses.sql": ( + "MODEL (description 'Test model order_statuses.', materialized table);\n" + "SELECT 'completed' AS order_status\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n" + 'SELECT order_status, __udf("is_completed_order")(order_status) AS is_completed ' + 'FROM __ref("order_statuses")\n' + ), + "tests/unit/orders_case.sql": ( + 'TEST (name "orders_case");\nWITH\n' + "__ref__order_statuses AS (SELECT 'completed' AS order_status),\n" + "__expected__orders AS (SELECT 'completed' AS order_status, TRUE AS is_completed)\n" + "SELECT 1\n" + ), +} + +_CALLEE_PATH: str = "functions/sql/status_matches.sql" +_CALLEE_HEADER: str = ( + "FUNCTION (description 'Whether an order status is the completed one.', " + "arguments (order_status VARCHAR), returns BOOLEAN);\n" +) +_CALLER_FILES: dict[str, str] = { + **_PROJECT_FILES, + _FUNCTION_PATH: _FUNCTION_HEADER + '__udf("status_matches")(order_status)', +} +_QUOTED_CALL_FILES: dict[str, str] = { + **_PROJECT_FILES, + _FUNCTION_PATH: _FUNCTION_HEADER + "order_status = 'completed'", + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n" + 'SELECT order_status, __udf("is_completed_order")(order_status) AS is_completed, ' + "length('main.is_completed_order(order_status)') AS formula_length " + "-- main.is_completed_order(order_status)\n" + 'FROM __ref("order_statuses")\n' + ), + "tests/unit/orders_case.sql": ( + 'TEST (name "orders_case");\nWITH\n' + "__ref__order_statuses AS (SELECT 'completed' AS order_status),\n" + "__expected__orders AS (SELECT 'completed' AS order_status, TRUE AS is_completed, " + "37 AS formula_length)\n" + "SELECT 1\n" + ), +} +_SECOND_CONNECTION_FAILS: dict[str, str] = { + **_PROJECT_FILES, + "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"].replace( + 'adapter = "duckdb"', 'adapter = "duckdb_one_session"' + ), + _FUNCTION_PATH: _FUNCTION_HEADER + "order_status = 'completed'", + "tests/unit/orders_again.sql": _PROJECT_FILES["tests/unit/orders_case.sql"].replace( + "orders_case", "orders_again" + ), + "adapters/duckdb_one_session.py": ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class DuckDbOneSessionAdapter(DuckDbAdapter):\n" + ' adapter_name = "duckdb_one_session"\n' + " macro_sessions = []\n\n" + " def execute(self, *, connection, sql):\n" + ' if sql.startswith("CREATE OR REPLACE TEMP MACRO"):\n' + " if not self.macro_sessions:\n" + " self.macro_sessions.append(connection)\n" + " if connection is not self.macro_sessions[0]:\n" + ' raise PermissionError("temporary macros are disabled on this session")\n' + " return super().execute(connection=connection, sql=sql)\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + TemporaryFunctionE2ETestCase( + description="edited function body decides the test without a rebuild", + built_body="order_status = 'completed'", + edited_body="order_status = 'shipped'", + expected_exit_code=1, + expected_stdout_fragment="FAIL=1", + expected_built_result=True, + ), + TemporaryFunctionE2ETestCase( + description="fixed function body passes before the built copy is replaced", + built_body="order_status = 'shipped'", + edited_body="order_status = 'completed'", + expected_exit_code=0, + expected_stdout_fragment="PASS=1", + expected_built_result=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_edited_function_when_testing_then_current_definition_runs_and_build_is_untouched( + test_case: TemporaryFunctionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_temporary_functions", + repo_files={**_PROJECT_FILES, _FUNCTION_PATH: _FUNCTION_HEADER + test_case.built_body}, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("build", "--no-tests", "--no-audits"), project_dir=project_dir + ) + _ = (project_dir / _FUNCTION_PATH).write_text( + _FUNCTION_HEADER + test_case.edited_body, encoding="utf-8" + ) + + tested: subprocess.CompletedProcess[str] = run_sqb(command=("test",), project_dir=project_dir) + + assert built.returncode == 0, built.stdout + built.stderr + built_rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "orders.duckdb", + sql="SELECT main.is_completed_order('completed')", + ) + assert tested.returncode == test_case.expected_exit_code, ( + test_case.description, + tested.stdout + tested.stderr, + ) + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description + assert built_rows == [(test_case.expected_built_result,)], test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + TemporaryFunctionE2ETestCase( + description="never-built function runs from its temporary copy", + built_body="order_status = 'completed'", + edited_body="order_status = 'completed'", + expected_exit_code=0, + expected_stdout_fragment="PASS=1", + expected_built_result=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_unbuilt_function_when_testing_then_test_passes_and_no_function_is_created( + test_case: TemporaryFunctionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_temporary_functions", + repo_files={**_PROJECT_FILES, _FUNCTION_PATH: _FUNCTION_HEADER + test_case.edited_body}, + ) + execute_duckdb(db_path=project_dir / "orders.duckdb", sql="SELECT 1") + + tested: subprocess.CompletedProcess[str] = run_sqb(command=("test",), project_dir=project_dir) + + function_rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "orders.duckdb", + sql=( + "SELECT count(*) > 0 FROM duckdb_functions() WHERE function_name = 'is_completed_order'" + ), + ) + assert tested.returncode == test_case.expected_exit_code, ( + test_case.description, + tested.stdout + tested.stderr, + ) + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description + assert function_rows == [(test_case.expected_built_result,)], test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + TemporaryCalleeE2ETestCase( + description="edited callee decides the caller's test without a rebuild", + build_first=True, + callee_body="order_status = 'shipped'", + expected_exit_code=1, + expected_stdout_fragment="FAIL=1", + ), + TemporaryCalleeE2ETestCase( + description="never-built caller and callee run from temporary copies", + build_first=False, + callee_body="order_status = 'completed'", + expected_exit_code=0, + expected_stdout_fragment="PASS=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_function_calling_another_when_testing_then_callee_current_definition_runs( + test_case: TemporaryCalleeE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_temporary_callee", + repo_files={**_CALLER_FILES, _CALLEE_PATH: _CALLEE_HEADER + "order_status = 'completed'"}, + ) + commands: dict[bool, tuple[tuple[str, ...], ...]] = { + True: (("build", "--no-tests", "--no-audits"),), + False: (), + } + built: tuple[subprocess.CompletedProcess[str], ...] = tuple( + run_sqb(command=command, project_dir=project_dir) + for command in commands[test_case.build_first] + ) + _ = (project_dir / _CALLEE_PATH).write_text( + _CALLEE_HEADER + test_case.callee_body, encoding="utf-8" + ) + + tested: subprocess.CompletedProcess[str] = run_sqb(command=("test",), project_dir=project_dir) + + assert all(result.returncode == 0 for result in built), test_case.description + assert tested.returncode == test_case.expected_exit_code, ( + test_case.description, + tested.stdout + tested.stderr, + ) + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + TemporaryFunctionConnectionE2ETestCase( + description="second test connection refusing the copies is a coded error", + expected_exit_code=1, + expected_stderr_fragment=( + "error[T006]: could not create the temporary copy of function " + "is_completed_order on test connection 2 of 2: " + "temporary macros are disabled on this session" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_connection_refusing_copies_when_testing_concurrently_then_reports_coded_error( + test_case: TemporaryFunctionConnectionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_temporary_functions", + repo_files=_SECOND_CONNECTION_FAILS, + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test", "--concurrency", "2"), project_dir=project_dir + ) + + output: str = tested.stdout + tested.stderr + assert tested.returncode == test_case.expected_exit_code, (test_case.description, output) + assert test_case.expected_stderr_fragment in tested.stderr, (test_case.description, output) + assert "Traceback" not in output, (test_case.description, output) + + +@pytest.mark.parametrize( + "test_case", + [ + TemporaryFunctionE2ETestCase( + description="function calls quoted in strings and comments keep their text", + built_body="order_status = 'completed'", + edited_body="order_status = 'completed'", + expected_exit_code=0, + expected_stdout_fragment="PASS=1", + expected_built_result=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_function_call_in_string_when_testing_then_only_code_calls_are_redirected( + test_case: TemporaryFunctionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_temporary_functions", + repo_files=_QUOTED_CALL_FILES, + ) + + tested: subprocess.CompletedProcess[str] = run_sqb(command=("test",), project_dir=project_dir) + + assert tested.returncode == test_case.expected_exit_code, ( + test_case.description, + tested.stdout + tested.stderr, + ) + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/_test_types.py index 0bb976dcc0..099e407f97 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/_test_types.py @@ -1,5 +1,9 @@ from dataclasses import dataclass +from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter +from sqlbuild.adapter.contract.models import TemporaryFunction +from sqlbuild.compiler.compile.types import FunctionLanguage + @dataclass(frozen=True) class StrictAdapterContractTestCase: @@ -25,6 +29,7 @@ class OptionalStrictMethodTestCase: description: str expected_optional_methods: frozenset[str] expected_output_relation: None + expected_temporary_function: None @dataclass(frozen=True) @@ -33,3 +38,12 @@ class StrictAdapterInlineSqlTestTestCase: expected_total: int expected_outcome: str expected_statement_count: int + + +@dataclass(frozen=True) +class TemporaryFunctionRenderingTestCase: + description: str + adapter: StrictAdapter + language: FunctionLanguage + return_columns: tuple[tuple[str, str], ...] + expected_temporary_function: TemporaryFunction | None diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index f1951d4d43..3abd653077 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -17,6 +17,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import FunctionDefinition, TemporaryFunction from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.pipeline._helpers.testing import run_test_pipeline from sqlbuild.executor.testing.models import SqlTestExecutionResult @@ -84,8 +85,15 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ [ OptionalStrictMethodTestCase( description="unit-test output storage is optional and defaults to inline", - expected_optional_methods=frozenset({"create_sql_test_output", "drop_sql_test_output"}), + expected_optional_methods=frozenset( + { + "create_sql_test_output", + "drop_sql_test_output", + "render_create_temporary_function", + } + ), expected_output_relation=None, + expected_temporary_function=None, ) ], ids=lambda case: case.description, @@ -100,10 +108,17 @@ def test_given_strict_adapter_when_reading_optional_methods_then_defaults_keep_i connection=None, sql="SELECT 1", name="orders_output" ) adapter.drop_sql_test_output(connection=None, relation="orders_output") + temporary: TemporaryFunction | None = adapter.render_create_temporary_function( + name="is_completed_order", + definition=FunctionDefinition( + destination="main.is_completed_order", arguments=(), returns="BOOLEAN", body_sql="TRUE" + ), + ) assert test_case.expected_optional_methods.isdisjoint(abstract_methods) assert test_case.expected_optional_methods <= frozenset(StrictAdapter.__dict__) assert relation is test_case.expected_output_relation + assert temporary is test_case.expected_temporary_function @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_temporary_function_rendering.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_temporary_function_rendering.py new file mode 100644 index 0000000000..3fe66e580b --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_temporary_function_rendering.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import pytest + +from sqlbuild._native import FunctionArgument, FunctionReturnColumn +from sqlbuild.adapter.contract.models import FunctionDefinition, TemporaryFunction +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.types import FunctionLanguage +from tests.unit.src.sqlbuild.adapter.contract.classes.strict_adapter._test_types import ( + TemporaryFunctionRenderingTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + TemporaryFunctionRenderingTestCase( + description="duckdb copies a sql function into a temporary macro", + adapter=DuckDbAdapter(), + language=FunctionLanguage.SQL, + return_columns=(), + expected_temporary_function=TemporaryFunction( + target='temp.main."is_completed_order"', + statements=( + 'CREATE OR REPLACE TEMP MACRO "is_completed_order"(order_status) AS (\n' + "order_status = 'completed'\n)", + ), + ), + ), + TemporaryFunctionRenderingTestCase( + description="duckdb copies a table function into a temporary table macro", + adapter=DuckDbAdapter(), + language=FunctionLanguage.SQL, + return_columns=(("order_id", "INTEGER"),), + expected_temporary_function=TemporaryFunction( + target='temp.main."is_completed_order"', + statements=( + 'CREATE OR REPLACE TEMP MACRO "is_completed_order"(order_status) AS TABLE\n' + "order_status = 'completed'", + ), + ), + ), + TemporaryFunctionRenderingTestCase( + description="duckdb leaves python functions to per-connection registration", + adapter=DuckDbAdapter(), + language=FunctionLanguage.PYTHON, + return_columns=(), + expected_temporary_function=None, + ), + TemporaryFunctionRenderingTestCase( + description="postgres creates the copy in the session pg_temp schema", + adapter=PostgresAdapter(), + language=FunctionLanguage.SQL, + return_columns=(), + expected_temporary_function=TemporaryFunction( + target='pg_temp."is_completed_order"', + statements=( + 'CREATE OR REPLACE FUNCTION pg_temp."is_completed_order"(order_status TEXT)\n' + "RETURNS BOOLEAN\n" + "LANGUAGE SQL AS $$\n" + "order_status = 'completed'\n" + "$$", + ), + ), + ), + TemporaryFunctionRenderingTestCase( + description="snowflake creates a prefixed temporary function in the current schema", + adapter=SnowflakeAdapter(), + language=FunctionLanguage.SQL, + return_columns=(), + expected_temporary_function=TemporaryFunction( + target='"__SQB_TEST__IS_COMPLETED_ORDER"', + statements=( + "CREATE OR REPLACE TEMPORARY FUNCTION " + '"__SQB_TEST__IS_COMPLETED_ORDER"(order_status TEXT)\n' + "RETURNS BOOLEAN\n" + "LANGUAGE SQL\n" + "AS $$\n" + "order_status = 'completed'\n" + "$$", + ), + ), + ), + TemporaryFunctionRenderingTestCase( + description="bigquery has no session to hold a temporary function", + adapter=BigQueryAdapter(), + language=FunctionLanguage.SQL, + return_columns=(), + expected_temporary_function=None, + ), + TemporaryFunctionRenderingTestCase( + description="databricks creates a prefixed session temporary function", + adapter=DatabricksAdapter(), + language=FunctionLanguage.SQL, + return_columns=(), + expected_temporary_function=TemporaryFunction( + target="`__sqb_test__is_completed_order`", + statements=( + "CREATE OR REPLACE TEMPORARY FUNCTION " + "`__sqb_test__is_completed_order`(order_status TEXT)\n" + "RETURNS BOOLEAN\n" + "RETURN order_status = 'completed'", + ), + ), + ), + TemporaryFunctionRenderingTestCase( + description="databricks keeps using built python functions", + adapter=DatabricksAdapter(), + language=FunctionLanguage.PYTHON, + return_columns=(), + expected_temporary_function=None, + ), + TemporaryFunctionRenderingTestCase( + description="sql server keeps using the built function", + adapter=SqlServerAdapter(), + language=FunctionLanguage.SQL, + return_columns=(), + expected_temporary_function=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_function_when_rendering_temporary_copy_then_adapter_returns_session_ddl( + test_case: TemporaryFunctionRenderingTestCase, +) -> None: + temporary: TemporaryFunction | None = test_case.adapter.render_create_temporary_function( + name="is_completed_order", + definition=FunctionDefinition( + destination="analytics.is_completed_order", + arguments=(FunctionArgument(name="order_status", type="TEXT"),), + returns="BOOLEAN", + body_sql="order_status = 'completed'", + return_columns=tuple( + FunctionReturnColumn(name=name, type=type_name) + for name, type_name in test_case.return_columns + ), + language=test_case.language, + ), + ) + + assert temporary == test_case.expected_temporary_function, test_case.description diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py index ea148bd256..7853c224f1 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py @@ -185,3 +185,20 @@ class ScenarioWorkerExitTestCase: description: str worker_count: int expected_returncode: int + + +@dataclass(frozen=True) +class TemporaryTestFunctionTestCase: + description: str + body_sql: str + test_count: int + max_concurrency: int + expected_outcomes: tuple[str, ...] + expected_error_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class FunctionCallRewriteTestCase: + description: str + sql: str + expected_sql: str diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 4b292bd609..9c59b2342d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -4,12 +4,14 @@ import threading from collections.abc import Callable +from dataclasses import replace from pathlib import Path from typing import Any, ClassVar from unittest.mock import MagicMock from sqlbuild._native import ( AuditPlanEntry, + ChainStep, CompiledObjectKey, CompiledProject, CompiledRelationLocation, @@ -17,6 +19,8 @@ CompiledSqlScenario, CompilePipelineResult, DiscoveredSqlScenarioFile, + FunctionArgument, + FunctionPlanEntry, PlanOutput, ScenarioExecutionPlan, ScenarioGraphPlan, @@ -25,6 +29,7 @@ SeedCsvSettings, SeedPlanEntry, SqlLexicalSyntax, + SqlTestPlanEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -320,3 +325,87 @@ def load_snapshot_failure(**_kwargs: object) -> ScenarioLocalSnapshotLoadResult: True: load_snapshot_success, False: load_snapshot_failure, }[test_case.load_error_message is None] + + +def order_status_function_test_plan(*, body_sql: str, test_count: int) -> PlanOutput: + """Plan of tests calling one unbuilt project function whose current body is `body_sql`.""" + + function_key: CompiledObjectKey = CompiledObjectKey( + resource_type=CompiledResourceType.UDF, name="is_completed_order" + ) + destination: CompiledRelationLocation = CompiledRelationLocation( + database=None, + schema="analytics", + name="is_completed_order", + qualified_name="analytics.is_completed_order", + ) + return PlanOutput( + test_entries=tuple( + SqlTestPlanEntry( + key=CompiledObjectKey( + resource_type=CompiledResourceType.SQL_TEST, name=f"test_orders_{index}" + ), + name=f"test_orders_{index}", + chain=( + ChainStep( + model_name="orders", + resolved_sql=( + "SELECT analytics.is_completed_order ('completed') AS is_completed" + ), + expected_cte_sql="SELECT TRUE AS is_completed", + ), + ), + function_deps=(function_key,), + ) + for index in range(test_count) + ), + function_entries=( + FunctionPlanEntry( + key=function_key, + name="is_completed_order", + relative_path=Path("functions/sql/is_completed_order.sql"), + destination=destination, + arguments=(FunctionArgument(name="order_status", type="VARCHAR"),), + returns="BOOLEAN", + body_sql=body_sql, + fingerprint_query_sql=body_sql, + fingerprint_destination=destination, + ), + ), + function_locations={"is_completed_order": destination}, + ) + + +def order_status_callee_test_plan(*, callee_body_sql: str, test_count: int) -> PlanOutput: + """Plan of tests calling `is_completed_order`, which calls `status_matches` with this body.""" + + plan: PlanOutput = order_status_function_test_plan( + body_sql="analytics.status_matches(order_status)", test_count=test_count + ) + callee_key: CompiledObjectKey = CompiledObjectKey( + resource_type=CompiledResourceType.UDF, name="status_matches" + ) + callee_destination: CompiledRelationLocation = CompiledRelationLocation( + database=None, + schema="analytics", + name="status_matches", + qualified_name="analytics.status_matches", + ) + return replace( + plan, + function_entries=( + *plan.function_entries, + FunctionPlanEntry( + key=callee_key, + name="status_matches", + relative_path=Path("functions/sql/status_matches.sql"), + destination=callee_destination, + arguments=(FunctionArgument(name="order_status", type="VARCHAR"),), + returns="BOOLEAN", + body_sql=callee_body_sql, + fingerprint_query_sql=callee_body_sql, + fingerprint_destination=callee_destination, + ), + ), + function_locations={**plan.function_locations, "status_matches": callee_destination}, + ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_temporary_functions.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_temporary_functions.py new file mode 100644 index 0000000000..b3057f1563 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_temporary_functions.py @@ -0,0 +1,75 @@ +"""Tests for rewriting test SQL onto temporary function copies.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.pipeline._helpers.temporary_functions import rewrite_function_calls +from tests.unit.src.sqlbuild.executor.pipeline._helpers._test_types import ( + FunctionCallRewriteTestCase, +) + +_REPLACEMENTS: dict[str, str] = { + "analytics.is_completed_order": 'temp.main."is_completed_order"', + '"warehouse"."analytics"."customer_orders"': 'temp.main."customer_orders"', +} + + +@pytest.mark.parametrize( + "test_case", + [ + FunctionCallRewriteTestCase( + description="scalar call moves to the temporary copy", + sql="SELECT analytics.is_completed_order(status) FROM orders", + expected_sql='SELECT temp.main."is_completed_order"(status) FROM orders', + ), + FunctionCallRewriteTestCase( + description="whitespace before the argument list still matches", + sql="SELECT analytics.is_completed_order (status)", + expected_sql='SELECT temp.main."is_completed_order" (status)', + ), + FunctionCallRewriteTestCase( + description="quoted table function target moves to the temporary copy", + sql='SELECT * FROM "warehouse"."analytics"."customer_orders"(7)', + expected_sql='SELECT * FROM temp.main."customer_orders"(7)', + ), + FunctionCallRewriteTestCase( + description="longer names and non-call mentions stay unchanged", + sql=( + "SELECT staging.analytics.is_completed_order(status), " + "analytics.is_completed_order_v2(status), " + "'analytics.is_completed_order' AS label" + ), + expected_sql=( + "SELECT staging.analytics.is_completed_order(status), " + "analytics.is_completed_order_v2(status), " + "'analytics.is_completed_order' AS label" + ), + ), + FunctionCallRewriteTestCase( + description="calls inside string literals and comments stay unchanged", + sql=( + "SELECT analytics.is_completed_order(status), " + "'analytics.is_completed_order(x)' AS label -- analytics.is_completed_order(y)\n" + "/* analytics.is_completed_order(z) */ FROM orders" + ), + expected_sql=( + 'SELECT temp.main."is_completed_order"(status), ' + "'analytics.is_completed_order(x)' AS label -- analytics.is_completed_order(y)\n" + "/* analytics.is_completed_order(z) */ FROM orders" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_deployed_function_calls_when_rewriting_then_only_calls_move_to_temporary_copies( + test_case: FunctionCallRewriteTestCase, +) -> None: + rewritten: str = rewrite_function_calls( + sql=test_case.sql, + replacements=_REPLACEMENTS, + syntax=DuckDbAdapter.sql_lexical_syntax.native_mapping, + ) + + assert rewritten == test_case.expected_sql, test_case.description diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py index 8690416f6c..386b1b60c2 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py @@ -32,9 +32,12 @@ SqlTestConcurrencyTestCase, SqlTestFunctionPreflightTestCase, SqlTestOperationLifecycleTestCase, + TemporaryTestFunctionTestCase, ) from tests.unit.src.sqlbuild.executor.pipeline._helpers.helpers import ( lifecycle_events_with_prefix, + order_status_callee_test_plan, + order_status_function_test_plan, ) @@ -132,6 +135,129 @@ def record_event(event: LifecycleEvent) -> None: ) +@pytest.mark.parametrize( + "test_case", + [ + TemporaryTestFunctionTestCase( + description="unbuilt function runs from its temporary copy", + body_sql="order_status = 'completed'", + test_count=1, + max_concurrency=1, + expected_outcomes=("pass",), + expected_error_fragments=(), + ), + TemporaryTestFunctionTestCase( + description="every worker connection holds the temporary copy", + body_sql="order_status = 'completed'", + test_count=2, + max_concurrency=2, + expected_outcomes=("pass", "pass"), + expected_error_fragments=(), + ), + TemporaryTestFunctionTestCase( + description="current definition decides the outcome", + body_sql="order_status = 'cancelled'", + test_count=1, + max_concurrency=1, + expected_outcomes=("fail",), + expected_error_fragments=(), + ), + TemporaryTestFunctionTestCase( + description="uncreatable copy reports why and asks for a build", + body_sql="order_status = = 'completed'", + test_count=1, + max_concurrency=1, + expected_outcomes=("error",), + expected_error_fragments=( + "its temporary copy for the test run could not be created", + "Run `sqb build` first", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unbuilt_function_when_running_tests_then_temporary_copy_holds_current_definition( + test_case: TemporaryTestFunctionTestCase, + tmp_path: Path, +) -> None: + results: tuple[SqlTestExecutionResult, ...] = run_test_pipeline( + plan=order_status_function_test_plan( + body_sql=test_case.body_sql, test_count=test_case.test_count + ), + connection_config={"database": str(tmp_path / "orders.duckdb")}, + adapter=DuckDbAdapter(), + max_concurrency=test_case.max_concurrency, + ) + + messages: str = " ".join(str(result.error_message) for result in results) + assert tuple(str(result.outcome) for result in results) == test_case.expected_outcomes, ( + test_case.description, + messages, + ) + assert all(fragment in messages for fragment in test_case.expected_error_fragments), ( + test_case.description, + messages, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + TemporaryTestFunctionTestCase( + description="callee's current definition decides the caller's test", + body_sql="order_status = 'cancelled'", + test_count=1, + max_concurrency=1, + expected_outcomes=("fail",), + expected_error_fragments=(), + ), + TemporaryTestFunctionTestCase( + description="callee copied on every worker connection", + body_sql="order_status = 'completed'", + test_count=2, + max_concurrency=2, + expected_outcomes=("pass", "pass"), + expected_error_fragments=(), + ), + TemporaryTestFunctionTestCase( + description="caller without a copyable callee reports the callee", + body_sql="order_status = = 'completed'", + test_count=1, + max_concurrency=1, + expected_outcomes=("error",), + expected_error_fragments=( + "function status_matches, which it calls, has no temporary copy", + "status_matches (its temporary copy for the test run could not be created", + "Run `sqb build` first", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unbuilt_callee_when_running_tests_then_callee_copy_holds_current_definition( + test_case: TemporaryTestFunctionTestCase, + tmp_path: Path, +) -> None: + results: tuple[SqlTestExecutionResult, ...] = run_test_pipeline( + plan=order_status_callee_test_plan( + callee_body_sql=test_case.body_sql, test_count=test_case.test_count + ), + connection_config={"database": str(tmp_path / "orders.duckdb")}, + adapter=DuckDbAdapter(), + max_concurrency=test_case.max_concurrency, + ) + + messages: str = " ".join(str(result.error_message) for result in results) + assert tuple(str(result.outcome) for result in results) == test_case.expected_outcomes, ( + test_case.description, + messages, + ) + assert all(fragment in messages for fragment in test_case.expected_error_fragments), ( + test_case.description, + messages, + ) + + @pytest.mark.parametrize( "test_case", ( diff --git a/website/src/content/docs/docs/concepts/adapters.mdx b/website/src/content/docs/docs/concepts/adapters.mdx index a05d0ed3a0..7e9c018df0 100644 --- a/website/src/content/docs/docs/concepts/adapters.mdx +++ b/website/src/content/docs/docs/concepts/adapters.mdx @@ -95,7 +95,7 @@ Set `sql_analysis_dialect_name` to the SQL dialect name that matches your engine For full control with no inherited defaults, subclass `StrictAdapter` instead. Every method is abstract and must be implemented explicitly. SQLBuild raises a clear error listing any unimplemented methods. -A few `StrictAdapter` methods are optional and have a documented default. `create_sql_test_output` and `drop_sql_test_output` let an adapter store a unit test's actual output once so every comparison reads it (see [Single-read comparison](/docs/concepts/testing#single-read-comparison)). By default `create_sql_test_output` returns `None`, and SQLBuild runs the test's inline comparison query instead, so adapters that do not implement them keep working unchanged. +A few `StrictAdapter` methods are optional and have a documented default. `create_sql_test_output` and `drop_sql_test_output` let an adapter store a unit test's actual output once so every comparison reads it (see [Single-read comparison](/docs/concepts/testing#single-read-comparison)). By default `create_sql_test_output` returns `None`, and SQLBuild runs the test's inline comparison query instead, so adapters that do not implement them keep working unchanged. `render_create_temporary_function` returns the call target and DDL for a session-scoped copy of a project function, which `sqb test` calls instead of the built function (see [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test)). By default it returns `None`, and tests call the built function. ### Discovery rules diff --git a/website/src/content/docs/docs/concepts/adapters/bigquery.mdx b/website/src/content/docs/docs/concepts/adapters/bigquery.mdx index 3c9ec55a5d..9bcc286861 100644 --- a/website/src/content/docs/docs/concepts/adapters/bigquery.mdx +++ b/website/src/content/docs/docs/concepts/adapters/bigquery.mdx @@ -66,3 +66,8 @@ NUMERIC '2.4700' BigQuery does not support arrays of arrays. A nested list or set requested with `render_as array` fails at compile time rather than producing invalid BigQuery SQL. Parenthesized `value_list` rendering remains available for ordinary scalar collections used with `IN`. See [Collections and Rendering](/docs/concepts/constants/collections-and-rendering#native-array-rendering) for declarations, rendering precedence, and the cross-adapter matrix. + +## Functions in unit tests + +BigQuery temporary functions exist only inside one script, so `sqb test` calls the built function. +Run `sqb build` before testing a new or changed function. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/adapters/databricks.mdx b/website/src/content/docs/docs/concepts/adapters/databricks.mdx index a1ef6f4f4a..c017c54238 100644 --- a/website/src/content/docs/docs/concepts/adapters/databricks.mdx +++ b/website/src/content/docs/docs/concepts/adapters/databricks.mdx @@ -46,3 +46,9 @@ The target's `database` selects the Unity Catalog catalog, and its `schema` sele On connect, SQLBuild runs `USE CATALOG` and `USE SCHEMA` statements from the active target to set the authoritative session namespace. + +## Functions in unit tests + +`sqb test` creates each SQL UDF and table function the selected tests call as a session +`TEMPORARY FUNCTION __sqb_test__`. Python UDFs cannot be temporary, so tests call the built +Python function. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index c42ac865a7..90b0e70a7d 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -57,3 +57,9 @@ DuckDB defaults to `staged` promotion: tables are materialized into a staging ta [settings] table_promotion_mode = "staged" ``` + +## Functions in unit tests + +`sqb test` copies each SQL UDF and table function the selected tests call into a `CREATE TEMP +MACRO` on every test connection, called as `temp.main.`, so tests use the current project +definition without a build. Python UDFs are registered on each test connection. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/adapters/motherduck.mdx b/website/src/content/docs/docs/concepts/adapters/motherduck.mdx index 380ae3bf0c..40963c1323 100644 --- a/website/src/content/docs/docs/concepts/adapters/motherduck.mdx +++ b/website/src/content/docs/docs/concepts/adapters/motherduck.mdx @@ -74,3 +74,9 @@ schema = "dev" ``` This lets you develop and test locally with zero MotherDuck compute cost, then deploy to MotherDuck in production. SQLBuild's [scenario replay](/docs/concepts/scenarios) also runs locally in DuckDB regardless of the production adapter. + +## Functions in unit tests + +`sqb test` copies each SQL UDF and table function the selected tests call into a session +`TEMP MACRO`, the same as [DuckDB](/docs/concepts/adapters/duckdb#functions-in-unit-tests), so the +functions in your MotherDuck database are not replaced. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/adapters/postgres.mdx b/website/src/content/docs/docs/concepts/adapters/postgres.mdx index 66752859c7..e0bb8c2153 100644 --- a/website/src/content/docs/docs/concepts/adapters/postgres.mdx +++ b/website/src/content/docs/docs/concepts/adapters/postgres.mdx @@ -61,3 +61,10 @@ connection = "postgres" database = "analytics" schema = "dev" ``` + +## Functions in unit tests + +`sqb test` creates each SQL function the selected tests call in the session's `pg_temp` schema and +calls it as `pg_temp.`, so tests use the current project definition without a build. Temporary +functions belong to one server session: behind a transaction-mode pooler, run `sqb test` through a +direct connection or a session-mode pool. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 9bac82061c..76d70017ad 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -124,3 +124,10 @@ not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60 cloud-services credits, contract adjustments, or multi-cluster billing. Run metadata and query IDs are stored under `target/executions//`; that statement ledger stores only an SQL digest, not SQL text. Executed SQL artifacts are stored separately under the sensitive `target/run/` tree. + +## Functions in unit tests + +`sqb test` creates each UDF and table function the selected tests call, SQL or Python, as a +`TEMPORARY FUNCTION __SQB_TEST__` in the session's current schema. The session needs a current +schema and permission to create functions there; otherwise tests call the built function. See +[Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx b/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx index 13f0132d91..7d9ef18ff7 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlserver.mdx @@ -81,3 +81,8 @@ connection = "sqlserver" database = "analytics" schema = "dev" ``` + +## Functions in unit tests + +SQL Server has no session-scoped functions, so `sqb test` calls the built function. Run `sqb build` +before testing a new or changed function. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 53bac26675..ef3f447579 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -565,6 +565,40 @@ __udf_expected__ AS ( UDFs are warehouse objects, so the function is created before the test runs. During `sqb build`, UDF tests run after the function is created but before any model that uses it. +### Functions in `sqb test` + +`sqb test` does not need a build to test the current version of a function. Before the tests run, +SQLBuild creates a session-scoped temporary copy of every project UDF and table function the +selected tests call, including functions those functions call, from the definitions in your +project. The tests call the copies, so an edited function is tested without `sqb build`, and the +built function in the target schema is never created, replaced or dropped. The copies disappear +when the test connections close. + +| Adapter | Temporary copy | +|---|---| +| DuckDB, MotherDuck | `CREATE TEMP MACRO`, called as `temp.main.`. Python UDFs are registered on each test connection. | +| PostgreSQL | SQL function in the session's `pg_temp` schema, called as `pg_temp.` | +| Snowflake | `CREATE TEMPORARY FUNCTION __sqb_test__` in the session's current schema, for SQL and Python UDFs and table functions | +| Databricks | `CREATE TEMPORARY FUNCTION __sqb_test__` for SQL functions | +| BigQuery, SQL Server | Not supported; tests call the built function | + +When an adapter cannot create a copy (a Databricks Python UDF, for example), or the warehouse +refuses to create it (the role lacks permission, or the Snowflake session has no current schema), +the test calls the built function instead. If that function has not been built either, the test +fails with an error that names the function and the reason no copy was created, and asks you to run +`sqb build` first. +A function whose callee has no copy calls the built functions too, so one test never mixes copies +and built functions. + +SQLBuild decides which copies exist on the first test connection and then creates the same copies +on every other test connection, including a connection that replaces one a timeout left unusable. +If one of those connections cannot create a copy, `sqb test` stops with error `T006`, which names +the function and the connection. + +PostgreSQL temporary functions belong to one server session. Behind a transaction-mode connection +pooler, a test's statements can reach a different server session that does not hold the copies, so +run `sqb test` for projects with functions through a direct connection or a session-mode pool. + ### Table function tests Test table functions by calling them in `__table_fn_actual__` and comparing against `__table_fn_expected__`: From da36af4703446dc8b7f1261fc60fbb2be19294a6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:39:39 +0100 Subject: [PATCH 12/34] ci: fail check-ci when generated skill reference pages are stale The bundled skill reference pages under src/sqlbuild/.agents/skills/sqlbuild/references/docs drifted from the website docs and nothing caught it. `make check-docs-skill`, now part of `make check-ci`, regenerates the pages in memory with the same generator as `make skills` and fails, listing every changed, missing or retired page, with `make skills` as the fix. Refs: CHI-629 --- Makefile | 8 ++- scripts/check_docs_skill.py | 15 +++++ scripts/docs_skill/_helpers/output.py | 20 +++++++ scripts/docs_skill/main/check.py | 41 +++++++++++++ .../unit/scripts/docs_skill/main/__init__.py | 0 .../scripts/docs_skill/main/_test_types.py | 9 +++ .../scripts/docs_skill/main/test_check.py | 59 +++++++++++++++++++ 7 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 scripts/check_docs_skill.py create mode 100644 scripts/docs_skill/main/check.py create mode 100644 tests/unit/scripts/docs_skill/main/__init__.py create mode 100644 tests/unit/scripts/docs_skill/main/_test_types.py create mode 100644 tests/unit/scripts/docs_skill/main/test_check.py diff --git a/Makefile b/Makefile index c0b1aaf9d9..b11102eca6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate \ +.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate check-docs-skill \ check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \ test-e2e-duckdb-build-incremental \ test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \ @@ -626,7 +626,7 @@ verify-quick: exit $$status -check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate rust-lint +check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate check-docs-skill rust-lint uv run ruff format --check . uv run ruff check . uv run ty check src tests @@ -646,6 +646,10 @@ check-release-gate: uv run python -m scripts.check_release_gate +check-docs-skill: + uv run python -m scripts.check_docs_skill + + check-pr-metadata: @test -n "$(PR_TITLE)" || { echo "PR_TITLE is required" >&2; exit 2; } @test -n "$(PR_BODY_FILE)" || { echo "PR_BODY_FILE is required" >&2; exit 2; } diff --git a/scripts/check_docs_skill.py b/scripts/check_docs_skill.py new file mode 100644 index 0000000000..91d6a8d69a --- /dev/null +++ b/scripts/check_docs_skill.py @@ -0,0 +1,15 @@ +"""Direct wrapper for the docs skill freshness check.""" + +from __future__ import annotations + +from scripts.docs_skill.main.check import check_docs_skill + + +def main(argv: list[str] | None = None) -> int: + """Check that the SQLBuild documentation skill is up to date.""" + + return check_docs_skill(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/docs_skill/_helpers/output.py b/scripts/docs_skill/_helpers/output.py index f036999ef0..4c07d7444b 100644 --- a/scripts/docs_skill/_helpers/output.py +++ b/scripts/docs_skill/_helpers/output.py @@ -25,3 +25,23 @@ def write_reference_pages(*, output_dir: Path, pages: dict[str, str]) -> None: if directory.is_dir() and not any(directory.iterdir()): directory.rmdir() print(f"Wrote {len(pages)} reference pages to {output_dir} ({len(stale_paths)} removed)") + + +def outdated_reference_pages(*, output_dir: Path, pages: dict[str, str]) -> tuple[str, ...]: + """Return generated pages that are missing, differ from `pages`, or are no longer generated.""" + + existing: dict[str, str] = { + path.relative_to(output_dir).as_posix(): path.read_text(encoding="utf-8") + for path in output_dir.rglob("*.md") + } + changed: set[str] = { + relative_path + for relative_path, contents in pages.items() + if existing.get(relative_path) != contents + } + stale: set[str] = { + relative_path + for relative_path, contents in existing.items() + if relative_path not in pages and GENERATED_MARKER in contents + } + return tuple(sorted(changed | stale)) diff --git a/scripts/docs_skill/main/check.py b/scripts/docs_skill/main/check.py new file mode 100644 index 0000000000..448ec6b068 --- /dev/null +++ b/scripts/docs_skill/main/check.py @@ -0,0 +1,41 @@ +"""Focused entry that fails when the bundled docs reference pages are out of date.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.docs_skill._helpers.output import outdated_reference_pages +from scripts.docs_skill._helpers.render import build_reference_pages +from scripts.docs_skill.constants import DEFAULT_DOCS_ROOT, DEFAULT_OUTPUT_DIR + + +def check_docs_skill(argv: list[str] | None = None) -> int: + """Compare freshly generated reference pages with the committed ones.""" + + args: argparse.Namespace = _parse_args(argv) + outdated: tuple[str, ...] = outdated_reference_pages( + output_dir=args.output_dir, + pages=build_reference_pages(docs_root=args.docs_root), + ) + if not outdated: + print(f"Docs skill reference pages in {args.output_dir} are up to date") + return 0 + print( + f"{len(outdated)} docs skill reference page(s) in {args.output_dir} are out of date " + "with the website docs. Run `make skills` and commit the result:", + file=sys.stderr, + ) + for relative_path in outdated: + print(f" {relative_path}", file=sys.stderr) + return 1 + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="Fail when bundled SQLBuild skill reference pages differ from the website docs." + ) + parser.add_argument("--docs-root", type=Path, default=DEFAULT_DOCS_ROOT) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) + return parser.parse_args(argv) diff --git a/tests/unit/scripts/docs_skill/main/__init__.py b/tests/unit/scripts/docs_skill/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/docs_skill/main/_test_types.py b/tests/unit/scripts/docs_skill/main/_test_types.py new file mode 100644 index 0000000000..81d866fbf7 --- /dev/null +++ b/tests/unit/scripts/docs_skill/main/_test_types.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DocsSkillCheckTestCase: + description: str + output_edits: dict[str, str] + expected_exit_code: int + expected_outdated: tuple[str, ...] diff --git a/tests/unit/scripts/docs_skill/main/test_check.py b/tests/unit/scripts/docs_skill/main/test_check.py new file mode 100644 index 0000000000..e26bff16c7 --- /dev/null +++ b/tests/unit/scripts/docs_skill/main/test_check.py @@ -0,0 +1,59 @@ +from pathlib import Path + +import pytest + +from scripts.docs_skill.constants import GENERATED_MARKER +from scripts.docs_skill.main.check import check_docs_skill +from scripts.docs_skill.main.generate import generate_docs_skill +from tests.unit.scripts.docs_skill.main._test_types import DocsSkillCheckTestCase + +_SIDEBAR_JSON: str = '[{"label":"Concepts","items":["docs/concepts/orders"]}]' +_ORDERS_PAGE: str = '---\ntitle: "Orders"\n---\n\nOrders body.\n' + + +@pytest.mark.parametrize( + "test_case", + [ + DocsSkillCheckTestCase( + description="freshly generated pages pass", + output_edits={}, + expected_exit_code=0, + expected_outdated=(), + ), + DocsSkillCheckTestCase( + description="hand-edited generated page fails", + output_edits={"concepts/orders.md": f"{GENERATED_MARKER}\nold orders body\n"}, + expected_exit_code=1, + expected_outdated=("concepts/orders.md",), + ), + DocsSkillCheckTestCase( + description="generated page with no docs source fails", + output_edits={"concepts/retired.md": f"{GENERATED_MARKER}\nretired\n"}, + expected_exit_code=1, + expected_outdated=("concepts/retired.md",), + ), + ], + ids=lambda case: case.description, +) +def test_given_reference_pages_when_checking_docs_skill_then_outdated_pages_fail_with_fix_command( + test_case: DocsSkillCheckTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + docs_root: Path = tmp_path / "src/content/docs/docs" + output_dir: Path = tmp_path / "references/docs" + (docs_root / "concepts").mkdir(parents=True) + (tmp_path / "src/sidebar.json").write_text(_SIDEBAR_JSON, encoding="utf-8") + (docs_root / "concepts/orders.mdx").write_text(_ORDERS_PAGE, encoding="utf-8") + arguments: list[str] = ["--docs-root", str(docs_root), "--output-dir", str(output_dir)] + _ = generate_docs_skill(arguments) + for relative_path, contents in test_case.output_edits.items(): + (output_dir / relative_path).write_text(contents, encoding="utf-8") + _ = capsys.readouterr() + + result: int = check_docs_skill(arguments) + + errors: str = capsys.readouterr().err + assert result == test_case.expected_exit_code, (test_case.description, errors) + assert all(f" {path}\n" in errors for path in test_case.expected_outdated), errors + assert ("make skills" in errors) is bool(test_case.expected_outdated), errors From 1a06ec4a7cc48861e973b70c2d2a093a08299499 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:45:10 +0100 Subject: [PATCH 13/34] fix(audits): point seed and source audits that read @model at @relation The seeds page attached `expression_is_true` without showing its definition, and the definition people copy from the playground reads `__ref("@model")`. An attached audit receives its owner kind's name argument (`model`, `source` or `seed`) plus `relation`, so on a seed it failed with a bare "missing argument 'model'". Passing the seed as `model` would not help: `__ref` resolves only models. The seeds page now shows a seed-compatible `@relation` definition and explains the owner arguments, the audits page documents them, and the missing-argument error names the argument the owner does receive and points at @relation. A DuckDB E2E builds the documented example. Refs: CHI-632 --- .../src/audits/_helpers/rendering.rs | 26 +++++- .../tests/test_render_attached_audit.rs | 17 ++++ .../references/docs/concepts/audits.md | 3 +- .../references/docs/concepts/seeds.md | 15 +++ .../cli/commands/main/seed/_test_types.py | 11 +++ .../main/seed/test_seed_yaml_audits.py | 93 +++++++++++++++++++ .../src/content/docs/docs/concepts/audits.mdx | 3 +- .../src/content/docs/docs/concepts/seeds.mdx | 15 +++ 8 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs b/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs index 5e9481fe81..84c1e1fe2e 100644 --- a/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs +++ b/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs @@ -2,6 +2,7 @@ use crate::audits::_helpers::parameters::{RenderStop, render_parameterized_sql}; use crate::audits::_helpers::policies::{resolve_run_scope, resolve_severity}; +use crate::audits::constants::{MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE}; use crate::audits::models::{ArgumentValue, AuditAttachment, PolicySource}; /// Severity first, then run scope, as Python resolves them after expanding the SQL. @@ -39,8 +40,10 @@ pub(crate) fn rendered( ) -> Result { render_parameterized_sql(sql, arguments).map_err(|stop| match stop { RenderStop::MissingArgument(name) => format!( - "{} is missing argument '{name}' for generic audit '{}'", - attachment.owner_label, attachment.definition_name + "{} is missing argument '{name}' for generic audit '{}'{}", + attachment.owner_label, + attachment.definition_name, + owner_argument_hint(attachment, &name) ), RenderStop::UnsupportedValue(name) => format!( "{} generic audit '{}' argument '{name}' uses an unsupported value", @@ -49,6 +52,25 @@ pub(crate) fn rendered( }) } +/// Why `@model` is missing on a seed or source: each owner passes its own kind's name argument. +fn owner_argument_hint(attachment: &AuditAttachment, missing: &str) -> String { + let owner_arguments: [&str; 3] = [MODEL_RESOURCE, SOURCE_RESOURCE, SEED_RESOURCE]; + if !owner_arguments.contains(&missing) { + return String::new(); + } + let Some((owner_kind, _)) = attachment + .implicit_arguments + .iter() + .find(|(argument, _)| owner_arguments.contains(&argument.as_str())) + else { + return String::new(); + }; + format!( + "; a {owner_kind} audit receives '{owner_kind}' and 'relation', not '{missing}', so read \ + the attached relation with @relation" + ) +} + /// `merge_audit_arguments`: an explicit argument may repeat an implicit one only as equal text. pub(crate) fn merged_arguments( attachment: &AuditAttachment, diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs b/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs index 9b571785ff..9d9981a31b 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs @@ -88,6 +88,23 @@ fn given_attached_audits_when_rendering_then_python_rendering_is_returned() { 'accepted_values'", ), }, + AttachedAuditTestCase { + description: "a seed audit reading @model is told to read @relation", + attachment: AuditAttachment { + owner_label: "seeds/waffle_types.yml".to_owned(), + sql_body: "SELECT * FROM __ref(\"@model\") WHERE NOT (price_cents >= 0)".to_owned(), + implicit_arguments: vec![ + ("seed".to_owned(), "waffle_types".to_owned()), + ("relation".to_owned(), "__seed(\"waffle_types\")".to_owned()), + ], + ..attachment(Vec::new(), None, None) + }, + expected_rendering: failed( + "seeds/waffle_types.yml is missing argument 'model' for generic audit \ + 'accepted_values'; a seed audit receives 'seed' and 'relation', not 'model', \ + so read the attached relation with @relation", + ), + }, AttachedAuditTestCase { description: "an opaque value inside a referenced list is Python's value error", attachment: attachment( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index 46f21a727d..be6e5ae7f3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -230,7 +230,8 @@ Generic audit SQL uses `@name` for parameter placeholders. These are resolved by | Parameter | Description | |-----------|-------------| | `@column` | The column name (auto-populated for column-level audits) | -| `@relation` | The target relation (auto-populated from the attached model or source) | +| `@relation` | The target relation (auto-populated from the attached model, source or seed) | +| `@model`, `@source`, `@seed` | The attached resource's name. Each is set only for its own kind, so an audit that attaches to more than one kind reads `@relation` | | `@'name'` | A quoted parameter passed from the audit declaration (e.g. `@'values'`) | | `@name` | An unquoted parameter passed from the audit declaration (e.g. `@expression`) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md index 34e3d3f4b1..bdf725e9be 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md @@ -102,6 +102,21 @@ seeds: severity: error ``` +`not_null` and `unique` are built in. `expression_is_true` is a project generic audit; one that +works on seeds reads the attached relation through `@relation`: + +```sql +-- audits/generic/expression_is_true.sql +AUDIT (); + +SELECT * +FROM @relation +WHERE NOT (@expression) +``` + +A seed audit receives `@seed` (the seed name) and `@relation`, but not `@model`, so a generic +audit written as `FROM __ref("@model")` works only on models. + During `sqb build`, seed audits run right after the seed loads. An `error` failure blocks every model that depends on the seed. See [Audits](audits.md#seed-audits). diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py index 30e69d645b..ac948917f0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py @@ -25,3 +25,14 @@ class SeedJsonOutputTestCase: expected_status: str expected_summary: dict[str, int] expected_assets: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class SeedYamlAuditE2ETestCase: + """Test case for the documented seed YAML audit example.""" + + description: str + audit_sql: str + seed_csv: str + expected_exit_code: int + expected_output_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py new file mode 100644 index 0000000000..9eda44997d --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py @@ -0,0 +1,93 @@ +"""The seed YAML audit example in the seeds documentation builds as documented.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.seed._test_types import SeedYamlAuditE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_SEED_YAML: str = """\ +seeds: + - name: waffle_types + description: Waffle types + columns: + - name: waffle_type_id + type: INTEGER + audits: + - not_null + - unique + - name: price_cents + type: INTEGER + audits: + - expression_is_true: + expression: "price_cents >= 0" + severity: error +""" +_RELATION_AUDIT_SQL: str = "AUDIT ();\n\nSELECT *\nFROM @relation\nWHERE NOT (@expression)\n" + + +@pytest.mark.parametrize( + "test_case", + [ + SeedYamlAuditE2ETestCase( + description="documented seed audits pass on valid rows", + audit_sql=_RELATION_AUDIT_SQL, + seed_csv="waffle_type_id,price_cents\n1,450\n2,525\n", + expected_exit_code=0, + expected_output_fragments=( + "expression_is_true (waffle_types)", + "PASS=4", + "FAIL=0", + ), + ), + SeedYamlAuditE2ETestCase( + description="documented seed expression audit fails a negative price", + audit_sql=_RELATION_AUDIT_SQL, + seed_csv="waffle_type_id,price_cents\n1,450\n2,-1\n", + expected_exit_code=1, + expected_output_fragments=("expression_is_true (waffle_types)", "FAIL=1"), + ), + SeedYamlAuditE2ETestCase( + description="model-only audit on a seed points at @relation", + audit_sql='AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n', + seed_csv="waffle_type_id,price_cents\n1,450\n", + expected_exit_code=1, + expected_output_fragments=( + "seeds/waffle_types.yml is missing argument 'model' for generic audit " + "'expression_is_true'; a seed audit receives 'seed' and 'relation', not 'model', " + "so read the attached relation with @relation", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_documented_seed_yaml_audits_when_building_then_audits_run_on_the_seed( + test_case: SeedYamlAuditE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="waffle_seed_audits", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "waffle_seed_audits"\nadapter = "duckdb"\n' + '[connection]\ndatabase = "waffles.duckdb"\n' + ), + "seeds/waffle_types.csv": test_case.seed_csv, + "seeds/waffle_types.yml": _SEED_YAML, + "audits/generic/expression_is_true.sql": test_case.audit_sql, + }, + ) + + built: subprocess.CompletedProcess[str] = run_sqb(command=("build",), project_dir=project_dir) + + output: str = built.stdout + built.stderr + assert built.returncode == test_case.expected_exit_code, (test_case.description, output) + assert all(fragment in output for fragment in test_case.expected_output_fragments), ( + test_case.description, + output, + ) diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 0dc17ff775..aed697b813 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -211,7 +211,8 @@ Generic audit SQL uses `@name` for parameter placeholders. These are resolved by | Parameter | Description | |-----------|-------------| | `@column` | The column name (auto-populated for column-level audits) | -| `@relation` | The target relation (auto-populated from the attached model or source) | +| `@relation` | The target relation (auto-populated from the attached model, source or seed) | +| `@model`, `@source`, `@seed` | The attached resource's name. Each is set only for its own kind, so an audit that attaches to more than one kind reads `@relation` | | `@'name'` | A quoted parameter passed from the audit declaration (e.g. `@'values'`) | | `@name` | An unquoted parameter passed from the audit declaration (e.g. `@expression`) | diff --git a/website/src/content/docs/docs/concepts/seeds.mdx b/website/src/content/docs/docs/concepts/seeds.mdx index cfbe20b4b9..bfbab4908e 100644 --- a/website/src/content/docs/docs/concepts/seeds.mdx +++ b/website/src/content/docs/docs/concepts/seeds.mdx @@ -90,6 +90,21 @@ seeds: severity: error ``` +`not_null` and `unique` are built in. `expression_is_true` is a project generic audit; one that +works on seeds reads the attached relation through `@relation`: + +```sql +-- audits/generic/expression_is_true.sql +AUDIT (); + +SELECT * +FROM @relation +WHERE NOT (@expression) +``` + +A seed audit receives `@seed` (the seed name) and `@relation`, but not `@model`, so a generic +audit written as `FROM __ref("@model")` works only on models. + During `sqb build`, seed audits run right after the seed loads. An `error` failure blocks every model that depends on the seed. See [Audits](/docs/concepts/audits#seed-audits). From 53d5282b807edf2fd02b6144c60543f5a465486a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:57:23 +0100 Subject: [PATCH 14/34] test: expect sqb plan to exit 1 for an on_schema_change fail finding CHI-429 makes sqb plan exit 1 when the plan reports an error finding, so the schema backfill case that plans an on_schema_change fail model now expects that exit code. --- .../e2e/src/sqlbuild/cli/commands/main/build/_test_types.py | 1 + .../cli/commands/main/build/test_schema_backfill_behavior.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 7b30f393d6..7816b2f698 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -717,6 +717,7 @@ class SchemaBackfillBuildE2ETestCase: schema_after_text: str command: tuple[str, ...] expected_exit_code: int + expected_plan_exit_code: int expected_reason: str expected_backfill_action: str | None = None expected_backfill_duration: str | None = None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_schema_backfill_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_schema_backfill_behavior.py index d647093272..5a35f0df47 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_schema_backfill_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_schema_backfill_behavior.py @@ -35,6 +35,7 @@ ), command=("plan", "--json", "--select", "order_status_index"), expected_exit_code=0, + expected_plan_exit_code=0, expected_reason="schema_changed", expected_backfill_action="bounded", expected_backfill_duration="7d", @@ -55,6 +56,7 @@ ), command=("plan", "--json", "--select", "order_status_index"), expected_exit_code=0, + expected_plan_exit_code=1, expected_reason="schema_changed", expected_backfill_action="bounded", expected_backfill_duration="7d", @@ -77,6 +79,7 @@ ), command=("plan", "--json", "--select", "order_status_index"), expected_exit_code=0, + expected_plan_exit_code=0, expected_reason="schema_changed", expected_backfill_action="bounded", expected_backfill_duration="7d", @@ -119,7 +122,7 @@ def test_given_schema_backfill_mutation_when_planning_then_expected_metadata_is_ command=test_case.command, project_dir=project_dir, ) - assert plan_result.returncode == test_case.expected_exit_code, ( + assert plan_result.returncode == test_case.expected_plan_exit_code, ( plan_result.stdout + plan_result.stderr ) payload: dict[str, object] = json.loads(plan_result.stdout) From f63b8a0abf0f0b9dcef259d02a3155174d3d02ff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:47:16 +0100 Subject: [PATCH 15/34] feat: add single-server ClickHouse adapter Adds a ClickHouse adapter over clickhouse-connect (HTTP, sessionless, per-statement query ids, interrupt via KILL QUERY). A SQLBuild schema is a ClickHouse database. Tables build aside and swap with EXCHANGE TABLES; merge, delete_insert, cursor windows and microbatches rewrite whole partitions with REPLACE PARTITION, so a rerun after a failure converges. Snapshots stage on a working copy and exchange in. Seeds stream as CSV. SQL functions deploy as __; Python and table functions are refused with coded errors. Historical snapshot SQL now aliases its key projections so engines that name qualified projections after the qualifier resolve them. --- .../src/lifecycle_events/constants.rs | 1 + .../semantic_shapes/source_bindings.rs | 11 +- fensu.toml | 10 + pyproject.toml | 4 + .../classes/historical_snapshot_sql.py | 9 +- src/sqlbuild/adapter/contract/types.py | 1 + src/sqlbuild/adapter/discovery/constants.py | 4 + src/sqlbuild/adapters/clickhouse/__init__.py | 1 + .../adapters/clickhouse/_helpers/__init__.py | 0 .../clickhouse/_helpers/partition_rewrite.py | 87 + .../clickhouse/_helpers/snapshot_rewrite.py | 123 + .../adapters/clickhouse/_helpers/type_text.py | 88 + .../adapters/clickhouse/classes/__init__.py | 1 + .../clickhouse/classes/clickhouse_adapter.py | 2639 +++++++++++++++++ .../classes/clickhouse_connection.py | 101 + .../clickhouse/classes/clickhouse_cursor.py | 56 + .../clickhouse/classes/partition_locks.py | 35 + src/sqlbuild/adapters/clickhouse/constants.py | 95 + src/sqlbuild/adapters/clickhouse/models.py | 38 + .../_helpers/materializations/full_refresh.py | 2 +- .../failure/config-unknown-adapter.json | 2 +- .../sqlbuild/adapters/clickhouse/__init__.py | 0 .../adapters/clickhouse/_test_types.py | 133 + .../sqlbuild/adapters/clickhouse/conftest.py | 71 + .../sqlbuild/adapters/clickhouse/helpers.py | 357 +++ .../adapters/clickhouse/test_client.py | 851 ++++++ .../snapshot_sql_rendering/test_rendering.py | 2 +- .../sqlbuild/adapters/clickhouse/__init__.py | 0 .../adapters/clickhouse/_test_types.py | 51 + .../adapters/clickhouse/test_helpers.py | 215 ++ uv.lock | 110 +- 31 files changed, 5091 insertions(+), 7 deletions(-) create mode 100644 src/sqlbuild/adapters/clickhouse/__init__.py create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/__init__.py create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/type_text.py create mode 100644 src/sqlbuild/adapters/clickhouse/classes/__init__.py create mode 100644 src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py create mode 100644 src/sqlbuild/adapters/clickhouse/classes/clickhouse_connection.py create mode 100644 src/sqlbuild/adapters/clickhouse/classes/clickhouse_cursor.py create mode 100644 src/sqlbuild/adapters/clickhouse/classes/partition_locks.py create mode 100644 src/sqlbuild/adapters/clickhouse/constants.py create mode 100644 src/sqlbuild/adapters/clickhouse/models.py create mode 100644 tests/integration/src/sqlbuild/adapters/clickhouse/__init__.py create mode 100644 tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py create mode 100644 tests/integration/src/sqlbuild/adapters/clickhouse/conftest.py create mode 100644 tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py create mode 100644 tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/__init__.py create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index a7e9135216..fed6366c4a 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -260,6 +260,7 @@ pub const OPERATION_STRATEGIES: [&str; 10] = [ ]; pub const OPERATION_ADAPTERS: [&str; 9] = [ "bigquery", + "clickhouse", "custom", "databricks", "duckdb", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index b56bcc6b6c..3d14cfc6d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -35,6 +35,8 @@ const REF_KIND: &str = "ref"; const SOURCE_KIND: &str = "source"; const UNKNOWN: &str = "UNKNOWN"; const GENERIC_DIALECT: &str = "generic"; +/// Dialects whose unquoted identifiers already bind with their exact case. +const EXACT_UNQUOTED_DIALECTS: [&str; 1] = ["clickhouse"]; type Object<'py> = Bound<'py, PyAny>; @@ -80,6 +82,8 @@ pub(crate) fn source_binding_diagnostics<'py>( let sensitive: bool = !dialect.is_none() && CASE_SENSITIVE_BINDING_DIALECTS.contains(&dialect.extract::()?.as_str()) && !ignore_case.is_truthy()?; + let exact_unquoted: bool = !dialect.is_none() + && EXACT_UNQUOTED_DIALECTS.contains(&dialect.extract::()?.as_str()); let mut changed: HashSet = HashSet::new(); for item in mapping_items(&columns)?.try_iter()? { let (name, value): (String, Object<'py>) = item?.extract()?; @@ -101,7 +105,12 @@ pub(crate) fn source_binding_diagnostics<'py>( } let quoted: Bound<'py, PyDict> = PyDict::new(py); for (column, column_type) in physical { - quoted.set_item(format!("\"{}\"", column.replace('"', "\"\"")), column_type)?; + let name: String = if exact_unquoted { + column + } else { + format!("\"{}\"", column.replace('"', "\"\"")) + }; + quoted.set_item(name, column_type)?; } shapes.set_item(name, quoted)?; } diff --git a/fensu.toml b/fensu.toml index e0ab3886b7..c4802e31b1 100644 --- a/fensu.toml +++ b/fensu.toml @@ -130,6 +130,11 @@ rule = "FFR309" path = "src/sqlbuild/adapters/bigquery/__init__.py" reason = "The BigQuery backend intentionally owns behavior through its adapter class rather than function entrypoints." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/adapters/clickhouse/__init__.py" +reason = "The ClickHouse backend intentionally owns behavior through its adapter class rather than function entrypoints." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/adapters/databricks/constants.py" @@ -205,6 +210,11 @@ rule = "FFR601" path = "src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py" reason = "The BigQuery adapter intentionally keeps overrideable backend behavior on one class." +[[targets.python.rule_exceptions]] +rule = "FFR601" +path = "src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py" +reason = "The ClickHouse adapter intentionally keeps overrideable backend behavior on one class." + [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapters/databricks/classes/databricks_adapter.py" diff --git a/pyproject.toml b/pyproject.toml index 1030d936f1..3324230487 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ postgres = [ sqlserver = [ "pymssql>=2.3.0", ] +clickhouse = [ + "clickhouse-connect>=0.8", +] ingestr = [ "ingestr>=1.0.0", ] @@ -110,6 +113,7 @@ markers = [ "databricks: tests that require a live Databricks warehouse", "postgres: tests that require a live PostgreSQL database", "sqlserver: tests that require a live Microsoft SQL Server database", + "clickhouse: tests that require a live ClickHouse server", "motherduck: tests that require live MotherDuck credentials", "performance: opt-in performance guard tests", "cold_compile_performance: fresh-process semantic compile scaling guards", diff --git a/src/sqlbuild/adapter/contract/classes/historical_snapshot_sql.py b/src/sqlbuild/adapter/contract/classes/historical_snapshot_sql.py index a20f2933db..dec67007a5 100644 --- a/src/sqlbuild/adapter/contract/classes/historical_snapshot_sql.py +++ b/src/sqlbuild/adapter/contract/classes/historical_snapshot_sql.py @@ -251,7 +251,8 @@ def _new_change_starts_sql(self) -> str: f"SELECT {self._key_sql(alias=None)}, MIN({SNAPSHOT_VERSION_START_COLUMN}) " f"AS __first_version_start FROM __changed_or_new GROUP BY {self._key_sql(alias=None)}" "), __reappearances AS (" - f"SELECT {classified_key_sql}, MIN(__classified.{observed_at}) AS __reappeared_at " + f"SELECT {self._key_projection_sql(alias='__classified')}, " + f"MIN(__classified.{observed_at}) AS __reappeared_at " "FROM __classified " f"JOIN __latest ON {self._key_condition(left='__classified', right='__latest')} " "LEFT JOIN __observed_group_sequence AS __closing_group " @@ -280,7 +281,8 @@ def _open_version_hard_deletes_sql(self, *, destination: str) -> str: present_condition: str = self._key_condition(left="__present", right="__target") observed_at: str = self.observed_at_column return ( - f"SELECT {target_key_sql}, MIN(__observed_groups.__observed_at) AS __close_at " + f"SELECT {self._key_projection_sql(alias='__target')}, " + "MIN(__observed_groups.__observed_at) AS __close_at " f"FROM {destination} AS __target " "JOIN __observed_group_sequence AS __observed_groups " f"ON __observed_groups.__observed_at > __target.{observed_at} " @@ -296,5 +298,8 @@ def _key_sql(self, *, alias: str | None) -> str: prefix: str = f"{alias}." if alias is not None else "" return ", ".join(f"{prefix}{column}" for column in self.unique_key) + def _key_projection_sql(self, *, alias: str) -> str: + return ", ".join(f"{alias}.{column} AS {column}" for column in self.unique_key) + def _key_condition(self, *, left: str, right: str) -> str: return " AND ".join(f"{left}.{column} = {right}.{column}" for column in self.unique_key) diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 0049d7c775..6ed7dd4f6b 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -25,6 +25,7 @@ class BuiltinAdapter(StrEnum): POSTGRES = "postgres" SQLSERVER = "sqlserver" SQLITE = "sqlite" + CLICKHOUSE = "clickhouse" class RelationReadStatus(StrEnum): diff --git a/src/sqlbuild/adapter/discovery/constants.py b/src/sqlbuild/adapter/discovery/constants.py index 9a816bfedb..53d89e6ebc 100644 --- a/src/sqlbuild/adapter/discovery/constants.py +++ b/src/sqlbuild/adapter/discovery/constants.py @@ -21,4 +21,8 @@ "sqlbuild.adapters.sqlserver.classes.sqlserver_adapter", "SqlServerAdapter", ), + "clickhouse": ( + "sqlbuild.adapters.clickhouse.classes.clickhouse_adapter", + "ClickHouseAdapter", + ), } diff --git a/src/sqlbuild/adapters/clickhouse/__init__.py b/src/sqlbuild/adapters/clickhouse/__init__.py new file mode 100644 index 0000000000..75263fd12f --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/__init__.py @@ -0,0 +1 @@ +"""ClickHouse integration.""" diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/__init__.py b/src/sqlbuild/adapters/clickhouse/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py b/src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py new file mode 100644 index 0000000000..5fff1d9a70 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py @@ -0,0 +1,87 @@ +"""Rewrite whole ClickHouse partitions: the write primitive behind merges and deletes.""" + +from __future__ import annotations + +import hashlib +from typing import Any + +from sqlbuild.adapters.clickhouse._helpers.type_text import quote_string +from sqlbuild.adapters.clickhouse.constants import ( + MAX_IDENTIFIER_LENGTH, + REWRITE_DELTA_INFIX, + REWRITE_STAGING_INFIX, +) +from sqlbuild.adapters.clickhouse.models import PartitionRewrite, RewriteExecutor +from sqlbuild.runtime.execution_limits.main.execution_deadlines_suspended import ( + execution_deadlines_suspended, +) + + +def rewrite_partitions(*, request: PartitionRewrite, executor: RewriteExecutor) -> int: + """Apply one rewrite and return the number of incoming rows.""" + + digest: str = hashlib.sha256( + f"{request.destination}\0{request.delta_insert_sql}\0{request.displaced_predicate}".encode() + ).hexdigest()[:12] + delta: str = artifact_relation( + relation=request.destination, infix=REWRITE_DELTA_INFIX, digest=digest + ) + staging: str = artifact_relation( + relation=request.destination, infix=REWRITE_STAGING_INFIX, digest=digest + ) + predicate: str = request.displaced_predicate.replace("{delta}", delta) + + def run(sql: str) -> Any: + executor.statement_recorder.record(sql) + return executor.execute(sql) + + try: + run(f"DROP TABLE IF EXISTS {delta}") + run(f"CREATE TABLE {delta} AS {request.destination}") + run(request.delta_insert_sql.replace("{delta}", delta)) + incoming: int = int(run(f"SELECT count() FROM {delta}").fetchone()[0]) + partitions: tuple[str, ...] = tuple( + sorted( + str(row[0]) + for row in run( + f"SELECT DISTINCT _partition_id FROM {delta} UNION DISTINCT " + f"SELECT DISTINCT _partition_id FROM {request.destination} " + f"WHERE ifNull({predicate}, 0)" + ).fetchall() + ) + ) + if not partitions: + return incoming + partition_list: str = ", ".join(quote_string(partition) for partition in partitions) + with executor.partition_locks.locked( + destination=request.destination, partitions=partitions + ): + run(f"DROP TABLE IF EXISTS {staging}") + run(f"CREATE TABLE {staging} AS {request.destination}") + run( + f"INSERT INTO {staging} SELECT * FROM {request.destination} " + f"WHERE _partition_id IN ({partition_list}) AND NOT ifNull({predicate}, 0)" + ) + run(f"INSERT INTO {staging} SELECT * FROM {delta}") + partition: str + for partition in partitions: + run( + f"ALTER TABLE {request.destination} REPLACE PARTITION ID " + f"{quote_string(partition)} FROM {staging}" + ) + return incoming + finally: + with execution_deadlines_suspended(): + run(f"DROP TABLE IF EXISTS {staging}") + run(f"DROP TABLE IF EXISTS {delta}") + + +def artifact_relation(*, relation: str, infix: str, digest: str) -> str: + """Return a sibling relation name bounded by the identifier limit.""" + + parts: list[str] = relation.rsplit(".", 1) + name: str = parts[-1].strip('"`') + suffix: str = f"{infix}{digest}" + fitted: str = name[: max(1, MAX_IDENTIFIER_LENGTH - len(suffix))] + suffix + quoted: str = '"' + fitted.replace('"', '""') + '"' + return quoted if len(parts) == 1 else f"{parts[0]}.{quoted}" diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py b/src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py new file mode 100644 index 0000000000..4499a06766 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py @@ -0,0 +1,123 @@ +"""Stage ClickHouse snapshot statements on a working copy and exchange it into place.""" + +from __future__ import annotations + +import re + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.clickhouse.models import SnapshotWorkspace + +_UPDATE_PREFIX: str = "UPDATE " +_TARGET_ALIAS: str = " AS __target SET " +_OFFSET_WINDOW_PATTERN: re.Pattern[str] = re.compile(r"\b(LAG|LEAD)\(([^()]+)\)(\s+OVER\b)") +_ESCAPE_CHARACTER: str = "\\" +_QUOTE_CHARACTERS: str = "'\"`" +_OPEN_PARENTHESIS: str = "(" +_CLOSE_PARENTHESIS: str = ")" +_WHERE_KEYWORD: str = " WHERE " +_FROM_KEYWORD: str = " FROM " + + +def nullable_offset_windows(statements: tuple[str, ...]) -> tuple[str, ...]: + """Make ``LAG``/``LEAD`` return NULL rather than the type default past the frame edge.""" + + return tuple( + _OFFSET_WINDOW_PATTERN.sub(r"\1(toNullable(\2))\3", statement) for statement in statements + ) + + +def rewrite_snapshot_statements( + *, workspace: SnapshotWorkspace, statements: tuple[str, ...] +) -> tuple[str, ...]: + """Translate statements written against ``workspace.work`` into a staged exchange.""" + + rewritten: list[str] = [ + f"DROP TABLE IF EXISTS {workspace.work}", + f"DROP TABLE IF EXISTS {workspace.rebuild}", + f"CREATE TABLE {workspace.work} AS {workspace.destination}", + f"INSERT INTO {workspace.work} SELECT * FROM {workspace.destination}", + ] + statement: str + for statement in nullable_offset_windows(statements): + if statement.lstrip().upper().startswith(_UPDATE_PREFIX): + rewritten.extend(_close_statements(workspace=workspace, statement=statement.strip())) + else: + rewritten.append(statement) + rewritten.extend( + ( + f"EXCHANGE TABLES {workspace.destination} AND {workspace.work}", + f"DROP TABLE IF EXISTS {workspace.work}", + ) + ) + return tuple(rewritten) + + +def _close_statements(*, workspace: SnapshotWorkspace, statement: str) -> tuple[str, ...]: + head: str = f"{_UPDATE_PREFIX}{workspace.work}{_TARGET_ALIAS}" + if not statement.startswith(head): + raise _unsupported(statement) + remainder: str = statement[len(head) :] + assignment_prefix: str = f"{workspace.valid_to_column} = " + if not remainder.startswith(assignment_prefix): + raise _unsupported(statement) + remainder = remainder[len(assignment_prefix) :] + where_at: int | None = _top_level_keyword(sql=remainder, keyword=_WHERE_KEYWORD) + if where_at is None: + raise _unsupported(statement) + head_sql: str = remainder[:where_at] + condition: str = remainder[where_at + len(_WHERE_KEYWORD) :] + from_at: int | None = _top_level_keyword(sql=head_sql, keyword=_FROM_KEYWORD) + expression: str = head_sql if from_at is None else head_sql[:from_at] + joined: str = "" if from_at is None else f", {head_sql[from_at + len(_FROM_KEYWORD) :]}" + valid_to: str = workspace.valid_to_column + row_hash: str = f"sipHash128(__target.* EXCEPT ({valid_to}))" + closed_sql: str = ( + f"SELECT {row_hash} AS __sqb_row, min({expression}) AS __sqb_valid_to, " + f"1 AS __sqb_closed FROM {workspace.work} AS __target{joined} WHERE {condition} " + "GROUP BY __sqb_row" + ) + return ( + f"CREATE TABLE {workspace.rebuild} AS {workspace.work}", + f"INSERT INTO {workspace.rebuild} SELECT __target.* REPLACE (" + f"if(__closed.__sqb_closed = 1 AND __target.{valid_to} IS NULL, " + f"CAST(__closed.__sqb_valid_to, toTypeName(__target.{valid_to})), " + f"__target.{valid_to}) AS {valid_to}) " + f"FROM {workspace.work} AS __target LEFT JOIN ({closed_sql}) AS __closed " + f"ON {row_hash} = __closed.__sqb_row", + f"EXCHANGE TABLES {workspace.work} AND {workspace.rebuild}", + f"DROP TABLE {workspace.rebuild}", + ) + + +def _top_level_keyword(*, sql: str, keyword: str) -> int | None: + """Return the first position of ``keyword`` outside parentheses and quotes.""" + + depth: int = 0 + quote: str | None = None + index: int = 0 + upper: str = sql.upper() + while index < len(sql): + character: str = sql[index] + if quote is not None: + if character == _ESCAPE_CHARACTER: + index += 2 + continue + if character == quote: + quote = None + elif character in _QUOTE_CHARACTERS: + quote = character + elif character == _OPEN_PARENTHESIS: + depth += 1 + elif character == _CLOSE_PARENTHESIS: + depth -= 1 + elif depth == 0 and upper.startswith(keyword, index): + return index + index += 1 + return None + + +def _unsupported(statement: str) -> AdapterUserError: + return AdapterUserError( + message="ClickHouse cannot stage this snapshot statement: " + statement[:200], + code="A709", + ) diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/type_text.py b/src/sqlbuild/adapters/clickhouse/_helpers/type_text.py new file mode 100644 index 0000000000..802dc7f102 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/type_text.py @@ -0,0 +1,88 @@ +"""ClickHouse type text helpers shared by inspection, diff and contracts.""" + +from __future__ import annotations + +from sqlbuild.adapters.clickhouse.constants import ( + DECIMAL_ALIAS, + DECIMAL_BASE_TYPE, + DECIMAL_PRECISION_BY_BASE, + LOW_CARDINALITY_WRAPPER, + NULLABLE_WRAPPER, + TYPE_ALIASES, + TYPE_WRAPPERS, + UNWRAPPABLE_TYPE_BASES, +) + + +def quote_string(value: str) -> str: + """Quote one string literal; ClickHouse treats backslash as an escape character.""" + + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + + +def strip_type_wrappers(type_sql: str) -> str: + """Remove the ``Nullable`` and ``LowCardinality`` wrappers that do not change values.""" + + stripped: str = type_sql.strip() + changed: bool = True + while changed: + changed = False + wrapper: str + for wrapper in TYPE_WRAPPERS: + inner: str | None = _unwrap(type_sql=stripped, wrapper=wrapper) + if inner is not None: + stripped = inner + changed = True + return stripped + + +def is_nullable_type(type_sql: str) -> bool: + """Return whether a ClickHouse column type admits NULL.""" + + stripped: str = type_sql.strip() + low_cardinality_inner: str | None = _unwrap(type_sql=stripped, wrapper=LOW_CARDINALITY_WRAPPER) + if low_cardinality_inner is not None: + stripped = low_cardinality_inner + return _unwrap(type_sql=stripped, wrapper=NULLABLE_WRAPPER) is not None + + +def canonical_type(type_sql: str) -> str: + """Map one ClickHouse type to the portable spelling the shared type system compares.""" + + stripped: str = strip_type_wrappers(type_sql) + base: str = _base_type(stripped) + params: str = stripped[len(base) :].strip() + alias: str | None = TYPE_ALIASES.get(base) + if alias is None: + return stripped + if alias == DECIMAL_ALIAS and params: + inner: list[str] = [part.strip() for part in params[1:-1].split(",")] + if base == DECIMAL_BASE_TYPE: + return f"{DECIMAL_ALIAS}({', '.join(inner)})" + return f"{DECIMAL_ALIAS}({DECIMAL_PRECISION_BY_BASE[base]}, {inner[0]})" + return alias + + +def nullable_type(type_sql: str) -> str: + """Wrap a scalar type in ``Nullable`` unless it already admits NULL or cannot be wrapped.""" + + stripped: str = type_sql.strip() + if is_nullable_type(stripped): + return stripped + if _base_type(stripped) in UNWRAPPABLE_TYPE_BASES: + return stripped + low_cardinality_inner: str | None = _unwrap(type_sql=stripped, wrapper=LOW_CARDINALITY_WRAPPER) + if low_cardinality_inner is not None: + return f"{LOW_CARDINALITY_WRAPPER}({NULLABLE_WRAPPER}({low_cardinality_inner}))" + return f"{NULLABLE_WRAPPER}({stripped})" + + +def _base_type(type_sql: str) -> str: + return type_sql.split("(", 1)[0].strip() + + +def _unwrap(*, type_sql: str, wrapper: str) -> str | None: + prefix: str = f"{wrapper}(" + if type_sql.startswith(prefix) and type_sql.endswith(")"): + return type_sql[len(prefix) : -1].strip() + return None diff --git a/src/sqlbuild/adapters/clickhouse/classes/__init__.py b/src/sqlbuild/adapters/clickhouse/classes/__init__.py new file mode 100644 index 0000000000..deeda21bb6 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/__init__.py @@ -0,0 +1 @@ +"""ClickHouse public classes.""" diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py new file mode 100644 index 0000000000..05b066b31c --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -0,0 +1,2639 @@ +"""ClickHouse adapter implementation for a single server.""" + +from __future__ import annotations + +import csv +import io +import json +import logging +import re +from collections.abc import Callable +from dataclasses import replace +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any, ClassVar + +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + SqlValueKind, + render_insert_source_freshness_records_sql, +) +from sqlbuild.adapter.contract.classes.base_adapter import ( + BaseAdapter, + _encode_typed_json, + _render_typed_value_list, + _typed_scalar_payload, + _validate_rectangular_typed_array, +) +from sqlbuild.adapter.contract.classes.historical_snapshot_sql import render_is_distinct_from +from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin +from sqlbuild.adapter.contract.constants import DIFF_LEFT_SIDE, DIFF_RIGHT_SIDE +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value +from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe +from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + CursorValue, + ExpressionInferenceProfile, + FunctionDefinition, + FunctionInfo, + MigrationStagePlan, + QueryResult, + RelationGrant, + RelationInfo, + RelationReadProbe, + RelationRowCountEstimate, + RowDiffColumnResult, + RowDiffCoverage, + RowDiffPreparedRelations, + RowDiffResult, + RowDiffSampleCell, + RowDiffSampleRow, + RowDiffSampling, + RowDiffTolerance, + RowDiffTolerances, + SchemaDiffResult, + SnapshotChangeTarget, + SnapshotSqlDialect, + TableFreshnessMetadata, + TableFreshnessRequest, +) +from sqlbuild.adapter.contract.types import ( + BuiltinAdapter, + CursorKind, + FrameworkType, + HistoricalSnapshotCloseStyle, + HistoricalSnapshotInsertStyle, + LoaderLogicalType, + MigrationTransfer, + PromotionStrategy, + RelationReadStatus, + SnapshotLatestVersionStyle, + SnapshotUpdateStyle, + StatementSizeLimit, + TablePromotionMode, +) +from sqlbuild.adapter.relations.main.get_columns_for_relations import ( + get_columns_for_relations_bulk, +) +from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family +from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.clickhouse._helpers.partition_rewrite import ( + artifact_relation, + rewrite_partitions, +) +from sqlbuild.adapters.clickhouse._helpers.snapshot_rewrite import ( + nullable_offset_windows, + rewrite_snapshot_statements, +) +from sqlbuild.adapters.clickhouse._helpers.type_text import ( + canonical_type, + nullable_type, + quote_string, +) +from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection +from sqlbuild.adapters.clickhouse.classes.partition_locks import PartitionLocks +from sqlbuild.adapters.clickhouse.constants import ( + CONNECTION_OPTION_KEYS, + DEFAULT_COLUMN_DEFAULT_KIND, + DEFAULT_DATABASE, + DEFAULT_HTTP_PORT, + DEFAULT_HTTPS_PORT, + DSN_OPTION, + EXCHANGE_UNSUPPORTED_CODES, + MAX_IDENTIFIER_LENGTH, + MAX_QUERY_SIZE_BYTES, + MINIMUM_SERVER_VERSION, + RELATION_NOT_FOUND_CODES, + RELATION_SEPARATOR, + SESSION_SETTINGS, + SNAPSHOT_REBUILD_INFIX, + SNAPSHOT_WORK_INFIX, + STATE_TABLE_ENGINE_KEYWORD, + TIMEOUT_OPTIONS, + VERIFY_OPTION, + VIEW_ENGINES, +) +from sqlbuild.adapters.clickhouse.models import ( + PartitionRewrite, + RewriteExecutor, + SnapshotWorkspace, +) +from sqlbuild.compiler.compile.types import FunctionLanguage +from sqlbuild.diagnostics.main.log_sql import log_sql +from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS + +_ERROR_CODE_PATTERN: re.Pattern[str] = re.compile(r"Code: (\d+)\.") +_BARE_BIGINT_PATTERN: re.Pattern[str] = re.compile(r"(\w+) BIGINT([,)])") +_NOT_NULL_COLUMN_PATTERN: re.Pattern[str] = re.compile( + r"Nullable\(((?:[^()]|\([^()]*\))*)\) NOT NULL" +) + + +class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): + """ClickHouse adapter that builds tables aside and rewrites whole partitions.""" + + adapter_name: ClassVar[str] = BuiltinAdapter.CLICKHOUSE.value + sql_analysis_dialect_name: ClassVar[str | None] = "clickhouse" + sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax( + backslash_escape_quotes=frozenset({"'", '"', "`"}), + line_comment_prefixes=frozenset({"--"}), + ) + max_identifier_length: ClassVar[int] = MAX_IDENTIFIER_LENGTH + relation_grants_supported: ClassVar[bool] = False + _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( + timestamp_type="Nullable(DateTime64(6, 'UTC'))", + distinct_condition=render_is_distinct_from, + update_style=SnapshotUpdateStyle.UPDATE_FROM, + latest_version=SnapshotLatestVersionStyle.DERIVED_TABLE, + historical_close=HistoricalSnapshotCloseStyle.UPDATE_FROM, + historical_insert=HistoricalSnapshotInsertStyle.INSERT_WITH, + ) + _partition_locks: ClassVar[PartitionLocks] = PartitionLocks() + + def connect(self, config: dict[str, Any]) -> _ClickHouseConnection: + try: + import clickhouse_connect + except ImportError as error: + raise AdapterUserError( + message="ClickHouse adapter requires optional dependency clickhouse-connect. " + "Install with: pip install 'sqlbuild[clickhouse]'", + code="A701", + ) from error + unknown: tuple[str, ...] = tuple(sorted(set(config) - CONNECTION_OPTION_KEYS)) + if unknown: + raise AdapterUserError( + message=f"ClickHouse connection has unsupported options: {', '.join(unknown)}", + code="A702", + help="Supported options: " + ", ".join(sorted(CONNECTION_OPTION_KEYS)), + ) + secure: bool = bool(config.get("secure", False)) + settings: dict[str, object] = dict(SESSION_SETTINGS) + settings.update(dict(config.get("settings") or {})) + client_options: dict[str, Any] = { + "host": str(config.get("host", "localhost")), + "port": int(config.get("port", DEFAULT_HTTPS_PORT if secure else DEFAULT_HTTP_PORT)), + "username": str(config.get("username", config.get("user", "default"))), + "password": str(config.get("password", "")), + "database": str(config.get("database", DEFAULT_DATABASE)), + "secure": secure, + "settings": settings, + "autogenerate_session_id": False, + } + if VERIFY_OPTION in config: + client_options[VERIFY_OPTION] = bool(config[VERIFY_OPTION]) + if DSN_OPTION in config: + client_options[DSN_OPTION] = str(config[DSN_OPTION]) + for timeout_key in TIMEOUT_OPTIONS: + if timeout_key in config: + client_options[timeout_key] = int(config[timeout_key]) + + def create_client() -> Any: + return clickhouse_connect.get_client(**client_options) + + client: Any = create_client() + version: str = str(client.server_version) + if _version_tuple(version) < MINIMUM_SERVER_VERSION: + client.close() + minimum: str = ".".join(str(part) for part in MINIMUM_SERVER_VERSION) + raise AdapterUserError( + message=f"ClickHouse server {version} is older than the supported {minimum}", + code="A703", + help=f"Upgrade the server to ClickHouse {minimum} or newer.", + ) + return _ClickHouseConnection(client=client, interrupt_client_factory=create_client) + + def _execute(self, *, connection: _ClickHouseConnection, sql: str) -> Any: + log_sql(logger=logging.getLogger("sqlbuild.adapter.clickhouse"), sql=sql) + return connection.execute(sql) + + def close(self, connection: _ClickHouseConnection) -> None: + connection.close() + + def supports_transactions(self) -> bool: + return False + + def begin(self, connection: Any) -> None: + del connection + + def commit(self, connection: Any) -> None: + del connection + + def rollback(self, connection: Any) -> None: + del connection + + def supports_transactional_ddl(self) -> bool: + return False + + def interrupt_connection(self, connection: Any) -> bool: + """Kill the connection's running queries by query ID from a side client.""" + + return bool(connection.cancel_active_statements()) + + def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: + cursor: Any = self.execute(connection=connection, sql=sql) + description: Any | None = getattr(cursor, "description", None) + if description is None: + return QueryResult() + columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + if limit is None: + return QueryResult(columns=columns, rows=tuple(cursor.fetchall())) + fetched: list[tuple[object, ...]] = cursor.fetchmany(limit + 1) + return QueryResult( + columns=columns, rows=tuple(fetched[:limit]), truncated=len(fetched) > limit + ) + + def supports_zero_copy_clone(self) -> bool: + return True + + def supports_durable_clone(self) -> bool: + return True + + def supports_relation_age_metadata(self) -> bool: + return False + + def supports_table_freshness_metadata(self) -> bool: + return False + + def supports_python_functions(self) -> bool: + return False + + def supports_table_functions(self) -> bool: + return False + + def supports_concurrent_microbatch_dml(self) -> bool: + """Batches rewrite partitions under per-partition locks, so disjoint windows run at once.""" + + return True + + def max_statement_size(self) -> StatementSizeLimit | None: + """The server's default ``max_query_size``; seeds never travel as SQL text.""" + + return (MAX_QUERY_SIZE_BYTES, "bytes") + + def lists_relations_authoritatively(self, *, database: str | None, schema: str) -> bool: + del database, schema + return True + + def default_table_promotion_mode(self) -> TablePromotionMode: + return TablePromotionMode.STAGED + + def default_promotion_strategy(self) -> PromotionStrategy: + return PromotionStrategy.ATOMIC_SWAP + + def requires_derived_table_aliases(self) -> bool: + return False + + def sql_analysis_dialect(self) -> str | None: + return self.sql_analysis_dialect_name + + def expression_inference_profile(self) -> ExpressionInferenceProfile: + return ExpressionInferenceProfile(sql_analysis_dialect=self.sql_analysis_dialect()) + + def default_schema(self) -> str: + return DEFAULT_DATABASE + + def default_database(self) -> str | None: + return None + + def render_qualified_name( + self, *, database: str | None, schema: str | None, name: str + ) -> str | None: + if database is not None: + raise AdapterUserError( + message=( + f"ClickHouse has no database level above schemas; got database '{database}'" + ), + code="A704", + help="Remove 'database' from the target; a SQLBuild schema is a ClickHouse " + "database.", + ) + if schema is not None: + return f"{schema}.{name}" + return None + + def render_identifier(self, name: str) -> str: + return '"' + name.replace("\\", "\\\\").replace('"', '\\"') + '"' + + def render_exact_identifier(self, name: str) -> str: + return self.render_identifier(name) + + def star_exclude_keyword(self) -> str: + return "EXCEPT" + + def render_final_relation(self, *, relation: str) -> str: + """Read with ``FINAL`` so replacing and collapsing engines return one row per key.""" + + return f"(SELECT * FROM {relation} FINAL)" + + def relation_names_match(self, *, left: str, right: str) -> bool: + return _unquote(left) == _unquote(right) + + def schema_exists(self, *, connection: Any, database: str | None, schema: str) -> bool: + del database + cursor: Any = self.execute( + connection=connection, + sql=f"SELECT 1 FROM system.databases WHERE name = {quote_string(schema)}", + ) + return cursor.fetchone() is not None + + def relation_exists( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> bool: + del database + cursor: Any = self.execute( + connection=connection, + sql="SELECT 1 FROM system.tables WHERE " + + _database_filter(schema) + + f" AND name = {quote_string(name)}", + ) + return cursor.fetchone() is not None + + def list_relations( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + del database + cursor: Any = self.execute( + connection=connection, + sql="SELECT database, name, engine FROM system.tables WHERE NOT is_temporary" + + _in_filter(column="database", values=schemas) + + _in_filter(column="name", values=names) + + " ORDER BY database, name", + ) + return tuple( + RelationInfo( + database=None, + schema=str(row[0]), + name=str(row[1]), + relation_type="VIEW" if str(row[2]) in VIEW_ENGINES else "BASE TABLE", + ) + for row in cursor.fetchall() + ) + + def list_functions( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[FunctionInfo, ...]: + del database + cursor: Any = self.execute( + connection=connection, + sql="SELECT name FROM system.functions WHERE origin = 'SQLUserDefined' ORDER BY name", + ) + functions: list[FunctionInfo] = [] + for row in cursor.fetchall(): + schema, separator, name = str(row[0]).partition("__") + if not separator: + continue + if schemas is not None and schema not in schemas: + continue + if names and name not in names: + continue + functions.append( + FunctionInfo(database=None, schema=schema, name=name, function_type="FUNCTION") + ) + return tuple(functions) + + def get_columns( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> tuple[ColumnInfo, ...]: + del database + cursor: Any = self.execute( + connection=connection, + sql="SELECT name, type FROM system.columns WHERE " + + _database_filter(schema) + + f" AND table = {quote_string(name)} ORDER BY position", + ) + return tuple(ColumnInfo(name=str(row[0]), type=str(row[1])) for row in cursor.fetchall()) + + def get_all_columns( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> dict[str, tuple[ColumnInfo, ...]]: + del database + cursor: Any = self.execute( + connection=connection, + sql="SELECT table, name, type FROM system.columns WHERE 1 = 1" + + _in_filter(column="database", values=schemas) + + _in_filter(column="table", values=names) + + " ORDER BY table, position", + ) + result: dict[str, list[ColumnInfo]] = {} + for row in cursor.fetchall(): + result.setdefault(str(row[0]), []).append( + ColumnInfo(name=str(row[1]), type=str(row[2])) + ) + return {table: tuple(columns) for table, columns in result.items()} + + def describe_relation(self, *, connection: Any, relation: str) -> tuple[ColumnInfo, ...]: + cursor: Any = self.execute(connection=connection, sql=f"DESCRIBE TABLE {relation}") + return tuple( + ColumnInfo(name=str(row[0]), type=str(row[1])) + for row in cursor.fetchall() + if not row[2] or str(row[2]) == DEFAULT_COLUMN_DEFAULT_KIND + ) + + def probe_relation_read(self, *, connection: Any, relation: str) -> RelationReadProbe: + return run_relation_read_probe( + execute=self.execute, + connection=connection, + relation=relation, + classify_not_found=_classify_relation_not_found, + ) + + def physical_relation_generation( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> str | None: + """The table UUID, which changes whenever the table is recreated or exchanged.""" + + del database + row: Any = self.execute( + connection=connection, + sql="SELECT toString(uuid) FROM system.tables WHERE " + + _database_filter(schema) + + f" AND name = {quote_string(name)}", + ).fetchone() + return None if row is None else str(row[0]) + + def estimate_relation_row_count( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> RelationRowCountEstimate: + del database + row: Any = self.execute( + connection=connection, + sql="SELECT engine, total_rows FROM system.tables WHERE " + + _database_filter(schema) + + f" AND name = {quote_string(name)}", + ).fetchone() + if row is None: + return RelationRowCountEstimate(row_count=None, detail="relation not found") + if row[1] is None: + return RelationRowCountEstimate( + row_count=None, detail=f"engine {row[0]} does not report row counts" + ) + return RelationRowCountEstimate(row_count=int(row[1])) + + def read_view_definition( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> str | None: + del database + row: Any = self.execute( + connection=connection, + sql="SELECT as_select FROM system.tables WHERE " + + _database_filter(schema) + + f" AND name = {quote_string(name)} AND engine = 'View'", + ).fetchone() + return None if row is None else str(row[0]) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + del connection, destination + return None + + def render_framework_type(self, type_name: FrameworkType) -> str: + match type_name: + case FrameworkType.STRING: + return "Nullable(String)" + case FrameworkType.INTEGER: + return "Nullable(Int64)" + case FrameworkType.TIMESTAMP: + return "Nullable(DateTime64(6, 'UTC'))" + + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + match type_name: + case LoaderLogicalType.BOOLEAN: + return "Nullable(Bool)" + case LoaderLogicalType.INTEGER: + return "Nullable(Int64)" + case LoaderLogicalType.FLOAT: + return "Nullable(Float64)" + case LoaderLogicalType.STRING: + return "Nullable(String)" + case LoaderLogicalType.TIMESTAMP: + return "Nullable(DateTime64(6, 'UTC'))" + case LoaderLogicalType.DATE: + return "Nullable(Date32)" + case LoaderLogicalType.JSON: + return "Nullable(String)" + + def render_typed_scalar(self, *, value: SqlValue) -> str: + payload: object = _typed_scalar_payload(value) + match value.kind: + case SqlValueKind.STRING: + return quote_string(str(payload)) + case SqlValueKind.BOOLEAN: + return "true" if payload else "false" + case SqlValueKind.NULL: + return "NULL" + case SqlValueKind.DECIMAL: + return f"toDecimal256({quote_string(str(payload))}, {_decimal_scale(payload)})" + case _: + return str(payload) + + def render_typed_array(self, *, value: SqlValue) -> str: + _validate_rectangular_typed_array(value=value, adapter_name=self.adapter_name) + return "[" + self._render_typed_array_items(value) + "]" + + def render_typed_object(self, *, value: SqlValue) -> str: + return f"CAST({quote_string(_encode_typed_json(value))} AS JSON)" + + def render_loader_value_literal( + self, *, value: object, logical_type: LoaderLogicalType | None + ) -> str: + if value is None: + return "NULL" + if logical_type == LoaderLogicalType.JSON: + return quote_string(json.dumps(value, sort_keys=True)) + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int | float | Decimal): + return str(value) + if isinstance(value, datetime | date): + return quote_string(value.isoformat()) + if isinstance(value, dict | list): + return quote_string(json.dumps(value, sort_keys=True)) + return quote_string(str(value)) + + def render_loader_rows_select( + self, + *, + rows: tuple[dict[str, object], ...], + column_names: tuple[str, ...], + column_sql_types: dict[str, str], + inferred_types: dict[str, LoaderLogicalType], + ) -> str: + """Render rows as a UNION ALL of selects; ClickHouse has no aliased VALUES list.""" + + if not rows: + return ( + "SELECT " + + ", ".join( + f"CAST(NULL AS {nullable_type(column_sql_types.get(name, 'String'))}) AS " + f"{self.render_identifier(name)}" + for name in column_names + ) + + " WHERE 1 = 0" + ) + return " UNION ALL ".join( + self._loader_row_select( + row=row, + column_names=column_names, + column_sql_types=column_sql_types, + inferred_types=inferred_types, + ) + for row in rows + ) + + def _loader_row_select( + self, + *, + row: dict[str, object], + column_names: tuple[str, ...], + column_sql_types: dict[str, str], + inferred_types: dict[str, LoaderLogicalType], + ) -> str: + projections: list[str] = [] + name: str + for name in column_names: + value_sql: str = self.render_loader_value_literal( + value=row.get(name), logical_type=inferred_types.get(name) + ) + sql_type: str | None = column_sql_types.get(name) + rendered: str = ( + value_sql if sql_type is None else f"CAST({value_sql} AS {nullable_type(sql_type)})" + ) + projections.append(f"{rendered} AS {self.render_identifier(name)}") + return "SELECT " + ", ".join(projections) + + def render_source_expression_cast( + self, *, expression: str, target_type: str, alias: str + ) -> str: + return f"CAST({expression} AS {nullable_type(target_type)}) AS {alias}" + + def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> str: + if cursor_type == CursorKind.INTEGER: + return value + if cursor_type == CursorKind.TIMESTAMP: + return f"parseDateTime64BestEffort({quote_string(value)}, 6, 'UTC')" + return quote_string(value) + + def build_cursor_filter( + self, + *, + cursor_column: str | None, + start_cursor: CursorValue | None, + end_cursor: CursorValue | None, + ) -> str: + if cursor_column is None or start_cursor is None: + return "" + clauses: list[str] = [f"{cursor_column} >= {_cursor_value_literal(start_cursor)}"] + if end_cursor is not None: + clauses.append(f"{cursor_column} < {_cursor_value_literal(end_cursor)}") + return " AND ".join(clauses) + + def render_max_cursor_at_or_before( + self, + *, + relation: str, + cursor_column: str, + maximum_allowed: str, + cursor_type: str | None, + is_date: bool, + ) -> str: + quoted_cursor: str = self.render_identifier(cursor_column) + maximum_literal: str = ( + f"toDate32({quote_string(maximum_allowed)})" + if is_date + else self.render_cursor_bound_literal(value=maximum_allowed, cursor_type=cursor_type) + ) + return ( + f"SELECT max({quoted_cursor}) FROM {relation} " + f"WHERE {quoted_cursor} <= {maximum_literal}" + ) + + def render_current_timestamp(self) -> str: + return "now64(6, 'UTC')" + + def normalize_row_diff_numeric_type(self, column_type: str) -> str | None: + return normalize_numeric_family(type_sql=canonical_type(column_type), dialect=None) + + def format_row_diff_decimal_sql(self, value: Decimal) -> str: + return f"toDecimal256({quote_string(format(value, 'f'))}, {_decimal_scale(value)})" + + def _render_row_diff_key_hash(self, *, keys: tuple[str, ...], alias: str, seed: int) -> str: + key_values: str = ", ".join(f"{alias}.{key}" for key in keys) + return f"sipHash64({seed}, {key_values})" + + def render_create_schema(self, *, database: str | None, schema: str) -> tuple[str, ...]: + del database + return (f"CREATE DATABASE IF NOT EXISTS {schema}",) + + def ensure_schema( + self, + *, + connection: Any, + database: str | None, + schema: str | None, + statement_recorder: StatementRecorder, + ) -> None: + if schema is None: + return + statements: tuple[str, ...] = self.render_create_schema(database=database, schema=schema) + statement_recorder.record_many(statements) + for statement in statements: + self.execute(connection=connection, sql=statement) + + def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, ...]: + return ( + f"DROP TABLE IF EXISTS {destination}", + f"CREATE TABLE {destination} ENGINE = MergeTree ORDER BY tuple() AS {sql}", + ) + + def render_create_view_as(self, *, destination: str, sql: str) -> tuple[str, ...]: + return (f"CREATE OR REPLACE VIEW {destination} AS {sql}",) + + def render_replace_view_keeping_grants( + self, *, destination: str, sql: str + ) -> tuple[str, ...] | None: + return self.render_create_view_as(destination=destination, sql=sql) + + def render_drop(self, *, destination: str, if_exists: bool = True) -> tuple[str, ...]: + exists_clause: str = " IF EXISTS" if if_exists else "" + return (f"DROP TABLE{exists_clause} {destination}",) + + def render_drop_view(self, *, destination: str, if_exists: bool = True) -> tuple[str, ...]: + exists_clause: str = " IF EXISTS" if if_exists else "" + return (f"DROP VIEW{exists_clause} {destination}",) + + def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: + return ( + f"RENAME TABLE {origin} TO {_qualify_like(destination=destination, origin=origin)}", + ) + + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return self.render_rename(origin=origin, destination=destination) + + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: + """Exchange two tables atomically; this needs an Atomic database engine.""" + + return (f"EXCHANGE TABLES {left} AND {right}",) + + def swap( + self, *, connection: Any, left: str, right: str, statement_recorder: StatementRecorder + ) -> None: + statements: tuple[str, ...] = self.render_swap(left=left, right=right) + statement_recorder.record_many(statements) + for statement in statements: + try: + self.execute(connection=connection, sql=statement) + except Exception as error: + if _error_code(error) in EXCHANGE_UNSUPPORTED_CODES: + raise AdapterUserError( + message=f"ClickHouse cannot exchange {left} and {right}: {error}", + code="A705", + help="Atomic promotion needs databases with the Atomic engine (the " + "default since ClickHouse 20.10).", + ) from error + raise + + def render_replace_table_from_relation( + self, *, destination: str, origin: str + ) -> tuple[str, ...]: + return ( + f"DROP TABLE IF EXISTS {destination}", + f"CREATE TABLE {destination} AS {origin}", + f"INSERT INTO {destination} SELECT * FROM {origin}", + ) + + def render_clone( + self, + *, + origin: str, + destination: str, + hard_copy: bool = False, + origin_is_transient: bool = False, + ) -> tuple[str, ...]: + """Clone by hard-linking every part; the clone keeps the engine, keys and settings.""" + + del origin_is_transient + if hard_copy: + return self.render_replace_table_from_relation(destination=destination, origin=origin) + return ( + f"DROP TABLE IF EXISTS {destination}", + f"CREATE TABLE {destination} AS {origin}", + f"ALTER TABLE {destination} ATTACH PARTITION ALL FROM {origin}", + ) + + def render_durable_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> tuple[str, ...]: + return self.render_clone( + origin=origin, destination=destination, origin_is_transient=origin_is_transient + ) + + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=( + f"CREATE TABLE {stage} AS {origin}", + f"INSERT INTO {stage} SELECT * FROM {origin}", + ), + ) + + def move_or_copy_relation( + self, + *, + connection: Any, + origin: str, + destination: str, + remove_origin: bool, + allow_copy_fallback: bool, + statement_recorder: StatementRecorder, + ) -> None: + """Move with RENAME TABLE, which also crosses databases; copy otherwise.""" + + statements: tuple[str, ...] + if remove_origin: + statements = self.render_rename(origin=origin, destination=destination) + else: + if not allow_copy_fallback: + raise AdapterUserError( + message="ClickHouse relation copy requires --allow-copy", code="A706" + ) + statements = self.render_replace_table_from_relation( + destination=destination, origin=origin + ) + statement_recorder.record_many(statements) + for statement in statements: + self.execute(connection=connection, sql=statement) + + def render_add_columns( + self, *, destination: str, columns: tuple[ColumnInfo, ...] + ) -> tuple[str, ...]: + return tuple( + f"ALTER TABLE {destination} ADD COLUMN {self.render_identifier(column.name)} " + f"{nullable_type(column.type)}" + for column in columns + ) + + def render_drop_columns( + self, *, destination: str, column_names: tuple[str, ...] + ) -> tuple[str, ...]: + return tuple( + f"ALTER TABLE {destination} DROP COLUMN {self.render_identifier(name)}" + for name in column_names + ) + + def render_alter_column_types( + self, *, destination: str, columns: tuple[ColumnInfo, ...] + ) -> tuple[str, ...]: + return tuple( + f"ALTER TABLE {destination} MODIFY COLUMN {self.render_identifier(column.name)} " + f"{column.type}" + for column in columns + ) + + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + return ( + f"ALTER TABLE {destination} RENAME COLUMN {self.render_identifier(old_name)} " + f"TO {self.render_identifier(new_name)}", + ) + + def render_udf_call(self, *, target: str, call_suffix_sql: str) -> str: + return f"{_mangled_function_name(target)}{call_suffix_sql}" + + def render_create_function( + self, + *, + destination: str, + arguments: tuple[Any, ...], + returns: str, + body_sql: str, + return_columns: tuple[Any, ...] = (), + language: FunctionLanguage = FunctionLanguage.SQL, + runtime_version: str | None = None, + entry_point: str | None = None, + packages: tuple[str, ...] = (), + ) -> tuple[str, ...]: + """Create a server-global SQL lambda named ``__``.""" + + del runtime_version, entry_point, packages + if language != FunctionLanguage.SQL: + raise AdapterUserError( + message="ClickHouse does not deploy Python functions; executable UDFs need " + "server configuration", + code="A707", + help="Install the function on the server and declare it as an external function.", + ) + if return_columns: + raise AdapterUserError( + message="ClickHouse does not support SQL table functions", code="A708" + ) + argument_names: str = ", ".join(str(argument.name) for argument in arguments) + body: str = body_sql.strip().removesuffix(";").strip() + if body.upper().startswith("SELECT "): + body = body[len("SELECT ") :].strip() + cast_body: str = f"CAST(({body}) AS {nullable_type(returns)})" if returns else f"({body})" + return ( + f"CREATE OR REPLACE FUNCTION {_mangled_function_name(destination)} " + f"AS ({argument_names}) -> {cast_body}", + ) + + def render_append( + self, *, destination: str, sql: str, columns: tuple[str, ...] | None = None + ) -> tuple[str, ...]: + if columns is None: + return (f"INSERT INTO {destination} {sql}",) + column_list: str = ", ".join(self.render_identifier(column) for column in columns) + return (f"INSERT INTO {destination} ({column_list}) {sql}",) + + def render_delete_insert( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + columns: tuple[str, ...] | None = None, + ) -> tuple[str, ...]: + request: PartitionRewrite = self._key_rewrite( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ) + return ( + request.delta_insert_sql, + f"-- rewrite partitions where {request.displaced_predicate}", + ) + + def render_delete_insert_cursor( + self, + *, + destination: str, + sql: str, + cursor_column: str, + cursor_start: str, + cursor_end: str, + columns: tuple[str, ...] | None = None, + cursor_type: str | None = None, + ) -> tuple[str, ...]: + request: PartitionRewrite = self._window_rewrite( + destination=destination, + sql=sql, + cursor_column=cursor_column, + cursor_start=cursor_start, + cursor_end=cursor_end, + columns=columns, + cursor_type=cursor_type, + ) + return ( + request.delta_insert_sql, + f"-- rewrite partitions where {request.displaced_predicate}", + ) + + def render_merge( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + source_columns: tuple[str, ...] = (), + exclude_columns: tuple[str, ...] = (), + ) -> tuple[str, ...]: + request: PartitionRewrite = self._merge_rewrite( + destination=destination, + sql=sql, + unique_key=unique_key, + source_columns=source_columns, + exclude_columns=exclude_columns, + ) + return ( + request.delta_insert_sql, + f"-- rewrite partitions where {request.displaced_predicate}", + ) + + def delete_insert( + self, + *, + connection: Any, + destination: str, + sql: str, + unique_key: str | tuple[str, ...], + columns: tuple[str, ...] | None = None, + statement_recorder: StatementRecorder, + ) -> int | None: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + return self._rewrite( + connection=connection, + request=self._key_rewrite( + destination=destination, sql=sql, unique_key=keys, columns=columns + ), + statement_recorder=statement_recorder, + ) + + def delete_insert_cursor( + self, + *, + connection: Any, + destination: str, + sql: str, + cursor_column: str, + cursor_start: str, + cursor_end: str, + columns: tuple[str, ...] | None = None, + statement_recorder: StatementRecorder, + cursor_type: str | None = None, + ) -> int | None: + return self._rewrite( + connection=connection, + request=self._window_rewrite( + destination=destination, + sql=sql, + cursor_column=cursor_column, + cursor_start=cursor_start, + cursor_end=cursor_end, + columns=columns, + cursor_type=cursor_type, + ), + statement_recorder=statement_recorder, + ) + + def merge( + self, + *, + connection: Any, + destination: str, + sql: str, + unique_key: str | tuple[str, ...], + statement_recorder: StatementRecorder, + exclude_columns: tuple[str, ...] = (), + ) -> int | None: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + source_columns: tuple[str, ...] = self.query_column_names(connection=connection, sql=sql) + return self._rewrite( + connection=connection, + request=self._merge_rewrite( + destination=destination, + sql=sql, + unique_key=keys, + source_columns=source_columns, + exclude_columns=exclude_columns, + ), + statement_recorder=statement_recorder, + ) + + def _rewrite( + self, *, connection: Any, request: PartitionRewrite, statement_recorder: StatementRecorder + ) -> int: + return rewrite_partitions( + request=request, + executor=RewriteExecutor( + execute=lambda sql: self.execute(connection=connection, sql=sql), + statement_recorder=statement_recorder, + partition_locks=self._partition_locks, + ), + ) + + def _key_rewrite( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + columns: tuple[str, ...] | None, + ) -> PartitionRewrite: + key_tuple: str = _key_tuple(adapter=self, keys=unique_key) + return PartitionRewrite( + destination=destination, + delta_insert_sql=self.render_append(destination="{delta}", sql=sql, columns=columns)[0], + displaced_predicate=f"{key_tuple} IN (SELECT {key_tuple} FROM {{delta}})", + ) + + def _window_rewrite( + self, + *, + destination: str, + sql: str, + cursor_column: str, + cursor_start: str, + cursor_end: str, + columns: tuple[str, ...] | None, + cursor_type: str | None, + ) -> PartitionRewrite: + quoted: str = self.render_identifier(cursor_column) + start: str = self.render_cursor_bound_literal(value=cursor_start, cursor_type=cursor_type) + end: str = self.render_cursor_bound_literal(value=cursor_end, cursor_type=cursor_type) + return PartitionRewrite( + destination=destination, + delta_insert_sql=self.render_append(destination="{delta}", sql=sql, columns=columns)[0], + displaced_predicate=f"({quoted} >= {start} AND {quoted} < {end})", + ) + + def _merge_rewrite( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + source_columns: tuple[str, ...], + exclude_columns: tuple[str, ...], + ) -> PartitionRewrite: + """Upsert: incoming rows win, except excluded columns keep matched target values.""" + + excluded: frozenset[str] = frozenset( + column.lower() for column in exclude_columns + ) - frozenset(key.lower() for key in unique_key) + key_tuple: str = _key_tuple(adapter=self, keys=unique_key) + projections: list[str] = [] + column: str + for column in source_columns: + quoted: str = self.render_identifier(column) + if column.lower() in excluded: + projections.append( + f"if(__target.__sqb_matched = 1, __target.{quoted}, __source.{quoted})" + ) + else: + projections.append(f"__source.{quoted}") + column_list: str = ", ".join(self.render_identifier(column) for column in source_columns) + source_sql: str = f"SELECT * FROM ({sql}) AS __source" + if excluded: + kept: str = ", ".join( + self.render_identifier(column) + for column in source_columns + if column.lower() in excluded + ) + key_list: str = ", ".join(self.render_identifier(key) for key in unique_key) + join_condition: str = " AND ".join( + f"__source.{self.render_identifier(key)} = __target.{self.render_identifier(key)}" + for key in unique_key + ) + source_sql = ( + f"SELECT {', '.join(projections)} FROM ({sql}) AS __source " + f"LEFT JOIN (SELECT {key_list}, {kept}, 1 AS __sqb_matched FROM {destination}) " + f"AS __target ON {join_condition}" + ) + return PartitionRewrite( + destination=destination, + delta_insert_sql=f"INSERT INTO {{delta}} ({column_list}) {source_sql}", + displaced_predicate=f"{key_tuple} IN (SELECT {key_tuple} FROM {{delta}})", + ) + + def render_create_initial_historical_timestamp_snapshot_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return nullable_offset_windows( + super().render_create_initial_historical_timestamp_snapshot_destination( + table_type=table_type, + destination=destination, + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ) + ) + + def render_create_initial_historical_timestamp_changes_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + ) -> tuple[str, ...]: + return nullable_offset_windows( + super().render_create_initial_historical_timestamp_changes_destination( + table_type=table_type, + destination=destination, + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + ) + ) + + def render_create_initial_historical_check_snapshot_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + unique_key: tuple[str, ...], + check_columns: tuple[str, ...], + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return nullable_offset_windows( + super().render_create_initial_historical_check_snapshot_destination( + table_type=table_type, + destination=destination, + origin=origin, + unique_key=unique_key, + check_columns=check_columns, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ) + ) + + def render_apply_timestamp_snapshot_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + observed_at_column: str | None, + valid_from_column: str, + valid_to_column: str, + initial_valid_from: str | None, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return self._staged_snapshot( + destination=destination, + valid_to_column=valid_to_column, + render=lambda work: super( + ClickHouseAdapter, self + ).render_apply_timestamp_snapshot_changes( + destination=work, + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + initial_valid_from=initial_valid_from, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ), + ) + + def render_apply_check_snapshot_changes( + self, + *, + target: SnapshotChangeTarget, + check_columns: tuple[str, ...], + updated_at_column: str | None, + observed_at_column: str | None, + initial_valid_from: str | None, + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return self._staged_snapshot( + destination=target.destination, + valid_to_column=target.valid_to_column, + render=lambda work: super(ClickHouseAdapter, self).render_apply_check_snapshot_changes( + target=replace(target, destination=work), + check_columns=check_columns, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + initial_valid_from=initial_valid_from, + invalidate_hard_deletes=invalidate_hard_deletes, + ), + ) + + def render_apply_historical_timestamp_snapshot_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return self._staged_snapshot( + destination=destination, + valid_to_column=valid_to_column, + render=lambda work: super( + ClickHouseAdapter, self + ).render_apply_historical_timestamp_snapshot_changes( + destination=work, + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ), + ) + + def render_apply_historical_timestamp_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + ) -> tuple[str, ...]: + return self._staged_snapshot( + destination=destination, + valid_to_column=valid_to_column, + render=lambda work: super( + ClickHouseAdapter, self + ).render_apply_historical_timestamp_changes( + destination=work, + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + ), + ) + + def render_apply_historical_check_snapshot_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + check_columns: tuple[str, ...], + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return self._staged_snapshot( + destination=destination, + valid_to_column=valid_to_column, + render=lambda work: super( + ClickHouseAdapter, self + ).render_apply_historical_check_snapshot_changes( + destination=work, + origin=origin, + unique_key=unique_key, + check_columns=check_columns, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ), + ) + + def _staged_snapshot( + self, + *, + destination: str, + valid_to_column: str, + render: Callable[[str], tuple[str, ...]], + ) -> tuple[str, ...]: + workspace: SnapshotWorkspace = SnapshotWorkspace( + destination=destination, + work=artifact_relation(relation=destination, infix=SNAPSHOT_WORK_INFIX, digest=""), + rebuild=artifact_relation( + relation=destination, infix=SNAPSHOT_REBUILD_INFIX, digest="" + ), + valid_to_column=valid_to_column, + ) + return rewrite_snapshot_statements(workspace=workspace, statements=render(workspace.work)) + + def load_seed( + self, + *, + connection: Any, + destination: str, + file_path: Path, + columns: tuple[ColumnInfo, ...], + csv_settings: SeedCsvSettings = DEFAULT_SEED_CSV_SETTINGS, + replace: bool = True, + infer_types: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + """Create the seed table and stream its rows as CSV, never as SQL text.""" + + del infer_types + if replace: + self.drop( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=statement_recorder, + ) + column_definitions: str = ", ".join( + f"{self.render_identifier(column.name)} {nullable_type(column.type)}" + for column in columns + ) + create_sql: str = ( + f"CREATE TABLE {destination} ({column_definitions}) ENGINE = MergeTree ORDER BY tuple()" + ) + statement_recorder.record(create_sql) + self.execute(connection=connection, sql=create_sql) + column_names: tuple[str, ...] = tuple(column.name for column in columns) + buffer: io.StringIO = io.StringIO() + writer: Any = csv.writer(buffer, lineterminator="\n", quoting=csv.QUOTE_MINIMAL) + row_count: int = 0 + with file_path.open("r", encoding=csv_settings.encoding or "utf-8", newline="") as seed: + reader: csv.DictReader[str] = csv.DictReader( + seed, + delimiter=csv_settings.delimiter or ",", + quotechar=csv_settings.quotechar or '"', + escapechar=csv_settings.escapechar, + doublequote=True if csv_settings.doublequote is None else csv_settings.doublequote, + skipinitialspace=bool(csv_settings.skipinitialspace), + ) + for row in reader: + values: list[str] = [] + for name in column_names: + value: object = normalize_seed_csv_value( + value=row.get(name), column_name=name, csv_settings=csv_settings + ) + values.append("\\N" if value is None else str(value)) + writer.writerow(values) + row_count += 1 + if row_count == 0: + return + statement_recorder.record(f"INSERT INTO {destination} FORMAT CSV") + connection.insert_csv( + table=destination, + column_names=column_names, + payload=buffer.getvalue().encode("utf-8"), + ) + + def diff_schema(self, *, connection: Any, left: str, right: str) -> SchemaDiffResult: + left_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=left + ) + right_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=right + ) + left_map: dict[str, str] = {column.name: column.type for column in left_columns} + right_map: dict[str, str] = {column.name: column.type for column in right_columns} + added: tuple[ColumnInfo, ...] = tuple( + ColumnInfo(name=name, type=type_sql) + for name, type_sql in right_map.items() + if name not in left_map + ) + removed: tuple[ColumnInfo, ...] = tuple( + ColumnInfo(name=name, type=type_sql) + for name, type_sql in left_map.items() + if name not in right_map + ) + changed: tuple[tuple[ColumnInfo, ColumnInfo], ...] = tuple( + (ColumnInfo(name=name, type=left_map[name]), ColumnInfo(name=name, type=type_sql)) + for name, type_sql in right_map.items() + if name in left_map + and not types_equal( + left=canonical_type(left_map[name]), right=canonical_type(type_sql), dialect=None + ) + ) + return SchemaDiffResult( + added_columns=added, + removed_columns=removed, + type_changed_columns=changed, + left_column_count=len(left_columns), + right_column_count=len(right_columns), + ) + + def validate_row_diff_keys( + self, + *, + connection: Any, + relation_sql: str, + relation_label: str, + keys: tuple[str, ...], + ) -> None: + if not keys: + raise AdapterUserError(message="row diff requires at least one unique_key column") + null_condition: str = " OR ".join(f"{key} IS NULL" for key in keys) + null_count_sql: str = ( + f"SELECT COUNT(*) FROM ({relation_sql}) AS __key_check WHERE {null_condition}" + ) + null_row: tuple[Any, ...] = self.execute( + connection=connection, sql=null_count_sql + ).fetchone() + if int(null_row[0]) > 0: + raise AdapterUserError( + message=f"row diff {relation_label} relation contains null unique_key values" + ) + + key_list: str = ", ".join(keys) + duplicate_count_sql: str = ( + f"SELECT COUNT(*) FROM (" + f"SELECT {key_list} FROM ({relation_sql}) AS __key_check " + f"GROUP BY {key_list} HAVING COUNT(*) > 1" + f") AS __duplicates" + ) + duplicate_row: tuple[Any, ...] = self.execute( + connection=connection, sql=duplicate_count_sql + ).fetchone() + if int(duplicate_row[0]) > 0: + raise AdapterUserError( + message=f"row diff {relation_label} relation contains duplicate unique_key values" + ) + + def build_row_diff_equal_expression( + self, + *, + column: str, + column_info: ColumnInfo, + tolerances: RowDiffTolerances | None, + ) -> str: + tolerance: RowDiffTolerance | None = self.resolve_row_diff_tolerance( + column=column, + column_type=column_info.type, + tolerances=tolerances, + ) + left_expression: str = f"__left.{column}" + right_expression: str = f"__right.{column}" + if tolerance is None: + return f"{left_expression} IS NOT DISTINCT FROM {right_expression}" + threshold_parts: list[str] = [] + if tolerance.absolute is not None: + threshold_parts.append(self.format_row_diff_decimal_sql(tolerance.absolute)) + if tolerance.relative is not None: + threshold_parts.append( + f"{self.format_row_diff_decimal_sql(tolerance.relative)} * " + f"GREATEST(ABS({left_expression}), ABS({right_expression}))" + ) + threshold_sql: str = threshold_parts[0] + if len(threshold_parts) > 1: + threshold_sql = f"GREATEST({', '.join(threshold_parts)})" + return ( + f"(({left_expression} IS NULL AND {right_expression} IS NULL) OR " + f"({left_expression} IS NOT NULL AND {right_expression} IS NOT NULL AND " + f"ABS({left_expression} - {right_expression}) <= {threshold_sql}))" + ) + + def resolve_row_diff_tolerance( + self, + *, + column: str, + column_type: str, + tolerances: RowDiffTolerances | None, + ) -> RowDiffTolerance | None: + if tolerances is None: + return None + column_tolerance: RowDiffTolerance | None = tolerances.by_column.get(column) + if column_tolerance is not None: + if self.normalize_row_diff_numeric_type(column_type) is None: + raise AdapterUserError( + message=f"row diff tolerance for non-numeric column '{column}' is invalid" + ) + self.validate_row_diff_tolerance( + column=column, + tolerance=column_tolerance, + ) + return column_tolerance + normalized_type: str | None = self.normalize_row_diff_numeric_type(column_type) + if normalized_type is None: + return None + type_tolerance: RowDiffTolerance | None = tolerances.by_type.get(normalized_type) + if type_tolerance is not None: + self.validate_row_diff_tolerance( + column=column, + tolerance=type_tolerance, + ) + return type_tolerance + + def validate_row_diff_tolerance(self, *, column: str, tolerance: RowDiffTolerance) -> None: + if tolerance.absolute is None and tolerance.relative is None: + raise AdapterUserError( + message=f"row diff tolerance for column '{column}' must define absolute or relative" + ) + + def inspect_row_diff_coverage( + self, + *, + connection: Any, + relation: str, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + ) -> RowDiffCoverage: + return self._inspect_row_diff_coverage( + connection=connection, + relation=relation, + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + + def diff_rows( + self, + *, + connection: Any, + left: str, + right: str, + unique_key: str | tuple[str, ...], + excluded_columns: tuple[str, ...] = (), + tolerances: RowDiffTolerances | None = None, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + ) -> RowDiffResult: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + sampling: RowDiffSampling | None = tolerances.sampling if tolerances is not None else None + left_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=left + ) + compare_columns: tuple[str, ...] = tuple( + col.name + for col in left_columns + if col.name not in keys and col.name not in excluded_columns + ) + left_columns_by_name: dict[str, ColumnInfo] = {col.name: col for col in left_columns} + cursor_filter: str = self.build_cursor_filter( + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + left_cte: str = f"SELECT * FROM {left}" + right_cte: str = f"SELECT * FROM {right}" + if cursor_filter: + left_cte += f" WHERE {cursor_filter}" + right_cte += f" WHERE {cursor_filter}" + self.validate_row_diff_keys( + connection=connection, relation_sql=left_cte, relation_label="left", keys=keys + ) + self.validate_row_diff_keys( + connection=connection, relation_sql=right_cte, relation_label="right", keys=keys + ) + prepared_relations: RowDiffPreparedRelations = self._build_row_diff_relation_ctes( + connection=connection, + left_sql=left_cte, + right_sql=right_cte, + keys=keys, + sampling=sampling, + inspect_population=True, + ) + join_condition: str = " AND ".join(f"__left.{k} = __right.{k}" for k in keys) + column_equal_expressions: dict[str, str] = { + col: self.build_row_diff_equal_expression( + column=col, + column_info=left_columns_by_name[col], + tolerances=tolerances, + ) + for col in compare_columns + } + column_tolerances: dict[str, RowDiffTolerance | None] = { + col: self.resolve_row_diff_tolerance( + column=col, + column_type=left_columns_by_name[col].type, + tolerances=tolerances, + ) + for col in compare_columns + } + equal_condition: str = "TRUE" + if compare_columns: + equal_condition = " AND ".join(column_equal_expressions.values()) + column_count_sql_parts: list[str] = [ + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL " + f"AND __right.{keys[0]} IS NOT NULL " + f"AND NOT ({column_equal_expressions[col]}) THEN 1 END) AS __{col}_mismatch_count" + for col in compare_columns + ] + column_count_sql: str = ( + ", " + ", ".join(column_count_sql_parts) if column_count_sql_parts else "" + ) + diff_sql: str = ( + f"WITH {prepared_relations.cte_sql} " + f"SELECT " + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL THEN 1 END) AS left_count, " + f"COUNT(CASE WHEN __right.{keys[0]} IS NOT NULL THEN 1 END) AS right_count, " + f"COUNT(*) AS joined, " + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL " + f"AND __right.{keys[0]} IS NOT NULL AND ({equal_condition}) THEN 1 END) AS equal, " + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL " + f"AND __right.{keys[0]} IS NOT NULL AND NOT ({equal_condition}) " + f"THEN 1 END) AS unequal, " + f"COUNT(CASE WHEN __right.{keys[0]} IS NULL THEN 1 END) AS left_only, " + f"COUNT(CASE WHEN __left.{keys[0]} IS NULL THEN 1 END) AS right_only" + f"{column_count_sql} FROM __left FULL OUTER JOIN __right ON {join_condition}" + ) + row: tuple[Any, ...] = self.execute(connection=connection, sql=diff_sql).fetchone() + column_results: tuple[RowDiffColumnResult, ...] = tuple( + RowDiffColumnResult( + name=col, + mismatched_count=int(row[index]), + tolerance=column_tolerances[col], + ) + for index, col in enumerate(compare_columns, start=7) + ) + return RowDiffResult( + left_count=int(row[0]), + right_count=int(row[1]), + joined_count=int(row[2]), + equal_count=int(row[3]), + unequal_count=int(row[4]), + left_only_count=int(row[5]), + right_only_count=int(row[6]), + column_results=column_results, + population_count=prepared_relations.population_count or int(row[2]), + compared_count=int(row[2]), + sampling=sampling, + ) + + def sample_unequal_rows( + self, + *, + connection: Any, + left: str, + right: str, + unique_key: str | tuple[str, ...], + excluded_columns: tuple[str, ...] = (), + tolerances: RowDiffTolerances | None = None, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + limit: int = 20, + ) -> tuple[RowDiffSampleRow, ...]: + sampling: RowDiffSampling | None = tolerances.sampling if tolerances is not None else None + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + left_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=left + ) + compare_columns: tuple[str, ...] = tuple( + col.name + for col in left_columns + if col.name not in keys and col.name not in excluded_columns + ) + left_columns_by_name: dict[str, ColumnInfo] = {col.name: col for col in left_columns} + cursor_filter: str = self.build_cursor_filter( + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + left_cte: str = f"SELECT * FROM {left}" + right_cte: str = f"SELECT * FROM {right}" + if cursor_filter: + left_cte += f" WHERE {cursor_filter}" + right_cte += f" WHERE {cursor_filter}" + self.validate_row_diff_keys( + connection=connection, relation_sql=left_cte, relation_label="left", keys=keys + ) + self.validate_row_diff_keys( + connection=connection, relation_sql=right_cte, relation_label="right", keys=keys + ) + prepared_relations: RowDiffPreparedRelations = self._build_row_diff_relation_ctes( + connection=connection, + left_sql=left_cte, + right_sql=right_cte, + keys=keys, + sampling=sampling, + inspect_population=False, + ) + column_equal_expressions: dict[str, str] = { + col: self.build_row_diff_equal_expression( + column=col, + column_info=left_columns_by_name[col], + tolerances=tolerances, + ) + for col in compare_columns + } + unequal_condition: str = ( + " OR ".join(f"NOT ({expr})" for expr in column_equal_expressions.values()) + if compare_columns + else "FALSE" + ) + join_condition: str = " AND ".join(f"__left.{k} = __right.{k}" for k in keys) + key_select_sql: str = ", ".join( + f"COALESCE(__left.{k}, __right.{k}) AS __key_{k}" for k in keys + ) + compare_select_sql: str = ", ".join( + f"__left.{col} AS __left_{col}, __right.{col} AS __right_{col}" + for col in compare_columns + ) + if compare_select_sql: + compare_select_sql = ", " + compare_select_sql + sample_sql: str = ( + f"WITH {prepared_relations.cte_sql} " + f"SELECT {key_select_sql}{compare_select_sql} " + f"FROM __left FULL OUTER JOIN __right ON {join_condition} " + f"WHERE __left.{keys[0]} IS NOT NULL AND __right.{keys[0]} IS NOT NULL " + f"AND ({unequal_condition}) " + f"ORDER BY {', '.join(f'__key_{k}' for k in keys)} LIMIT {limit}" + ) + rows: list[tuple[Any, ...]] = self.execute(connection=connection, sql=sample_sql).fetchall() + samples: list[RowDiffSampleRow] = [] + for row in rows: + key_values: tuple[tuple[str, object], ...] = tuple( + (k, row[i]) for i, k in enumerate(keys) + ) + changed_cells: list[RowDiffSampleCell] = [] + for col_index, col in enumerate(compare_columns): + left_i: int = len(keys) + (col_index * 2) + left_val: object = row[left_i] + right_val: object = row[left_i + 1] + if left_val != right_val: + changed_cells.append( + RowDiffSampleCell(name=col, left_value=left_val, right_value=right_val) + ) + samples.append( + RowDiffSampleRow(key_values=key_values, changed_cells=tuple(changed_cells)) + ) + return tuple(samples) + + def sample_side_only_rows( + self, + *, + connection: Any, + left: str, + right: str, + unique_key: str | tuple[str, ...], + side: str, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + limit: int = 20, + sampling: RowDiffSampling | None = None, + ) -> tuple[tuple[tuple[str, object], ...], ...]: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + cursor_filter: str = self.build_cursor_filter( + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + left_cte: str = f"SELECT * FROM {left}" + right_cte: str = f"SELECT * FROM {right}" + if cursor_filter: + left_cte += f" WHERE {cursor_filter}" + right_cte += f" WHERE {cursor_filter}" + self.validate_row_diff_keys( + connection=connection, relation_sql=left_cte, relation_label="left", keys=keys + ) + self.validate_row_diff_keys( + connection=connection, relation_sql=right_cte, relation_label="right", keys=keys + ) + prepared_relations: RowDiffPreparedRelations = self._build_row_diff_relation_ctes( + connection=connection, + left_sql=left_cte, + right_sql=right_cte, + keys=keys, + sampling=sampling, + inspect_population=False, + ) + join_condition: str = " AND ".join(f"__left.{k} = __right.{k}" for k in keys) + key_select_sql: str = ", ".join( + f"COALESCE(__left.{k}, __right.{k}) AS __key_{k}" for k in keys + ) + if side == DIFF_LEFT_SIDE: + side_condition: str = f"__left.{keys[0]} IS NOT NULL AND __right.{keys[0]} IS NULL" + elif side == DIFF_RIGHT_SIDE: + side_condition = f"__right.{keys[0]} IS NOT NULL AND __left.{keys[0]} IS NULL" + else: + raise AdapterUserError(message="sample_side_only_rows side must be 'left' or 'right'") + sample_sql: str = ( + f"WITH {prepared_relations.cte_sql} " + f"SELECT {key_select_sql} " + f"FROM __left FULL OUTER JOIN __right ON {join_condition} " + f"WHERE {side_condition} " + f"ORDER BY {', '.join(f'__key_{k}' for k in keys)} LIMIT {limit}" + ) + rows: list[tuple[Any, ...]] = self.execute(connection=connection, sql=sample_sql).fetchall() + samples: list[tuple[tuple[str, object], ...]] = [] + for row in rows: + sample: list[tuple[str, object]] = [] + for index, key in enumerate(keys): + sample.append((key, row[index])) + samples.append(tuple(sample)) + return tuple(samples) + + def render_create_fingerprint_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_fingerprint_table_sql(database=database, schema=schema) + ) + + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_source_freshness_table_sql(database=database, schema=schema) + ) + + def render_create_node_result_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_node_result_table_sql(database=database, schema=schema) + ) + + def render_create_audit_result_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_audit_result_table_sql(database=database, schema=schema) + ) + + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_janitor_event_table_sql(database=database, schema=schema) + ) + + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_migration_state_table_sql(database=database, schema=schema) + ) + + def render_create_old_name_view_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + return _state_table_ddl( + super().render_create_old_name_view_state_table_sql(database=database, schema=schema) + ) + + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + return _state_table_ddl( + super().render_create_column_migration_state_table_sql(database=database, schema=schema) + ) + + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_microbatch_state_table_sql(database=database, schema=schema) + ) + + def render_create_fingerprint_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + del database, schema + return () + + def render_create_source_freshness_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + del database, schema + return () + + def render_create_node_result_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + del database, schema + return () + + def render_create_audit_result_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + del database, schema + return () + + def render_prune_fingerprint_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + from sqlbuild._native import FINGERPRINT_TABLE_NAME + + return _prune_history_sql( + table=f"{schema}.{FINGERPRINT_TABLE_NAME}", + partition_columns=("node_type", "node_name"), + order_columns=("ts", "run_id"), + retain_versions=retain_versions, + ) + + def render_prune_source_freshness_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME + + return _prune_history_sql( + table=f"{schema}.{SOURCE_FRESHNESS_TABLE_NAME}", + partition_columns=("source_name", "target_database", "target_schema", "target_name"), + order_columns=("observed_at", "run_id"), + retain_versions=retain_versions, + ) + + def add_columns( + self, + *, + connection: Any, + destination: str, + columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_add_columns( + destination=destination, columns=columns + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def alter_column_types( + self, + *, + connection: Any, + destination: str, + columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_alter_column_types( + destination=destination, columns=columns + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def append( + self, + *, + connection: Any, + destination: str, + sql: str, + columns: tuple[str, ...] | None = None, + statement_recorder: StatementRecorder, + ) -> int | None: + statements: tuple[str, ...] = self.render_append( + destination=destination, sql=sql, columns=columns + ) + statement_recorder.record_many(statements) + affected: int | None = None + stmt: str + for stmt in statements: + cursor: Any = self.execute(connection=connection, sql=stmt) + affected = self.affected_row_count(cursor=cursor) + return affected + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del connection, database, schema, name + return () + + def clone( + self, + *, + connection: Any, + origin: str, + destination: str, + hard_copy: bool = False, + origin_is_transient: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_clone( + origin=origin, + destination=destination, + hard_copy=hard_copy, + origin_is_transient=origin_is_transient, + ) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def create_function( + self, + *, + connection: Any, + definition: FunctionDefinition, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_create_function( + destination=definition.destination, + arguments=definition.arguments, + returns=definition.returns, + body_sql=definition.body_sql, + return_columns=definition.return_columns, + language=definition.language, + runtime_version=definition.runtime_version, + entry_point=definition.entry_point, + packages=definition.packages, + ) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def create_table_as( + self, + *, + connection: Any, + destination: str, + sql: str, + config: dict[str, Any] | None = None, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_create_table_as(destination=destination, sql=sql) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def create_view_as( + self, + *, + connection: Any, + destination: str, + sql: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_create_view_as(destination=destination, sql=sql) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def drop( + self, + *, + connection: Any, + destination: str, + if_exists: bool = True, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_drop(destination=destination, if_exists=if_exists) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def drop_columns( + self, + *, + connection: Any, + destination: str, + column_names: tuple[str, ...], + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_drop_columns( + destination=destination, column_names=column_names + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def drop_view( + self, + *, + connection: Any, + destination: str, + if_exists: bool = True, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_drop_view( + destination=destination, if_exists=if_exists + ) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def durable_clone( + self, + *, + connection: Any, + origin: str, + destination: str, + origin_is_transient: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_durable_clone( + origin=origin, destination=destination, origin_is_transient=origin_is_transient + ) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def get_columns_for_relations( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]]: + """Fetch columns keyed by fully qualified physical relation identity.""" + + return get_columns_for_relations_bulk( + adapter=self, connection=connection, relations=relations + ) + + def get_table_freshness_metadata( + self, + *, + connection: Any, + database: str | None, + schema: str | None, + name: str, + ) -> TableFreshnessMetadata: + raise AdapterUserError( + message=f"adapter '{self.adapter_name}' does not support table freshness metadata" + ) + + def get_tables_freshness_metadata( + self, + *, + connection: Any, + requests: tuple[TableFreshnessRequest, ...], + ) -> dict[TableFreshnessRequest, TableFreshnessMetadata]: + results: dict[TableFreshnessRequest, TableFreshnessMetadata] = {} + request: TableFreshnessRequest + for request in requests: + try: + results[request] = self.get_table_freshness_metadata( + connection=connection, + database=request.database, + schema=request.schema, + name=request.name, + ) + except AdapterUserError as error: + results[request] = TableFreshnessMetadata.unavailable(message=error.message) + return results + + def list_dependent_view_names( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + """Return qualified names of views bound to this relation.""" + + del connection, database, schema, name + return () + + def maximum_identifier_length(self) -> int: + """Return the maximum unqualified identifier length supported by the adapter.""" + + return self.max_identifier_length + + def persists_python_functions(self) -> bool: + return True + + def python_functions_inherit_default_namespace(self) -> bool: + return True + + def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: + """Return column names produced by a SQL query without materializing full rows.""" + + cursor: Any = self.execute( + connection=connection, sql=f"SELECT * FROM ({sql}) AS __describe_source LIMIT 0" + ) + description: Any | None = getattr(cursor, "description", None) + if description is None: + return () + return tuple(str(column[0]) for column in description) + + def read_relation_grants( + self, + *, + connection: Any, + database: str | None, + schema: str, + name: str, + relation_type: str, + ) -> tuple[RelationGrant, ...]: + """Return the privileges granted or denied on a relation and its columns.""" + + del connection, database, schema, name, relation_type + return () + + def rename( + self, + *, + connection: Any, + origin: str, + destination: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename(origin=origin, destination=destination) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def rename_view( + self, + *, + connection: Any, + origin: str, + destination: str, + statement_recorder: StatementRecorder, + ) -> None: + """Rename a view in place, keeping its privileges.""" + + statements: tuple[str, ...] = self.render_rename_view( + origin=origin, destination=destination + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def render_create_initial_snapshot_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + snapshot_strategy: str | None, + updated_at_column: str | None, + observed_at_column: str | None, + valid_from_column: str, + valid_to_column: str, + initial_valid_from: str | None, + ) -> tuple[str, ...]: + return self.render_create_table_as( + destination=destination, + sql=SnapshotSql(dialect=self._snapshot_sql_dialect).initial_select_sql( + origin=origin, + snapshot_strategy=snapshot_strategy, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + initial_valid_from=initial_valid_from, + current_timestamp=self.render_current_timestamp(), + ), + ) + + def render_insert_source_freshness_records_sql( + self, + *, + database: str | None, + schema: str, + records: tuple[SourceFreshnessRecord, ...], + ) -> str: + """Render DML that appends source freshness records.""" + + return render_insert_source_freshness_records_sql( + database=database, + schema=schema, + records=records, + render_qualified_name=self.render_qualified_name, + ) + + def render_read_latest_fingerprints_sql( + self, + *, + database: str | None, + schema: str, + ) -> str: + """Render SQL that reads latest fingerprint rows per identity.""" + + from sqlbuild._native import ( + build_read_latest_sql, + ) + + return build_read_latest_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + ) + + def render_read_latest_source_freshness_sql( + self, + *, + database: str | None, + schema: str, + ) -> str: + """Render SQL that reads latest source freshness rows per identity.""" + + from sqlbuild._native import build_source_freshness_read_latest_sql + + return build_source_freshness_read_latest_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + ) + + def render_relation_grants( + self, *, grants: tuple[RelationGrant, ...], destination: str, columns: tuple[str, ...] + ) -> tuple[str, ...]: + """Render grants onto ``destination``, keeping column grants only for ``columns``.""" + + del grants, destination, columns + return () + + def render_relation_revokes( + self, *, grants: tuple[RelationGrant, ...], destination: str + ) -> tuple[str, ...]: + """Render statements that remove grants from ``destination``.""" + + del grants, destination + return () + + def render_seed_select_before_cursor( + self, + *, + origin: str, + cursor_column: str, + cursor_end_exclusive: str, + cursor_type: str | None, + ) -> str: + return self._render_seed_select_before_cursor_impl( + origin=origin, + cursor_column=cursor_column, + cursor_end_exclusive=cursor_end_exclusive, + cursor_type=cursor_type, + ) + + def render_set_difference_operator(self) -> str: + """Render the generic SQL set-difference operator.""" + + return "EXCEPT" + + def render_source_expression_cast_subquery( + self, *, source_relation: str, projections: tuple[str, ...] + ) -> str: + """Render a generic type-enforced source expression table factor.""" + + projection_clause: str = ", ".join(projections) + return f"(SELECT {projection_clause} FROM {source_relation} AS __source_expression)" + + def render_source_expression_relation(self, *, expression: str) -> str: + """Render a generic source expression as a SQL table factor.""" + + stripped_expression: str = expression.strip().removesuffix(";").strip() + if stripped_expression.startswith("("): + return stripped_expression + lowered: str = stripped_expression.lower() + if lowered.startswith(("select", "with", "values")): + return f"({stripped_expression})" + return stripped_expression + + def render_source_freshness_max_query( + self, *, column: str, source_relation: str, source_is_subquery: bool, where_sql: str + ) -> str: + """Render a max-column source freshness query over a source table factor.""" + + del source_is_subquery + return ( + f"SELECT MAX({self.render_identifier(column)}) AS data_version " + f"FROM {source_relation}{where_sql}" + ) + + def render_source_relation_cast_subquery( + self, + *, + source_relation: str, + cast_projections: tuple[str, ...], + cast_column_names: tuple[str, ...], + all_columns_cast: bool, + ) -> str: + """Render a generic type-enforced source relation table factor.""" + + cast_clause: str = ", ".join(cast_projections) + if all_columns_cast: + return f"(SELECT {cast_clause} FROM {source_relation})" + exclude_list: str = ", ".join(self.render_identifier(name) for name in cast_column_names) + return ( + f"(SELECT * {self.star_exclude_keyword()} ({exclude_list}), {cast_clause} " + f"FROM {source_relation})" + ) + + def render_table_function_call(self, *, target: str, call_suffix_sql: str) -> str: + return f"{target}{call_suffix_sql}" + + def render_typed_value_list(self, *, value: SqlValue) -> str: + """Render scalar collection members as a parenthesized SQL value list.""" + + return _render_typed_value_list(value=value, render_scalar=self.render_typed_scalar) + + def replace_table_from_relation( + self, + *, + connection: Any, + destination: str, + origin: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_replace_table_from_relation( + destination=destination, + origin=origin, + ) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def supports_unqualified_function_fingerprints(self) -> bool: + return False + + def view_definition_matches( + self, *, connection: Any, database: str | None, schema: str, name: str, sql: str + ) -> bool: + """Return whether the view's stored definition is ``sql`` as this warehouse stores it.""" + + definition: str | None = self.read_view_definition( + connection=connection, database=database, schema=schema, name=name + ) + return definition is not None and same_view_definition(definition=definition, sql=sql) + + def views_bind_to_relation_identity(self) -> bool: + """Return whether views follow a renamed relation instead of re-resolving its name.""" + + return False + + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + + +def _state_table_ddl(sql: str) -> str: + """Make NOT NULL columns plain types and give the append-only table an explicit engine.""" + + ddl: str = _NOT_NULL_COLUMN_PATTERN.sub(r"\1", sql.strip().removesuffix(";")) + ddl = _BARE_BIGINT_PATTERN.sub(r"\1 Nullable(Int64)\2", ddl) + if STATE_TABLE_ENGINE_KEYWORD in ddl.upper().rsplit(")", 1)[-1]: + return ddl + return f"{ddl} ENGINE = MergeTree ORDER BY tuple()" + + +def _version_tuple(version: str) -> tuple[int, ...]: + parts: list[int] = [] + for part in version.split(".")[:2]: + digits: str = "".join(character for character in part if character.isdigit()) + parts.append(int(digits or 0)) + return tuple(parts) + + +def _error_code(error: BaseException) -> int | None: + current: BaseException | None = error + while current is not None: + match: re.Match[str] | None = _ERROR_CODE_PATTERN.search(str(current)) + if match is not None: + return int(match.group(1)) + current = current.__cause__ or getattr(current, "original_error", None) + return None + + +def _classify_relation_not_found(error: BaseException) -> RelationReadStatus | None: + return RelationReadStatus.MISSING if _error_code(error) in RELATION_NOT_FOUND_CODES else None + + +def _database_filter(schema: str | None) -> str: + if schema is None: + return "database = currentDatabase()" + return f"database = {quote_string(schema)}" + + +def _in_filter(*, column: str, values: tuple[str, ...] | None) -> str: + if values is None: + return "" + if not values: + return " AND 0" + return f" AND {column} IN ({', '.join(quote_string(value) for value in values)})" + + +def _unquote(relation: str) -> str: + return relation.replace('"', "").replace("`", "") + + +def _split_relation(relation: str) -> tuple[str | None, str]: + parts: list[str] = relation.rsplit(".", 1) + name: str = parts[-1].strip().strip('"`') + if len(parts) == 1: + return None, name + return parts[0].strip().strip('"`'), name + + +def _qualify_like(*, destination: str, origin: str) -> str: + if RELATION_SEPARATOR in destination or RELATION_SEPARATOR not in origin: + return destination + return f"{origin.rsplit(RELATION_SEPARATOR, 1)[0]}{RELATION_SEPARATOR}{destination}" + + +def _mangled_function_name(target: str) -> str: + """Name a UDF ``__``; ClickHouse SQL functions are server-global.""" + + database, name = _split_relation(target) + return name if database is None else f"{database}__{name}" + + +def _key_tuple(*, adapter: ClickHouseAdapter, keys: tuple[str, ...]) -> str: + rendered: str = ", ".join(adapter.render_identifier(key) for key in keys) + return f"({rendered})" if len(keys) > 1 else rendered + + +def _decimal_scale(value: object) -> int: + exponent: object = Decimal(str(value)).as_tuple().exponent + return max(0, -exponent) if isinstance(exponent, int) else 0 + + +def _cursor_value_literal(cursor: CursorValue) -> str: + if cursor.kind == CursorKind.INTEGER: + return str(cursor.value) + value: object = cursor.value + text: str = value.isoformat() if isinstance(value, datetime) else str(value) + return f"parseDateTime64BestEffort({quote_string(text)}, 6, 'UTC')" + + +def _prune_history_sql( + *, + table: str, + partition_columns: tuple[str, ...], + order_columns: tuple[str, ...], + retain_versions: int, +) -> str: + identity: str = ", ".join((*partition_columns, *order_columns)) + partition: str = ", ".join(partition_columns) + order: str = ", ".join(f"{column} DESC" for column in order_columns) + return ( + f"DELETE FROM {table} WHERE ({identity}) IN (" + f"SELECT {identity} FROM (SELECT {identity}, row_number() OVER (" + f"PARTITION BY {partition} ORDER BY {order}) AS __sqlbuild_history_rank FROM {table}) " + f"WHERE __sqlbuild_history_rank > {retain_versions}) SETTINGS transform_null_in = 1" + ) diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_connection.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_connection.py new file mode 100644 index 0000000000..ed7c44d711 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_connection.py @@ -0,0 +1,101 @@ +"""ClickHouse connection wrapper over one clickhouse-connect HTTP client.""" + +from __future__ import annotations + +import math +import threading +import uuid +from collections.abc import Sequence +from typing import Any + +from sqlbuild.adapters.clickhouse.classes.clickhouse_cursor import ClickHouseCursor +from sqlbuild.runtime.execution_limits.classes.item_deadline import ItemDeadline +from sqlbuild.runtime.execution_limits.main.current_execution_deadline import ( + current_execution_deadline, +) +from sqlbuild.runtime.execution_limits.models import ExecutionDeadline +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle + +_ROW_COUNT_KEYWORDS: tuple[str, ...] = ("INSERT",) + + +class _ClickHouseConnection: + """Run statements without a server session so worker threads never serialize.""" + + def __init__(self, *, client: Any, interrupt_client_factory: Any) -> None: + self.client: Any = client + self._interrupt_client_factory: Any = interrupt_client_factory + self._running: set[str] = set() + self._running_lock: threading.Lock = threading.Lock() + + def execute(self, sql: str) -> ClickHouseCursor: + with StatementLifecycle(adapter="clickhouse", sql=sql, intent="execute") as lifecycle: + query_id: str = uuid.uuid4().hex + lifecycle.submitted(query_id=query_id) + try: + cursor: ClickHouseCursor = self._run(sql=sql, query_id=query_id) + except Exception as error: + lifecycle.failed(error=error, query_id=query_id) + raise + if cursor.description is not None: + lifecycle.completed(query_id=query_id, row_count=len(cursor._rows)) + else: + lifecycle.completed( + query_id=query_id, + affected_rows=cursor.rowcount if cursor.rowcount >= 0 else None, + ) + return cursor + + def _run(self, *, sql: str, query_id: str) -> ClickHouseCursor: + deadline: ExecutionDeadline | ItemDeadline | None = current_execution_deadline() + settings: dict[str, object] = {"query_id": query_id} + if deadline is not None: + deadline.enforce() + settings["max_execution_time"] = max(1, math.ceil(deadline.remaining_seconds)) + with self._running_lock: + self._running.add(query_id) + try: + result: Any = self.client.query(sql, settings=settings) + finally: + with self._running_lock: + self._running.discard(query_id) + summary: dict[str, Any] = dict(getattr(result, "summary", None) or {}) + rowcount: int = -1 + if sql.lstrip().upper().startswith(_ROW_COUNT_KEYWORDS): + rowcount = int(summary.get("written_rows", 0) or 0) + return ClickHouseCursor( + column_names=tuple(result.column_names or ()), + column_types=tuple(column.name for column in (result.column_types or ())), + rows=result.result_rows or (), + rowcount=rowcount, + query_id=query_id, + ) + + def insert_csv(self, *, table: str, column_names: Sequence[str], payload: bytes) -> None: + """Bulk insert CSV text; ClickHouse parses each column into its declared type.""" + + self.client.raw_insert( + table=table, + column_names=list(column_names), + insert_block=payload, + settings={"input_format_csv_empty_as_default": 0}, + fmt="CSV", + ) + + def cancel_active_statements(self) -> bool: + """Kill this connection's running queries from a separate client.""" + + with self._running_lock: + running: tuple[str, ...] = tuple(self._running) + if not running: + return True + killer: Any = self._interrupt_client_factory() + try: + identifiers: str = ", ".join(f"'{query_id}'" for query_id in running) + killer.command(f"KILL QUERY WHERE query_id IN ({identifiers}) ASYNC") + finally: + killer.close() + return True + + def close(self) -> None: + self.client.close() diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_cursor.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_cursor.py new file mode 100644 index 0000000000..6312754d99 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_cursor.py @@ -0,0 +1,56 @@ +"""DB-API shaped cursor over one fully read ClickHouse result.""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from typing import Any + + +class ClickHouseCursor: + """A DB-API shaped cursor over one fully read ClickHouse result.""" + + def __init__( + self, + *, + column_names: Sequence[str], + column_types: Sequence[str], + rows: Sequence[Sequence[Any]], + rowcount: int, + query_id: str, + ) -> None: + self.description: tuple[tuple[str, str, None, None, None, None, None], ...] | None = ( + tuple( + (name, type_name, None, None, None, None, None) + for name, type_name in zip(column_names, column_types, strict=True) + ) + if column_names + else None + ) + self._rows: list[tuple[Any, ...]] = [tuple(row) for row in rows] + self._index: int = 0 + self.rowcount: int = rowcount + self.query_id: str = query_id + + def fetchone(self) -> tuple[Any, ...] | None: + if self._index >= len(self._rows): + return None + row: tuple[Any, ...] = self._rows[self._index] + self._index += 1 + return row + + def fetchmany(self, size: int = 1) -> list[tuple[Any, ...]]: + rows: list[tuple[Any, ...]] = self._rows[self._index : self._index + size] + self._index += len(rows) + return rows + + def fetchall(self) -> list[tuple[Any, ...]]: + rows: list[tuple[Any, ...]] = self._rows[self._index :] + self._index = len(self._rows) + return rows + + def close(self) -> None: + self._rows = [] + + def __iter__(self) -> Iterator[tuple[Any, ...]]: + while (row := self.fetchone()) is not None: + yield row diff --git a/src/sqlbuild/adapters/clickhouse/classes/partition_locks.py b/src/sqlbuild/adapters/clickhouse/classes/partition_locks.py new file mode 100644 index 0000000000..80a57b2ba8 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/partition_locks.py @@ -0,0 +1,35 @@ +"""In-process locks that serialize rewrites of one ClickHouse partition.""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager + + +class PartitionLocks: + """Hand out one lock per destination partition and acquire them in a deadlock-free order.""" + + def __init__(self) -> None: + self._guard: threading.Lock = threading.Lock() + self._locks: dict[tuple[str, str], threading.Lock] = {} + + @contextmanager + def locked(self, *, destination: str, partitions: tuple[str, ...]) -> Iterator[None]: + """Hold the locks of sorted ``partitions`` of ``destination`` while the body runs.""" + + with self._guard: + locks: list[threading.Lock] = [ + self._locks.setdefault((destination, partition), threading.Lock()) + for partition in partitions + ] + acquired: list[threading.Lock] = [] + try: + lock: threading.Lock + for lock in locks: + lock.acquire() + acquired.append(lock) + yield + finally: + for lock in reversed(acquired): + lock.release() diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py new file mode 100644 index 0000000000..8c1e76527b --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -0,0 +1,95 @@ +"""ClickHouse adapter constants.""" + +DEFAULT_DATABASE: str = "default" +DEFAULT_HTTP_PORT: int = 8123 +DEFAULT_HTTPS_PORT: int = 8443 +MAX_QUERY_SIZE_BYTES: int = 262_144 +MINIMUM_SERVER_VERSION: tuple[int, ...] = (25, 8) +MAX_IDENTIFIER_LENGTH: int = 200 +SESSION_SETTINGS: dict[str, object] = { + "join_use_nulls": 1, + "cast_keep_nullable": 1, + "default_table_engine": "MergeTree", + "union_default_mode": "DISTINCT", + "except_default_mode": "DISTINCT", + "intersect_default_mode": "DISTINCT", + "mutations_sync": 2, + "lightweight_deletes_sync": 2, + "date_time_input_format": "best_effort", + "cast_string_to_date_time_mode": "best_effort", +} +CONNECTION_OPTION_KEYS: frozenset[str] = frozenset( + { + "host", + "port", + "user", + "username", + "password", + "database", + "secure", + "verify", + "settings", + "connect_timeout", + "send_receive_timeout", + "dsn", + } +) +VIEW_ENGINES: frozenset[str] = frozenset({"View", "MaterializedView", "LiveView", "WindowView"}) +RELATION_NOT_FOUND_CODES: frozenset[int] = frozenset({60, 81}) +ATOMIC_DATABASE_ENGINES: frozenset[str] = frozenset({"Atomic", "Replicated"}) +NULLABLE_WRAPPER: str = "Nullable" +TYPE_WRAPPERS: tuple[str, ...] = ("Nullable", "LowCardinality") +TYPE_ALIASES: dict[str, str] = { + "Int8": "TINYINT", + "Int16": "SMALLINT", + "Int32": "INTEGER", + "Int64": "BIGINT", + "Int128": "HUGEINT", + "Int256": "HUGEINT", + "UInt8": "SMALLINT", + "UInt16": "INTEGER", + "UInt32": "BIGINT", + "UInt64": "UBIGINT", + "UInt128": "HUGEINT", + "UInt256": "HUGEINT", + "Float32": "REAL", + "Float64": "DOUBLE", + "BFloat16": "REAL", + "String": "VARCHAR", + "FixedString": "VARCHAR", + "UUID": "UUID", + "Bool": "BOOLEAN", + "Date": "DATE", + "Date32": "DATE", + "DateTime": "TIMESTAMP", + "DateTime64": "TIMESTAMP", + "Decimal": "DECIMAL", + "Decimal32": "DECIMAL", + "Decimal64": "DECIMAL", + "Decimal128": "DECIMAL", + "Decimal256": "DECIMAL", + "JSON": "JSON", +} +REWRITE_STAGING_INFIX: str = "__sqb_rw_" +REWRITE_DELTA_INFIX: str = "__sqb_rd_" +SNAPSHOT_WORK_INFIX: str = "__sqb_snap_work" +SNAPSHOT_REBUILD_INFIX: str = "__sqb_snap_rebuild" +LOW_CARDINALITY_WRAPPER: str = "LowCardinality" +DECIMAL_ALIAS: str = "DECIMAL" +DECIMAL_BASE_TYPE: str = "Decimal" +DECIMAL_PRECISION_BY_BASE: dict[str, int] = { + "Decimal32": 9, + "Decimal64": 18, + "Decimal128": 38, + "Decimal256": 76, +} +UNWRAPPABLE_TYPE_BASES: frozenset[str] = frozenset( + {"Array", "Map", "Tuple", "Nested", "JSON", "Object", "Variant", "Dynamic"} +) +VERIFY_OPTION: str = "verify" +DSN_OPTION: str = "dsn" +TIMEOUT_OPTIONS: tuple[str, ...] = ("connect_timeout", "send_receive_timeout") +DEFAULT_COLUMN_DEFAULT_KIND: str = "DEFAULT" +EXCHANGE_UNSUPPORTED_CODES: frozenset[int] = frozenset({1, 48}) +STATE_TABLE_ENGINE_KEYWORD: str = "ENGINE" +RELATION_SEPARATOR: str = "." diff --git a/src/sqlbuild/adapters/clickhouse/models.py b/src/sqlbuild/adapters/clickhouse/models.py new file mode 100644 index 0000000000..d9800b5bfd --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/models.py @@ -0,0 +1,38 @@ +"""ClickHouse adapter request models.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapters.clickhouse.classes.partition_locks import PartitionLocks + + +@dataclass(frozen=True) +class PartitionRewrite: + """Incoming delta rows and the target rows they displace; ``{delta}`` names the delta.""" + + destination: str + delta_insert_sql: str + displaced_predicate: str + + +@dataclass(frozen=True) +class RewriteExecutor: + """The adapter operations and process-wide partition locks a rewrite needs.""" + + execute: Callable[[str], Any] + statement_recorder: StatementRecorder + partition_locks: PartitionLocks + + +@dataclass(frozen=True) +class SnapshotWorkspace: + """The live snapshot and the two working relations that stand in for it.""" + + destination: str + work: str + rebuild: str + valid_to_column: str diff --git a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py index 012aa5470d..f5d8c71f22 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py @@ -140,7 +140,7 @@ def promote_staged_relation( statement_recorder=statement_recorder, ) return - if adapter.adapter_name == BuiltinAdapter.SNOWFLAKE: + if adapter.adapter_name in (BuiltinAdapter.SNOWFLAKE, BuiltinAdapter.CLICKHOUSE): adapter.swap( connection=connection, left=target_qualified, diff --git a/tests/goldens/compiler/failure/config-unknown-adapter.json b/tests/goldens/compiler/failure/config-unknown-adapter.json index c834d3788e..8ebcdfe9d3 100644 --- a/tests/goldens/compiler/failure/config-unknown-adapter.json +++ b/tests/goldens/compiler/failure/config-unknown-adapter.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "C601", - "message": "unknown adapter 'warehouse9000'. Available built-in adapters: bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlite, sqlserver. Project-local adapters are discovered from adapters/**/*.py." + "message": "unknown adapter 'warehouse9000'. Available built-in adapters: bigquery, clickhouse, databricks, duckdb, motherduck, postgres, snowflake, sqlite, sqlserver. Project-local adapters are discovered from adapters/**/*.py." } ] } diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/__init__.py b/tests/integration/src/sqlbuild/adapters/clickhouse/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py new file mode 100644 index 0000000000..aa9da8188a --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -0,0 +1,133 @@ +from dataclasses import dataclass + +from sqlbuild.adapter.contract.models import ColumnInfo + + +@dataclass(frozen=True) +class ClickHouseQueryTestCase: + description: str + sql: str + expected_columns: tuple[str, ...] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseInspectionTestCase: + description: str + ddl: str + expected_columns: tuple[ColumnInfo, ...] + expected_relation_type: str + + +@dataclass(frozen=True) +class ClickHouseSeedTestCase: + description: str + csv_contents: str + columns: tuple[ColumnInfo, ...] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRewriteTestCase: + description: str + strategy: str + partition_by: str + initial_rows: tuple[tuple[int, str, int, str], ...] + incoming_rows: tuple[tuple[int, str, int, str], ...] + unique_key: tuple[str, ...] = ("order_id",) + exclude_columns: tuple[str, ...] = () + window: tuple[str, str] | None = None + expected_leftover_tables: tuple[tuple[object, ...], ...] = () + + +@dataclass(frozen=True) +class ClickHouseRewriteFaultTestCase: + description: str + fail_statement_number: int + expected_target_unchanged: bool + + +@dataclass(frozen=True) +class ClickHouseFunctionTestCase: + description: str + arguments: tuple[tuple[str, str], ...] + returns: str + body_sql: str + call_sql: str + expected_value: object + + +@dataclass(frozen=True) +class ClickHouseQualifiedNameTestCase: + description: str + database: str + schema: str + name: str + expected_code: str + + +@dataclass(frozen=True) +class ClickHouseSwapTestCase: + description: str + target_sql: str + staging_sql: str + expected_view_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseStateTableTestCase: + description: str + expected_engines: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseConcurrentWindowTestCase: + description: str + day_count: int + start_offsets: tuple[int, ...] + window_days: int + expected_error_count: int + expected_summary: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseCloneTestCase: + description: str + partition_by: str + expected_summary: tuple[tuple[object, ...], ...] + expected_partition_key: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRefusedFunctionTestCase: + description: str + body_sql: str + expected_code: str + + +@dataclass(frozen=True) +class ClickHouseInterruptTestCase: + description: str + sql: str + delay_seconds: float + expected_outcome: tuple[bool, bool, int] + expected_max_seconds: float + + +@dataclass(frozen=True) +class ClickHouseCursorFilterTestCase: + description: str + rows_sql: str + start: str + end: str + expected_count: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseColumnMigrationTestCase: + description: str + added: tuple[ColumnInfo, ...] + widened: tuple[ColumnInfo, ...] + renamed: tuple[str, str] + dropped: tuple[str, ...] + expected_columns: tuple[ColumnInfo, ...] diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/conftest.py b/tests/integration/src/sqlbuild/adapters/clickhouse/conftest.py new file mode 100644 index 0000000000..eec139c6a0 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/conftest.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import os +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from tests.integration.src.sqlbuild.adapters.clickhouse.helpers import ( + CLICKHOUSE_IMAGE, + build_unique_schema_name, +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + test_dir: Path = Path(__file__).resolve().parent + for item in items: + if not Path(str(item.path)).resolve().is_relative_to(test_dir): + continue + item.add_marker(pytest.mark.real_warehouse) + item.add_marker(pytest.mark.clickhouse) + + +@pytest.fixture(scope="session") +def clickhouse_config() -> Iterator[dict[str, object]]: + port: str | None = os.environ.get("SQLBUILD_TEST_CLICKHOUSE_PORT") + if port is not None: + yield { + "host": os.environ.get("SQLBUILD_TEST_CLICKHOUSE_HOST", "localhost"), + "port": int(port), + "user": os.environ.get("SQLBUILD_TEST_CLICKHOUSE_USER", "default"), + "password": os.environ.get("SQLBUILD_TEST_CLICKHOUSE_PASSWORD", ""), + } + return + from testcontainers.clickhouse import ClickHouseContainer + + try: + with ClickHouseContainer(CLICKHOUSE_IMAGE) as container: + yield { + "host": container.get_container_host_ip(), + "port": int(container.get_exposed_port(8123)), + "user": container.username, + "password": container.password, + } + except Exception as exc: + pytest.skip(f"Docker not available for ClickHouse testcontainer: {exc}") + + +@pytest.fixture +def adapter() -> ClickHouseAdapter: + return ClickHouseAdapter() + + +@pytest.fixture +def clickhouse_schema() -> str: + return build_unique_schema_name(prefix="sqb_integration") + + +@pytest.fixture +def connection( + adapter: ClickHouseAdapter, clickhouse_config: dict[str, object], clickhouse_schema: str +) -> Iterator[Any]: + conn: Any = adapter.connect(dict(clickhouse_config)) + adapter.execute(connection=conn, sql=f"CREATE DATABASE IF NOT EXISTS {clickhouse_schema}") + try: + yield conn + finally: + adapter.execute(connection=conn, sql=f"DROP DATABASE IF EXISTS {clickhouse_schema} SYNC") + adapter.close(conn) diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py new file mode 100644 index 0000000000..a3d369d265 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py @@ -0,0 +1,357 @@ +"""Shared helpers for ClickHouse integration tests.""" + +from __future__ import annotations + +import threading +import time +import uuid +from collections.abc import Callable +from datetime import date +from typing import Any + +import duckdb + +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from tests.integration.src.sqlbuild.adapters.clickhouse._test_types import ( + ClickHouseRewriteTestCase, +) + +CLICKHOUSE_IMAGE: str = "clickhouse/clickhouse-server:26.1" +ORDERS_COLUMNS: str = "order_id Int64, customer String, amount Int64, order_date Date" +ORDER_COLUMN_NAMES: tuple[str, ...] = ("order_id", "customer", "amount", "order_date") +INITIAL_ORDERS: tuple[tuple[int, str, int, str], ...] = ( + (1, "ada", 10, "2026-01-05"), + (2, "bo", 20, "2026-01-20"), + (3, "cy", 30, "2026-02-03"), + (4, "di", 40, "2026-02-17"), + (5, "ed", 50, "2026-03-01"), +) + + +class InjectedFailure(RuntimeError): + """Raised by ``FailingExecute`` in place of one chosen statement.""" + + +class FailingExecute: + """Wrap ``adapter.execute`` and fail the n-th executed statement (1-based).""" + + def __init__(self, *, execute: Callable[..., Any], fail_statement_number: int) -> None: + self._execute: Callable[..., Any] = execute + self._failures: dict[int, Callable[[str], None]] = { + fail_statement_number: _raise_injected_failure + } + self._count: int = 0 + + def __call__(self, *, connection: Any, sql: str) -> Any: + self._count += 1 + self._failures.get(self._count, _ignore_statement)(sql) + return self._execute(connection=connection, sql=sql) + + +def build_unique_schema_name(*, prefix: str) -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +def fetch_rows( + *, adapter: ClickHouseAdapter, connection: Any, sql: str +) -> tuple[tuple[object, ...], ...]: + cursor: Any = adapter.execute(connection=connection, sql=sql) + return tuple(tuple(row) for row in cursor.fetchall()) + + +def execute_all( + *, adapter: ClickHouseAdapter, connection: Any, statements: tuple[str, ...] +) -> None: + for statement in statements: + adapter.execute(connection=connection, sql=statement) + + +def recorded_statements(recorder: StatementRecorder) -> tuple[str, ...]: + return tuple(event.content for event in recorder.snapshot()) + + +def create_orders( + *, + adapter: ClickHouseAdapter, + connection: Any, + relation: str, + partition_by: str, + rows: tuple[tuple[int, str, int, str], ...], +) -> None: + adapter.execute( + connection=connection, + sql=f"CREATE TABLE {relation} ({ORDERS_COLUMNS}) ENGINE = MergeTree " + f"PARTITION BY {partition_by} ORDER BY order_id", + ) + adapter.execute( + connection=connection, + sql=f"INSERT INTO {relation} VALUES " + ", ".join(_row_sql(row) for row in rows), + ) + + +def order_rows_as_dates( + rows: tuple[tuple[int, str, int, str], ...], +) -> tuple[tuple[object, ...], ...]: + return tuple((row[0], row[1], row[2], date.fromisoformat(row[3])) for row in rows) + + +def oracle_rows(test_case: ClickHouseRewriteTestCase) -> tuple[tuple[object, ...], ...]: + """Apply the same write on DuckDB with plain DELETE, UPDATE and INSERT statements.""" + + database: Any = duckdb.connect() + database.execute( + "CREATE TABLE orders (order_id BIGINT, customer VARCHAR, amount BIGINT, order_date DATE)" + ) + database.execute( + "CREATE TABLE incoming (order_id BIGINT, customer VARCHAR, amount BIGINT, order_date DATE)" + ) + for table, rows in (("orders", test_case.initial_rows), ("incoming", test_case.incoming_rows)): + for row in rows: + database.execute(f"INSERT INTO {table} VALUES (?, ?, ?, ?)", list(row)) + statement: str + for statement in _ORACLE_STATEMENTS[test_case.strategy](test_case): + database.execute(statement) + rows: list[tuple[object, ...]] = database.execute( + "SELECT order_id, customer, amount, order_date FROM orders " + "ORDER BY order_id, customer, amount" + ).fetchall() + database.close() + return tuple(tuple(row) for row in rows) + + +def apply_write( + *, + adapter: ClickHouseAdapter, + connection: Any, + test_case: ClickHouseRewriteTestCase, + relation: str, +) -> None: + _WRITERS[test_case.strategy]( + adapter=adapter, + connection=connection, + test_case=test_case, + relation=relation, + sql=_incoming_sql(test_case.incoming_rows), + ) + + +def run_concurrent_cursor_windows( + *, + adapter: ClickHouseAdapter, + config: dict[str, object], + relation: str, + first_day: date, + start_offsets: tuple[int, ...], + window_days: int, +) -> list[BaseException]: + """Rewrite one cursor window per thread, each on its own connection, and collect errors.""" + + errors: list[BaseException] = [] + + def run_window(start_offset: int) -> None: + worker: Any = adapter.connect(dict(config)) + try: + start: date = date.fromordinal(first_day.toordinal() + start_offset) + end: date = date.fromordinal(start.toordinal() + window_days) + adapter.delete_insert_cursor( + connection=worker, + destination=relation, + sql=( + f"SELECT event_id, event_day, toInt64(1) AS version FROM {relation} " + f"WHERE event_day >= '{start}' AND event_day < '{end}'" + ), + cursor_column="event_day", + cursor_start=start.isoformat(), + cursor_end=end.isoformat(), + statement_recorder=StatementRecorder(), + cursor_type="timestamp", + ) + except BaseException as error: + errors.append(error) + finally: + adapter.close(worker) + + threads: list[threading.Thread] = [ + threading.Thread(target=run_window, args=(offset,)) for offset in start_offsets + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return errors + + +def interrupt_running_query( + *, adapter: ClickHouseAdapter, connection: Any, sql: str, delay_seconds: float +) -> tuple[bool, bool, int, float]: + """Interrupt ``sql`` mid-flight; return (interrupted, finished, error count, seconds).""" + + errors: list[BaseException] = [] + started: float = time.monotonic() + + def run() -> None: + try: + adapter.execute(connection=connection, sql=sql) + except BaseException as error: + errors.append(error) + + thread: threading.Thread = threading.Thread(target=run) + thread.start() + time.sleep(delay_seconds) + interrupted: bool = adapter.interrupt_connection(connection) + thread.join(timeout=20) + return interrupted, not thread.is_alive(), len(errors), time.monotonic() - started + + +def _row_sql(row: tuple[int, str, int, str]) -> str: + return f"({row[0]}, '{row[1]}', {row[2]}, '{row[3]}')" + + +def _incoming_sql(rows: tuple[tuple[int, str, int, str], ...]) -> str: + empty: str = ( + "SELECT toInt64(0) AS order_id, '' AS customer, toInt64(0) AS amount, " + "toDate('2026-01-01') AS order_date WHERE 0" + ) + return " UNION ALL ".join( + ( + empty, + *( + f"SELECT toInt64({row[0]}) AS order_id, '{row[1]}' AS customer, " + f"toInt64({row[2]}) AS amount, toDate('{row[3]}') AS order_date" + for row in rows + ), + ) + ) + + +def _raise_injected_failure(sql: str) -> None: + raise InjectedFailure(sql) + + +def _ignore_statement(sql: str) -> None: + del sql + + +def _append_oracle(test_case: ClickHouseRewriteTestCase) -> tuple[str, ...]: + del test_case + return ("INSERT INTO orders SELECT * FROM incoming",) + + +def _delete_insert_oracle(test_case: ClickHouseRewriteTestCase) -> tuple[str, ...]: + keys: str = ", ".join(test_case.unique_key) + return ( + f"DELETE FROM orders WHERE ({keys}) IN (SELECT {keys} FROM incoming)", + "INSERT INTO orders SELECT * FROM incoming", + ) + + +def _cursor_oracle(test_case: ClickHouseRewriteTestCase) -> tuple[str, ...]: + assert test_case.window is not None + return ( + f"DELETE FROM orders WHERE order_date >= '{test_case.window[0]}' " + f"AND order_date < '{test_case.window[1]}'", + "INSERT INTO orders SELECT * FROM incoming", + ) + + +def _merge_oracle(test_case: ClickHouseRewriteTestCase) -> tuple[str, ...]: + updated_columns: list[str] = sorted( + set(ORDER_COLUMN_NAMES[1:]) - set(test_case.exclude_columns) + ) + updated: str = ", ".join(f"{column} = incoming.{column}" for column in updated_columns) + return ( + f"UPDATE orders SET {updated} FROM incoming WHERE orders.order_id = incoming.order_id", + "INSERT INTO orders SELECT * FROM incoming WHERE order_id NOT IN " + "(SELECT order_id FROM orders)", + ) + + +def _write_append( + *, + adapter: ClickHouseAdapter, + connection: Any, + test_case: ClickHouseRewriteTestCase, + relation: str, + sql: str, +) -> None: + del test_case + adapter.append( + connection=connection, + destination=relation, + sql=sql, + columns=ORDER_COLUMN_NAMES, + statement_recorder=StatementRecorder(), + ) + + +def _write_delete_insert( + *, + adapter: ClickHouseAdapter, + connection: Any, + test_case: ClickHouseRewriteTestCase, + relation: str, + sql: str, +) -> None: + adapter.delete_insert( + connection=connection, + destination=relation, + sql=sql, + unique_key=test_case.unique_key, + columns=ORDER_COLUMN_NAMES, + statement_recorder=StatementRecorder(), + ) + + +def _write_cursor( + *, + adapter: ClickHouseAdapter, + connection: Any, + test_case: ClickHouseRewriteTestCase, + relation: str, + sql: str, +) -> None: + assert test_case.window is not None + adapter.delete_insert_cursor( + connection=connection, + destination=relation, + sql=sql, + cursor_column="order_date", + cursor_start=test_case.window[0], + cursor_end=test_case.window[1], + columns=ORDER_COLUMN_NAMES, + statement_recorder=StatementRecorder(), + cursor_type="timestamp", + ) + + +def _write_merge( + *, + adapter: ClickHouseAdapter, + connection: Any, + test_case: ClickHouseRewriteTestCase, + relation: str, + sql: str, +) -> None: + adapter.merge( + connection=connection, + destination=relation, + sql=sql, + unique_key=test_case.unique_key, + statement_recorder=StatementRecorder(), + exclude_columns=test_case.exclude_columns, + ) + + +_ORACLE_STATEMENTS: dict[str, Callable[[ClickHouseRewriteTestCase], tuple[str, ...]]] = { + "append": _append_oracle, + "delete_insert": _delete_insert_oracle, + "cursor": _cursor_oracle, + "merge": _merge_oracle, +} +_WRITERS: dict[str, Callable[..., None]] = { + "append": _write_append, + "delete_insert": _write_delete_insert, + "cursor": _write_cursor, + "merge": _write_merge, +} diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py new file mode 100644 index 0000000000..95dbff7901 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py @@ -0,0 +1,851 @@ +from __future__ import annotations + +from datetime import date, datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import ColumnInfo, CursorValue, QueryResult, RelationInfo +from sqlbuild.adapter.contract.types import RelationReadStatus +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from sqlbuild.compiler.compile.types import FunctionLanguage +from tests.integration.src.sqlbuild.adapters.clickhouse._test_types import ( + ClickHouseCloneTestCase, + ClickHouseColumnMigrationTestCase, + ClickHouseConcurrentWindowTestCase, + ClickHouseCursorFilterTestCase, + ClickHouseFunctionTestCase, + ClickHouseInspectionTestCase, + ClickHouseInterruptTestCase, + ClickHouseQualifiedNameTestCase, + ClickHouseQueryTestCase, + ClickHouseRefusedFunctionTestCase, + ClickHouseRewriteFaultTestCase, + ClickHouseRewriteTestCase, + ClickHouseSeedTestCase, + ClickHouseStateTableTestCase, + ClickHouseSwapTestCase, +) +from tests.integration.src.sqlbuild.adapters.clickhouse.helpers import ( + INITIAL_ORDERS, + FailingExecute, + InjectedFailure, + apply_write, + create_orders, + execute_all, + fetch_rows, + interrupt_running_query, + oracle_rows, + order_rows_as_dates, + recorded_statements, + run_concurrent_cursor_windows, +) + +_PARTITIONED: str = "toYYYYMM(order_date)" +_UNPARTITIONED: str = "tuple()" +_FAULT_WRITE: ClickHouseRewriteTestCase = ClickHouseRewriteTestCase( + description="three partitions", + strategy="delete_insert", + partition_by=_PARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=( + (1, "ada", 11, "2026-01-05"), + (3, "cy", 31, "2026-02-03"), + (5, "ed", 51, "2026-03-01"), + ), +) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseQueryTestCase( + description="returns named rows", + sql="SELECT 1 AS id, 'hello' AS name", + expected_columns=("id", "name"), + expected_rows=((1, "hello"),), + ), + ClickHouseQueryTestCase( + description="pins distinct set operations and outer join nulls", + sql=( + "SELECT a, b.y FROM (SELECT 1 AS a UNION SELECT 1 UNION SELECT 2) AS l " + "LEFT JOIN (SELECT 1 AS x, 'v' AS y) AS b ON l.a = b.x ORDER BY a" + ), + expected_columns=("a", "y"), + expected_rows=((1, "v"), (2, None)), + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_when_querying_then_clickhouse_returns_rows( + test_case: ClickHouseQueryTestCase, adapter: ClickHouseAdapter, connection: Any +) -> None: + result: QueryResult = adapter.query(connection=connection, sql=test_case.sql, limit=None) + + assert result.columns == test_case.expected_columns + assert tuple(tuple(row) for row in result.rows) == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseInspectionTestCase( + description="table columns in position order", + ddl="CREATE TABLE {target} (id Int64, total Nullable(Decimal(10, 2))) " + "ENGINE = MergeTree ORDER BY id", + expected_columns=( + ColumnInfo(name="id", type="Int64"), + ColumnInfo(name="total", type="Nullable(Decimal(10, 2))"), + ), + expected_relation_type="BASE TABLE", + ), + ClickHouseInspectionTestCase( + description="view columns", + ddl="CREATE VIEW {target} AS SELECT toInt64(1) AS id, 'x' AS label", + expected_columns=( + ColumnInfo(name="id", type="Int64"), + ColumnInfo(name="label", type="String"), + ), + expected_relation_type="VIEW", + ), + ], + ids=lambda case: case.description, +) +def test_given_relation_when_inspecting_then_clickhouse_reports_columns_and_kind( + test_case: ClickHouseInspectionTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + target: str = f"{clickhouse_schema}.orders" + adapter.execute(connection=connection, sql=test_case.ddl.format(target=target)) + + relations: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, database=None, schemas=(clickhouse_schema,) + ) + + assert relations == ( + RelationInfo( + database=None, + schema=clickhouse_schema, + name="orders", + relation_type=test_case.expected_relation_type, + ), + ) + assert adapter.describe_relation(connection=connection, relation=target) == ( + test_case.expected_columns + ) + assert adapter.get_columns( + connection=connection, database=None, schema=clickhouse_schema, name="orders" + ) == (test_case.expected_columns) + assert adapter.relation_exists( + connection=connection, database=None, schema=clickhouse_schema, name="orders" + ) + assert adapter.schema_exists(connection=connection, database=None, schema=clickhouse_schema) + assert ( + adapter.probe_relation_read(connection=connection, relation=target).status + == RelationReadStatus.READABLE + ) + assert ( + adapter.probe_relation_read(connection=connection, relation=f"{target}_missing").status + == RelationReadStatus.MISSING + ) + assert ( + adapter.probe_relation_read(connection=connection, relation="sqb_missing_db.orders").status + == RelationReadStatus.MISSING + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseQualifiedNameTestCase( + description="database qualifier refused", + database="warehouse", + schema="sales", + name="orders", + expected_code="A704", + ), + ], + ids=lambda case: case.description, +) +def test_given_database_in_target_when_qualifying_then_clickhouse_refuses_with_code( + test_case: ClickHouseQualifiedNameTestCase, adapter: ClickHouseAdapter +) -> None: + with pytest.raises(AdapterUserError) as error: + adapter.render_qualified_name( + database=test_case.database, schema=test_case.schema, name=test_case.name + ) + + assert error.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseSeedTestCase( + description="typed seed round-trips nulls, quotes and backslashes", + csv_contents=( + "id,label,amount,ordered_at,is_gift,ship_date\n" + '1,"plain",1.50,2026-01-02T03:04:05.123456,true,2026-01-03\n' + '2,"it\'s ""quoted""",,2026-01-02 00:00:00,false,\n' + '3,"back\\slash",2.25,,,1950-06-01\n' + ), + columns=( + ColumnInfo(name="id", type="Int64"), + ColumnInfo(name="label", type="String"), + ColumnInfo(name="amount", type="Decimal(10, 2)"), + ColumnInfo(name="ordered_at", type="DateTime64(6, 'UTC')"), + ColumnInfo(name="is_gift", type="Bool"), + ColumnInfo(name="ship_date", type="Date32"), + ), + expected_rows=( + (1, "plain", "1.5", "2026-01-02 03:04:05.123456", True, date(2026, 1, 3)), + (2, 'it\'s "quoted"', None, "2026-01-02 00:00:00.000000", False, None), + (3, "back\\slash", "2.25", None, None, date(1950, 6, 1)), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_seed_file_when_loading_then_clickhouse_streams_typed_rows( + test_case: ClickHouseSeedTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, + tmp_path: Path, +) -> None: + seed_path: Path = tmp_path / "orders.csv" + seed_path.write_text(test_case.csv_contents, encoding="utf-8") + target: str = f"{clickhouse_schema}.orders" + recorder: StatementRecorder = StatementRecorder() + + adapter.load_seed( + connection=connection, + destination=target, + file_path=seed_path, + columns=test_case.columns, + statement_recorder=recorder, + ) + + rows: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql=( + "SELECT id, label, toString(amount), " + "toString(ordered_at), is_gift, ship_date FROM " + target + " ORDER BY id" + ), + ) + assert rows == test_case.expected_rows + assert all("VALUES" not in statement for statement in recorded_statements(recorder)) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseSwapTestCase( + description="dependent view sees the exchanged table", + target_sql="SELECT 1 AS id", + staging_sql="SELECT number AS id FROM numbers(5)", + expected_view_rows=((5,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_staged_table_when_swapping_then_clickhouse_exchanges_atomically( + test_case: ClickHouseSwapTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + recorder: StatementRecorder = StatementRecorder() + target: str = f"{clickhouse_schema}.orders" + staging: str = f"{clickhouse_schema}.orders__staging" + adapter.create_table_as( + connection=connection, + destination=target, + sql=test_case.target_sql, + statement_recorder=recorder, + ) + adapter.create_table_as( + connection=connection, + destination=staging, + sql=test_case.staging_sql, + statement_recorder=recorder, + ) + adapter.create_view_as( + connection=connection, + destination=f"{clickhouse_schema}.orders_view", + sql=f"SELECT count() AS n FROM {target}", + statement_recorder=recorder, + ) + before: str | None = adapter.physical_relation_generation( + connection=connection, database=None, schema=clickhouse_schema, name="orders" + ) + + adapter.swap(connection=connection, left=target, right=staging, statement_recorder=recorder) + + after: str | None = adapter.physical_relation_generation( + connection=connection, database=None, schema=clickhouse_schema, name="orders" + ) + assert before is not None and after is not None and before != after + assert fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT n FROM {clickhouse_schema}.orders_view", + ) == (test_case.expected_view_rows) + assert f"EXCHANGE TABLES {target} AND {staging}" in recorded_statements(recorder) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseStateTableTestCase( + description="every state table is a MergeTree", expected_engines=(("MergeTree",),) + ), + ], + ids=lambda case: case.description, +) +def test_given_framework_state_ddl_when_creating_then_clickhouse_accepts_every_table( + test_case: ClickHouseStateTableTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + statements: tuple[str, ...] = ( + adapter.render_create_fingerprint_table_sql(database=None, schema=clickhouse_schema), + adapter.render_create_source_freshness_table_sql(database=None, schema=clickhouse_schema), + adapter.render_create_node_result_table_sql(database=None, schema=clickhouse_schema), + adapter.render_create_audit_result_table_sql(database=None, schema=clickhouse_schema), + adapter.render_create_janitor_event_table_sql(database=None, schema=clickhouse_schema), + adapter.render_create_migration_state_table_sql(database=None, schema=clickhouse_schema), + adapter.render_create_old_name_view_state_table_sql( + database=None, schema=clickhouse_schema + ), + adapter.render_create_column_migration_state_table_sql( + database=None, schema=clickhouse_schema + ), + adapter.render_create_microbatch_state_table_sql(database=None, schema=clickhouse_schema), + ) + + execute_all(adapter=adapter, connection=connection, statements=statements) + execute_all(adapter=adapter, connection=connection, statements=statements) + execute_all( + adapter=adapter, + connection=connection, + statements=( + adapter.render_read_latest_fingerprints_sql(database=None, schema=clickhouse_schema), + adapter.render_prune_fingerprint_history_sql( + database=None, schema=clickhouse_schema, retain_versions=1 + ), + adapter.render_prune_source_freshness_history_sql( + database=None, schema=clickhouse_schema, retain_versions=1 + ), + ), + ) + + engines: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT DISTINCT engine FROM system.tables WHERE database = '{clickhouse_schema}'", + ) + assert engines == test_case.expected_engines + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRewriteTestCase( + description="append keeps every row (partitioned)", + strategy="append", + partition_by=_PARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((6, "fa", 60, "2026-03-09"),), + ), + ClickHouseRewriteTestCase( + description="delete_insert replaces keys moving between partitions (partitioned)", + strategy="delete_insert", + partition_by=_PARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=( + (2, "bo", 21, "2026-03-02"), + (4, "di", 41, "2026-02-17"), + (7, "gu", 70, "2026-04-01"), + ), + ), + ClickHouseRewriteTestCase( + description="merge upserts and keeps excluded columns (partitioned)", + strategy="merge", + partition_by=_PARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((1, "ann", 11, "2026-01-05"), (8, "ho", 80, "2026-01-09")), + exclude_columns=("customer",), + ), + ClickHouseRewriteTestCase( + description="cursor window replaces only the window (partitioned)", + strategy="cursor", + partition_by=_PARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((9, "iv", 90, "2026-01-21"),), + window=("2026-01-15", "2026-02-10"), + ), + ClickHouseRewriteTestCase( + description="empty window clears its rows (partitioned)", + strategy="cursor", + partition_by=_PARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=(), + window=("2026-02-01", "2026-03-01"), + ), + ClickHouseRewriteTestCase( + description="append keeps every row (unpartitioned)", + strategy="append", + partition_by=_UNPARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((6, "fa", 60, "2026-03-09"),), + ), + ClickHouseRewriteTestCase( + description="delete_insert replaces keys moving between partitions (unpartitioned)", + strategy="delete_insert", + partition_by=_UNPARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=( + (2, "bo", 21, "2026-03-02"), + (4, "di", 41, "2026-02-17"), + (7, "gu", 70, "2026-04-01"), + ), + ), + ClickHouseRewriteTestCase( + description="merge upserts and keeps excluded columns (unpartitioned)", + strategy="merge", + partition_by=_UNPARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((1, "ann", 11, "2026-01-05"), (8, "ho", 80, "2026-01-09")), + exclude_columns=("customer",), + ), + ClickHouseRewriteTestCase( + description="cursor window replaces only the window (unpartitioned)", + strategy="cursor", + partition_by=_UNPARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((9, "iv", 90, "2026-01-21"),), + window=("2026-01-15", "2026-02-10"), + ), + ClickHouseRewriteTestCase( + description="empty window clears its rows (unpartitioned)", + strategy="cursor", + partition_by=_UNPARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=(), + window=("2026-02-01", "2026-03-01"), + ), + ], + ids=lambda case: case.description, +) +def test_given_incremental_write_when_rewriting_partitions_then_rows_match_duckdb_oracle( + test_case: ClickHouseRewriteTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + relation: str = f"{clickhouse_schema}.orders" + create_orders( + adapter=adapter, + connection=connection, + relation=relation, + partition_by=test_case.partition_by, + rows=test_case.initial_rows, + ) + apply_write(adapter=adapter, connection=connection, test_case=test_case, relation=relation) + + rows: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT order_id, customer, amount, order_date FROM {relation} " + "ORDER BY order_id, customer, amount", + ) + assert rows == oracle_rows(test_case) + leftovers: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT name FROM system.tables WHERE database = '{clickhouse_schema}' " + "AND name != 'orders'", + ) + assert leftovers == test_case.expected_leftover_tables + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRewriteFaultTestCase( + description="crash while staging the delta", + fail_statement_number=3, + expected_target_unchanged=True, + ), + ClickHouseRewriteFaultTestCase( + description="crash while building the partition staging", + fail_statement_number=9, + expected_target_unchanged=True, + ), + ClickHouseRewriteFaultTestCase( + description="crash before the first partition swap", + fail_statement_number=10, + expected_target_unchanged=True, + ), + ClickHouseRewriteFaultTestCase( + description="crash between partition swaps", + fail_statement_number=11, + expected_target_unchanged=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_failure_at_rewrite_step_when_rerunning_then_target_converges( + test_case: ClickHouseRewriteFaultTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + relation: str = f"{clickhouse_schema}.orders" + create_orders( + adapter=adapter, + connection=connection, + relation=relation, + partition_by=_FAULT_WRITE.partition_by, + rows=_FAULT_WRITE.initial_rows, + ) + monkeypatch.setattr( + adapter, + "execute", + FailingExecute( + execute=adapter.execute, fail_statement_number=test_case.fail_statement_number + ), + ) + with pytest.raises(InjectedFailure): + apply_write( + adapter=adapter, connection=connection, test_case=_FAULT_WRITE, relation=relation + ) + select_sql: str = ( + f"SELECT order_id, customer, amount, order_date FROM {relation} ORDER BY order_id" + ) + after_crash: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, connection=connection, sql=select_sql + ) + initial: tuple[tuple[object, ...], ...] = order_rows_as_dates(_FAULT_WRITE.initial_rows) + assert (after_crash == initial) is test_case.expected_target_unchanged + + apply_write(adapter=adapter, connection=connection, test_case=_FAULT_WRITE, relation=relation) + + assert fetch_rows(adapter=adapter, connection=connection, sql=select_sql) == oracle_rows( + _FAULT_WRITE + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseConcurrentWindowTestCase( + description="four two-week windows over two months", + day_count=56, + start_offsets=(0, 14, 28, 42), + window_days=14, + expected_error_count=0, + expected_summary=((56, 56, 56),), + ), + ], + ids=lambda case: case.description, +) +def test_given_four_concurrent_windows_sharing_partitions_when_rewriting_then_no_update_is_lost( + test_case: ClickHouseConcurrentWindowTestCase, + adapter: ClickHouseAdapter, + clickhouse_config: dict[str, object], + connection: Any, + clickhouse_schema: str, +) -> None: + relation: str = f"{clickhouse_schema}.events" + adapter.execute( + connection=connection, + sql=f"CREATE TABLE {relation} (event_id Int64, event_day Date, version Int64) " + "ENGINE = MergeTree PARTITION BY toYYYYMM(event_day) ORDER BY event_id", + ) + adapter.execute( + connection=connection, + sql=f"INSERT INTO {relation} SELECT number, toDate('2026-01-01') + number, 0 " + f"FROM numbers({test_case.day_count})", + ) + + errors: list[BaseException] = run_concurrent_cursor_windows( + adapter=adapter, + config=clickhouse_config, + relation=relation, + first_day=date(2026, 1, 1), + start_offsets=test_case.start_offsets, + window_days=test_case.window_days, + ) + + assert len(errors) == test_case.expected_error_count, errors + assert fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT count(), sum(version), uniqExact(event_id) FROM {relation}", + ) == (test_case.expected_summary) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseCloneTestCase( + description="monthly partitions survive the clone", + partition_by=_PARTITIONED, + expected_summary=((5, 150),), + expected_partition_key=(("toYYYYMM(order_date)",),), + ), + ], + ids=lambda case: case.description, +) +def test_given_table_when_cloning_then_clickhouse_hard_links_parts( + test_case: ClickHouseCloneTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + origin: str = f"{clickhouse_schema}.orders" + clone: str = f"{clickhouse_schema}.orders_clone" + create_orders( + adapter=adapter, + connection=connection, + relation=origin, + partition_by=test_case.partition_by, + rows=INITIAL_ORDERS, + ) + + adapter.durable_clone( + connection=connection, + origin=origin, + destination=clone, + statement_recorder=StatementRecorder(), + ) + adapter.drop(connection=connection, destination=origin, statement_recorder=StatementRecorder()) + + assert fetch_rows( + adapter=adapter, connection=connection, sql=f"SELECT count(), sum(amount) FROM {clone}" + ) == (test_case.expected_summary) + partition_key: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql="SELECT partition_key FROM system.tables " + f"WHERE database = '{clickhouse_schema}' AND name = 'orders_clone'", + ) + assert partition_key == test_case.expected_partition_key + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseFunctionTestCase( + description="SQL lambda named after its schema", + arguments=(("amount", "Int64"), ("rate", "Float64")), + returns="Float64", + body_sql="amount * rate", + call_sql="(10, 0.5)", + expected_value=5.0, + ) + ], + ids=lambda case: case.description, +) +def test_given_sql_function_when_creating_then_clickhouse_mangles_server_global_name( + test_case: ClickHouseFunctionTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + destination: str = f"{clickhouse_schema}.discounted" + statements: tuple[str, ...] = adapter.render_create_function( + destination=destination, + arguments=tuple( + SimpleNamespace(name=name, type=kind) for name, kind in test_case.arguments + ), + returns=test_case.returns, + body_sql=test_case.body_sql, + ) + try: + execute_all(adapter=adapter, connection=connection, statements=statements) + call: str = adapter.render_udf_call(target=destination, call_suffix_sql=test_case.call_sql) + + assert fetch_rows(adapter=adapter, connection=connection, sql=f"SELECT {call}") == ( + (test_case.expected_value,), + ) + assert [ + function.name + for function in adapter.list_functions( + connection=connection, database=None, schemas=(clickhouse_schema,) + ) + ] == ["discounted"] + finally: + adapter.execute( + connection=connection, + sql=f"DROP FUNCTION IF EXISTS {clickhouse_schema}__discounted", + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRefusedFunctionTestCase( + description="python function refused", + body_sql="def score(): return 1.0", + expected_code="A707", + ), + ], + ids=lambda case: case.description, +) +def test_given_python_function_when_rendering_then_clickhouse_refuses_with_code( + test_case: ClickHouseRefusedFunctionTestCase, adapter: ClickHouseAdapter +) -> None: + with pytest.raises(AdapterUserError) as error: + adapter.render_create_function( + destination="sales.score", + arguments=(), + returns="Float64", + body_sql=test_case.body_sql, + language=FunctionLanguage.PYTHON, + ) + + assert error.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseInterruptTestCase( + description="slow scan is killed", + sql="SELECT sleepEachRow(1) FROM numbers(30) SETTINGS max_block_size = 1, " + "function_sleep_max_microseconds_per_block = 0", + delay_seconds=1.5, + expected_outcome=(True, True, 1), + expected_max_seconds=20.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_long_query_when_interrupting_then_clickhouse_kills_it( + test_case: ClickHouseInterruptTestCase, adapter: ClickHouseAdapter, connection: Any +) -> None: + interrupted, finished, error_count, seconds = interrupt_running_query( + adapter=adapter, + connection=connection, + sql=test_case.sql, + delay_seconds=test_case.delay_seconds, + ) + + assert (interrupted, finished, error_count) == test_case.expected_outcome + assert seconds < test_case.expected_max_seconds + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseCursorFilterTestCase( + description="sub-second row inside a UTC window", + rows_sql="('2026-01-01 00:00:00.5'), ('2026-01-02 00:00:00')", + start="2026-01-01T00:00:00+00:00", + end="2026-01-01T12:00:00+00:00", + expected_count=((1,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_cursor_values_when_filtering_then_clickhouse_parses_iso_timestamps( + test_case: ClickHouseCursorFilterTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + relation: str = f"{clickhouse_schema}.events" + adapter.execute( + connection=connection, + sql=f"CREATE TABLE {relation} (at DateTime64(6, 'UTC')) ENGINE = MergeTree ORDER BY at", + ) + adapter.execute( + connection=connection, sql=f"INSERT INTO {relation} VALUES {test_case.rows_sql}" + ) + condition: str = adapter.build_cursor_filter( + cursor_column="at", + start_cursor=CursorValue(kind="timestamp", value=datetime.fromisoformat(test_case.start)), + end_cursor=CursorValue(kind="timestamp", value=datetime.fromisoformat(test_case.end)), + ) + + assert fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT count() FROM {relation} WHERE {condition}", + ) == (test_case.expected_count) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseColumnMigrationTestCase( + description="add, widen, rename and drop in place", + added=(ColumnInfo(name="note", type="String"),), + widened=(ColumnInfo(name="amount", type="Int64"),), + renamed=("customer", "customer_name"), + dropped=("order_date",), + expected_columns=( + ColumnInfo(name="order_id", type="Int64"), + ColumnInfo(name="customer_name", type="String"), + ColumnInfo(name="amount", type="Int64"), + ColumnInfo(name="note", type="Nullable(String)"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_table_when_migrating_columns_then_clickhouse_alters_in_place( + test_case: ClickHouseColumnMigrationTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + relation: str = f"{clickhouse_schema}.orders" + adapter.execute( + connection=connection, + sql=f"CREATE TABLE {relation} (order_id Int64, customer String, amount Int32, " + "order_date Date) ENGINE = MergeTree ORDER BY order_id", + ) + recorder: StatementRecorder = StatementRecorder() + + adapter.add_columns( + connection=connection, + destination=relation, + columns=test_case.added, + statement_recorder=recorder, + ) + adapter.alter_column_types( + connection=connection, + destination=relation, + columns=test_case.widened, + statement_recorder=recorder, + ) + adapter.rename_column( + connection=connection, + destination=relation, + old_name=test_case.renamed[0], + new_name=test_case.renamed[1], + statement_recorder=recorder, + ) + adapter.drop_columns( + connection=connection, + destination=relation, + column_names=test_case.dropped, + statement_recorder=recorder, + ) + + assert adapter.describe_relation(connection=connection, relation=relation) == ( + test_case.expected_columns + ) diff --git a/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/test_rendering.py b/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/test_rendering.py index f297d819fb..fbd18f79dd 100644 --- a/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/test_rendering.py +++ b/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/test_rendering.py @@ -31,7 +31,7 @@ ) _REAPPEARANCE_APPLY_FRAGMENTS: tuple[str, ...] = ( "AS __latest_rn FROM target_table) AS __latest_ranked WHERE __latest_rn = 1", - "__reappearances AS (SELECT __classified.customer_id, " + "__reappearances AS (SELECT __classified.customer_id AS customer_id, " "MIN(__classified.observed_at) AS __reappeared_at", "WHERE __latest.valid_to IS NOT NULL AND __closing_group.__observed_at IS NULL", "OR __classified.observed_at < __first_new_starts.__first_version_start", diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/__init__.py b/tests/unit/src/sqlbuild/adapters/clickhouse/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py new file mode 100644 index 0000000000..aec5c0d0fe --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CanonicalTypeTestCase: + description: str + type_sql: str + expected_type: str + + +@dataclass(frozen=True) +class NullableTypeTestCase: + description: str + type_sql: str + expected_type: str + + +@dataclass(frozen=True) +class QuoteStringTestCase: + description: str + value: str + expected_literal: str + + +@dataclass(frozen=True) +class ArtifactRelationTestCase: + description: str + relation: str + expected_prefix: str + expected_name_length: int + + +@dataclass(frozen=True) +class SnapshotCloseRewriteTestCase: + description: str + statement: str + expected_close_select: str + + +@dataclass(frozen=True) +class UnsupportedSnapshotStatementTestCase: + description: str + statement: str + expected_code: str + + +@dataclass(frozen=True) +class NullableOffsetWindowTestCase: + description: str + statement: str + expected_statement: str diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py new file mode 100644 index 0000000000..3a76620376 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.clickhouse._helpers.partition_rewrite import artifact_relation +from sqlbuild.adapters.clickhouse._helpers.snapshot_rewrite import ( + nullable_offset_windows, + rewrite_snapshot_statements, +) +from sqlbuild.adapters.clickhouse._helpers.type_text import ( + canonical_type, + nullable_type, + quote_string, +) +from sqlbuild.adapters.clickhouse.models import SnapshotWorkspace +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import ( + ArtifactRelationTestCase, + CanonicalTypeTestCase, + NullableOffsetWindowTestCase, + NullableTypeTestCase, + QuoteStringTestCase, + SnapshotCloseRewriteTestCase, + UnsupportedSnapshotStatementTestCase, +) + +_WORKSPACE: SnapshotWorkspace = SnapshotWorkspace( + destination="sales.customers", + work="sales.customers__w", + rebuild="sales.customers__r", + valid_to_column="valid_to", +) + + +@pytest.mark.parametrize( + "test_case", + [ + CanonicalTypeTestCase("nullable integer", "Nullable(Int64)", "BIGINT"), + CanonicalTypeTestCase( + "low cardinality string", "LowCardinality(Nullable(String))", "VARCHAR" + ), + CanonicalTypeTestCase("parameterized decimal", "Decimal(18, 4)", "DECIMAL(18, 4)"), + CanonicalTypeTestCase("sized decimal", "Decimal64(2)", "DECIMAL(18, 2)"), + CanonicalTypeTestCase("timestamp", "DateTime64(6, 'UTC')", "TIMESTAMP"), + CanonicalTypeTestCase("unknown type kept", "Array(Int64)", "Array(Int64)"), + ], + ids=lambda case: case.description, +) +def test_given_clickhouse_type_when_canonicalizing_then_shared_spelling_returns( + test_case: CanonicalTypeTestCase, +) -> None: + assert canonical_type(test_case.type_sql) == test_case.expected_type + + +@pytest.mark.parametrize( + "test_case", + [ + NullableTypeTestCase("scalar wraps", "Int64", "Nullable(Int64)"), + NullableTypeTestCase("already nullable", "Nullable(String)", "Nullable(String)"), + NullableTypeTestCase( + "low cardinality wraps inside", + "LowCardinality(String)", + "LowCardinality(Nullable(String))", + ), + NullableTypeTestCase("arrays cannot wrap", "Array(Int64)", "Array(Int64)"), + ], + ids=lambda case: case.description, +) +def test_given_column_type_when_making_nullable_then_clickhouse_rules_apply( + test_case: NullableTypeTestCase, +) -> None: + assert nullable_type(test_case.type_sql) == test_case.expected_type + + +@pytest.mark.parametrize( + "test_case", + [ + QuoteStringTestCase( + description="backslash and quote both escape", + value="it's a\\b", + expected_literal="'it\\'s a\\\\b'", + ), + ], + ids=lambda case: case.description, +) +def test_given_backslash_and_quote_when_quoting_then_both_escape( + test_case: QuoteStringTestCase, +) -> None: + assert quote_string(test_case.value) == test_case.expected_literal + + +@pytest.mark.parametrize( + "test_case", + [ + ArtifactRelationTestCase( + description="long names are cut to the identifier limit", + relation="sales." + "x" * 300, + expected_prefix='sales."x', + expected_name_length=200, + ), + ], + ids=lambda case: case.description, +) +def test_given_long_name_when_naming_artifact_then_identifier_limit_holds( + test_case: ArtifactRelationTestCase, +) -> None: + relation: str = artifact_relation(relation=test_case.relation, infix="__sqb_rw_", digest="abc") + + assert relation.startswith(test_case.expected_prefix) + assert len(relation.split(".", 1)[1].strip('"')) == test_case.expected_name_length + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotCloseRewriteTestCase( + description="joined close keeps its source", + statement=( + "UPDATE sales.customers__w AS __target SET valid_to = __source.updated_at " + "FROM (SELECT * FROM src WHERE x = 1) AS __source " + "WHERE __target.customer_id = __source.customer_id AND __target.valid_to IS NULL" + ), + expected_close_select=( + "SELECT sipHash128(__target.* EXCEPT (valid_to)) AS __sqb_row, " + "min(__source.updated_at) AS __sqb_valid_to, 1 AS __sqb_closed " + "FROM sales.customers__w AS __target, " + "(SELECT * FROM src WHERE x = 1) AS __source WHERE " + "__target.customer_id = __source.customer_id AND __target.valid_to IS NULL " + "GROUP BY __sqb_row" + ), + ), + SnapshotCloseRewriteTestCase( + description="hard delete close has no join", + statement=( + "UPDATE sales.customers__w AS __target SET valid_to = now64(6, 'UTC') " + "WHERE __target.valid_to IS NULL AND NOT EXISTS (SELECT 1 FROM src AS __source " + "WHERE __source.customer_id = __target.customer_id)" + ), + expected_close_select=( + "SELECT sipHash128(__target.* EXCEPT (valid_to)) AS __sqb_row, " + "min(now64(6, 'UTC')) AS __sqb_valid_to, 1 AS __sqb_closed " + "FROM sales.customers__w AS __target " + "WHERE __target.valid_to IS NULL AND NOT EXISTS (SELECT 1 FROM src AS __source " + "WHERE __source.customer_id = __target.customer_id) GROUP BY __sqb_row" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_snapshot_close_when_rewriting_then_working_copy_is_rebuilt_and_exchanged( + test_case: SnapshotCloseRewriteTestCase, +) -> None: + statements: tuple[str, ...] = rewrite_snapshot_statements( + workspace=_WORKSPACE, + statements=("INSERT INTO sales.customers__w SELECT 1", test_case.statement), + ) + + assert statements[:5] == ( + "DROP TABLE IF EXISTS sales.customers__w", + "DROP TABLE IF EXISTS sales.customers__r", + "CREATE TABLE sales.customers__w AS sales.customers", + "INSERT INTO sales.customers__w SELECT * FROM sales.customers", + "INSERT INTO sales.customers__w SELECT 1", + ) + assert test_case.expected_close_select in statements[6] + assert statements[-4:] == ( + "EXCHANGE TABLES sales.customers__w AND sales.customers__r", + "DROP TABLE sales.customers__r", + "EXCHANGE TABLES sales.customers AND sales.customers__w", + "DROP TABLE IF EXISTS sales.customers__w", + ) + + +@pytest.mark.parametrize( + "test_case", + [ + UnsupportedSnapshotStatementTestCase( + description="update of another relation", + statement="UPDATE other SET valid_to = 1 WHERE 1", + expected_code="A709", + ), + ], + ids=lambda case: case.description, +) +def test_given_unrecognized_update_when_rewriting_then_coded_error_names_it( + test_case: UnsupportedSnapshotStatementTestCase, +) -> None: + with pytest.raises(AdapterUserError) as error: + rewrite_snapshot_statements(workspace=_WORKSPACE, statements=(test_case.statement,)) + + assert error.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + NullableOffsetWindowTestCase( + description="lag and lead arguments become nullable", + statement=( + "SELECT LAG(__observed_at) OVER (ORDER BY __observed_at), " + "LEAD(__new_changes.updated_at) OVER (PARTITION BY k ORDER BY x), lag_value FROM t" + ), + expected_statement=( + "SELECT LAG(toNullable(__observed_at)) OVER (ORDER BY __observed_at), " + "LEAD(toNullable(__new_changes.updated_at)) OVER (PARTITION BY k ORDER BY x), " + "lag_value FROM t" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_offset_windows_when_rewriting_then_arguments_become_nullable( + test_case: NullableOffsetWindowTestCase, +) -> None: + assert nullable_offset_windows((test_case.statement,)) == (test_case.expected_statement,) diff --git a/uv.lock b/uv.lock index ad50c7550a..a36bbcec0e 100644 --- a/uv.lock +++ b/uv.lock @@ -274,6 +274,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, ] +[[package]] +name = "backports-zstd" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/9c/13569626440e88f09d16f43ec1c2aa0d10a523be2811414580d1cfb7c9f3/backports_zstd-1.8.0.tar.gz", hash = "sha256:9dae4f4c481716e3db473d667457b4f508ff7459c0931b567a5c9677fb3db316", size = 1006566, upload-time = "2026-10-10T16:36:40.642Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/03/3c303d6f3066f84f2c52acfc38852546a836596dd9a2bc7add83bd96b527/backports_zstd-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6e024aee6bfd04094fce60133b0e6bd0f8027cdb2823157880bc87f1ffdfee21", size = 439718, upload-time = "2026-10-10T16:34:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/92/31/1e73b2835c78a9067ecba390b0eea032f827fc0b2f8bf2c8656992c30dc8/backports_zstd-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d810d83c8a703f424ed2a49aa271078c91b530da2d8c104bd88207e68d116de8", size = 368337, upload-time = "2026-10-10T16:34:58.287Z" }, + { url = "https://files.pythonhosted.org/packages/85/43/b0cc88c7d13a544f6d38f288fd96e1595395dad31f49fad2619f06b96d95/backports_zstd-1.8.0-cp312-cp312-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:d057948e8cffa19f0cc8668e06fd502ad8a69f398e91a426b39dcc5eeb197c2f", size = 509148, upload-time = "2026-10-10T16:34:59.951Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/81cc731a0408c3cba05a44ece00476305dbe1a52e27a4c323c98685f7015/backports_zstd-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6aa762cf369d9bfca1e013eaad562f8e129d71b7a82f0c459870d6d21651bcb3", size = 478911, upload-time = "2026-10-10T16:35:01.791Z" }, + { url = "https://files.pythonhosted.org/packages/df/63/dc62779cabb725a8974a2d303bfe0d7cd5b8987fab79ab445c48efcfb2e4/backports_zstd-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b9d6c4ca7d927fd094badcf9174ee5c82ddb4855fe14658806c8c8a07d4a165", size = 584283, upload-time = "2026-10-10T16:35:03.666Z" }, + { url = "https://files.pythonhosted.org/packages/e5/12/5e8ce29119d78845cd3351bcd79baa16a30aa8c19f8c359a1719a15d97b3/backports_zstd-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d85b8ce50aea247289be183f853e67c106959c4048ce286b26c4663b06bb6d", size = 643167, upload-time = "2026-10-10T16:35:05.342Z" }, + { url = "https://files.pythonhosted.org/packages/3f/08/a9d59fb9e20215ede0c8ea4d729373dc0592aee45776cdd86c92c3c6242c/backports_zstd-1.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f9e9aa28a44db1897fb637f037175566f3b75890d4bae6cae7ba34f1df1e0804", size = 496867, upload-time = "2026-10-10T16:35:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/e8/b8/abcd2be476a47dd236500c405df32aa81902c54750b26c626f190bbef6b9/backports_zstd-1.8.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2c431f3cdc7eb663a42574e27a8604a18181ea4e193504f222d8e61c6f5f8b78", size = 571623, upload-time = "2026-10-10T16:35:09.014Z" }, + { url = "https://files.pythonhosted.org/packages/03/ce/31e668dcdfe017b3240f49c3ef67b108224d3f66d90e9f26caecafc3c29c/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e0431230a67e8f07210efe654abda9844a55c3bf57d74e60425d9d65770b1de4", size = 484948, upload-time = "2026-10-10T16:35:10.974Z" }, + { url = "https://files.pythonhosted.org/packages/5a/98/d9122b7531830ceb0f62adb88694bb8cc414a27d1d03539c44dd96fa7a63/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9b62b6c8c5a43b294d4358c2016bfbc507cc574315ffa75346ccf0b621746461", size = 512635, upload-time = "2026-10-10T16:35:12.658Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f0/168c6d0c93a3ad6568d0b0ac2f732efc9132b2839d4e6759e61f5239107d/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:869ab7e5421873dfbdbf646d52b4e8d711093972819c06c6daf3249a1ec6e0e7", size = 588696, upload-time = "2026-10-10T16:35:14.595Z" }, + { url = "https://files.pythonhosted.org/packages/22/32/b8eacce542dae88df98f923e81c079a01b66b7fbdf103e319f6fb1df2dfa/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ec1a796429674ebc0e2d48feb3b6658bf49d3ae840b0c0e14ad50c4d6b7341fe", size = 568895, upload-time = "2026-10-10T16:35:16.287Z" }, + { url = "https://files.pythonhosted.org/packages/dd/16/8abede9513ec8fd584e36159b1dce82042a97214e69f53f08605b245999f/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:775b701a576769df053cfb7d9456b06223b40e329c010be6cc178fe9e404a3d2", size = 633610, upload-time = "2026-10-10T16:35:18.014Z" }, + { url = "https://files.pythonhosted.org/packages/6d/74/4e82ed15ae212b0fc0cd8f82c5bbf6a9dd584b6b37df0c3485663c6ad105/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ab77a2e6e21c57e8341bb7656c71d1a1653151ebe787b3f092ce86a02543eb52", size = 501407, upload-time = "2026-10-10T16:35:19.688Z" }, + { url = "https://files.pythonhosted.org/packages/bd/02/7e86774e0a3c2457d23939acbb32bdb019e6bdec48892986255faa262c3d/backports_zstd-1.8.0-cp312-cp312-win32.whl", hash = "sha256:f99b44c2c13fc60f65ad568bf7401d9540370f996b1040793a34988324e3b712", size = 293065, upload-time = "2026-10-10T16:35:21.309Z" }, + { url = "https://files.pythonhosted.org/packages/a5/78/2f497fd2bbf46099e46650f75467967d21f25bb921c894d28d493bbfb7e4/backports_zstd-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:1eddf59fedaf19dd3a8e9c597add7eb6f0d51d4467a0924b2dcd2c118ed18ff5", size = 330563, upload-time = "2026-10-10T16:35:22.968Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2c/3a1a91cea5b98e24cb54ecf142a72246d2e1efa5efe41504388188598951/backports_zstd-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:2b3247a7a916b90f155b4133eedaceadd0c37b4149ee32e4d74fe512a14be89b", size = 322189, upload-time = "2026-10-10T16:35:24.494Z" }, + { url = "https://files.pythonhosted.org/packages/66/a8/7a04f1daaa42936ec3d98f213b4698b18053d1154f2aee1d067c4121fe3a/backports_zstd-1.8.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4e92ff4ce96b3c61d25900875b6cf1ee249349b8e419abd80893ec9b8026444e", size = 401586, upload-time = "2026-10-10T16:35:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c2/d26216501b3e13583084e11106ade1779b280f3304c75d84d2dfb9e5d609/backports_zstd-1.8.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:0c2e652b4fbc2e6b7bd05a09b6eab3a51bfaed9e7fca1bc81d763dc47361e2ff", size = 455589, upload-time = "2026-10-10T16:35:28.174Z" }, + { url = "https://files.pythonhosted.org/packages/df/66/372b138fa7e7be4d6aff343a55dd77e492867cb5de701899b5aa01722836/backports_zstd-1.8.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915d3e7e57194b5cee33f10cf2d9f5c4f7658c8a167236f9ba5501520cf133e8", size = 358662, upload-time = "2026-10-10T16:35:29.819Z" }, + { url = "https://files.pythonhosted.org/packages/7a/26/0b89de2f83088f89e10ea3f4a5badef9bc95098bdd39a3031362da48dc60/backports_zstd-1.8.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e6f8483b795a09c0e0fbacca4fa844242bc6d5fc64b8a6ee99f88ad8af27b08", size = 367357, upload-time = "2026-10-10T16:35:31.649Z" }, + { url = "https://files.pythonhosted.org/packages/74/01/5239b39d3f65ba80e2129b9273bf736245e4a1c03b8a317ed399c4fe10dd/backports_zstd-1.8.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1fe4b06a019aa4cdf87af320eef56a4bdbdb924ead36a7a918645d72edece966", size = 447892, upload-time = "2026-10-10T16:35:33.534Z" }, + { url = "https://files.pythonhosted.org/packages/b5/13/e4eceee62d144f68944addb0179368d626f96d3644d965620774f1f5e463/backports_zstd-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:49c4006cdf41c15ffcc74f10d9a6485be841106cd4d5aa7ea7bf1075cc37fb83", size = 439240, upload-time = "2026-10-10T16:35:35.351Z" }, + { url = "https://files.pythonhosted.org/packages/1f/5f/996aceebbbc4eebc05d99fe1714b1b0930260eac5171e8ebc3a952390c0d/backports_zstd-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fa862d24b7fb392279a95bc9acc1f0ede8a25de9efbed03fb305ceac2f6abb0", size = 367710, upload-time = "2026-10-10T16:35:37.004Z" }, + { url = "https://files.pythonhosted.org/packages/93/0b/c373a7f92df9df1f9e0657ea0dd86c45444b8414db616b3d38b62f90075c/backports_zstd-1.8.0-cp313-cp313-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:9af83a6d7dc67896fd91bcd4c2cd182ba97d7cca2b09a94373a5fef154001d98", size = 508347, upload-time = "2026-10-10T16:35:38.683Z" }, + { url = "https://files.pythonhosted.org/packages/b4/36/07dca77032300047efd09808d49ab9d1fff8657553adbc8e0e6405aba864/backports_zstd-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a808ba1371231c00a2b71f03840a727088e287d0ee1dfb3230958950f21f421", size = 478416, upload-time = "2026-10-10T16:35:40.504Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a9/bb96724619a1dcc3a9e3138d15a6f7a2fc40b581926db4ac00e424af79c1/backports_zstd-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6cc15051c282ac2585a2425d22f416ae2deb5afb441b22831b349b02fd58a782", size = 583888, upload-time = "2026-10-10T16:35:42.159Z" }, + { url = "https://files.pythonhosted.org/packages/cd/6d/65e6e437eb54b5be2ce7248ac236d82a771a672457c950e7f96849699274/backports_zstd-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a23d38d7b9ca93403acd3c2c306af6e547a24d150c25ac2d7a8acd751fbd968", size = 644796, upload-time = "2026-10-10T16:35:43.882Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6d/3c422b33d40aaca6e9d9fdd47f1a047ac499de749c887ab3dab62f731fb2/backports_zstd-1.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44a9004f9e809ea56910d326d21946650369db59eb86edc0c76840f21530704c", size = 493385, upload-time = "2026-10-10T16:35:45.576Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b9/ea08e2c2b8a7bfabff359852e4d7a9cbc2cde09715907250c0e53432fbe9/backports_zstd-1.8.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ff307f3f0ef3b7f40ccfce42c0704fddc99cd30bca451330f42466db1981be9", size = 568613, upload-time = "2026-10-10T16:35:47.394Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/775cb7317f1f693c7f3e96fa5cf5426b461616b52730a72f978f31b334b0/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6c8572e27c5f0b9d11020d3f597bf3c35fe0f5ae6f99156dc52b0bd937ba8908", size = 484237, upload-time = "2026-10-10T16:35:49.496Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f8/c31798a8911390fb0d4f058f65cba2e54141d6394c35430b1d495d121667/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cc1d9d3660c40abe4095de80f43ce4c955d08f7d9803d3da97176aa61b76d923", size = 511865, upload-time = "2026-10-10T16:35:51.223Z" }, + { url = "https://files.pythonhosted.org/packages/68/df/0ff79b6a2d7f5c10d3ebc7e23b5281f51130feb4db8afadac98ba5131c18/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:83cea5cdd70e1d74382be6deeeda1db79aedd1a06af4f8a8fbafba9eedae5230", size = 588422, upload-time = "2026-10-10T16:35:53.371Z" }, + { url = "https://files.pythonhosted.org/packages/19/a7/d5dbad63911fc3040253dc209a7aac8921e928fe64f3fcde051066aa5a75/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e74eb204b9d7798fc57393202c443fc2ec84283d82387168baeb763f8beb224d", size = 566480, upload-time = "2026-10-10T16:35:55.459Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b9/621e734eb144d56c7632b763c0ce3fa196839fc0f82830244206a9d37d8d/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:515497b3d49dd6d7a84fb16a0a0007bc460b4a7e1f55e70f33315c66d3844e8e", size = 635191, upload-time = "2026-10-10T16:35:57.307Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/1b6709f13f2a22a1d72e15f114ab62e852db33ba0f8840c7d102523bcdb6/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6283c90997038abf46c8a0bb75afb4dc6cbf061421802fda0afc382fe4b348b3", size = 497769, upload-time = "2026-10-10T16:35:59.395Z" }, + { url = "https://files.pythonhosted.org/packages/de/52/cd0a82fd52ae159a0316d2257156968c356cab81062d6050af48a4e8a3d6/backports_zstd-1.8.0-cp313-cp313-win32.whl", hash = "sha256:9d76a3193a3a4a6b1249021e7ecf72e4cabc1dca611c6fb41db1c0b5d2faf741", size = 292645, upload-time = "2026-10-10T16:36:01.439Z" }, + { url = "https://files.pythonhosted.org/packages/12/0e/5c5a916cea73b455850083ccf76078de655face3dfe4126848570c57a6dd/backports_zstd-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:b583990d554cc6f6141c5c43b6db3c7da87a214253e08339d917ee3baa3021b6", size = 330247, upload-time = "2026-10-10T16:36:03.058Z" }, + { url = "https://files.pythonhosted.org/packages/86/3c/7297d87eed9254f6b4823c05b37aa07ec2a99bc5f195760dc574e925eecf/backports_zstd-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:0600e166cb00739a26de74ee1696221a53a4d5dc1f96a0bdeb6b307c1626c15c", size = 322066, upload-time = "2026-10-10T16:36:04.932Z" }, + { url = "https://files.pythonhosted.org/packages/1f/c8/dba9e5905e83ac955c1c19b797f59f5335a351664a7b25a709929d63dfbc/backports_zstd-1.8.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:f710d03f84d74f11737735f846b44ef1545cadb73ef47bcd3d0e124f253dd763", size = 413972, upload-time = "2026-10-10T16:36:28.92Z" }, + { url = "https://files.pythonhosted.org/packages/93/11/8ee691bfd2c8292a573a0378a616372aa01ed9e6001d5778ae666a239265/backports_zstd-1.8.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:2b11fb8b9c798657c97ad3165893f146c300e2f7f800e9c54c0d2143052c1486", size = 344652, upload-time = "2026-10-10T16:36:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/19/33/86bb2cd5c6e827adba98fb091ccecb29dae3bb33e0406f8e08be7bdbe70b/backports_zstd-1.8.0-pp312-pypy312_pp80-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:ec7351d3e6ea92338dc4e0e53c876d2e2092e07ad3a2083088e0160200efdd15", size = 422892, upload-time = "2026-10-10T16:36:32.708Z" }, + { url = "https://files.pythonhosted.org/packages/42/a2/629f5e9c3edd2a31f7dd65b8097241b5036f98105efac251a12c1a8f7cb5/backports_zstd-1.8.0-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63ae348b629121eeb967244fecd254f41b4b3a63d074c252f4d7777f5d17c71c", size = 396431, upload-time = "2026-10-10T16:36:34.842Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f6/9c223e9cccc5a797c17475fde1a8a78ada0dcdd39be2302f4605e565c0ce/backports_zstd-1.8.0-pp312-pypy312_pp80-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:163b5c36321bf5652b6e4aeb04d3644ddbf9c1881a82322e376e5be3532af26b", size = 416398, upload-time = "2026-10-10T16:36:36.706Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e3/2eb6f517c9a6746a735b49ba4ab3ed3df6c4ec9072169805547ae590e296/backports_zstd-1.8.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:3f0288db18a64f4f4146f4526456ff62b2edb625b2d43956e764885edd3f1da2", size = 404272, upload-time = "2026-10-10T16:36:38.766Z" }, +] + [[package]] name = "boto3" version = "1.42.93" @@ -462,6 +515,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, ] +[[package]] +name = "clickhouse-connect" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-zstd", marker = "python_full_version < '3.14'" }, + { name = "certifi" }, + { name = "lz4" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/5a/102a18e1e41a4316437f63fc9a9cb72e34e8c07fe2df7edf69e0def429b5/clickhouse_connect-1.11.0.tar.gz", hash = "sha256:73a71e82a8b7ef02d2c50aa0fb38dee4aed4fa14cdd2e6af0e2efbe6d856b6f9", size = 267397, upload-time = "2026-10-16T17:41:49.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/dd/8f22d4210fff6238232ccc2be34f388bf1092844a0dc58f0f19dbb8ee704/clickhouse_connect-1.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7fe7e632996b97e486d2df652cdd76c17c10876157e435866f0d870eb596642", size = 489547, upload-time = "2026-10-16T17:40:11.713Z" }, + { url = "https://files.pythonhosted.org/packages/f0/40/beb4edafeb4371fc8da84da993bbf20f7a946831005f851e3c667ad0b46a/clickhouse_connect-1.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ac2e79a4f1e814c1ed01e55139ee17a0ee99f3ad709dd20a972ddd5ed6637ca0", size = 487097, upload-time = "2026-10-16T17:40:13.248Z" }, + { url = "https://files.pythonhosted.org/packages/e9/2d/9964b8c7669c4261cffe162385eb5301970c6dbcb28e87b8d94701a43f66/clickhouse_connect-1.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9a8c6973aa5842251da8dcf8a688d120e49b4179295947974aeb6855c6659f3", size = 1493361, upload-time = "2026-10-16T17:40:14.902Z" }, + { url = "https://files.pythonhosted.org/packages/01/32/5988d094f3c23e913910147e9769e5913f32a0b39fcc19d413e7d437cf27/clickhouse_connect-1.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c803a3a30c55de6e53efc09f7524ddd75f143e2847fd86e98b86b289ddd4b", size = 1515867, upload-time = "2026-10-16T17:40:17.016Z" }, + { url = "https://files.pythonhosted.org/packages/2a/25/f2a02e96365d8318c67e8b65c33953c93844b627ee87d63933162df211ef/clickhouse_connect-1.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bca58be4e8aff5fd317f5cfe2a7d5f7cc89050e0bf5a06a0b370ee1070f5a443", size = 1470552, upload-time = "2026-10-16T17:40:18.771Z" }, + { url = "https://files.pythonhosted.org/packages/be/84/8811cb0b83137ea9272b30592b7949eb10144f1c23ddaae15d2873698017/clickhouse_connect-1.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3df737e301f79a009f0c8a46be1999ac0d176f9934b0e7ebcce7f488176263ab", size = 1512194, upload-time = "2026-10-16T17:40:20.528Z" }, + { url = "https://files.pythonhosted.org/packages/d6/89/3b7dce45fd4294c0aa7328dfd476217c5c2c5e4ee484c9598ad908283cba/clickhouse_connect-1.11.0-cp312-cp312-win32.whl", hash = "sha256:feb3e17850d768a45ff8aa064ce53f89ce79c7be870abc608d3df775aed48ba4", size = 437172, upload-time = "2026-10-16T17:40:22.348Z" }, + { url = "https://files.pythonhosted.org/packages/f4/00/5ffb096ecc7fe24e79280077bf1fd440a59cf508473d6dc472b966ac418a/clickhouse_connect-1.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:a7a527d8a7e82bad75022f809f380b5a40c7acfb6431565a5473e7d3d6fac66d", size = 458737, upload-time = "2026-10-16T17:40:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/08/37/fe6f5b17c0c7767733694fcdebd54ace5b470c2520d6b3032bf593766f01/clickhouse_connect-1.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:222df58e745c7f94cb5527074e1a3c1326e9bee7008c51a5268e97da27a4d8fe", size = 447911, upload-time = "2026-10-16T17:40:25.343Z" }, + { url = "https://files.pythonhosted.org/packages/a5/49/3649d3100f04b18657e014902f936dd44a71e64fa77387e38f934b646844/clickhouse_connect-1.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:43af3766a0ce18b6a9e93f04a45fee645851f53464508a763e403bfa86b2ad07", size = 488788, upload-time = "2026-10-16T17:40:26.881Z" }, + { url = "https://files.pythonhosted.org/packages/68/8c/2eb648fb7af5ce26b47ca05115a416e25c633989c77d815ad8ea863ea6ff/clickhouse_connect-1.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d828c96cef7836d2a4ff5d918e78e40010d04940a1bd1da4b89a9c25ed0bdd8f", size = 486001, upload-time = "2026-10-16T17:40:28.675Z" }, + { url = "https://files.pythonhosted.org/packages/79/aa/a7ab521399ca1b0d79c5ca8208f92f727988908cb5f12866bdd8487a84e4/clickhouse_connect-1.11.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d380fd944e404d90cd34dd4e3d54d1e799d3ebdcf2b5c07b121154020b7cb75", size = 1476207, upload-time = "2026-10-16T17:40:30.28Z" }, + { url = "https://files.pythonhosted.org/packages/8c/58/d5082bb9993020570f6380e8fcc031ce00c404867b7fe4181b9e0b8c32c5/clickhouse_connect-1.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee88c01423f57582a98f8d3e8e4a8ee71e3a9ff2718e6c8a6b079b597b3471f8", size = 1498131, upload-time = "2026-10-16T17:40:32.186Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/d2a34852d2b15d070dcce1a515cc36ef237ace59d8d0486178e77f1f0b1f/clickhouse_connect-1.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f767f04b07378177885c150f0cf56fac0a61b3531c466d265f65af1a615f6acd", size = 1453981, upload-time = "2026-10-16T17:40:34.002Z" }, + { url = "https://files.pythonhosted.org/packages/7b/6c/94976487ad1fc4fff2e3f8b1a0d0d23bad147743052edfe1088cbf35cfc0/clickhouse_connect-1.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:41650c0cfce30b3ce151b0cecb727fc9d3a28d53be44ff6d416374dfa5412159", size = 1495195, upload-time = "2026-10-16T17:40:35.549Z" }, + { url = "https://files.pythonhosted.org/packages/5c/dc/b4ac3777236172ba7f0fe71d2b81fbf49082c3e4c147e0adbadf9c3f780d/clickhouse_connect-1.11.0-cp313-cp313-win32.whl", hash = "sha256:9afc1d2adbdfb8ef9386fd2657bb6c493c23b32ff5f63e91354bc9dbe1757514", size = 436527, upload-time = "2026-10-16T17:40:37.135Z" }, + { url = "https://files.pythonhosted.org/packages/35/b5/efb3634c36ec08b154aa2af1cd5321d7b5b5ebed4f1e88c8bb96082f1432/clickhouse_connect-1.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:d3e79aaf25f3521981da648ea9fb85c5ee8873d25a9b31016376f7c6e2acd8b3", size = 457247, upload-time = "2026-10-16T17:40:38.775Z" }, + { url = "https://files.pythonhosted.org/packages/0e/61/b90bb01f12b983de853ba359ee6ec32045654b1feb6cbe809f8aba2d9211/clickhouse_connect-1.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:0216cdfa21c62480ee0e39de8c838788f8cdcbd747bdb90e15270ca7aef72603", size = 446975, upload-time = "2026-10-16T17:40:40.301Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0b/0a68a8b2809dc1fa0e606bbe387868785ad9c4f5325bf91ec797ebb0afa4/clickhouse_connect-1.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8d8477c782bcc14b2b10c9012589a8e324d65f162f2c8a08f6dabb5a69c01bf3", size = 489702, upload-time = "2026-10-16T17:40:41.726Z" }, + { url = "https://files.pythonhosted.org/packages/86/76/92fbfc4a7c13d07a8a1d555e2281f8fa0fffce51bed02498dff1057c8c6c/clickhouse_connect-1.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf8817b75961641674b00ac7798035d6cb57458b9560a7db9e6cdf9a76488fd4", size = 487630, upload-time = "2026-10-16T17:40:43.253Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d1/edfe03a8e82d7d891dcebc9b868b9da1fd08d97c619ef8055c837c2cfb3e/clickhouse_connect-1.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6d56e3d1f67bcf59672000385a822bcd3b934e858a334eafa7200f3cf1ad0f8", size = 1473417, upload-time = "2026-10-16T17:40:45.078Z" }, + { url = "https://files.pythonhosted.org/packages/f3/00/48fc68b74765fade6b567cc80b8d52f39dee496ac6c5ac79eb9171c76fe1/clickhouse_connect-1.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fbe36f21e793210112797938abbff54bfd3010367a2e51f7f941156885999a0", size = 1484398, upload-time = "2026-10-16T17:40:46.708Z" }, + { url = "https://files.pythonhosted.org/packages/2e/26/828ee3070f6ea2b1a4a965518717108ea3d895a9443bd1a7817a6af171ec/clickhouse_connect-1.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:77e84e429205335e3081a59942bce6cb9a993fecf1855818f40a568156c5dd74", size = 1450515, upload-time = "2026-10-16T17:40:48.222Z" }, + { url = "https://files.pythonhosted.org/packages/83/33/4cd4418055dfe5d75183e3a00d25ae13ee051e3cff510a8ec80533ec5b49/clickhouse_connect-1.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f2cb149ba92ca0b2c49eeaa894d711dba7e91e557772752e0ed5ef067e570bc8", size = 1482823, upload-time = "2026-10-16T17:40:49.801Z" }, + { url = "https://files.pythonhosted.org/packages/77/0b/a1fd38786b2ad73cbc71c6244cb778270100acb9673259f6aa6cf91408d9/clickhouse_connect-1.11.0-cp314-cp314-win32.whl", hash = "sha256:56e717869758b327ee1faeee1fe8d90b80d781c4157453b456ea5b1c8bcf2cea", size = 440162, upload-time = "2026-10-16T17:40:51.566Z" }, + { url = "https://files.pythonhosted.org/packages/94/3e/fe4ab09a3cbd99ce64be5b205b0cecbf8f881f263f07f64b3f71d24bdff3/clickhouse_connect-1.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:13f5b62e15a636e264aeb36d687ec2dbb263106bc0148a6fb121124f103475f7", size = 461991, upload-time = "2026-10-16T17:40:53.285Z" }, + { url = "https://files.pythonhosted.org/packages/da/8f/62e46b25599d8cdfc756555bb3dcefe438ee1e8c885e3e02eb24078bc969/clickhouse_connect-1.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:d8d8058bd3d0b9b1ba259d27e992d4a3c345d819cb930b1d44a3034dd724eb53", size = 452399, upload-time = "2026-10-16T17:40:55.021Z" }, + { url = "https://files.pythonhosted.org/packages/29/92/e79eb1efe0e4018a1843d5d5e790cd2ffd3591e98718346c858579437f58/clickhouse_connect-1.11.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a353bd619da1b71ccbe9272d43580dfff856ffa09f8f4c59b9d1dd2326565ebc", size = 502160, upload-time = "2026-10-16T17:40:56.629Z" }, + { url = "https://files.pythonhosted.org/packages/86/7a/2506610c3ad8a77f61deacb9dd2d89b1b988a759862f7b984d56ac674399/clickhouse_connect-1.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3586234146a7fefc9261fed2f23b01dc3a315befe8c1a2993bf375d516a807b9", size = 504101, upload-time = "2026-10-16T17:40:58.505Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b1/c317eeb7bae3cc43ffb23e6c9b2b0c6484ce38245219e41ab7ba9b5529a1/clickhouse_connect-1.11.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64dad7b517c90e13e264f8eee8bcf35c73ec0d8c204a96c742fe5c926205ff01", size = 1561598, upload-time = "2026-10-16T17:41:00.17Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c7/eeb9184c14cfb075655dfd81bf190c32d1a25eaca7924ca036b8f26442fe/clickhouse_connect-1.11.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:12ca174f5e8968eb15c1567d431882ec28f3509508f4c221fa1ebc26223a9f80", size = 1543992, upload-time = "2026-10-16T17:41:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/8c/90/ed7244af43e66f1e55945c4651b55d336ce9ea81082083d7ac14a24d5396/clickhouse_connect-1.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:079cd29857f6d8e810cf68ee51a576fc4242fa5c735b8631943a97dc8568966e", size = 1516021, upload-time = "2026-10-16T17:41:04.186Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f9/2fbf2fac7dc87452585b0cccd7c6f741c8dc8919d9f4f425e4d76c15e26e/clickhouse_connect-1.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ebd20dfbc955ddc37758e002d21c14c5f36db07da53361ddf8de0f498488c315", size = 1533114, upload-time = "2026-10-16T17:41:05.926Z" }, + { url = "https://files.pythonhosted.org/packages/de/6c/0cffbc363045f11fdee7c3611ec2fdf033880a0c19fb5ca49e01d49b42b2/clickhouse_connect-1.11.0-cp314-cp314t-win32.whl", hash = "sha256:139199ef4363bdefd8363b7e1bc346d0ed492347cc283756a65b7012176c1a41", size = 454385, upload-time = "2026-10-16T17:41:07.792Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b5/9bdf7ddff064dca181f4180333000825fbc8ced7e94139641137cdb54dd4/clickhouse_connect-1.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b9dc9dc71c82e77507a92b40b993d1c469b65973983ed4d8be072921cb0ccf70", size = 476863, upload-time = "2026-10-16T17:41:09.344Z" }, + { url = "https://files.pythonhosted.org/packages/d7/56/ce7005cd472b6da2bf477689136b30e4c4b4b682f2480d2c5ddcd0093bf6/clickhouse_connect-1.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:4fca0769c78bc28e0812f2f7711dcea702c3a16aed994cb28c4f17c771f470ac", size = 465887, upload-time = "2026-10-16T17:41:10.961Z" }, +] + [[package]] name = "cloudpickle" version = "3.1.2" @@ -4154,6 +4258,9 @@ dependencies = [ bigquery = [ { name = "google-cloud-bigquery" }, ] +clickhouse = [ + { name = "clickhouse-connect" }, +] dagster = [ { name = "dagster" }, { name = "dagster-webserver" }, @@ -4209,6 +4316,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "clickhouse-connect", marker = "extra == 'clickhouse'", specifier = ">=0.8" }, { name = "dagster", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "dagster-webserver", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "databricks-sql-connector", extras = ["pyarrow"], marker = "extra == 'databricks'", specifier = ">=4.1.2" }, @@ -4229,7 +4337,7 @@ requires-dist = [ { name = "snowflake-connector-python", marker = "extra == 'snowflake'", specifier = ">=4.7.1,<5" }, { name = "sqlbuild-cli", directory = "crates/sqlbuild-cli" }, ] -provides-extras = ["dbt", "snowflake", "bigquery", "databricks", "dagster", "postgres", "sqlserver", "ingestr", "dlt", "rivers"] +provides-extras = ["dbt", "snowflake", "bigquery", "databricks", "dagster", "postgres", "sqlserver", "clickhouse", "ingestr", "dlt", "rivers"] [package.metadata.requires-dev] dev = [ From 1496c5da58353c73ebcf52a1aaf747b41126b9ce Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:47:23 +0100 Subject: [PATCH 16/34] feat: read ReplacingMergeTree sources with the final source option Sources accept 'final: true'. Every generated read of such a source (model references, source audits and freshness) is rendered natively as (SELECT * FROM FINAL) so replacing engines return one row per key. Adapters whose SQL dialect is not ClickHouse refuse the option with A010, and expression sources cannot set it. --- .../src/yaml_entries/_helpers/sources.rs | 8 ++- .../src/yaml_entries/models.rs | 3 + .../src/yaml_entries/tests/helpers.rs | 5 +- .../tests/test_parse_sources_document.rs | 28 +++++++++ .../src/bindings/_helpers/auditing/render.rs | 9 ++- .../_helpers/contract_records/constants.rs | 1 + .../_helpers/discovery/yaml_entries.rs | 1 + .../_helpers/freshness/observation.rs | 4 +- .../_helpers/planner_resolution/sources.rs | 4 +- .../_helpers/references/source_relations.rs | 62 ++++++++++++++++++- .../references/docs/concepts/sources.md | 1 + src/sqlbuild/_native.pyi | 2 + .../clickhouse/classes/clickhouse_adapter.py | 5 -- .../compiler/failure/source-unknown-key.json | 2 +- .../adapters/clickhouse/_test_types.py | 8 +++ .../adapters/clickhouse/test_final_reads.py | 61 ++++++++++++++++++ .../content/docs/docs/concepts/sources.mdx | 1 + 17 files changed, 188 insertions(+), 17 deletions(-) create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs index af199d8c9d..b917e30008 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/sources.rs @@ -14,7 +14,7 @@ use crate::yaml_entries::_helpers::integrations::source_integration_loader; use crate::yaml_entries::main::integration_loader_name::integration_loader_name; use crate::yaml_entries::models::{ParsedIntegrationLoader, ParsedSourceEntry, ParsedYamlColumn}; -const ENTRY_KEYS: [&str; 18] = [ +const ENTRY_KEYS: [&str; 19] = [ "audits", "columns", "contract", @@ -22,6 +22,7 @@ const ENTRY_KEYS: [&str; 18] = [ "database", "description", "expression", + "final", "freshness", "ingestr", "load_batch_size", @@ -137,6 +138,8 @@ fn with_settings( parsed.database = optional_text(entry, "database", site)?; parsed.schema = optional_text(entry, "schema", site)?; parsed.table = optional_text(entry, "table", site)?; + parsed.read_final = optional_bool(entry, "final", site)? + .is_some_and(|node| node.kind() == NodeKind::Bool(true)); parsed.freshness = freshness(entry, &shape.columns, path)?; parsed.write_strategy = write_strategy(entry, site)?; parsed.load_batch_size = positive_int(entry, "load_batch_size", path)?; @@ -226,6 +229,9 @@ fn validate_source( relation_keys.join(", ") ))); } + if entry.expression.is_some() && entry.read_final { + return Err(error("cannot define final with expression")); + } if entry.expression.is_some() && entry.type_enforcement == Some(true) && !entry diff --git a/crates/sqlbuild-discovery/src/yaml_entries/models.rs b/crates/sqlbuild-discovery/src/yaml_entries/models.rs index 62ca7010ee..23d3105ef4 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/models.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/models.rs @@ -133,6 +133,8 @@ pub struct ParsedSourceEntry { pub database: Option, pub schema: Option, pub table: Option, + /// Whether reads of the source add `FINAL`, so replacing engines return one row per key. + pub read_final: bool, pub loader: Option, pub managed: bool, pub integration_loader: Option>, @@ -160,6 +162,7 @@ impl ParsedSourceEntry { database: None, schema: None, table: None, + read_final: false, loader: None, managed: false, integration_loader: None, diff --git a/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs b/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs index ef1080e235..62e5b95790 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/tests/helpers.rs @@ -63,12 +63,13 @@ fn seed_line(seed: &ParsedSeedEntry) -> String { fn source_line(source: &ParsedSourceEntry) -> String { format!( - "{} loader={} managed={} schema={} table={}", + "{} loader={} managed={} schema={} table={}{}", source.name.python_str(), source.loader.as_deref().unwrap_or("-"), source.managed, shown(source.schema.as_ref()), - shown(source.table.as_ref()) + shown(source.table.as_ref()), + if source.read_final { " final" } else { "" } ) } diff --git a/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs b/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs index 3f50cadc1c..382737cc62 100644 --- a/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs +++ b/crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs @@ -26,6 +26,34 @@ fn given_sources_documents_when_parsing_then_matches_python_sources_and_errors() ]), expected_sources: lines(&["orders loader=orders managed=true schema=raw table=orders"]), }, + SourcesDocumentTestCase { + description: "final reads a table through its deduplicating form", + document: sources(vec![ + ("name", text("customers")), + ("schema", text("raw")), + ("final", Node::Bool(true)), + ]), + expected_sources: lines(&["customers loader=- managed=false schema=raw table=- final"]), + }, + SourcesDocumentTestCase { + description: "final false is a plain read", + document: sources(vec![ + ("name", text("customers")), + ("final", Node::Bool(false)), + ]), + expected_sources: lines(&["customers loader=- managed=false schema=- table=-"]), + }, + SourcesDocumentTestCase { + description: "final on an expression source", + document: sources(vec![ + ("name", text("customers")), + ("expression", text("SELECT 1 AS id")), + ("final", Node::Bool(true)), + ]), + expected_sources: parse_error( + "/project/sources/shop.yml source 'customers' cannot define final with expression", + ), + }, SourcesDocumentTestCase { description: "merge without a unique key", document: sources(vec![ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs index 488d5e3075..d4c929b39c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs @@ -8,7 +8,9 @@ use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_referenc use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; -use crate::bindings::_helpers::references::source_relations::render_source_relation; +use crate::bindings::_helpers::references::source_relations::{ + render_source_read, render_source_relation, +}; const REF_FUNCTION: &str = "__ref"; const SEED_FUNCTION: &str = "__seed"; @@ -65,7 +67,10 @@ pub(crate) fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult< let Some(entry) = lookup(&sources, name)? else { return Ok(None); }; - render_source_relation(&entry, adapter.as_ref()).map(Some) + match adapter.as_ref() { + Some(adapter) => render_source_read(&entry, adapter).map(Some), + None => render_source_relation(&entry, None).map(Some), + } }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 8030fa854d..159055abea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -683,6 +683,7 @@ const R_SOURCE_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("database", FieldDefault::None), FieldSpec::new("schema", FieldDefault::None), FieldSpec::new("table", FieldDefault::None), + FieldSpec::new("final", FieldDefault::False), FieldSpec::new("loader", FieldDefault::None), FieldSpec::new("managed", FieldDefault::False), FieldSpec::new("integration_loader", FieldDefault::None), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 0f354cfde5..b2b19a4c73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -296,6 +296,7 @@ fn source_entry<'py>( fields.set_item("database", optional(py, source.database.as_ref()))?; fields.set_item("schema", optional(py, source.schema.as_ref()))?; fields.set_item("table", optional(py, source.table.as_ref()))?; + fields.set_item("final", source.read_final)?; fields.set_item("loader", source.loader.as_deref())?; fields.set_item("managed", source.managed)?; fields.set_item("integration_loader", integration)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs index d13ee85465..5f12c69fac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::references::source_relations::render_source_relation; +use crate::bindings::_helpers::references::source_relations::render_source_read; use crate::bindings::_helpers::state_rows::freshness::{normalize, version_hash}; use crate::bindings::_helpers::state_rows::timestamps::{in_utc, offset_aware}; @@ -200,7 +200,7 @@ fn observe_configured_inner<'py>( } else { format!(" WHERE {}", filter.str()?) }; - let relation: String = render_source_relation(source, Some(&observer.adapter))?; + let relation: String = render_source_read(source, &observer.adapter)?; call_with( &native_field(&observer.adapter, "render_source_freshness_max_query")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 292a1bd99b..aa2bbb39de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::planner_strategy::values::input_error; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::ErrorId; -use crate::bindings::_helpers::references::source_relations::render_source_relation; +use crate::bindings::_helpers::references::source_relations::render_source_read; use crate::bindings::_helpers::type_system::equality::types_equal; use pyo3::IntoPyObject; use pyo3::prelude::{ @@ -246,7 +246,7 @@ pub(crate) fn render_source_read_relation( adapter: &Bound<'_, PyAny>, source_entry: &Bound<'_, PyAny>, ) -> PyResult { - render_source_relation(source_entry, Some(adapter)) + render_source_read(source_entry, adapter) } fn resolve_one( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs index 7fc39f8452..9b7fccfde0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs @@ -1,10 +1,23 @@ //! A declared source rendered as the SQL table factor that reads it. +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::native_host::adapter::AdapterHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; +/// The SQL analysis dialect whose engines read replacing tables with `FINAL`. +const FINAL_READ_DIALECT: &str = "clickhouse"; +/// Error code for a `final` source on another adapter. +const FINAL_UNSUPPORTED_CODE: &str = "A010"; +/// Guidance for a `final` source on another adapter. +const FINAL_UNSUPPORTED_HELP: &str = + "Remove 'final' from the source; only ClickHouse reads sources with FINAL."; + /// Leading words of a source expression that is a query rather than a relation. const QUERY_PREFIXES: [&str; 3] = ["select", "with", "values"]; @@ -72,6 +85,51 @@ pub(crate) fn render_source_relation<'py>( .join(".")) } +/// A source as the table factor that reads it, deduplicated with `FINAL` when `final`. +pub(crate) fn render_source_read<'py>( + entry: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult { + let relation: String = render_source_relation(entry, Some(adapter))?; + if !native_field(entry, "final")?.is_truthy()? { + return Ok(relation); + } + let dialect: Option = native_field(adapter, "sql_analysis_dialect_name")?.extract()?; + if dialect.as_deref() != Some(FINAL_READ_DIALECT) { + return Err(final_unsupported( + entry.py(), + &native_field(adapter, "adapter_name")?.str()?.to_string(), + )); + } + Ok(format!("(SELECT * FROM {relation} FINAL)")) +} + +/// The coded error for a `final` source on an adapter that cannot read with `FINAL`. +fn final_unsupported(py: Python<'_>, adapter_name: &str) -> PyErr { + match host::entry(py, AdapterHost::AdapterUserError).and_then(|class| { + call_with( + &class, + &[ + ( + "message", + PyString::new( + py, + &format!( + "adapter '{adapter_name}' does not support the source option 'final'" + ), + ) + .into_any(), + ), + ("code", PyString::new(py, FINAL_UNSUPPORTED_CODE).into_any()), + ("help", PyString::new(py, FINAL_UNSUPPORTED_HELP).into_any()), + ], + ) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_source_relation, module)?)?; Ok(()) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md index 9f3aa79008..347d21adf8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md @@ -281,6 +281,7 @@ Use [`sqb freshness`](../cli/freshness.md) to observe source freshness on demand | `schema` | Physical source schema (optional). For managed sources, overrides target `loader_schema`; otherwise managed loaders fall back to `loader_schema`, then target `schema`. | | `table` | Target table name (defaults to `name` if omitted) | | `expression` | Inline SQL expression (alternative to table reference) | +| `final` | ClickHouse only. Set to `true` to read the source with `FINAL`, so a `ReplacingMergeTree` table returns one row per key (see [ClickHouse](adapters/clickhouse.md#reading-replacingmergetree-sources)). Other adapters reject it with `A010`. | | `managed` | Set to `true` to bind the source to the `@loader` function of the same name (see [Loaders](python-nodes/loaders.md)) | | `write_strategy` | How the loader writes data: `table`, `append`, `delete_insert`, or `merge` (requires `managed: true`) | | `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 67501a09dd..57739c3139 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14399,6 +14399,7 @@ class SourceEntry(NativeRecord): database: str | None schema: str | None table: str | None + final: bool loader: str | None managed: bool integration_loader: IntegrationLoaderConfig | None @@ -14420,6 +14421,7 @@ class SourceEntry(NativeRecord): database: str | None = ..., schema: str | None = ..., table: str | None = ..., + final: bool = ..., loader: str | None = ..., managed: bool = ..., integration_loader: IntegrationLoaderConfig | None = ..., diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 05b066b31c..605a27d729 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -329,11 +329,6 @@ def render_exact_identifier(self, name: str) -> str: def star_exclude_keyword(self) -> str: return "EXCEPT" - def render_final_relation(self, *, relation: str) -> str: - """Read with ``FINAL`` so replacing and collapsing engines return one row per key.""" - - return f"(SELECT * FROM {relation} FINAL)" - def relation_names_match(self, *, left: str, right: str) -> bool: return _unquote(left) == _unquote(right) diff --git a/tests/goldens/compiler/failure/source-unknown-key.json b/tests/goldens/compiler/failure/source-unknown-key.json index b458236a1f..63ad391a74 100644 --- a/tests/goldens/compiler/failure/source-unknown-key.json +++ b/tests/goldens/compiler/failure/source-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D006", - "message": "/sources/raw.yml source has unknown keys: freshnes; allowed keys: audits, columns, contract, cursor_column, database, description, expression, freshness, ingestr, load_batch_size, managed, meta, name, schema, table, type_enforcement, unique_key, write_strategy", + "message": "/sources/raw.yml source has unknown keys: freshnes; allowed keys: audits, columns, contract, cursor_column, database, description, expression, final, freshness, ingestr, load_batch_size, managed, meta, name, schema, table, type_enforcement, unique_key, write_strategy", "help": "did you mean 'freshness'?" } ] diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index aec5c0d0fe..774954d20b 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -49,3 +49,11 @@ class NullableOffsetWindowTestCase: description: str statement: str expected_statement: str + + +@dataclass(frozen=True) +class FinalSourceReadTestCase: + description: str + adapter_name: str + expected_relation: str | None + expected_code: str | None diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py new file mode 100644 index 0000000000..a1a392a98f --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_final_reads.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import pytest + +from sqlbuild._native import SourceEntry, render_source_read_relation +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.discovery.main.builtins import builtin_adapter_classes +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import FinalSourceReadTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + FinalSourceReadTestCase( + description="clickhouse reads the table with FINAL", + adapter_name="clickhouse", + expected_relation="(SELECT * FROM raw.customers FINAL)", + expected_code=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_final_source_when_rendering_on_clickhouse_then_read_is_deduplicated( + test_case: FinalSourceReadTestCase, +) -> None: + adapter: object = builtin_adapter_classes()[test_case.adapter_name]() + entry: SourceEntry = SourceEntry(name="customers", schema="raw", table="customers", final=True) + + assert render_source_read_relation(adapter=adapter, source_entry=entry) == ( + test_case.expected_relation + ) + + +@pytest.mark.parametrize( + "test_case", + [ + FinalSourceReadTestCase( + description="duckdb refuses final", + adapter_name="duckdb", + expected_relation=None, + expected_code="A010", + ), + FinalSourceReadTestCase( + description="postgres refuses final", + adapter_name="postgres", + expected_relation=None, + expected_code="A010", + ), + ], + ids=lambda case: case.description, +) +def test_given_final_source_when_rendering_on_other_adapter_then_coded_error( + test_case: FinalSourceReadTestCase, +) -> None: + adapter: object = builtin_adapter_classes()[test_case.adapter_name]() + entry: SourceEntry = SourceEntry(name="customers", schema="raw", table="customers", final=True) + + with pytest.raises(AdapterUserError) as error: + render_source_read_relation(adapter=adapter, source_entry=entry) + + assert error.value.code == test_case.expected_code diff --git a/website/src/content/docs/docs/concepts/sources.mdx b/website/src/content/docs/docs/concepts/sources.mdx index 774abd7ece..cca96b7b90 100644 --- a/website/src/content/docs/docs/concepts/sources.mdx +++ b/website/src/content/docs/docs/concepts/sources.mdx @@ -267,6 +267,7 @@ Use [`sqb freshness`](/docs/cli/freshness) to observe source freshness on demand | `schema` | Physical source schema (optional). For managed sources, overrides target `loader_schema`; otherwise managed loaders fall back to `loader_schema`, then target `schema`. | | `table` | Target table name (defaults to `name` if omitted) | | `expression` | Inline SQL expression (alternative to table reference) | +| `final` | ClickHouse only. Set to `true` to read the source with `FINAL`, so a `ReplacingMergeTree` table returns one row per key (see [ClickHouse](/docs/concepts/adapters/clickhouse#reading-replacingmergetree-sources)). Other adapters reject it with `A010`. | | `managed` | Set to `true` to bind the source to the `@loader` function of the same name (see [Loaders](/docs/concepts/python-nodes/loaders)) | | `write_strategy` | How the loader writes data: `table`, `append`, `delete_insert`, or `merge` (requires `managed: true`) | | `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | From 33c4af5ebd6f5a847156f015ea908b257d2af450 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:49:18 +0100 Subject: [PATCH 17/34] test: run ClickHouse integration and CLI tests in CI and document the adapter Adds end-to-end CLI coverage on a real ClickHouse server (views, tables, seeds, functions, merge, concurrent microbatches, final sources, query, diff, clone and the snapshot matrix), a ClickHouse CI job and test-real-clickhouse target, and the adapter documentation page. --- .github/workflows/verify.yml | 54 ++- Makefile | 11 + .../sqlbuild/references/docs/CONTENTS.md | 1 + .../references/docs/concepts/adapters.md | 7 +- .../docs/concepts/adapters/clickhouse.md | 137 ++++++ .../docs/concepts/project-configuration.md | 2 +- .../cli/commands/main/clickhouse/__init__.py | 0 .../commands/main/clickhouse/_test_types.py | 48 +++ .../cli/commands/main/clickhouse/conftest.py | 43 ++ .../cli/commands/main/clickhouse/helpers.py | 194 +++++++++ .../main/clickhouse/test_clickhouse.py | 396 ++++++++++++++++++ .../clickhouse/test_clickhouse_commands.py | 177 ++++++++ .../content/docs/docs/concepts/adapters.mdx | 7 +- .../docs/concepts/adapters/clickhouse.mdx | 127 ++++++ .../docs/concepts/project-configuration.mdx | 2 +- website/src/sidebar.json | 3 +- 16 files changed, 1195 insertions(+), 14 deletions(-) create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/__init__.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_commands.py create mode 100644 website/src/content/docs/docs/concepts/adapters/clickhouse.mdx diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 56abc70ac3..2d8fc4e0d6 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -23,6 +23,7 @@ jobs: compiler_differential: ${{ steps.scope.outputs.compiler_differential }} duckdb: ${{ steps.scope.outputs.duckdb }} postgres: ${{ steps.scope.outputs.postgres }} + clickhouse: ${{ steps.scope.outputs.clickhouse }} performance: ${{ steps.scope.outputs.performance }} format_performance: ${{ steps.scope.outputs.format_performance }} rules_performance: ${{ steps.scope.outputs.rules_performance }} @@ -50,11 +51,12 @@ jobs: echo "checks=$6" echo "wheels=$7" echo "compiler_differential=$8" + echo "clickhouse=$9" } >> "${GITHUB_OUTPUT}" } if [[ "${EVENT_NAME}" != "pull_request" && "${EVENT_NAME}" != "merge_group" ]]; then - write_scope true true true true true true true true + write_scope true true true true true true true true true exit 0 fi @@ -66,7 +68,7 @@ jobs: if [[ -z "${BASE_SHA}" || -z "${HEAD_SHA}" ]] || \ ! git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}" > "${changed_paths}"; then echo "Could not diff ${BASE_SHA:-}...${HEAD_SHA:-}; running every suite" >&2 - write_scope true true true true true true true true + write_scope true true true true true true true true true exit 0 fi @@ -112,12 +114,13 @@ jobs: && "${HEAD_REF}" == release-please--branches--* ]] && release_metadata_only; then echo "Release Please pull request changes only release metadata; skipping the suites" \ | tee -a "${GITHUB_STEP_SUMMARY}" - write_scope false false false false false false false false + write_scope false false false false false false false false false exit 0 fi duckdb=false postgres=false + clickhouse=false performance=false format_performance=false rules_performance=false @@ -182,6 +185,9 @@ jobs: tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*) postgres=true ;; + src/sqlbuild/adapters/clickhouse/*|tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/*|tests/integration/src/sqlbuild/adapters/clickhouse/*) + clickhouse=true + ;; tests/e2e/*) duckdb=true ;; @@ -201,7 +207,7 @@ jobs: done < "${changed_paths}" if [[ "${changed}" != "true" ]]; then - write_scope true true true true true true true true + write_scope true true true true true true true true true exit 0 fi @@ -213,6 +219,7 @@ jobs: if [[ "${broad}" == "true" ]]; then duckdb=true postgres=true + clickhouse=true format_performance=true compiler_differential=true fi @@ -224,7 +231,8 @@ jobs: "${rules_performance}" \ "${checks}" \ "${wheels}" \ - "${compiler_differential}" + "${compiler_differential}" \ + "${clickhouse}" lockfile: name: Lockfile consistency @@ -853,6 +861,39 @@ jobs: -m "real_warehouse and postgres" -vv --color=yes -n auto --dist loadfile + clickhouse: + name: ClickHouse integration and E2E tests + needs: [changes, lockfile] + if: needs.changes.outputs.clickhouse == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras + - name: Verify Docker + run: docker info + - name: Run ClickHouse tests + env: + PYTHONUNBUFFERED: "1" + TESTCONTAINERS_RYUK_DISABLED: "true" + run: >- + uv run pytest tests + -m "real_warehouse and clickhouse" + -vv --color=yes -n auto --dist loadfile + wheels: name: Wheels needs: [changes] @@ -891,6 +932,7 @@ jobs: cache_compile_performance, rules_performance, postgres, + clickhouse, wheels, ] runs-on: ubuntu-24.04 @@ -913,6 +955,7 @@ jobs: CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }} RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }} POSTGRES_RESULT: ${{ needs.postgres.result }} + CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }} WHEELS_RESULT: ${{ needs.wheels.result }} run: | if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then @@ -937,6 +980,7 @@ jobs: "${CACHE_COMPILE_PERFORMANCE_RESULT}" \ "${RULES_PERFORMANCE_RESULT}" \ "${POSTGRES_RESULT}" \ + "${CLICKHOUSE_RESULT}" \ "${WHEELS_RESULT}"; do if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then echo "A selected verification suite did not pass: ${result}" >&2 diff --git a/Makefile b/Makefile index b11102eca6..aadd9d9307 100644 --- a/Makefile +++ b/Makefile @@ -153,6 +153,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ tests/e2e/src/sqlbuild/cli/commands/main/bigquery \ tests/e2e/src/sqlbuild/cli/commands/main/check \ + tests/e2e/src/sqlbuild/cli/commands/main/clickhouse \ tests/e2e/src/sqlbuild/cli/commands/main/compile \ tests/e2e/src/sqlbuild/cli/commands/main/databricks \ tests/e2e/src/sqlbuild/cli/commands/main/dbt \ @@ -459,6 +460,16 @@ test-real-postgres: exit $$status +test-real-clickhouse: + @mkdir -p /tmp/opencode + @log="/tmp/opencode/test-real-clickhouse-$$(date +%Y%m%d-%H%M%S).log"; \ + echo "Logging to $$log"; \ + env PYTHONUNBUFFERED=1 TESTCONTAINERS_RYUK_DISABLED=true SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest tests -m "real_warehouse and clickhouse" -vv --color=yes -n auto --dist loadfile 2>&1 | tee "$$log"; \ + status=$${PIPESTATUS[0]}; \ + echo "TEST_REAL_CLICKHOUSE_EXIT=$$status (log: $$log)" | tee -a "$$log"; \ + exit $$status + + test-real-sqlserver: @mkdir -p /tmp/opencode @log="/tmp/opencode/test-real-sqlserver-$$(date +%Y%m%d-%H%M%S).log"; \ diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 336a3a65dc..f007592421 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -35,6 +35,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [BigQuery](concepts/adapters/bigquery.md) (`concepts/adapters/bigquery`) - BigQuery adapter configuration for SQLBuild. - [Databricks](concepts/adapters/databricks.md) (`concepts/adapters/databricks`) - Databricks adapter configuration for SQLBuild. - [SQL Server](concepts/adapters/sqlserver.md) (`concepts/adapters/sqlserver`) - Microsoft SQL Server adapter configuration for SQLBuild. +- [ClickHouse](concepts/adapters/clickhouse.md) (`concepts/adapters/clickhouse`) - ClickHouse adapter configuration for SQLBuild. - [Sources](concepts/sources.md) (`concepts/sources`) - Declare external data inputs for your pipeline. - [Seeds](concepts/seeds.md) (`concepts/seeds`) - Load static CSV data into your pipeline as tables. - [Overview](concepts/models.md) (`concepts/models`) - SQL model anatomy, references, dependencies, and the model documentation guide. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md index 32a57fc8e7..35f105f766 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md @@ -25,7 +25,7 @@ SQLBuild uses adapters to connect to different database engines. Snowflake is th | [Databricks](adapters/databricks.md) | Beta | `sqlbuild[databricks]` | | [SQL Server](adapters/sqlserver.md) | Beta | `sqlbuild[sqlserver]` | | [SQLite](adapters/sqlite.md) | Beta | included by default | -| ClickHouse | Planned | | +| [ClickHouse](adapters/clickhouse.md) | Beta | `sqlbuild[clickhouse]` | | Redshift | Planned | | | Trino | Planned | | | Spark | Planned | | @@ -109,7 +109,7 @@ A few `StrictAdapter` methods are optional and have a documented default. `creat - SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) - Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) - Adapter names must be unique across all adapter files - duplicates raise an error -- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`) +- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`, `clickhouse`) ### Adapter class hierarchy @@ -122,7 +122,8 @@ StrictAdapter (fully abstract - all methods must be implemented) ├── BigQueryAdapter ├── DatabricksAdapter ├── PostgresAdapter - └── SqlServerAdapter + ├── SqlServerAdapter + └── ClickHouseAdapter ``` `StrictAdapter` composes four mixins that define the full adapter contract: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md new file mode 100644 index 0000000000..0696857d1d --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -0,0 +1,137 @@ + + +# ClickHouse + +> ClickHouse adapter configuration for SQLBuild. + +Online: https://sqlbuild.com/docs/concepts/adapters/clickhouse/ + +## Contents + +- Connection config +- Schemas are databases +- How SQLBuild writes +- Reading ReplacingMergeTree sources +- Functions +- Clones +- Settings SQLBuild pins +- Limits + +The ClickHouse adapter is in beta and supports a single server. Clusters (`ON CLUSTER`, replicated and distributed tables) are not supported yet. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). + +ClickHouse requires the optional `clickhouse-connect` dependency: + +```bash +pip install 'sqlbuild[clickhouse]' +# or +uv pip install 'sqlbuild[clickhouse]' +``` + +SQLBuild talks to ClickHouse over HTTP or HTTPS. The oldest supported server is ClickHouse 25.8; older servers fail to connect with `A703`. + +## Connection config + +```toml +adapter = "clickhouse" +default_target = "dev" + +[connections.clickhouse] +host = "localhost" +port = 8123 +user = "default" +password = "${ENV:CLICKHOUSE_PASSWORD}" + +[targets.dev] +connection = "clickhouse" +schema = "analytics_dev" +``` + +| Field | Description | +|-------|-------------| +| `host` | Server hostname (default: `localhost`) | +| `port` | HTTP port (default: `8123`, or `8443` with `secure = true`) | +| `user` | User name (default: `default`). Also accepts `username`. | +| `password` | Password | +| `database` | Database for unqualified names (default: `default`) | +| `secure` | Use HTTPS (default: `false`) | +| `verify` | Verify the server certificate over HTTPS (default: `true`) | +| `settings` | Extra ClickHouse settings applied to every query, as a table | +| `connect_timeout`, `send_receive_timeout` | Client timeouts in seconds | + +Any other key fails with `A702`. + +## Schemas are databases + +ClickHouse has no level between a database and a table, so a SQLBuild **schema** is a ClickHouse **database**. Set `schema` on targets, models and sources; setting `database` fails with `A704`. SQLBuild creates model schemas with `CREATE DATABASE IF NOT EXISTS`. + +Identifiers are case-sensitive in ClickHouse, quoted or not. + +## How SQLBuild writes + +SQLBuild never relies on background merges for correctness, and it never updates or deletes rows in place. + +- **Tables** build in a staging table and replace the live table with `EXCHANGE TABLES`, which is atomic. This needs databases with the `Atomic` engine, the default; other engines fail with `A705`. +- **Views** use `CREATE OR REPLACE VIEW`. +- **Incremental `append`** inserts the new rows. +- **Incremental `merge`, `delete_insert` and cursor windows** rewrite whole partitions. SQLBuild stages the incoming rows, finds the partitions they touch and the partitions holding the rows they replace, rebuilds those partitions in a staging table, and swaps each one in with `ALTER TABLE ... REPLACE PARTITION`. Each swap is atomic. If a build stops before the swaps the table is untouched; if it stops between swaps, some partitions are already new, and rerunning the build recomputes the rest and converges. +- **Microbatches** use the same partition rewrite for each batch. Concurrent batches (`batch_concurrency`) rewrite different partitions in parallel; batches of one `sqb` process that share a partition take turns. + +### One writer per model + +A partition rewrite reads the partition, rebuilds it and swaps it in. ClickHouse has no transaction that spans those steps, and SQLBuild only serializes rewrites of the same partition inside one `sqb` process. If two `sqb` processes write the same model at the same time and touch the same partition, the later swap wins and the rows the other process wrote to that partition are lost. + +Run each model from one process at a time: one scheduled job per model or per project, with no overlapping runs. Concurrency within a run (`--concurrency`, `batch_concurrency`) is safe. +- **Snapshots** apply their changes to a private copy of the snapshot table and exchange it in. Closing a version rebuilds the copy rather than updating rows, so the live snapshot changes only at the final exchange. + +Tables SQLBuild creates use `ENGINE = MergeTree ORDER BY tuple()` with no partitioning, so on a normal incremental run a partition rewrite rewrites the whole table. The cost of a rewrite is proportional to the partitions it touches. + +`merge` matches rows on `unique_key` exactly as other adapters do: incoming rows replace matched rows, `merge_exclude_columns` keep their stored values, and unmatched rows are inserted. + +## Reading ReplacingMergeTree sources + +A `ReplacingMergeTree` table can hold several versions of a row until the server merges them in the background. Set `final true` on the source so every read SQLBuild generates for it uses `FINAL` and sees one version per key: + +```yaml +sources: + - name: customers + description: Customer master data. + schema: raw + table: customers + final: true +``` + +## Functions + +SQL functions deploy as ClickHouse SQL user-defined functions (`CREATE FUNCTION name AS (args) -> expression`). ClickHouse function names are global to the server, not scoped to a database, so SQLBuild names each function `__`. A function `half_amount` in schema `analytics` is created as `analytics__half_amount`, and `__udf("half_amount")` renders that name. + +SQLBuild does not deploy Python functions (`A707`) or SQL table functions (`A708`) to ClickHouse: executable functions need server configuration files. Install them on the server and call them directly. + +## Clones + +`sqb clone` creates the clone with `CREATE TABLE ... AS` and attaches every part of the origin with `ALTER TABLE ... ATTACH PARTITION ALL FROM`, which hard-links the data instead of copying it. The clone keeps the origin's engine, keys and settings and stays valid after the origin is dropped. + +## Settings SQLBuild pins + +Every query runs with these settings so SQL behaves the same as on other adapters: + +| Setting | Value | Why | +|---------|-------|-----| +| `join_use_nulls` | `1` | Outer joins return `NULL` for missing rows | +| `cast_keep_nullable` | `1` | `CAST` keeps `NULL` values | +| `union_default_mode`, `except_default_mode`, `intersect_default_mode` | `DISTINCT` | Bare `UNION`, `EXCEPT` and `INTERSECT` follow standard SQL | +| `default_table_engine` | `MergeTree` | Tables created without an engine clause are MergeTree tables | +| `mutations_sync`, `lightweight_deletes_sync` | `2` | Statements finish before the next one starts | +| `date_time_input_format`, `cast_string_to_date_time_mode` | `best_effort` | ISO 8601 timestamps with `T` and offsets parse | + +Values under `settings` in the connection override these. + +ClickHouse returns a type's default value (`0`, the epoch, an empty string) from `lag` and `lead` past the edge of a window frame unless the argument is `Nullable`. SQLBuild's generated snapshot SQL wraps those arguments in `toNullable` so they return `NULL` as on other adapters. Write the same in your own models when you rely on `NULL` there. + +## Limits + +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- No transactions. Each step above is designed so a rerun converges instead. +- Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). +- Model physical settings (`ENGINE`, `ORDER BY`, `PARTITION BY`) are not configurable yet. +- Seeds load through the HTTP insert interface as CSV, so large seeds never hit the `max_query_size` limit. +- Retention and time travel are not available. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 9be589d4d1..b47766fc67 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -91,7 +91,7 @@ materialized = "view" | Field | Description | |-------|-------------| | `name` | Project name. Used in fingerprint tracking and manifest generation. | -| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, or `sqlserver`. See [Adapters](adapters.md). | +| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`, or `clickhouse`. See [Adapters](adapters.md). | ### Optional fields diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py new file mode 100644 index 0000000000..91af7138ad --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ClickHouseBuildE2ETestCase: + description: str + expected_customer_rows: tuple[tuple[object, ...], ...] + expected_order_totals: tuple[tuple[object, ...], ...] + expected_name_rows: tuple[tuple[object, ...], ...] + expected_daily_rows_after_first_build: tuple[tuple[object, ...], ...] + expected_daily_rows_after_second_build: tuple[tuple[object, ...], ...] + expected_latest_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseSnapshotE2ETestCase: + description: str + expected_current_rows_after_initial_build: tuple[tuple[object, ...], ...] + expected_current_rows_after_recovery: tuple[tuple[object, ...], ...] + expected_historical_timestamp_rows: tuple[tuple[object, ...], ...] + expected_historical_check_rows: tuple[tuple[object, ...], ...] + expected_failure_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseSnapshotApplyE2ETestCase: + description: str + expected_current_check_rows: tuple[tuple[object, ...], ...] + expected_current_delete_rows: tuple[tuple[object, ...], ...] + expected_historical_timestamp_rows: tuple[tuple[object, ...], ...] + expected_historical_check_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseDiffE2ETestCase: + description: str + command: tuple[str, ...] + mutation_sql: tuple[str, ...] + expected_stdout_fragments: tuple[str, ...] + expected_return_code: int + expected_absent_stdout_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ClickHouseCloneE2ETestCase: + description: str + expected_stdout_fragments: tuple[str, ...] + expected_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py new file mode 100644 index 0000000000..531933d58e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import os +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import CLICKHOUSE_IMAGE + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + test_dir: Path = Path(__file__).resolve().parent + for item in items: + if not Path(str(item.path)).resolve().is_relative_to(test_dir): + continue + item.add_marker(pytest.mark.real_warehouse) + item.add_marker(pytest.mark.clickhouse) + + +@pytest.fixture(scope="session") +def clickhouse_e2e_config() -> Iterator[dict[str, object]]: + port: str | None = os.environ.get("SQLBUILD_TEST_CLICKHOUSE_PORT") + if port is not None: + yield { + "host": os.environ.get("SQLBUILD_TEST_CLICKHOUSE_HOST", "localhost"), + "port": int(port), + "user": os.environ.get("SQLBUILD_TEST_CLICKHOUSE_USER", "default"), + "password": os.environ.get("SQLBUILD_TEST_CLICKHOUSE_PASSWORD", ""), + } + return + from testcontainers.clickhouse import ClickHouseContainer + + try: + with ClickHouseContainer(CLICKHOUSE_IMAGE) as container: + yield { + "host": container.get_container_host_ip(), + "port": int(container.get_exposed_port(8123)), + "user": container.username, + "password": container.password, + } + except Exception as exc: + pytest.skip(f"Docker not available for ClickHouse testcontainer: {exc}") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py new file mode 100644 index 0000000000..b325c8a56c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -0,0 +1,194 @@ +"""Helpers for ClickHouse CLI E2E tests.""" + +from __future__ import annotations + +import uuid +from textwrap import dedent +from typing import Any + +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter + +CLICKHOUSE_IMAGE: str = "clickhouse/clickhouse-server:26.1" + + +def build_unique_schema_name(*, prefix: str) -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +def build_clickhouse_project_toml( + *, + project_name: str, + schema_name: str, + config: dict[str, object], + extra: str = "", +) -> str: + return ( + f'name = "{project_name}"\n' + 'adapter = "clickhouse"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + f'host = "{config["host"]}"\n' + f"port = {config['port']}\n" + f'user = "{config["user"]}"\n' + f'password = "{config["password"]}"\n\n' + "[targets.dev]\n" + f'schema = "{schema_name}"\n\n' + "[defaults]\n" + 'materialized = "table"\n' + extra + ) + + +def execute_clickhouse_sql(*, config: dict[str, object], statements: tuple[str, ...]) -> None: + adapter: ClickHouseAdapter = ClickHouseAdapter() + connection: Any = adapter.connect(dict(config)) + try: + for statement in statements: + adapter.execute(connection=connection, sql=statement) + finally: + adapter.close(connection) + + +def fetch_clickhouse_rows(*, config: dict[str, object], sql: str) -> tuple[tuple[object, ...], ...]: + adapter: ClickHouseAdapter = ClickHouseAdapter() + connection: Any = adapter.connect(dict(config)) + try: + cursor: Any = adapter.execute(connection=connection, sql=sql) + return tuple(tuple(row) for row in cursor.fetchall()) + finally: + adapter.close(connection) + + +def create_clickhouse_schema(*, config: dict[str, object], schema_name: str) -> None: + execute_clickhouse_sql( + config=config, statements=(f"CREATE DATABASE IF NOT EXISTS {schema_name}",) + ) + + +def cleanup_clickhouse_schema(*, config: dict[str, object], schema_name: str) -> None: + rows: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, + sql=( + "SELECT name FROM system.functions WHERE origin = 'SQLUserDefined' " + f"AND startsWith(name, '{schema_name}__')" + ), + ) + execute_clickhouse_sql( + config=config, + statements=( + *(f"DROP FUNCTION IF EXISTS {row[0]}" for row in rows), + f"DROP DATABASE IF EXISTS {schema_name} SYNC", + ), + ) + + +def build_two_target_toml( + *, project_name: str, prod_schema: str, dev_schema: str, config: dict[str, object] +) -> str: + return ( + f'name = "{project_name}"\n' + 'adapter = "clickhouse"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + f'host = "{config["host"]}"\n' + f"port = {config['port']}\n" + f'user = "{config["user"]}"\n' + f'password = "{config["password"]}"\n\n' + "[targets.prod]\n" + f'schema = "{prod_schema}"\n\n' + "[targets.prod.clone]\n" + "allow_as_clone_destination = true\n\n" + "[targets.dev]\n" + f'schema = "{dev_schema}"\n\n' + "[targets.dev.clone]\n" + "allow_as_clone_origin = true\n" + ) + + +def build_orders_project_files(*, schema_name: str, config: dict[str, object]) -> dict[str, str]: + return { + "sqlbuild_project.toml": build_clickhouse_project_toml( + project_name="clickhouse_orders", + schema_name=schema_name, + config=config, + extra="\n[settings]\nconcurrency = 4\nmicrobatch_concurrency = true\n", + ), + "sources/raw.yml": dedent( + f""" + sources: + - name: raw_events + description: Synthetic order events. + schema: {schema_name} + table: raw_events + - name: raw_customers + description: Customer versions kept by a replacing engine. + schema: {schema_name} + table: raw_customers + final: true + """ + ).lstrip(), + "seeds/regions.csv": "customer_id,region_name\n1,north\n2,south\n3,east\n", + "seeds/regions.yml": dedent( + """ + seeds: + - name: regions + description: Customer regions. + columns: + - name: customer_id + type: Int64 + - name: region_name + type: String + """ + ).lstrip(), + "functions/sql/half_amount.sql": ( + 'FUNCTION (\n description "Half of an amount",\n' + " arguments (amount Int64),\n returns Float64,\n);\n\namount / 2\n" + ), + "models/stg_events.sql": ( + "MODEL (description 'Staged events.', materialized view);\n\n" + "SELECT event_id, customer_id, amount, event_time\n" + 'FROM __source("raw_events")\n' + ), + "models/customer_totals.sql": ( + "MODEL (description 'One row per customer.');\n\n" + "SELECT e.customer_id AS customer_id, r.region_name AS region_name,\n" + " sum(e.amount) AS total_amount,\n" + ' __udf("half_amount")(sum(e.amount)) AS half_total\n' + 'FROM __ref("stg_events") AS e\n' + 'JOIN __seed("regions") AS r ON e.customer_id = r.customer_id\n' + "GROUP BY e.customer_id, r.region_name\n" + ), + "models/customer_names.sql": ( + "MODEL (description 'One row per customer name.');\n\n" + 'SELECT customer_id, customer_name FROM __source("raw_customers")\n' + ), + "models/customer_latest.sql": ( + "MODEL (description 'Latest activity per customer.',\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [customer_id],\n" + ");\n\n" + "SELECT customer_id, count() AS event_count, max(event_time) AS last_event_time\n" + 'FROM __ref("stg_events")\n' + "GROUP BY customer_id\n" + ), + "models/daily_events.sql": ( + "MODEL (description 'Events by day.',\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " cursor_watermark_mode all,\n" + " cursor event_time,\n" + " cursor_type timestamp,\n" + " cursor_grain hour,\n" + " cursor_inputs (\n" + " raw_events (column event_time, roles [filter, watermark]),\n" + " ),\n" + " batch_size 1d,\n" + " batch_concurrency 4,\n" + ");\n\n" + "SELECT event_id, customer_id, amount, event_time\n" + 'FROM __source("raw_events")\n' + "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse.py new file mode 100644 index 0000000000..440593cc8f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseBuildE2ETestCase, + ClickHouseSnapshotApplyE2ETestCase, + ClickHouseSnapshotE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + build_clickhouse_project_toml, + build_orders_project_files, + build_unique_schema_name, + cleanup_clickhouse_schema, + create_clickhouse_schema, + execute_clickhouse_sql, + fetch_clickhouse_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + build_current_check_customers_model_sql, + build_current_customers_model_sql, + build_current_delete_customers_model_sql, + build_historical_check_daily_model_sql, + build_historical_timestamp_extracts_model_sql, + build_real_warehouse_existing_snapshot_project_files, + build_real_warehouse_snapshot_project_files, + prepare_inline_project, + run_sqb, + stringify_warehouse_rows, +) + +_FIRST_EVENTS: str = ( + "(1, 1, 100, '2026-01-01 08:00:00'), (2, 2, 200, '2026-01-01 09:00:00'), " + "(3, 1, 50, '2026-01-02 10:00:00'), (4, 3, 300, '2026-01-02 11:00:00'), " + "(5, 2, 25, '2026-01-03 12:00:00'), (6, 3, 75, '2026-01-04 13:00:00')" +) +_LATER_EVENTS: str = "(7, 1, 10, '2026-01-05 08:00:00'), (8, 2, 20, '2026-01-06 09:00:00')" + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseBuildE2ETestCase( + description="views, tables, seeds, functions, merge and concurrent microbatches", + expected_customer_rows=( + ("1", "north", "150", "75.0"), + ("2", "south", "225", "112.5"), + ("3", "east", "375", "187.5"), + ), + expected_order_totals=(("6", "750"),), + expected_name_rows=(("1", "ada lovelace"), ("2", "bo"), ("3", "cy")), + expected_daily_rows_after_first_build=( + ("2026-01-01", "2"), + ("2026-01-02", "2"), + ("2026-01-03", "1"), + ("2026-01-04", "1"), + ), + expected_daily_rows_after_second_build=( + ("2026-01-01", "2"), + ("2026-01-02", "2"), + ("2026-01-03", "1"), + ("2026-01-04", "1"), + ("2026-01-05", "1"), + ("2026-01-06", "1"), + ), + expected_latest_rows=( + ("1", "3", "2026-01-05 08:00:00"), + ("2", "3", "2026-01-06 09:00:00"), + ("3", "2", "2026-01-04 13:00:00"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_orders_project_when_building_twice_on_clickhouse_then_every_strategy_lands( + tmp_path: Path, + test_case: ClickHouseBuildE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_orders") + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_orders", + repo_files=build_orders_project_files(schema_name=schema_name, config=config), + ) + create_clickhouse_schema(config=config, schema_name=schema_name) + execute_clickhouse_sql( + config=config, + statements=( + f"CREATE TABLE {schema_name}.raw_events (event_id Int64, customer_id Int64, " + "amount Int64, event_time DateTime64(6, 'UTC')) ENGINE = MergeTree ORDER BY event_id", + f"INSERT INTO {schema_name}.raw_events VALUES {_FIRST_EVENTS}", + f"CREATE TABLE {schema_name}.raw_customers (customer_id Int64, " + "customer_name String, version Int64) ENGINE = ReplacingMergeTree(version) " + "ORDER BY customer_id", + f"SYSTEM STOP MERGES {schema_name}.raw_customers", + f"INSERT INTO {schema_name}.raw_customers VALUES (1, 'ada', 1), (2, 'bo', 1)", + f"INSERT INTO {schema_name}.raw_customers VALUES (1, 'ada lovelace', 2), (3, 'cy', 1)", + ), + ) + try: + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql=f"SELECT customer_id, region_name, total_amount, half_total " + f"FROM {schema_name}.customer_totals ORDER BY customer_id", + ) + ) == (test_case.expected_customer_rows) + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql=f"SELECT count(), sum(amount) FROM {schema_name}.stg_events", + ) + ) == (test_case.expected_order_totals) + assert fetch_clickhouse_rows( + config=config, sql=f"SELECT count() FROM {schema_name}.raw_customers" + ) == ((4,),) + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql=f"SELECT customer_id, customer_name FROM {schema_name}.customer_names " + "ORDER BY customer_id", + ) + ) == (test_case.expected_name_rows) + daily_sql: str = ( + f"SELECT toDate(event_time) AS day, count() FROM {schema_name}.daily_events " + "GROUP BY day ORDER BY day" + ) + assert stringify_warehouse_rows(fetch_clickhouse_rows(config=config, sql=daily_sql)) == ( + test_case.expected_daily_rows_after_first_build + ) + + execute_clickhouse_sql( + config=config, + statements=(f"INSERT INTO {schema_name}.raw_events VALUES {_LATER_EVENTS}",), + ) + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert second.returncode == 0, second.stdout + second.stderr + assert "Concurrent microbatch execution is enabled" in second.stdout + second.stderr + assert stringify_warehouse_rows(fetch_clickhouse_rows(config=config, sql=daily_sql)) == ( + test_case.expected_daily_rows_after_second_build + ) + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql=f"SELECT customer_id, event_count, toString(toDateTime(last_event_time)) " + f"FROM {schema_name}.customer_latest ORDER BY customer_id", + ) + ) == (test_case.expected_latest_rows) + artifacts: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, + sql=f"SELECT name FROM system.tables WHERE database = '{schema_name}' " + "AND (name LIKE '%__sqb_%' OR name LIKE '__sqb_%')", + ) + assert artifacts == () + state_engines: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, + sql=f"SELECT DISTINCT engine FROM system.tables WHERE database = '{schema_name}' " + "AND startsWith(name, '_sqlbuild_')", + ) + assert state_engines == (("MergeTree",),) + + query: subprocess.CompletedProcess[str] = run_sqb( + command=( + "--no-color", + "query", + f"SELECT sum(total_amount) AS revenue FROM {schema_name}.customer_totals", + ), + project_dir=project_dir, + ) + assert query.returncode == 0, query.stdout + query.stderr + assert "revenue" in query.stdout and "780" in query.stdout + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseSnapshotE2ETestCase( + description="executes the snapshot matrix on clickhouse", + expected_current_rows_after_initial_build=(("1", "10", "basic", "2026-01-01", None),), + expected_current_rows_after_recovery=( + ("1", "10", "basic", "2026-01-01", "2026-01-02"), + ("1", "10", "pro", "2026-01-02", None), + ), + expected_historical_timestamp_rows=( + ("1", "basic", "2026-01-01", "2026-01-03"), + ("1", "pro", "2026-01-03", None), + ("2", "trial", "2026-01-02", None), + ), + expected_historical_check_rows=( + ("1", "active", "2026-01-01", "2026-01-03"), + ("1", "paused", "2026-01-03", None), + ("2", "active", "2026-01-01", "2026-01-02"), + ("2", "active", "2026-01-03", None), + ), + expected_failure_fragments=( + "current_customer_snapshot", + "delta audit for 'current_customer_snapshot' failed before target update", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_snapshot_project_when_building_on_clickhouse_then_scd2_history_is_valid( + tmp_path: Path, + test_case: ClickHouseSnapshotE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_snapshot") + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_snapshot_project", + repo_files=build_real_warehouse_snapshot_project_files( + project_toml=build_clickhouse_project_toml( + project_name="clickhouse_snapshot_project", + schema_name=schema_name, + config=config, + ) + ), + ) + create_clickhouse_schema(config=config, schema_name=schema_name) + current_sql: str = ( + "SELECT customer_id, region_id, plan, toDate(effective_from), toDate(effective_to) " + f"FROM {schema_name}.current_customer_snapshot " + "ORDER BY customer_id, region_id, effective_from" + ) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--concurrency", "4"), project_dir=project_dir + ) + assert initial.returncode == 0, initial.stdout + initial.stderr + assert stringify_warehouse_rows(fetch_clickhouse_rows(config=config, sql=current_sql)) == ( + test_case.expected_current_rows_after_initial_build + ) + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql="SELECT customer_id, plan, toDate(valid_from), toDate(valid_to) " + f"FROM {schema_name}.historical_customer_snapshot ORDER BY customer_id, valid_from", + ) + ) == (test_case.expected_historical_timestamp_rows) + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql="SELECT customer_id, status, toDate(valid_from), toDate(valid_to) " + f"FROM {schema_name}.historical_membership_snapshot " + "ORDER BY customer_id, valid_from", + ) + ) == (test_case.expected_historical_check_rows) + + (project_dir / "models" / "current_customers.sql").write_text( + build_current_customers_model_sql(plan="blocked", updated_at="2026-01-02 00:00:00"), + encoding="utf-8", + ) + failure: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "+current_customer_snapshot"), + project_dir=project_dir, + ) + assert failure.returncode == 1, failure.stdout + failure.stderr + for fragment in test_case.expected_failure_fragments: + assert fragment in failure.stdout + failure.stderr + assert stringify_warehouse_rows(fetch_clickhouse_rows(config=config, sql=current_sql)) == ( + test_case.expected_current_rows_after_initial_build + ) + + (project_dir / "models" / "current_customers.sql").write_text( + build_current_customers_model_sql(plan="pro", updated_at="2026-01-02 00:00:00"), + encoding="utf-8", + ) + recovery: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "+current_customer_snapshot"), + project_dir=project_dir, + ) + assert recovery.returncode == 0, recovery.stdout + recovery.stderr + assert stringify_warehouse_rows(fetch_clickhouse_rows(config=config, sql=current_sql)) == ( + test_case.expected_current_rows_after_recovery + ) + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseSnapshotApplyE2ETestCase( + description="applies existing-target snapshot changes on clickhouse", + expected_current_check_rows=( + ("1", "active", "False"), + ("1", "paused", "True"), + ("2", "active", "True"), + ), + expected_current_delete_rows=( + ("1", "basic", "False"), + ("1", "pro", "True"), + ("2", "trial", "False"), + ), + expected_historical_timestamp_rows=( + ("1", "basic", "2026-01-01", "2026-01-03"), + ("1", "pro", "2026-01-03", None), + ("2", "trial", "2026-01-01", "2026-01-04"), + ), + expected_historical_check_rows=( + ("1", "active", "2026-01-01", "2026-01-03"), + ("1", "paused", "2026-01-03", None), + ("2", "active", "2026-01-01", "2026-01-02"), + ("2", "active", "2026-01-03", None), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_existing_snapshot_targets_when_building_on_clickhouse_then_changes_apply( + tmp_path: Path, + test_case: ClickHouseSnapshotApplyE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_snapshot_apply") + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_snapshot_apply_project", + repo_files=build_real_warehouse_existing_snapshot_project_files( + project_toml=build_clickhouse_project_toml( + project_name="clickhouse_snapshot_apply_project", + schema_name=schema_name, + config=config, + ) + ), + ) + create_clickhouse_schema(config=config, schema_name=schema_name) + try: + initial: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--concurrency", "4"), project_dir=project_dir + ) + assert initial.returncode == 0, initial.stdout + initial.stderr + models: Path = project_dir / "models" + (models / "current_check_customers.sql").write_text( + build_current_check_customers_model_sql(changed=True), encoding="utf-8" + ) + (models / "current_delete_customers.sql").write_text( + build_current_delete_customers_model_sql(changed=True), encoding="utf-8" + ) + (models / "historical_timestamp_extracts.sql").write_text( + build_historical_timestamp_extracts_model_sql(changed=True), encoding="utf-8" + ) + (models / "historical_check_daily.sql").write_text( + build_historical_check_daily_model_sql(changed=True), encoding="utf-8" + ) + + applied: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--concurrency", "4"), project_dir=project_dir + ) + assert applied.returncode == 0, applied.stdout + applied.stderr + for name, columns, expected in ( + ("current_check_snapshot", "status", test_case.expected_current_check_rows), + ("current_delete_snapshot", "plan", test_case.expected_current_delete_rows), + ): + rows: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, + sql=f"SELECT customer_id, {columns}, valid_to IS NULL FROM {schema_name}.{name} " + f"ORDER BY customer_id, {columns}, valid_to IS NULL", + ) + assert stringify_warehouse_rows( + tuple((row[0], row[1], bool(row[2])) for row in rows) + ) == (expected) + for name, columns, expected in ( + ("historical_timestamp_snapshot", "plan", test_case.expected_historical_timestamp_rows), + ("historical_check_snapshot", "status", test_case.expected_historical_check_rows), + ): + rows = fetch_clickhouse_rows( + config=config, + sql=f"SELECT customer_id, {columns}, toDate(valid_from), toDate(valid_to) " + f"FROM {schema_name}.{name} ORDER BY customer_id, valid_from", + ) + assert stringify_warehouse_rows(rows) == expected + leftovers: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, + sql=f"SELECT name FROM system.tables WHERE database = '{schema_name}' " + "AND name LIKE '%__sqb_snap%'", + ) + assert leftovers == () + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_commands.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_commands.py new file mode 100644 index 0000000000..e86a871ea2 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_commands.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseCloneE2ETestCase, + ClickHouseDiffE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + build_two_target_toml, + build_unique_schema_name, + cleanup_clickhouse_schema, + create_clickhouse_schema, + execute_clickhouse_sql, + fetch_clickhouse_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_FACT_ORDERS_SQL: str = ( + "MODEL (description 'One row per order.',\n" + " materialized table,\n" + " unique_key [order_id],\n" + " cursor order_id,\n" + " cursor_type integer,\n" + ");\n\n" + "SELECT toInt64(1) AS order_id, toInt64(100) AS amount_cents\n" +) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseDiffE2ETestCase( + description="schema-only diff ignores row drift", + command=("--no-color", "diff", "prod:dev", "--schema-only", "--select", "fact_orders"), + mutation_sql=("ALTER TABLE {dev} UPDATE amount_cents = 105 WHERE order_id = 1",), + expected_stdout_fragments=("fact_orders", "No schema differences."), + expected_return_code=0, + ), + ClickHouseDiffE2ETestCase( + description="full diff reports row mismatch", + command=("--no-color", "diff", "prod:dev", "--full", "--select", "fact_orders"), + mutation_sql=("ALTER TABLE {dev} UPDATE amount_cents = 105 WHERE order_id = 1",), + expected_stdout_fragments=("amount_cents", "mismatches=1", "order_id=1 | 100 -> 105"), + expected_return_code=1, + ), + ClickHouseDiffE2ETestCase( + description="bounded diff filters rows below integer cursor bound", + command=("--no-color", "diff", "prod:dev", "--bounded", "2", "--select", "fact_orders"), + mutation_sql=( + "ALTER TABLE {dev} UPDATE amount_cents = 105 WHERE order_id = 1", + "ALTER TABLE {dev} UPDATE amount_cents = 205 WHERE order_id = 2", + ), + expected_stdout_fragments=("amount_cents", "mismatches=1", "order_id=2 | 200 -> 205"), + expected_return_code=1, + expected_absent_stdout_fragments=("order_id=1 | 100 -> 105",), + ), + ), + ids=lambda case: case.description, +) +def test_given_clickhouse_targets_when_running_diff_then_all_modes_execute_end_to_end( + test_case: ClickHouseDiffE2ETestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], +) -> None: + config: dict[str, object] = clickhouse_e2e_config + prod_schema: str = build_unique_schema_name(prefix="sqb_diff_prod") + dev_schema: str = build_unique_schema_name(prefix="sqb_diff_dev") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_diff_project", + repo_files={ + "sqlbuild_project.toml": build_two_target_toml( + project_name="clickhouse_diff_project", + prod_schema=prod_schema, + dev_schema=dev_schema, + config=config, + ), + "models/fact_orders.sql": _FACT_ORDERS_SQL, + }, + ) + try: + for schema in (prod_schema, dev_schema): + create_clickhouse_schema(config=config, schema_name=schema) + execute_clickhouse_sql( + config=config, + statements=( + f"CREATE TABLE {schema}.fact_orders (order_id Int64, amount_cents Int64) " + "ENGINE = MergeTree ORDER BY order_id", + f"INSERT INTO {schema}.fact_orders VALUES (1, 100), (2, 200), (3, 300)", + ), + ) + execute_clickhouse_sql( + config=config, + statements=tuple( + mutation.format(dev=f"{dev_schema}.fact_orders") + for mutation in test_case.mutation_sql + ), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout, result.stdout + result.stderr + for fragment in test_case.expected_absent_stdout_fragments: + assert fragment not in result.stdout, result.stdout + result.stderr + finally: + cleanup_clickhouse_schema(config=config, schema_name=dev_schema) + cleanup_clickhouse_schema(config=config, schema_name=prod_schema) + + +@pytest.mark.parametrize( + "test_case", + ( + ClickHouseCloneE2ETestCase( + description="clone hard-links tables and recreates views", + expected_stdout_fragments=("fact_orders", "order_summary"), + expected_rows=((1, 100),), + ), + ), + ids=lambda case: case.description, +) +def test_given_built_dev_target_when_cloning_to_prod_then_clickhouse_clones_relations( + test_case: ClickHouseCloneE2ETestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], +) -> None: + config: dict[str, object] = clickhouse_e2e_config + prod_schema: str = build_unique_schema_name(prefix="sqb_clone_prod") + dev_schema: str = build_unique_schema_name(prefix="sqb_clone_dev") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_clone_project", + repo_files={ + "sqlbuild_project.toml": build_two_target_toml( + project_name="clickhouse_clone_project", + prod_schema=prod_schema, + dev_schema=dev_schema, + config=config, + ), + "models/fact_orders.sql": _FACT_ORDERS_SQL, + "models/order_summary.sql": ( + "MODEL (description 'Order summary.', materialized view);\n\n" + 'SELECT order_id, amount_cents FROM __ref("fact_orders")\n' + ), + }, + ) + try: + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + + cloned: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "clone", "--from", "dev", "--to", "prod"), + project_dir=project_dir, + ) + + assert cloned.returncode == 0, cloned.stdout + cloned.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in cloned.stdout, cloned.stdout + cloned.stderr + assert ( + fetch_clickhouse_rows( + config=config, + sql=f"SELECT order_id, amount_cents FROM {prod_schema}.order_summary", + ) + == test_case.expected_rows + ) + finally: + cleanup_clickhouse_schema(config=config, schema_name=dev_schema) + cleanup_clickhouse_schema(config=config, schema_name=prod_schema) diff --git a/website/src/content/docs/docs/concepts/adapters.mdx b/website/src/content/docs/docs/concepts/adapters.mdx index 7e9c018df0..0ab7a25970 100644 --- a/website/src/content/docs/docs/concepts/adapters.mdx +++ b/website/src/content/docs/docs/concepts/adapters.mdx @@ -18,7 +18,7 @@ SQLBuild uses adapters to connect to different database engines. Snowflake is th | [Databricks](/docs/concepts/adapters/databricks) | Beta | `sqlbuild[databricks]` | | [SQL Server](/docs/concepts/adapters/sqlserver) | Beta | `sqlbuild[sqlserver]` | | [SQLite](/docs/concepts/adapters/sqlite) | Beta | included by default | -| ClickHouse | Planned | | +| [ClickHouse](/docs/concepts/adapters/clickhouse) | Beta | `sqlbuild[clickhouse]` | | Redshift | Planned | | | Trino | Planned | | | Spark | Planned | | @@ -102,7 +102,7 @@ A few `StrictAdapter` methods are optional and have a documented default. `creat - SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) - Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) - Adapter names must be unique across all adapter files - duplicates raise an error -- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`) +- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`, `clickhouse`) ### Adapter class hierarchy @@ -115,7 +115,8 @@ StrictAdapter (fully abstract - all methods must be implemented) ├── BigQueryAdapter ├── DatabricksAdapter ├── PostgresAdapter - └── SqlServerAdapter + ├── SqlServerAdapter + └── ClickHouseAdapter ``` `StrictAdapter` composes four mixins that define the full adapter contract: diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx new file mode 100644 index 0000000000..e6d8f17e74 --- /dev/null +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -0,0 +1,127 @@ +--- +title: "ClickHouse" +description: "ClickHouse adapter configuration for SQLBuild." +--- + +import Note from '../../../../../components/mintlify/Note.astro'; + + +The ClickHouse adapter is in beta and supports a single server. Clusters (`ON CLUSTER`, replicated and distributed tables) are not supported yet. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). + + +ClickHouse requires the optional `clickhouse-connect` dependency: + +```bash +pip install 'sqlbuild[clickhouse]' +# or +uv pip install 'sqlbuild[clickhouse]' +``` + +SQLBuild talks to ClickHouse over HTTP or HTTPS. The oldest supported server is ClickHouse 25.8; older servers fail to connect with `A703`. + +## Connection config + +```toml +adapter = "clickhouse" +default_target = "dev" + +[connections.clickhouse] +host = "localhost" +port = 8123 +user = "default" +password = "${ENV:CLICKHOUSE_PASSWORD}" + +[targets.dev] +connection = "clickhouse" +schema = "analytics_dev" +``` + +| Field | Description | +|-------|-------------| +| `host` | Server hostname (default: `localhost`) | +| `port` | HTTP port (default: `8123`, or `8443` with `secure = true`) | +| `user` | User name (default: `default`). Also accepts `username`. | +| `password` | Password | +| `database` | Database for unqualified names (default: `default`) | +| `secure` | Use HTTPS (default: `false`) | +| `verify` | Verify the server certificate over HTTPS (default: `true`) | +| `settings` | Extra ClickHouse settings applied to every query, as a table | +| `connect_timeout`, `send_receive_timeout` | Client timeouts in seconds | + +Any other key fails with `A702`. + +## Schemas are databases + +ClickHouse has no level between a database and a table, so a SQLBuild **schema** is a ClickHouse **database**. Set `schema` on targets, models and sources; setting `database` fails with `A704`. SQLBuild creates model schemas with `CREATE DATABASE IF NOT EXISTS`. + +Identifiers are case-sensitive in ClickHouse, quoted or not. + +## How SQLBuild writes + +SQLBuild never relies on background merges for correctness, and it never updates or deletes rows in place. + +- **Tables** build in a staging table and replace the live table with `EXCHANGE TABLES`, which is atomic. This needs databases with the `Atomic` engine, the default; other engines fail with `A705`. +- **Views** use `CREATE OR REPLACE VIEW`. +- **Incremental `append`** inserts the new rows. +- **Incremental `merge`, `delete_insert` and cursor windows** rewrite whole partitions. SQLBuild stages the incoming rows, finds the partitions they touch and the partitions holding the rows they replace, rebuilds those partitions in a staging table, and swaps each one in with `ALTER TABLE ... REPLACE PARTITION`. Each swap is atomic. If a build stops before the swaps the table is untouched; if it stops between swaps, some partitions are already new, and rerunning the build recomputes the rest and converges. +- **Microbatches** use the same partition rewrite for each batch. Concurrent batches (`batch_concurrency`) rewrite different partitions in parallel; batches of one `sqb` process that share a partition take turns. + +### One writer per model + +A partition rewrite reads the partition, rebuilds it and swaps it in. ClickHouse has no transaction that spans those steps, and SQLBuild only serializes rewrites of the same partition inside one `sqb` process. If two `sqb` processes write the same model at the same time and touch the same partition, the later swap wins and the rows the other process wrote to that partition are lost. + +Run each model from one process at a time: one scheduled job per model or per project, with no overlapping runs. Concurrency within a run (`--concurrency`, `batch_concurrency`) is safe. +- **Snapshots** apply their changes to a private copy of the snapshot table and exchange it in. Closing a version rebuilds the copy rather than updating rows, so the live snapshot changes only at the final exchange. + +Tables SQLBuild creates use `ENGINE = MergeTree ORDER BY tuple()` with no partitioning, so on a normal incremental run a partition rewrite rewrites the whole table. The cost of a rewrite is proportional to the partitions it touches. + +`merge` matches rows on `unique_key` exactly as other adapters do: incoming rows replace matched rows, `merge_exclude_columns` keep their stored values, and unmatched rows are inserted. + +## Reading ReplacingMergeTree sources + +A `ReplacingMergeTree` table can hold several versions of a row until the server merges them in the background. Set `final true` on the source so every read SQLBuild generates for it uses `FINAL` and sees one version per key: + +```yaml +sources: + - name: customers + description: Customer master data. + schema: raw + table: customers + final: true +``` + +## Functions + +SQL functions deploy as ClickHouse SQL user-defined functions (`CREATE FUNCTION name AS (args) -> expression`). ClickHouse function names are global to the server, not scoped to a database, so SQLBuild names each function `__`. A function `half_amount` in schema `analytics` is created as `analytics__half_amount`, and `__udf("half_amount")` renders that name. + +SQLBuild does not deploy Python functions (`A707`) or SQL table functions (`A708`) to ClickHouse: executable functions need server configuration files. Install them on the server and call them directly. + +## Clones + +`sqb clone` creates the clone with `CREATE TABLE ... AS` and attaches every part of the origin with `ALTER TABLE ... ATTACH PARTITION ALL FROM`, which hard-links the data instead of copying it. The clone keeps the origin's engine, keys and settings and stays valid after the origin is dropped. + +## Settings SQLBuild pins + +Every query runs with these settings so SQL behaves the same as on other adapters: + +| Setting | Value | Why | +|---------|-------|-----| +| `join_use_nulls` | `1` | Outer joins return `NULL` for missing rows | +| `cast_keep_nullable` | `1` | `CAST` keeps `NULL` values | +| `union_default_mode`, `except_default_mode`, `intersect_default_mode` | `DISTINCT` | Bare `UNION`, `EXCEPT` and `INTERSECT` follow standard SQL | +| `default_table_engine` | `MergeTree` | Tables created without an engine clause are MergeTree tables | +| `mutations_sync`, `lightweight_deletes_sync` | `2` | Statements finish before the next one starts | +| `date_time_input_format`, `cast_string_to_date_time_mode` | `best_effort` | ISO 8601 timestamps with `T` and offsets parse | + +Values under `settings` in the connection override these. + +ClickHouse returns a type's default value (`0`, the epoch, an empty string) from `lag` and `lead` past the edge of a window frame unless the argument is `Nullable`. SQLBuild's generated snapshot SQL wraps those arguments in `toNullable` so they return `NULL` as on other adapters. Write the same in your own models when you rely on `NULL` there. + +## Limits + +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- No transactions. Each step above is designed so a rerun converges instead. +- Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). +- Model physical settings (`ENGINE`, `ORDER BY`, `PARTITION BY`) are not configurable yet. +- Seeds load through the HTTP insert interface as CSV, so large seeds never hit the `max_query_size` limit. +- Retention and time travel are not available. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index afaa92fa35..19e4b3324f 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -68,7 +68,7 @@ materialized = "view" | Field | Description | |-------|-------------| | `name` | Project name. Used in fingerprint tracking and manifest generation. | -| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, or `sqlserver`. See [Adapters](/docs/concepts/adapters). | +| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`, or `clickhouse`. See [Adapters](/docs/concepts/adapters). | ### Optional fields diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 5f1dc92e98..05f688d4f2 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -47,7 +47,8 @@ "docs/concepts/adapters/sqlite", "docs/concepts/adapters/bigquery", "docs/concepts/adapters/databricks", - "docs/concepts/adapters/sqlserver" + "docs/concepts/adapters/sqlserver", + "docs/concepts/adapters/clickhouse" ] }, "docs/concepts/sources", From 0f52bae8b86fb244fe6231ea0149c97f770d3c8e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 03:17:30 +0100 Subject: [PATCH 18/34] fix: keep microsecond microbatch state timestamps on ClickHouse Microbatch events order by created_at then event ID when choosing the latest completion and requirement. ClickHouse's TIMESTAMP is DateTime, so casting state literals to it cut them to whole seconds and let events from the same second order by ID instead of time. Adapters may now name the type state timestamps cast to; ClickHouse uses DateTime64(6, 'UTC'). --- .../_helpers/contracts/state_literals.rs | 22 +++++++-- .../_helpers/microbatches/statements.rs | 19 +++++--- .../_helpers/state_rows/state_tables.rs | 2 +- src/sqlbuild/_native.pyi | 9 +++- .../adapter/contract/classes/base_adapter.py | 1 + .../clickhouse/classes/clickhouse_adapter.py | 2 + src/sqlbuild/adapters/clickhouse/constants.py | 1 + .../microbatches/classes/direct_store.py | 1 + .../adapters/clickhouse/_test_types.py | 8 ++++ .../sqlbuild/adapters/clickhouse/helpers.py | 41 +++++++++++++++- .../adapters/clickhouse/test_client.py | 47 ++++++++++++++++++- 11 files changed, 137 insertions(+), 16 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs index 9d4ad26874..2a2b4e9d68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs @@ -18,15 +18,18 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_state::freshness::constants::TEXT_TIMESTAMP_STATE_VALUE_TYPE as TEXT_TIMESTAMP_TYPE; const DATE_TYPE: &str = "date"; +/// The portable SQL type state timestamps cast to unless the warehouse names a finer one. +const SQL_TIMESTAMP_TYPE: &str = "TIMESTAMP"; const JSON_TYPE: &str = "json"; -/// `render_state_sql_literal(*, value, declared_type)`: `value` as its state column's literal. +/// `value` as its state column's literal, timestamps cast to `timestamp_type` when given. #[pyfunction] -#[pyo3(signature = (*, value, declared_type))] +#[pyo3(signature = (*, value, declared_type, timestamp_type = None))] pub(crate) fn render_state_sql_literal( py: Python<'_>, value: &Bound<'_, PyAny>, declared_type: &str, + timestamp_type: Option<&str>, ) -> PyResult { if value.is_none() { return Ok(typed_null(declared_type).to_owned()); @@ -60,7 +63,9 @@ pub(crate) fn render_state_sql_literal( } Ok(if value.is_truthy()? { "TRUE" } else { "FALSE" }.to_owned()) } - TIMESTAMP_TYPE | TEXT_TIMESTAMP_TYPE => timestamp_literal(value, declared_type), + TIMESTAMP_TYPE | TEXT_TIMESTAMP_TYPE => { + timestamp_literal(value, declared_type, timestamp_type) + } DATE_TYPE => { let datetime = py.import(DATETIME_MODULE)?; if value.is_instance(&native_field(&datetime, "datetime")?)? @@ -81,7 +86,11 @@ pub(crate) fn render_state_sql_literal( } } -fn timestamp_literal(value: &Bound<'_, PyAny>, declared_type: &str) -> PyResult { +fn timestamp_literal( + value: &Bound<'_, PyAny>, + declared_type: &str, + timestamp_type: Option<&str>, +) -> PyResult { let py: Python<'_> = value.py(); let datetime = py.import(DATETIME_MODULE)?; if !value.is_instance(&native_field(&datetime, "datetime")?)? { @@ -103,7 +112,10 @@ fn timestamp_literal(value: &Bound<'_, PyAny>, declared_type: &str) -> PyResult< if declared_type == TEXT_TIMESTAMP_TYPE { return Ok(rendered); } - Ok(format!("CAST({rendered} AS TIMESTAMP)")) + Ok(format!( + "CAST({rendered} AS {})", + timestamp_type.unwrap_or(SQL_TIMESTAMP_TYPE) + )) } fn json_literal(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs index 5afe9922ae..2619ade2c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs @@ -53,20 +53,26 @@ impl Layout { .map_or("", |index| self.declared[index].as_str()) } - /// One event's column literals in column order. - fn literals(&self, event: &Bound<'_, PyAny>) -> PyResult> { + /// One event's column literals in column order, timestamps cast to `timestamp_type`. + fn literals( + &self, + event: &Bound<'_, PyAny>, + timestamp_type: Option<&str>, + ) -> PyResult> { let py: Python<'_> = event.py(); let values: Bound<'_, PyAny> = host::entry(py, ExecutorHost::MicrobatchEventCodec)? .call_method1("values", (event,))?; values .try_iter()? .zip(&self.declared) - .map(|(value, declared)| render_state_sql_literal(py, &value?, declared)) + .map(|(value, declared)| { + render_state_sql_literal(py, &value?, declared, timestamp_type) + }) .collect() } fn literal(&self, column: &str, value: &Bound<'_, PyAny>) -> PyResult { - render_state_sql_literal(value.py(), value, self.declared(column)) + render_state_sql_literal(value.py(), value, self.declared(column), None) } } @@ -120,16 +126,17 @@ fn events_table<'py>( /// Build one guarded set-based insert for events in the same state table. #[pyfunction(name = "build_microbatch_insert_many_sql")] -#[pyo3(signature = (*, events, render_qualified_name))] +#[pyo3(signature = (*, events, render_qualified_name, timestamp_type = None))] fn insert_many( events: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, + timestamp_type: Option<&str>, ) -> PyResult { let (listed, table) = events_table(events, render_qualified_name, EMPTY_INSERT)?; let layout: Layout = Layout::new(events.py())?; let rows: Vec> = listed .iter() - .map(|event| layout.literals(event)) + .map(|event| layout.literals(event, timestamp_type)) .collect::>()?; Ok(insert_many_sql(&table, &layout.columns, &rows)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index eb11cae7ff..e6d61f9339 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -124,7 +124,7 @@ fn render_insert_source_freshness_records_sql<'py>( let row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, declared)| { - render_state_sql_literal(py, &native_field(&record, column)?, declared) + render_state_sql_literal(py, &native_field(&record, column)?, declared, None) }) .collect::>()?; rows.push(row); diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 57739c3139..8061f43fc5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4721,7 +4721,9 @@ def normalize_sql_value( explicit_type: str | None = None, limits: SqlValueLimits | None = None, ) -> SqlValue: ... -def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... +def render_state_sql_literal( + *, value: object | None, declared_type: StateSqlValueType, timestamp_type: str | None = None +) -> str: ... def format_diagnostics_console_record(record: logging.LogRecord, *, use_color: bool) -> str: ... def interpreter_identity() -> tuple[str, str, str, str]: ... def project_microbatch_coverage( @@ -4750,7 +4752,10 @@ def deterministic_microbatch_event_id( completion_reason: str, ) -> str: ... def build_microbatch_insert_many_sql( - *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] + *, + events: tuple[MicrobatchEvent, ...], + render_qualified_name: Callable[..., str | None], + timestamp_type: str | None = None, ) -> str: ... def build_microbatch_existing_event_ids_sql( *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index fafb71c79f..58df5e5cc2 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -115,6 +115,7 @@ class BaseAdapter(RetentionAdapterMixin, StrictAdapter): statement_limit_excludes_queueing: ClassVar[bool] = False metadata_inspection_concurrency: ClassVar[int] = 1 supports_session_warehouse: ClassVar[bool] = False + state_timestamp_type: ClassVar[str | None] = None _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", distinct_condition=render_is_distinct_from, diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 605a27d729..679d4dfab3 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -118,6 +118,7 @@ SNAPSHOT_REBUILD_INFIX, SNAPSHOT_WORK_INFIX, STATE_TABLE_ENGINE_KEYWORD, + STATE_TIMESTAMP_TYPE, TIMEOUT_OPTIONS, VERIFY_OPTION, VIEW_ENGINES, @@ -149,6 +150,7 @@ class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): ) max_identifier_length: ClassVar[int] = MAX_IDENTIFIER_LENGTH relation_grants_supported: ClassVar[bool] = False + state_timestamp_type: ClassVar[str | None] = STATE_TIMESTAMP_TYPE _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="Nullable(DateTime64(6, 'UTC'))", distinct_condition=render_is_distinct_from, diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index 8c1e76527b..df5262dd1e 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -93,3 +93,4 @@ EXCHANGE_UNSUPPORTED_CODES: frozenset[int] = frozenset({1, 48}) STATE_TABLE_ENGINE_KEYWORD: str = "ENGINE" RELATION_SEPARATOR: str = "." +STATE_TIMESTAMP_TYPE: str = "DateTime64(6, 'UTC')" diff --git a/src/sqlbuild/microbatches/classes/direct_store.py b/src/sqlbuild/microbatches/classes/direct_store.py index 1fb05f220f..2184bc8bae 100644 --- a/src/sqlbuild/microbatches/classes/direct_store.py +++ b/src/sqlbuild/microbatches/classes/direct_store.py @@ -72,6 +72,7 @@ def write_many(self, events: tuple[MicrobatchEvent, ...]) -> MicrobatchWriteResu sql=build_microbatch_insert_many_sql( events=missing, render_qualified_name=self._adapter.render_qualified_name, + timestamp_type=self._adapter.state_timestamp_type, ), ) inserted += len(missing) diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py index aa9da8188a..ce7910fd63 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from datetime import datetime from sqlbuild.adapter.contract.models import ColumnInfo @@ -131,3 +132,10 @@ class ClickHouseColumnMigrationTestCase: renamed: tuple[str, str] dropped: tuple[str, ...] expected_columns: tuple[ColumnInfo, ...] + + +@dataclass(frozen=True) +class ClickHouseStateTimestampTestCase: + description: str + created_at: datetime + expected_created_at: tuple[tuple[object, ...], ...] diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py index a3d369d265..324870a3ed 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py @@ -6,13 +6,20 @@ import time import uuid from collections.abc import Callable -from datetime import date +from datetime import date, datetime from typing import Any import duckdb from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope +from sqlbuild.microbatches.types import ( + MicrobatchCompletionType, + MicrobatchFingerprintStatus, + MicrobatchRecordType, + MicrobatchRunType, +) from tests.integration.src.sqlbuild.adapters.clickhouse._test_types import ( ClickHouseRewriteTestCase, ) @@ -49,6 +56,38 @@ def __call__(self, *, connection: Any, sql: str) -> Any: return self._execute(connection=connection, sql=sql) +def microbatch_completion(*, schema: str, created_at: datetime) -> MicrobatchEvent: + return MicrobatchEvent( + event_id="event-1", + record_type=MicrobatchRecordType.PARTITION_COMPLETION, + scope=MicrobatchScope( + scope_kind="direct_logical", + scope_key=f"clickhouse:{schema}.orders", + model_name="orders", + target_database=None, + target_schema=schema, + target_name="orders", + physical_generation_id="*", + ), + origin_run_id="origin-run", + execution_run_id="execution-run", + run_type=MicrobatchRunType.REPLAY_ON_CHANGE, + completion_type=MicrobatchCompletionType.RECOVERY, + run_start="0", + run_end="1", + partition_start="0", + partition_end="1", + batch_size="1", + cursor_column="batch_id", + cursor_type="integer", + model_version_hash="F2", + definition_hash="definition", + fingerprint_status=MicrobatchFingerprintStatus.KNOWN, + rows_affected=0, + created_at=created_at, + ) + + def build_unique_schema_name(*, prefix: str) -> str: return f"{prefix}_{uuid.uuid4().hex[:10]}" diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py index 95dbff7901..5c532b3db3 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py @@ -1,12 +1,13 @@ from __future__ import annotations -from datetime import date, datetime +from datetime import UTC, date, datetime from pathlib import Path from types import SimpleNamespace from typing import Any import pytest +from sqlbuild._native import build_microbatch_insert_many_sql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ColumnInfo, CursorValue, QueryResult, RelationInfo @@ -28,6 +29,7 @@ ClickHouseRewriteTestCase, ClickHouseSeedTestCase, ClickHouseStateTableTestCase, + ClickHouseStateTimestampTestCase, ClickHouseSwapTestCase, ) from tests.integration.src.sqlbuild.adapters.clickhouse.helpers import ( @@ -39,6 +41,7 @@ execute_all, fetch_rows, interrupt_running_query, + microbatch_completion, oracle_rows, order_rows_as_dates, recorded_statements, @@ -849,3 +852,45 @@ def test_given_table_when_migrating_columns_then_clickhouse_alters_in_place( assert adapter.describe_relation(connection=connection, relation=relation) == ( test_case.expected_columns ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseStateTimestampTestCase( + description="microbatch event times keep microseconds", + created_at=datetime(2026, 1, 1, 0, 0, 0, 123456, tzinfo=UTC), + expected_created_at=((datetime(2026, 1, 1, 0, 0, 0, 123456),),), + ), + ], + ids=lambda case: case.description, +) +def test_given_microbatch_event_when_publishing_state_then_clickhouse_keeps_microseconds( + test_case: ClickHouseStateTimestampTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + adapter.execute( + connection=connection, + sql=adapter.render_create_microbatch_state_table_sql( + database=None, schema=clickhouse_schema + ), + ) + + adapter.execute( + connection=connection, + sql=build_microbatch_insert_many_sql( + events=( + microbatch_completion(schema=clickhouse_schema, created_at=test_case.created_at), + ), + render_qualified_name=adapter.render_qualified_name, + timestamp_type=adapter.state_timestamp_type, + ), + ) + + assert fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT created_at FROM {clickhouse_schema}._sqlbuild_microbatches", + ) == (test_case.expected_created_at) From 2fe2eb645ef45b04d3232b693471e50d47d74adb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:28:52 +0100 Subject: [PATCH 19/34] feat: configure ClickHouse table engine, sorting and partitioning per model Table and incremental models accept engine, order_by, partition_by, primary_key and table_settings in the MODEL header. Compile validates their shapes (MergeTree-family single-server engines, primary key a prefix of the sorting key, scalar settings) and rejects them on views and snapshots; planning rejects them on other adapters with A011. The keys join the model's version identity, the plan entry carries them to every target-creating CREATE TABLE, and ClickHouse renders them as table clauses. An E2E shows an incremental window rewriting only its partition. --- .../src/planner_identity/constants.rs | 7 +- .../sqlbuild-config/src/project/constants.rs | 7 +- .../_helpers/materialization.rs | 151 +++++++++++++++++- .../src/model_validation/constants.rs | 25 +++ .../main/validate_model_config.rs | 4 +- .../src/model_validation/tests.rs | 2 + .../src/model_validation/tests/helpers.rs | 7 + .../tests/test_physical_storage.rs | 116 ++++++++++++++ .../bindings/_helpers/discovery/constants.rs | 7 +- .../_helpers/planner_records/constants.rs | 1 + .../_helpers/planner_strategy/constants.rs | 4 + .../planner_strategy/model_entries.rs | 39 +++++ .../docs/concepts/adapters/clickhouse.md | 40 ++++- .../docs/concepts/models/configuration.md | 1 + src/sqlbuild/_native.pyi | 2 + .../clickhouse/_helpers/table_layout.py | 46 ++++++ .../clickhouse/classes/clickhouse_adapter.py | 15 +- src/sqlbuild/adapters/clickhouse/constants.py | 3 + .../run/_helpers/execution/staging.py | 1 + .../_helpers/materializations/microbatch.py | 5 +- .../_helpers/validation/type_enforcement.py | 7 +- src/sqlbuild/executor/run/main/_execute.py | 2 + .../commands/main/clickhouse/_test_types.py | 10 ++ .../cli/commands/main/clickhouse/helpers.py | 56 ++++++- .../main/clickhouse/test_clickhouse_layout.py | 116 ++++++++++++++ .../adapters/clickhouse/_test_types.py | 7 + .../adapters/clickhouse/test_table_layout.py | 48 ++++++ .../docs/concepts/adapters/clickhouse.mdx | 39 ++++- .../docs/concepts/models/configuration.mdx | 1 + 29 files changed, 755 insertions(+), 14 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/table_layout.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py diff --git a/crates/sqlbuild-analysis/src/planner_identity/constants.rs b/crates/sqlbuild-analysis/src/planner_identity/constants.rs index 3faff6781e..23d471a561 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/constants.rs @@ -1,7 +1,7 @@ //! Planner identity constants shared with the Python planner. /// Model config keys whose values participate in version identity, in Python's insertion order after `cursor_inputs`. -pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 34] = [ +pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 39] = [ "append_cursor_inclusive", "batch_size", "check_columns", @@ -36,6 +36,11 @@ pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 34] = [ "updated_at", "valid_from_column", "valid_to_column", + "engine", + "order_by", + "partition_by", + "primary_key", + "table_settings", ]; /// Model config key of cursor inputs, recorded before every other identity key. pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs"; diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index a8e53fead1..dc1c783554 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -94,7 +94,7 @@ pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "unaccounted_partition_policy", ]; /// The MODEL header keys, which `[path_defaults]` entries may set. -pub(crate) const MODEL_HEADER_KEYS: [&str; 68] = [ +pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "alias", "append_cursor_inclusive", "audit_factories", @@ -121,6 +121,7 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 68] = [ "description", "dynamic_columns", "enabled", + "engine", "enums", "full_refresh", "historical_input", @@ -141,9 +142,12 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 68] = [ "observed_at", "old_name_view", "on_schema_change", + "order_by", + "partition_by", "placeholders", "post_hooks", "pre_hooks", + "primary_key", "replay_on_change", "row_diff_exclude_columns", "row_diff_sample_rows", @@ -155,6 +159,7 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 68] = [ "snapshot_strategy", "sql_analysis", "sql_validation", + "table_settings", "table_type", "tags", "time_travel_retention", diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index 01c6543312..d715e32167 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -9,7 +9,7 @@ use sqlbuild_core::text::main::is_word_char::is_word_char; use crate::errors::ConfigError; use crate::model_validation::_helpers::config::python_lower; use crate::model_validation::_helpers::config::{ - ConfigView, one_of, sorted_values, text_if_string, + ConfigView, model_header_help, non_blank_string_check, one_of, sorted_values, text_if_string, }; use crate::model_validation::constants::{ BUILTIN_MATERIALIZATIONS, CONTRACT_POLICIES, CUSTOM_MATERIALIZATION_DISALLOWED_KEYS, @@ -21,6 +21,11 @@ use crate::model_validation::constants::{ CHAINED_HELP, CLAIMED_HELP, MATERIALIZATION_HELP, MIGRATABLE_MATERIALIZATIONS, ORIGIN_HELP, QUALIFIED_NAME_SEPARATOR, }; +use crate::model_validation::constants::{ + ENGINE_FAMILY_SUFFIX, ENGINE_KEY, ORDER_BY_KEY, PARTITION_BY_KEY, PHYSICAL_STORAGE_KEYS, + PHYSICAL_STORAGE_MATERIALIZATIONS, PRIMARY_KEY_KEY, TABLE_SETTINGS_KEY, + UNSUPPORTED_ENGINE_PREFIXES, +}; use crate::model_validation::models::{ ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; @@ -381,3 +386,147 @@ pub(crate) fn chained_problem( } Ok(()) } + +/// Check the physical layout keys a table-backed model may set. +pub(crate) fn check_physical_storage(config: &ConfigView<'_, N>) -> Check { + let Some(first) = config.first_present(&PHYSICAL_STORAGE_KEYS) else { + return Ok(()); + }; + let materialized: Option = config.string("materialized")?; + if !materialized + .as_deref() + .is_some_and(|value| one_of(value, &PHYSICAL_STORAGE_MATERIALIZATIONS)) + { + return Err(config.error(format!( + "{first} is only valid for table and incremental models" + ))); + } + check_engine(config)?; + let order_by: Option> = expressions(config, ORDER_BY_KEY)?; + if let Some(value) = config.get(PARTITION_BY_KEY) + && !non_blank_string_check(value) + { + return Err(config.error(format!( + "{PARTITION_BY_KEY} must be a non-empty SQL expression string" + ))); + } + check_primary_key(config, order_by.as_deref())?; + check_table_settings(config) +} + +fn check_engine(config: &ConfigView<'_, N>) -> Check { + let Some(value) = config.get(ENGINE_KEY) else { + return Ok(()); + }; + let engine: Option = text_if_string(value).map(|text| text.trim().to_owned()); + let name: Option<&str> = engine + .as_deref() + .and_then(|engine| engine_name(engine)) + .filter(|name| name.ends_with(ENGINE_FAMILY_SUFFIX)); + match name { + Some(name) + if !UNSUPPORTED_ENGINE_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) => + { + Ok(()) + } + _ => Err(ValidationStop::Error( + config + .config_error(format!( + "{ENGINE_KEY} '{}' must be a single-server MergeTree-family table engine", + value.python_str() + )) + .with_help(model_header_help( + "to name a MergeTree-family engine", + "engine 'ReplacingMergeTree(updated_at)'", + )), + )), + } +} + +/// The engine name before its parenthesized arguments, when the text is shaped like one. +fn engine_name(engine: &str) -> Option<&str> { + let (name, rest) = match engine.find('(') { + Some(open) => (&engine[..open], &engine[open..]), + None => (engine, ""), + }; + let named: bool = !name.is_empty() + && name + .chars() + .all(|character| character.is_ascii_alphanumeric() || character == '_'); + let arguments: bool = rest.is_empty() || rest.ends_with(')'); + (named && arguments).then_some(name) +} + +/// The expressions of a string or a list of strings, failing for anything else. +fn expressions( + config: &ConfigView<'_, N>, + key: &str, +) -> Result>, ValidationStop> { + let Some(value) = config.get(key) else { + return Ok(None); + }; + let items: Vec = match value.kind() { + NodeKind::Str => vec![value.clone()], + NodeKind::List | NodeKind::Tuple => value.items(), + _ => Vec::new(), + }; + let texts: Vec = items + .iter() + .filter(|item| non_blank_string_check(*item)) + .filter_map(text_if_string) + .map(|text| text.trim().to_owned()) + .collect(); + if texts.is_empty() || texts.len() != items.len() { + return Err(config.error(format!( + "{key} must be a SQL expression string or a list of them" + ))); + } + Ok(Some(texts)) +} + +fn check_primary_key( + config: &ConfigView<'_, N>, + order_by: Option<&[String]>, +) -> Check { + let Some(primary_key) = expressions(config, PRIMARY_KEY_KEY)? else { + return Ok(()); + }; + let prefix: bool = order_by.is_some_and(|order_by| order_by.starts_with(&primary_key)); + if prefix { + return Ok(()); + } + Err(config.error(format!( + "{PRIMARY_KEY_KEY} must be a prefix of {ORDER_BY_KEY}" + ))) +} + +fn check_table_settings(config: &ConfigView<'_, N>) -> Check { + let Some(value) = config.get(TABLE_SETTINGS_KEY) else { + return Ok(()); + }; + let valid: bool = value.kind() == NodeKind::Map + && value.entries().iter().all(|(key, setting)| { + key.with_text(setting_name).unwrap_or(false) + && matches!( + setting.kind(), + NodeKind::Str | NodeKind::Bool(_) | NodeKind::Int { .. } + ) + }); + if valid { + return Ok(()); + } + Err(config.error(format!( + "{TABLE_SETTINGS_KEY} must map setting names to strings, integers or booleans" + ))) +} + +fn setting_name(name: &str) -> bool { + name.chars() + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && name + .chars() + .all(|character| character.is_ascii_alphanumeric() || character == '_') +} diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index 341d1f0cf4..fdef85e14e 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -277,3 +277,28 @@ pub(crate) const CLAIMED_HELP: &str = pub(crate) const CHAINED_HELP: &str = "a chain or swap cannot be renamed in place safely; give the new column \ a name no other renamed column uses, or rebuild the model with \ --full-refresh"; + +/// The table engine key. +pub const ENGINE_KEY: &str = "engine"; +/// The sorting key. +pub const ORDER_BY_KEY: &str = "order_by"; +/// The partitioning key. +pub const PARTITION_BY_KEY: &str = "partition_by"; +/// The primary key, a prefix of the sorting key. +pub const PRIMARY_KEY_KEY: &str = "primary_key"; +/// The table settings key. +pub const TABLE_SETTINGS_KEY: &str = "table_settings"; +/// Physical table layout keys, in the order their errors are checked. +pub const PHYSICAL_STORAGE_KEYS: [&str; 5] = [ + ENGINE_KEY, + ORDER_BY_KEY, + PARTITION_BY_KEY, + PRIMARY_KEY_KEY, + TABLE_SETTINGS_KEY, +]; +/// Materializations whose tables SQLBuild creates with the model's physical layout. +pub const PHYSICAL_STORAGE_MATERIALIZATIONS: [&str; 2] = ["table", "incremental"]; +/// The name suffix every supported table engine shares. +pub const ENGINE_FAMILY_SUFFIX: &str = "MergeTree"; +/// Engine name prefixes of replicated and shared engines, which need a cluster. +pub const UNSUPPORTED_ENGINE_PREFIXES: [&str; 2] = ["Replicated", "Shared"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs index 5b261cbea5..5b299b390a 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs @@ -4,7 +4,8 @@ use crate::model_validation::_helpers::config::ConfigView; use crate::model_validation::_helpers::incremental::check_incremental; use crate::model_validation::_helpers::materialization::{ check_contract, check_custom_materialization, check_execution_limit, check_migration, - check_non_incremental, check_placeholders, check_project_capability, check_storage_policies, + check_non_incremental, check_physical_storage, check_placeholders, check_project_capability, + check_storage_policies, }; use crate::model_validation::_helpers::references::check_references; use crate::model_validation::_helpers::snapshot::check_snapshot; @@ -29,6 +30,7 @@ pub fn validate_model_config( check_snapshot(&config, facts)?; check_custom_materialization(&config, project)?; check_storage_policies(&config, facts)?; + check_physical_storage(&config)?; check_migration(&config, facts)?; check_placeholders(&config, project, facts) } diff --git a/crates/sqlbuild-model-config/src/model_validation/tests.rs b/crates/sqlbuild-model-config/src/model_validation/tests.rs index ebddf13611..a8b4c370ec 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests.rs @@ -2,6 +2,8 @@ mod column_migrations; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_physical_storage.rs"] +mod physical_storage; #[path = "tests/test_storage_overrides.rs"] mod storage_overrides; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs index b42549c7cb..45308d1651 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs @@ -132,3 +132,10 @@ pub(super) fn column_migration_message( .err() .map(|error| error.message) } + +/// A table config with `extra` keys. +pub(super) fn table(extra: Vec<(&'static str, Value)>) -> Vec<(&'static str, Value)> { + let mut config: Vec<(&'static str, Value)> = vec![("materialized", Value::Str("table"))]; + config.extend(extra); + config +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs new file mode 100644 index 0000000000..e03994b1c2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs @@ -0,0 +1,116 @@ +use crate::model_validation::tests::helpers::{ + incremental, map, snapshot, strings, table, validation_outcome, +}; +use crate::model_validation::tests::test_types::ValidateTestCase; +use crate::tests::test_types::Value; + +#[test] +fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_are_checked() { + let test_cases = [ + ValidateTestCase { + description: "a complete layout on a table", + config: table(vec![ + ("engine", Value::Str("ReplacingMergeTree(updated_at)")), + ("order_by", strings(&["customer_id", "toDate(ordered_at)"])), + ("partition_by", Value::Str("toYYYYMM(ordered_at)")), + ("primary_key", strings(&["customer_id"])), + ( + "table_settings", + map(vec![ + ("index_granularity", Value::Int(4096)), + ("allow_nullable_key", Value::Bool(true)), + ]), + ), + ]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a single sorting expression on an incremental model", + config: incremental(vec![("order_by", Value::Str("order_id"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "layout on a view", + config: vec![ + ("materialized", Value::Str("view")), + ("partition_by", Value::Str("toYYYYMM(ordered_at)")), + ], + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': partition_by is only valid for table and incremental models", + }, + ValidateTestCase { + description: "layout on a snapshot", + config: snapshot(vec![("order_by", Value::Str("customer_id"))]), + references: &["ref:customers"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': order_by is only valid for table and incremental models", + }, + ValidateTestCase { + description: "a replicated engine", + config: table(vec![( + "engine", + Value::Str("ReplicatedMergeTree('/tables/orders', '{replica}')"), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'ReplicatedMergeTree('/tables/orders', '{replica}')' must be a single-server MergeTree-family table engine", + }, + ValidateTestCase { + description: "an engine outside the MergeTree family", + config: table(vec![("engine", Value::Str("Log"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'Log' must be a single-server MergeTree-family table engine", + }, + ValidateTestCase { + description: "a blank sorting expression", + config: table(vec![("order_by", strings(&["customer_id", " "]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': order_by must be a SQL expression string or a list of them", + }, + ValidateTestCase { + description: "a partition expression that is not a string", + config: table(vec![("partition_by", Value::Int(3))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': partition_by must be a non-empty SQL expression string", + }, + ValidateTestCase { + description: "a primary key that is not a sorting prefix", + config: table(vec![ + ("order_by", strings(&["customer_id", "ordered_at"])), + ("primary_key", strings(&["ordered_at"])), + ]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': primary_key must be a prefix of order_by", + }, + ValidateTestCase { + description: "a setting with a list value", + config: table(vec![( + "table_settings", + map(vec![("index_granularity", strings(&["8192"]))]), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': table_settings must map setting names to strings, integers or booleans", + }, + ]; + + for test_case in test_cases { + let outcome = + validation_outcome(test_case.config, test_case.references, test_case.query_sql); + + assert_eq!( + outcome, test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs index e7019c3865..6f14cf3cd6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs @@ -1,6 +1,6 @@ //! Header keys native discovery accepts, sorted by code point as Python sorts them. -pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 68] = [ +pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 73] = [ "alias", "append_cursor_inclusive", "audit_factories", @@ -27,6 +27,7 @@ pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 68] = [ "description", "dynamic_columns", "enabled", + "engine", "enums", "full_refresh", "historical_input", @@ -47,9 +48,12 @@ pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 68] = [ "observed_at", "old_name_view", "on_schema_change", + "order_by", + "partition_by", "placeholders", "post_hooks", "pre_hooks", + "primary_key", "replay_on_change", "row_diff_exclude_columns", "row_diff_sample_rows", @@ -61,6 +65,7 @@ pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 68] = [ "snapshot_strategy", "sql_analysis", "sql_validation", + "table_settings", "table_type", "tags", "time_travel_retention", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index d035277161..669ab180a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -628,6 +628,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("snapshot_full_refresh", FieldDefault::None), FieldSpec::new("snapshot_schema_change", FieldDefault::None), FieldSpec::new("table_type", FieldDefault::Text("transient")), + FieldSpec::new("physical_storage", FieldDefault::EmptyDict), FieldSpec::new("on_schema_change", FieldDefault::None), FieldSpec::new("type_enforcement", FieldDefault::False), FieldSpec::new("declared_columns", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index a2bc231951..7ee3feb5c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -21,3 +21,7 @@ pub(crate) const DIRECT_TRANSFORM: &str = "direct"; pub(crate) const MODEL_REFERENCE_KIND: &str = "model"; pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; +/// The SQL analysis dialect whose adapter lays tables out by the physical storage keys. +pub(crate) const PHYSICAL_STORAGE_DIALECT: &str = "clickhouse"; +/// The error code for physical storage keys on an adapter without them. +pub(crate) const PHYSICAL_STORAGE_UNSUPPORTED_CODE: &str = "A011"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index c028a82e37..e16356d4e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -11,6 +11,7 @@ use sqlbuild_analysis::planner_strategy::models::{CursorTypeCheck, LookbackConfi use sqlbuild_model_config::execution_limits::constants::MODEL_MAX_DURATION_KEY; use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; use sqlbuild_model_config::execution_limits::models::ExecutionLimit; +use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; @@ -23,6 +24,7 @@ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_strategy::constants::{ CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, + PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; use crate::bindings::_helpers::planner_strategy::cursor_types::{ @@ -302,6 +304,42 @@ fn logical_ddl<'py>( Ok(statements.join(DDL_SEPARATOR)) } +/// The model's physical table layout, refused on adapters whose tables have none. +fn storage_fields<'py>( + request: &ModelEntryDecisions<'py>, + values: &Bound<'py, PyAny>, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = values.py(); + let storage: Bound<'py, PyDict> = PyDict::new(py); + let mut keys: Vec<&str> = Vec::new(); + for key in PHYSICAL_STORAGE_KEYS { + let value: Bound<'py, PyAny> = mapping_get(values, key)?; + if !value.is_none() { + storage.set_item(key, value)?; + keys.push(key); + } + } + if !keys.is_empty() { + let dialect: Option = + native_field(&request.adapter, "sql_analysis_dialect_name")?.extract()?; + if dialect.as_deref() != Some(PHYSICAL_STORAGE_DIALECT) { + return Err(input_error( + py, + ErrorId::CompileInput, + &format!( + "model '{}' sets {}, but adapter '{}' has no physical table layout settings", + model_reads::model_name(&request.model)?, + keys.join(", "), + native_field(&request.adapter, "adapter_name")?.str()? + ), + Some(PHYSICAL_STORAGE_UNSUPPORTED_CODE), + )); + } + } + entry.set_item("physical_storage", storage) +} + /// The custom materialization name, config and placeholders of a custom model. fn custom_fields<'py>( values: &Bound<'py, PyAny>, @@ -597,6 +635,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( entry.set_item("unique_key", unique_key)?; entry.set_item("merge_exclude_columns", merge_exclude)?; entry.set_item("table_type", table_type)?; + storage_fields(request, &values, &entry)?; entry.set_item("on_schema_change", policy)?; entry.set_item("type_enforcement", enforcement)?; contract_fields(model, &values, &entry)?; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index 0696857d1d..f9c36065bf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/clickhouse/ - Connection config - Schemas are databases - How SQLBuild writes +- Physical layout - Reading ReplacingMergeTree sources - Functions - Clones @@ -83,10 +84,45 @@ A partition rewrite reads the partition, rebuilds it and swaps it in. ClickHouse Run each model from one process at a time: one scheduled job per model or per project, with no overlapping runs. Concurrency within a run (`--concurrency`, `batch_concurrency`) is safe. - **Snapshots** apply their changes to a private copy of the snapshot table and exchange it in. Closing a version rebuilds the copy rather than updating rows, so the live snapshot changes only at the final exchange. -Tables SQLBuild creates use `ENGINE = MergeTree ORDER BY tuple()` with no partitioning, so on a normal incremental run a partition rewrite rewrites the whole table. The cost of a rewrite is proportional to the partitions it touches. +The cost of a rewrite is proportional to the partitions it touches. Without a `partition_by`, a table is one partition, so every incremental run rewrites the whole table; partition large incremental models by the column their new rows arrive on. `merge` matches rows on `unique_key` exactly as other adapters do: incoming rows replace matched rows, `merge_exclude_columns` keep their stored values, and unmatched rows are inserted. +## Physical layout + +Table and incremental models set their table engine, sorting key, partitioning, primary key and table settings in the MODEL header: + +```sql +MODEL ( + description 'One row per order event.', + materialized incremental, + incremental_strategy delete_insert, + unique_key [event_id], + cursor event_time, + cursor_type timestamp, + cursor_inputs (raw_events event_time), + engine 'ReplacingMergeTree(version)', + order_by [event_id, event_time], + partition_by 'toYYYYMM(event_time)', + primary_key [event_id], + table_settings (index_granularity 4096), +); +``` + +| Field | Value | Default | +|-------|-------|---------| +| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | +| `partition_by` | A partitioning expression | No partitioning | +| `primary_key` | A prefix of `order_by` | The sorting key | +| `table_settings` | Setting names mapped to strings, integers or booleans | None | + +Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table, whose layout only changes on `--full-refresh`. + +ClickHouse rejects nullable sorting and partitioning keys by default. Wrap the column in `assumeNotNull()` or `coalesce()` in the expression, or set `table_settings (allow_nullable_key true)`. + ## Reading ReplacingMergeTree sources A `ReplacingMergeTree` table can hold several versions of a row until the server merges them in the background. Set `final true` on the source so every read SQLBuild generates for it uses `FINAL` and sees one version per key: @@ -132,6 +168,6 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from - Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). -- Model physical settings (`ENGINE`, `ORDER BY`, `PARTITION BY`) are not configurable yet. +- Snapshots always use an unsorted `MergeTree` table. - Seeds load through the HTTP insert interface as CSV, so large seeds never hit the `max_query_size` limit. - Retention and time travel are not available. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index c5f6b291c4..7a5fa287d8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -150,6 +150,7 @@ When the parser cannot read a model, the `P001` syntax error shows the exact MOD |-------|-------------| | `table_type` | Snowflake table type: `permanent`, `transient`, or `inherit` | | `time_travel_retention` | Managed time travel retention such as `7d`, `inherit`, or `disabled` | +| `engine`, `order_by`, `partition_by`, `primary_key`, `table_settings` | ClickHouse physical layout of table and incremental models; see [ClickHouse](../adapters/clickhouse.md#physical-layout) | | `run_despite_unchanged` | Removed with virtual environments. It is rejected in MODEL headers, project defaults, and path defaults. | Table promotion mode is a project setting rather than a `MODEL()` field. Staged promotion is the default. Immediate promotion is incompatible with model type enforcement and exact contracts; see [Materializations](materializations.md#table). diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8061f43fc5..4736fbce1d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -15710,6 +15710,7 @@ class ModelPlanEntry(NativeRecord): snapshot_full_refresh: str | None snapshot_schema_change: str | None table_type: str + physical_storage: dict[str, object] on_schema_change: OnSchemaChange | None type_enforcement: bool declared_columns: tuple[ColumnInfo, ...] @@ -15794,6 +15795,7 @@ class ModelPlanEntry(NativeRecord): snapshot_full_refresh: str | None = ..., snapshot_schema_change: str | None = ..., table_type: str = ..., + physical_storage: dict[str, object] = ..., on_schema_change: OnSchemaChange | None = ..., type_enforcement: bool = ..., declared_columns: tuple[ColumnInfo, ...] = ..., diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/table_layout.py b/src/sqlbuild/adapters/clickhouse/_helpers/table_layout.py new file mode 100644 index 0000000000..98d5e9e86c --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/table_layout.py @@ -0,0 +1,46 @@ +"""Render a model's physical table layout as ClickHouse table clauses.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any, cast + +from sqlbuild.adapters.clickhouse._helpers.type_text import quote_string +from sqlbuild.adapters.clickhouse.constants import DEFAULT_TABLE_ENGINE, EMPTY_SORTING_KEY + + +def render_table_layout(layout: dict[str, Any]) -> str: + """Return the ENGINE, ORDER BY, PARTITION BY, PRIMARY KEY and SETTINGS clauses.""" + + clauses: list[str] = [ + f"ENGINE = {str(layout.get('engine') or DEFAULT_TABLE_ENGINE).strip()}", + f"ORDER BY {_key_tuple(layout.get('order_by')) or EMPTY_SORTING_KEY}", + ] + partition_by: object = layout.get("partition_by") + if partition_by: + clauses.append(f"PARTITION BY {str(partition_by).strip()}") + primary_key: str | None = _key_tuple(layout.get("primary_key")) + if primary_key: + clauses.append(f"PRIMARY KEY {primary_key}") + settings: dict[str, object] = dict(layout.get("table_settings") or {}) + if settings: + clauses.append( + "SETTINGS " + + ", ".join(f"{name} = {_setting_value(value)}" for name, value in settings.items()) + ) + return " ".join(clauses) + + +def _key_tuple(value: object) -> str | None: + items: Iterable[object] = (value,) if isinstance(value, str) else cast(Any, value) or () + expressions: list[str] = [str(item) for item in items] + expressions = [expression.strip() for expression in expressions if expression.strip()] + return f"({', '.join(expressions)})" if expressions else None + + +def _setting_value(value: object) -> str: + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, int): + return str(value) + return quote_string(str(value)) diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 679d4dfab3..7d3996db13 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -94,6 +94,7 @@ nullable_offset_windows, rewrite_snapshot_statements, ) +from sqlbuild.adapters.clickhouse._helpers.table_layout import render_table_layout from sqlbuild.adapters.clickhouse._helpers.type_text import ( canonical_type, nullable_type, @@ -112,6 +113,7 @@ MAX_IDENTIFIER_LENGTH, MAX_QUERY_SIZE_BYTES, MINIMUM_SERVER_VERSION, + PHYSICAL_STORAGE_CONFIG_KEY, RELATION_NOT_FOUND_CODES, RELATION_SEPARATOR, SESSION_SETTINGS, @@ -700,9 +702,14 @@ def ensure_schema( self.execute(connection=connection, sql=statement) def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, ...]: + return self._render_create_table_with_layout(destination=destination, sql=sql, layout={}) + + def _render_create_table_with_layout( + self, *, destination: str, sql: str, layout: dict[str, Any] + ) -> tuple[str, ...]: return ( f"DROP TABLE IF EXISTS {destination}", - f"CREATE TABLE {destination} ENGINE = MergeTree ORDER BY tuple() AS {sql}", + f"CREATE TABLE {destination} {render_table_layout(layout)} AS {sql}", ) def render_create_view_as(self, *, destination: str, sql: str) -> tuple[str, ...]: @@ -2095,7 +2102,11 @@ def create_table_as( config: dict[str, Any] | None = None, statement_recorder: StatementRecorder, ) -> None: - statements: tuple[str, ...] = self.render_create_table_as(destination=destination, sql=sql) + statements: tuple[str, ...] = self._render_create_table_with_layout( + destination=destination, + sql=sql, + layout=dict((config or {}).get(PHYSICAL_STORAGE_CONFIG_KEY) or {}), + ) statement_recorder.record_many(statements) stmt: str for stmt in statements: diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index df5262dd1e..6f0d5574a3 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -94,3 +94,6 @@ STATE_TABLE_ENGINE_KEYWORD: str = "ENGINE" RELATION_SEPARATOR: str = "." STATE_TIMESTAMP_TYPE: str = "DateTime64(6, 'UTC')" +PHYSICAL_STORAGE_CONFIG_KEY: str = "physical_storage" +DEFAULT_TABLE_ENGINE: str = "MergeTree" +EMPTY_SORTING_KEY: str = "tuple()" diff --git a/src/sqlbuild/executor/run/_helpers/execution/staging.py b/src/sqlbuild/executor/run/_helpers/execution/staging.py index bab6eeabba..605aa1fc38 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/staging.py +++ b/src/sqlbuild/executor/run/_helpers/execution/staging.py @@ -51,6 +51,7 @@ def create_staging_relation( config={ "table_type": context.entry.table_type, "inferred_column_types": inferred_column_types(context.entry), + "physical_storage": context.entry.physical_storage, }, statement_recorder=statement_recorder, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 6a0b2a2c1f..d42b325d1f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -3008,7 +3008,10 @@ def _apply_microbatch_dml( connection=context.connection, destination=targets.target_qualified, sql=f"SELECT * FROM {targets.delta_qualified}", - config={"table_type": context.entry.table_type}, + config={ + "table_type": context.entry.table_type, + "physical_storage": context.entry.physical_storage, + }, statement_recorder=state.statement_recorder, ) else: diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index 41eb23fe1a..41d5599c25 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -27,6 +27,7 @@ def enforce_types_staged( declared_columns: tuple[ColumnInfo, ...], table_type: str, statement_recorder: StatementRecorder, + physical_storage: dict[str, object] | None = None, ) -> None: """Inspect staging columns and rebuild with casts for declared types.""" @@ -74,7 +75,11 @@ def enforce_types_staged( connection=connection, destination=enforced_qualified, sql=f"SELECT {projection_sql} FROM {staging_qualified}", - config={"table_type": table_type, "column_types": declared_map}, + config={ + "table_type": table_type, + "column_types": declared_map, + "physical_storage": physical_storage or {}, + }, statement_recorder=statement_recorder, ) adapter.drop( diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 3ba6b40c38..df4d2f90ef 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -244,6 +244,7 @@ def _staged_lifecycle( staging_table=targets.staging_table, declared_columns=declared_columns, table_type=entry.table_type, + physical_storage=entry.physical_storage, statement_recorder=statement_recorder, ) except Exception as exc: @@ -425,6 +426,7 @@ def _immediate_lifecycle( config={ "table_type": entry.table_type, "inferred_column_types": inferred_column_types(entry), + "physical_storage": entry.physical_storage, }, statement_recorder=statement_recorder, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 91af7138ad..02facb4d7a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -46,3 +46,13 @@ class ClickHouseCloneE2ETestCase: description: str expected_stdout_fragments: tuple[str, ...] expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHousePhysicalLayoutE2ETestCase: + description: str + first_events: str + later_events: str + expected_create_fragments: tuple[str, ...] + expected_partition_rewrites: dict[str, bool] + expected_totals: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index b325c8a56c..8f85206a3a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -4,7 +4,7 @@ import uuid from textwrap import dedent -from typing import Any +from typing import Any, cast from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter @@ -192,3 +192,57 @@ def build_orders_project_files(*, schema_name: str, config: dict[str, object]) - "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" ), } + + +def build_partitioned_events_project_files( + *, schema_name: str, config: dict[str, object] +) -> dict[str, str]: + return { + "sqlbuild_project.toml": build_clickhouse_project_toml( + project_name="clickhouse_layout", schema_name=schema_name, config=config + ), + "sources/raw.yml": dedent( + f""" + sources: + - name: raw_events + description: Synthetic order events. + schema: {schema_name} + table: raw_events + """ + ).lstrip(), + "models/events.sql": ( + "MODEL (description 'One row per event.',\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key [event_id],\n" + " cursor event_time,\n" + " cursor_type timestamp,\n" + " cursor_grain hour,\n" + " cursor_inputs (\n" + " raw_events event_time,\n" + " ),\n" + " engine 'ReplacingMergeTree(version)',\n" + " order_by [event_id, event_time],\n" + " partition_by 'toYYYYMM(event_time)',\n" + " primary_key [event_id],\n" + " table_settings (index_granularity 4096),\n" + ");\n\n" + "SELECT event_id, amount, event_time, toInt64(1) AS version\n" + 'FROM __source("raw_events")\n' + "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" + ), + } + + +def active_parts_by_partition( + *, config: dict[str, object], schema_name: str, table: str +) -> dict[str, tuple[str, ...]]: + rows: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, + sql=( + "SELECT partition, arraySort(groupArray(name)) FROM system.parts " + f"WHERE active AND database = '{schema_name}' AND table = '{table}' " + "GROUP BY partition ORDER BY partition" + ), + ) + return {str(row[0]): tuple(map(str, cast(list[object], row[1]))) for row in rows} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py new file mode 100644 index 0000000000..d7e038ea7e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHousePhysicalLayoutE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + active_parts_by_partition, + build_partitioned_events_project_files, + build_unique_schema_name, + cleanup_clickhouse_schema, + create_clickhouse_schema, + execute_clickhouse_sql, + fetch_clickhouse_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, + stringify_warehouse_rows, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHousePhysicalLayoutE2ETestCase( + description="a March window rewrites only March", + first_events=( + "(1, 10, '2026-01-05 08:00:00'), (2, 20, '2026-01-20 09:00:00'), " + "(3, 30, '2026-02-03 10:00:00'), (4, 40, '2026-02-17 11:00:00'), " + "(5, 50, '2026-03-01 12:00:00')" + ), + later_events="(6, 60, '2026-03-09 08:00:00'), (7, 70, '2026-03-20 09:00:00')", + expected_create_fragments=( + "ENGINE = ReplacingMergeTree(version)", + "PARTITION BY toYYYYMM(event_time)", + "PRIMARY KEY event_id", + "ORDER BY (event_id, event_time)", + "index_granularity = 4096", + ), + expected_partition_rewrites={"202601": False, "202602": False, "202603": True}, + expected_totals=(("7", "280"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_partitioned_layout_when_building_incrementally_then_only_touched_partitions_change( + tmp_path: Path, + test_case: ClickHousePhysicalLayoutE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_layout") + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_layout", + repo_files=build_partitioned_events_project_files(schema_name=schema_name, config=config), + ) + create_clickhouse_schema(config=config, schema_name=schema_name) + execute_clickhouse_sql( + config=config, + statements=( + f"CREATE TABLE {schema_name}.raw_events (event_id Int64, amount Int64, " + "event_time DateTime64(6, 'UTC')) ENGINE = MergeTree ORDER BY event_id", + f"INSERT INTO {schema_name}.raw_events VALUES {test_case.first_events}", + ), + ) + try: + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + create_sql: str = str( + fetch_clickhouse_rows( + config=config, + sql="SELECT create_table_query FROM system.tables " + f"WHERE database = '{schema_name}' AND name = 'events'", + )[0][0] + ) + assert [fragment in create_sql for fragment in test_case.expected_create_fragments] == [ + True + ] * len(test_case.expected_create_fragments), create_sql + execute_clickhouse_sql( + config=config, statements=(f"SYSTEM STOP MERGES {schema_name}.events",) + ) + before: dict[str, tuple[str, ...]] = active_parts_by_partition( + config=config, schema_name=schema_name, table="events" + ) + execute_clickhouse_sql( + config=config, + statements=(f"INSERT INTO {schema_name}.raw_events VALUES {test_case.later_events}",), + ) + + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert second.returncode == 0, second.stdout + second.stderr + after: dict[str, tuple[str, ...]] = active_parts_by_partition( + config=config, schema_name=schema_name, table="events" + ) + assert {partition: after[partition] != parts for partition, parts in before.items()} == ( + test_case.expected_partition_rewrites + ) + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql=f"SELECT count(), sum(amount) FROM {schema_name}.events FINAL", + ) + ) == (test_case.expected_totals) + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index 774954d20b..5da6716d68 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -57,3 +57,10 @@ class FinalSourceReadTestCase: adapter_name: str expected_relation: str | None expected_code: str | None + + +@dataclass(frozen=True) +class TableLayoutTestCase: + description: str + layout: dict[str, object] + expected_clauses: str diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py new file mode 100644 index 0000000000..ff381b1932 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.clickhouse._helpers.table_layout import render_table_layout +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import TableLayoutTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + TableLayoutTestCase( + description="no layout keeps an unsorted MergeTree", + layout={}, + expected_clauses="ENGINE = MergeTree ORDER BY tuple()", + ), + TableLayoutTestCase( + description="every clause", + layout={ + "engine": "ReplacingMergeTree(version)", + "order_by": ["customer_id", "toDate(ordered_at)"], + "partition_by": "toYYYYMM(ordered_at)", + "primary_key": ["customer_id"], + "table_settings": { + "index_granularity": 4096, + "allow_nullable_key": True, + "storage_policy": "default", + }, + }, + expected_clauses=( + "ENGINE = ReplacingMergeTree(version) ORDER BY (customer_id, toDate(ordered_at)) " + "PARTITION BY toYYYYMM(ordered_at) PRIMARY KEY (customer_id) " + "SETTINGS index_granularity = 4096, allow_nullable_key = 1, " + "storage_policy = 'default'" + ), + ), + TableLayoutTestCase( + description="a single sorting expression", + layout={"order_by": "order_id"}, + expected_clauses="ENGINE = MergeTree ORDER BY (order_id)", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_layout_when_rendering_then_clickhouse_table_clauses_result( + test_case: TableLayoutTestCase, +) -> None: + assert render_table_layout(test_case.layout) == test_case.expected_clauses diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index e6d8f17e74..1bd12dd634 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -73,10 +73,45 @@ A partition rewrite reads the partition, rebuilds it and swaps it in. ClickHouse Run each model from one process at a time: one scheduled job per model or per project, with no overlapping runs. Concurrency within a run (`--concurrency`, `batch_concurrency`) is safe. - **Snapshots** apply their changes to a private copy of the snapshot table and exchange it in. Closing a version rebuilds the copy rather than updating rows, so the live snapshot changes only at the final exchange. -Tables SQLBuild creates use `ENGINE = MergeTree ORDER BY tuple()` with no partitioning, so on a normal incremental run a partition rewrite rewrites the whole table. The cost of a rewrite is proportional to the partitions it touches. +The cost of a rewrite is proportional to the partitions it touches. Without a `partition_by`, a table is one partition, so every incremental run rewrites the whole table; partition large incremental models by the column their new rows arrive on. `merge` matches rows on `unique_key` exactly as other adapters do: incoming rows replace matched rows, `merge_exclude_columns` keep their stored values, and unmatched rows are inserted. +## Physical layout + +Table and incremental models set their table engine, sorting key, partitioning, primary key and table settings in the MODEL header: + +```sql +MODEL ( + description 'One row per order event.', + materialized incremental, + incremental_strategy delete_insert, + unique_key [event_id], + cursor event_time, + cursor_type timestamp, + cursor_inputs (raw_events event_time), + engine 'ReplacingMergeTree(version)', + order_by [event_id, event_time], + partition_by 'toYYYYMM(event_time)', + primary_key [event_id], + table_settings (index_granularity 4096), +); +``` + +| Field | Value | Default | +|-------|-------|---------| +| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | +| `partition_by` | A partitioning expression | No partitioning | +| `primary_key` | A prefix of `order_by` | The sorting key | +| `table_settings` | Setting names mapped to strings, integers or booleans | None | + +Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table, whose layout only changes on `--full-refresh`. + +ClickHouse rejects nullable sorting and partitioning keys by default. Wrap the column in `assumeNotNull()` or `coalesce()` in the expression, or set `table_settings (allow_nullable_key true)`. + ## Reading ReplacingMergeTree sources A `ReplacingMergeTree` table can hold several versions of a row until the server merges them in the background. Set `final true` on the source so every read SQLBuild generates for it uses `FINAL` and sees one version per key: @@ -122,6 +157,6 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from - Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). -- Model physical settings (`ENGINE`, `ORDER BY`, `PARTITION BY`) are not configurable yet. +- Snapshots always use an unsorted `MergeTree` table. - Seeds load through the HTTP insert interface as CSV, so large seeds never hit the `max_query_size` limit. - Retention and time travel are not available. diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index bd6d51ac94..c964396771 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -135,6 +135,7 @@ When the parser cannot read a model, the `P001` syntax error shows the exact MOD |-------|-------------| | `table_type` | Snowflake table type: `permanent`, `transient`, or `inherit` | | `time_travel_retention` | Managed time travel retention such as `7d`, `inherit`, or `disabled` | +| `engine`, `order_by`, `partition_by`, `primary_key`, `table_settings` | ClickHouse physical layout of table and incremental models; see [ClickHouse](/docs/concepts/adapters/clickhouse#physical-layout) | | `run_despite_unchanged` | Removed with virtual environments. It is rejected in MODEL headers, project defaults, and path defaults. | Table promotion mode is a project setting rather than a `MODEL()` field. Staged promotion is the default. Immediate promotion is incompatible with model type enforcement and exact contracts; see [Materializations](/docs/concepts/models/materializations#table). From 03bae4c654fa11a49ee3ac3c6c3379165f2d403c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:31:56 +0100 Subject: [PATCH 20/34] feat: warn when an incremental model's ClickHouse layout change awaits a full refresh Changing engine, order_by, partition_by, primary_key or table_settings on an incremental model changes its version, but runs keep writing into the existing table with its old layout. The plan now carries a PHYSICAL_LAYOUT_PENDING warning naming the changed keys and the full refresh that applies them, in text and JSON output. Plan decisions are unchanged. --- .../src/planner_strategy/constants.rs | 12 +++ .../src/planner_strategy/main/mod.rs | 1 + .../main/physical_layout_warning.rs | 51 ++++++++++ .../src/planner_strategy/tests.rs | 2 + .../tests/test_physical_layout_warning.rs | 76 +++++++++++++++ .../src/planner_strategy/tests/test_types.rs | 9 ++ .../_helpers/planner_strategy/constants.rs | 2 + .../planner_strategy/model_entries.rs | 36 ++++++- .../docs/concepts/adapters/clickhouse.md | 2 +- .../commands/main/clickhouse/_test_types.py | 11 +++ .../cli/commands/main/clickhouse/helpers.py | 2 +- .../main/clickhouse/test_clickhouse_layout.py | 95 +++++++++++++++++++ .../docs/concepts/adapters/clickhouse.mdx | 2 +- 13 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/main/physical_layout_warning.rs create mode 100644 crates/sqlbuild-analysis/src/planner_strategy/tests/test_physical_layout_warning.rs diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs index 2eabec0ab1..1176dc46a1 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -86,3 +86,15 @@ pub const PERMANENT_TABLE_TYPE: &str = "permanent"; pub const TRANSIENT_TABLE_DDL: &str = "TRANSIENT TABLE"; /// The permanent table DDL replacing the transient keyword. pub const PERMANENT_TABLE_DDL: &str = "TABLE"; +/// The warning code of an incremental model whose table keeps an outdated physical layout. +pub const PHYSICAL_LAYOUT_PENDING_WARNING_CODE: &str = "PHYSICAL_LAYOUT_PENDING"; +/// Model config keys of a table's physical layout, in the order warnings name them. +pub const PHYSICAL_LAYOUT_CONFIG_KEYS: [&str; 5] = [ + "engine", + "order_by", + "partition_by", + "primary_key", + "table_settings", +]; +/// The version metadata member holding the model config. +pub const METADATA_CONFIG_MEMBER: &str = "config"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs index b7532df463..7d2258959c 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs @@ -13,6 +13,7 @@ pub mod microbatch_limit_warning; pub mod microbatch_lookback; pub mod model_plan_action; pub mod model_warnings; +pub mod physical_layout_warning; pub mod schema_actions; pub mod singular_attachment; pub mod table_type_ddl; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/physical_layout_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/physical_layout_warning.rs new file mode 100644 index 0000000000..6ece279bb1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/physical_layout_warning.rs @@ -0,0 +1,51 @@ +//! The warning of an incremental model whose existing table keeps an outdated physical layout. + +use serde_json::Value; + +use crate::planner_strategy::constants::{ + METADATA_CONFIG_MEMBER, PHYSICAL_LAYOUT_CONFIG_KEYS, PHYSICAL_LAYOUT_PENDING_WARNING_CODE, +}; +use crate::planner_strategy::models::{MaterializationType, WarningRow, WarningSeverity}; + +/// The warning of changed layout keys on an incremental table the plan does not recreate. +pub fn physical_layout_warning( + model_name: &str, + (materialization, table_recreated): (MaterializationType, bool), + (previous_metadata_json, current_metadata_json): (Option<&str>, Option<&str>), +) -> Option { + if materialization != MaterializationType::Incremental || table_recreated { + return None; + } + let previous: Value = config(previous_metadata_json?)?; + let current: Value = config(current_metadata_json?)?; + let changed: Vec<&str> = PHYSICAL_LAYOUT_CONFIG_KEYS + .into_iter() + .filter(|key| layout_value(&previous, key) != layout_value(¤t, key)) + .collect(); + if changed.is_empty() { + return None; + } + Some(WarningRow { + severity: WarningSeverity::Warning, + message: format!( + "{model_name}: {} changed, but the existing table keeps its old physical layout; \ + a full refresh (`sqb build {model_name} --full-refresh`) rebuilds it with the new \ + layout", + changed.join(", ") + ), + code: Some(PHYSICAL_LAYOUT_PENDING_WARNING_CODE), + }) +} + +/// The recorded config, or none for metadata that is not a JSON object with one. +fn config(metadata_json: &str) -> Option { + match serde_json::from_str::(metadata_json) { + Ok(mut metadata) => Some(metadata.get_mut(METADATA_CONFIG_MEMBER)?.take()), + Err(_) => None, + } +} + +/// A key's value, with an absent key read as null. +fn layout_value<'a>(config: &'a Value, key: &str) -> &'a Value { + config.get(key).unwrap_or(&Value::Null) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs index 9cc302c56a..80d6b312c9 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs @@ -2,6 +2,8 @@ mod helpers; #[path = "tests/test_model_entry.rs"] mod model_entry; +#[path = "tests/test_physical_layout_warning.rs"] +mod physical_layout_warning; #[path = "tests/test_planner_strategy.rs"] mod planner_strategy; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_physical_layout_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_physical_layout_warning.rs new file mode 100644 index 0000000000..4d14a424c8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_physical_layout_warning.rs @@ -0,0 +1,76 @@ +use crate::planner_strategy::main::physical_layout_warning::physical_layout_warning; +use crate::planner_strategy::models::MaterializationType; +use crate::planner_strategy::tests::test_types::PhysicalLayoutWarningTestCase; + +const MONTHLY: &str = + r#"{"config": {"materialized": "incremental", "partition_by": "toYYYYMM(ordered_at)"}}"#; +const WEEKLY_SORTED: &str = r#"{"config": {"materialized": "incremental", "order_by": ["order_id"], "partition_by": "toMonday(ordered_at)"}}"#; +const MONTHLY_RETAGGED: &str = r#"{"config": {"materialized": "incremental", "partition_by": "toYYYYMM(ordered_at)", "tags": ["daily"]}}"#; + +#[test] +fn given_changed_layout_when_planning_then_only_kept_incremental_tables_warn() { + let test_cases = [ + PhysicalLayoutWarningTestCase { + description: "incremental run keeps the old layout", + materialization: MaterializationType::Incremental, + table_recreated: false, + previous_metadata_json: Some(MONTHLY), + current_metadata_json: WEEKLY_SORTED, + expected_message: Some( + "orders: order_by, partition_by changed, but the existing table keeps its old \ + physical layout; a full refresh (`sqb build orders --full-refresh`) rebuilds it \ + with the new layout", + ), + }, + PhysicalLayoutWarningTestCase { + description: "a rebuilt incremental table takes the new layout", + materialization: MaterializationType::Incremental, + table_recreated: true, + previous_metadata_json: Some(MONTHLY), + current_metadata_json: WEEKLY_SORTED, + expected_message: None, + }, + PhysicalLayoutWarningTestCase { + description: "a table model is always rebuilt", + materialization: MaterializationType::Table, + table_recreated: true, + previous_metadata_json: Some(MONTHLY), + current_metadata_json: WEEKLY_SORTED, + expected_message: None, + }, + PhysicalLayoutWarningTestCase { + description: "other config changes", + materialization: MaterializationType::Incremental, + table_recreated: false, + previous_metadata_json: Some(MONTHLY), + current_metadata_json: MONTHLY_RETAGGED, + expected_message: None, + }, + PhysicalLayoutWarningTestCase { + description: "no recorded build", + materialization: MaterializationType::Incremental, + table_recreated: false, + previous_metadata_json: None, + current_metadata_json: WEEKLY_SORTED, + expected_message: None, + }, + ]; + + for test_case in test_cases { + let warning = physical_layout_warning( + "orders", + (test_case.materialization, test_case.table_recreated), + ( + test_case.previous_metadata_json, + Some(test_case.current_metadata_json), + ), + ); + + assert_eq!( + warning.as_ref().map(|row| row.message.as_str()), + test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs index 325815169d..611b3fa01e 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs @@ -103,3 +103,12 @@ pub(super) struct TableTypeDdlTestCase { pub(super) table_type: &'static str, pub(super) expected_ddl: &'static str, } + +pub(super) struct PhysicalLayoutWarningTestCase { + pub(super) description: &'static str, + pub(super) materialization: MaterializationType, + pub(super) table_recreated: bool, + pub(super) previous_metadata_json: Option<&'static str>, + pub(super) current_metadata_json: &'static str, + pub(super) expected_message: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index 7ee3feb5c9..eff32c12da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -25,3 +25,5 @@ pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; pub(crate) const PHYSICAL_STORAGE_DIALECT: &str = "clickhouse"; /// The error code for physical storage keys on an adapter without them. pub(crate) const PHYSICAL_STORAGE_UNSUPPORTED_CODE: &str = "A011"; +/// Plan action values that create the model's table anew or leave it untouched. +pub(crate) const TABLE_RECREATING_ACTIONS: [&str; 2] = ["create_table", "skip"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index e16356d4e3..37aacdcee8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -5,7 +5,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_lookback; +use sqlbuild_analysis::planner_strategy::main::physical_layout_warning::physical_layout_warning; use sqlbuild_analysis::planner_strategy::main::table_type_ddl::table_type_ddl; use sqlbuild_analysis::planner_strategy::models::{CursorTypeCheck, LookbackConfig}; use sqlbuild_model_config::execution_limits::constants::MODEL_MAX_DURATION_KEY; @@ -24,7 +26,7 @@ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_strategy::constants::{ CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, - PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, + PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, TABLE_RECREATING_ACTIONS, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; use crate::bindings::_helpers::planner_strategy::cursor_types::{ @@ -36,7 +38,9 @@ use crate::bindings::_helpers::planner_strategy::models::ModelEntryDecisions; use crate::bindings::_helpers::planner_strategy::strategy::{ model_warning_records, resolve_schema_actions, warning_inputs, }; -use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; +use crate::bindings::_helpers::planner_strategy::values::{ + input_error, optional_string, plan_warning, text, +}; use crate::bindings::_helpers::records::construct::{enum_from_value, new_record, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -400,9 +404,37 @@ fn model_warnings<'py>( if let Some(cursor_warning) = cursor_type_warning_record(&name, &check)? { warnings.push(cursor_warning); } + if let Some(layout_warning) = layout_warning(request, &name)? { + warnings.push(layout_warning); + } PyTuple::new(py, warnings) } +/// The warning of an incremental model whose table keeps a physical layout it no longer declares. +fn layout_warning<'py>( + request: &ModelEntryDecisions<'py>, + name: &Bound<'py, PyAny>, +) -> PyResult>> { + let action: String = text(&native_field(&request.action, "value")?)?; + let metadata = |field: &str| -> PyResult> { + optional_string(&native_field(&request.change_result, field)?) + }; + let (previous, current) = ( + metadata("previous_metadata_json")?, + metadata("fingerprint_metadata_json")?, + ); + physical_layout_warning( + &text(name)?, + ( + materialization_type(Some(&text(&request.materialization_type)?)), + TABLE_RECREATING_ACTIONS.contains(&action.as_str()), + ), + (previous.as_deref(), current.as_deref()), + ) + .map(|row| plan_warning(name, row)) + .transpose() +} + /// The cursor bounds the logical DDL renders: placeholders for runtime-owned windows, else the batch range or cursor bounds. fn ddl_cursor_bounds<'py>(request: &ModelEntryDecisions<'py>) -> PyResult> { let py: Python<'py> = request.model.py(); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index f9c36065bf..a86f5b174d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -119,7 +119,7 @@ MODEL ( Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. -The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table, whose layout only changes on `--full-refresh`. +The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. ClickHouse rejects nullable sorting and partitioning keys by default. Wrap the column in `assumeNotNull()` or `coalesce()` in the expression, or set `table_settings (allow_nullable_key true)`. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 02facb4d7a..a0cbdc7e59 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -56,3 +56,14 @@ class ClickHousePhysicalLayoutE2ETestCase: expected_create_fragments: tuple[str, ...] expected_partition_rewrites: dict[str, bool] expected_totals: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseLayoutChangeE2ETestCase: + description: str + first_partition_by: str + changed_partition_by: str + full_refresh_args: tuple[str, ...] + expected_warning_codes: tuple[str, ...] + expected_full_refresh_warning_codes: tuple[str, ...] + expected_rebuilt_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 8f85206a3a..a04079ad6a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -195,7 +195,7 @@ def build_orders_project_files(*, schema_name: str, config: dict[str, object]) - def build_partitioned_events_project_files( - *, schema_name: str, config: dict[str, object] + *, schema_name: str, config: dict[str, object], partition_by: str = "toYYYYMM(event_time)" ) -> dict[str, str]: return { "sqlbuild_project.toml": build_clickhouse_project_toml( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py index d7e038ea7e..732a93ae46 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_layout.py @@ -1,11 +1,13 @@ from __future__ import annotations +import json import subprocess from pathlib import Path import pytest from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseLayoutChangeE2ETestCase, ClickHousePhysicalLayoutE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( @@ -114,3 +116,96 @@ def test_given_partitioned_layout_when_building_incrementally_then_only_touched_ ) == (test_case.expected_totals) finally: cleanup_clickhouse_schema(config=config, schema_name=schema_name) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseLayoutChangeE2ETestCase( + description="a changed partitioning warns until a full refresh applies it", + first_partition_by="toYYYYMM(event_time)", + changed_partition_by="toMonday(event_time)", + full_refresh_args=( + "--full-refresh", + "--start-cursor-ts", + "2026-01-01T00:00:00", + "--end-cursor-ts", + "2026-02-01T00:00:00", + ), + expected_warning_codes=("PHYSICAL_LAYOUT_PENDING",), + expected_full_refresh_warning_codes=(), + expected_rebuilt_fragment="PARTITION BY toMonday(event_time)", + ) + ], + ids=lambda case: case.description, +) +def test_given_changed_layout_when_planning_incremental_then_warns_until_full_refresh( + tmp_path: Path, + test_case: ClickHouseLayoutChangeE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_layout_change") + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_layout", + repo_files=build_partitioned_events_project_files( + schema_name=schema_name, config=config, partition_by=test_case.first_partition_by + ), + ) + create_clickhouse_schema(config=config, schema_name=schema_name) + execute_clickhouse_sql( + config=config, + statements=( + f"CREATE TABLE {schema_name}.raw_events (event_id Int64, amount Int64, " + "event_time DateTime64(6, 'UTC')) ENGINE = MergeTree ORDER BY event_id", + f"INSERT INTO {schema_name}.raw_events VALUES (1, 10, '2026-01-05 08:00:00')", + ), + ) + try: + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + model_path: Path = project_dir / "models" / "events.sql" + model_path.write_text( + model_path.read_text().replace( + test_case.first_partition_by, test_case.changed_partition_by + ) + ) + + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--json"), project_dir=project_dir + ) + full_refresh_plan: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan", "--json", *test_case.full_refresh_args), + project_dir=project_dir, + ) + rebuild: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", *test_case.full_refresh_args), project_dir=project_dir + ) + + assert (plan.returncode, full_refresh_plan.returncode, rebuild.returncode) == (0, 0, 0), ( + plan.stderr + full_refresh_plan.stderr + rebuild.stdout + rebuild.stderr + ) + assert tuple( + filter(None, (warning.get("code") for warning in json.loads(plan.stdout)["warnings"])) + ) == (test_case.expected_warning_codes) + assert tuple( + filter( + None, + ( + warning.get("code") + for warning in json.loads(full_refresh_plan.stdout)["warnings"] + ), + ) + ) == (test_case.expected_full_refresh_warning_codes) + assert test_case.expected_rebuilt_fragment in str( + fetch_clickhouse_rows( + config=config, + sql="SELECT create_table_query FROM system.tables " + f"WHERE database = '{schema_name}' AND name = 'events'", + )[0][0] + ) + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 1bd12dd634..6ca48f7897 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -108,7 +108,7 @@ MODEL ( Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. -The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table, whose layout only changes on `--full-refresh`. +The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. ClickHouse rejects nullable sorting and partitioning keys by default. Wrap the column in `assumeNotNull()` or `coalesce()` in the expression, or set `table_settings (allow_nullable_key true)`. From 99b813e1d1abd525fbaaf8fa706743f93b2f1927 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:34:41 +0100 Subject: [PATCH 21/34] fix: compare ClickHouse primary keys with sorting keys element by element A sorting key written as one tuple string such as 'customer_id, ordered_at' was compared whole against the primary key, so a valid prefix was rejected. Both keys are now split at their top-level commas, outside quotes and brackets and after removing enclosing parentheses, before the prefix check. --- .../_helpers/materialization.rs | 71 ++++++++++++++++++- .../tests/test_physical_storage.rs | 38 ++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index d715e32167..0500b3c29b 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -493,7 +493,9 @@ fn check_primary_key( let Some(primary_key) = expressions(config, PRIMARY_KEY_KEY)? else { return Ok(()); }; - let prefix: bool = order_by.is_some_and(|order_by| order_by.starts_with(&primary_key)); + let primary_key: Vec = key_elements(&primary_key); + let prefix: bool = + order_by.is_some_and(|order_by| key_elements(order_by).starts_with(&primary_key)); if prefix { return Ok(()); } @@ -502,6 +504,73 @@ fn check_primary_key( ))) } +/// The key's element expressions: each text unwrapped and split at its top-level commas. +fn key_elements(texts: &[String]) -> Vec { + texts + .iter() + .flat_map(|text| top_level_items(unwrapped(text.trim()))) + .collect() +} + +/// `text` without one pair of parentheses enclosing all of it. +fn unwrapped(text: &str) -> &str { + let inner: Option<&str> = text + .strip_prefix('(') + .and_then(|rest| rest.strip_suffix(')')); + match inner { + Some(inner) if balanced(inner) => inner, + _ => text, + } +} + +/// Whether every bracket in `text` outside quotes closes within it. +fn balanced(text: &str) -> bool { + let mut depth: i64 = 0; + let mut quote: Option = None; + for character in text.chars() { + match (quote, character) { + (Some(open), _) if character == open => quote = None, + (Some(_), _) => {} + (None, '\'' | '"' | '`') => quote = Some(character), + (None, '(' | '[') => depth += 1, + (None, ')' | ']') => { + depth -= 1; + if depth < 0 { + return false; + } + } + (None, _) => {} + } + } + depth == 0 && quote.is_none() +} + +/// `text` split at commas outside quotes and brackets, each item trimmed. +fn top_level_items(text: &str) -> Vec { + let mut items: Vec = Vec::new(); + let mut current: String = String::new(); + let mut depth: i64 = 0; + let mut quote: Option = None; + for character in text.chars() { + match (quote, character) { + (Some(open), _) if character == open => quote = None, + (Some(_), _) => {} + (None, '\'' | '"' | '`') => quote = Some(character), + (None, '(' | '[') => depth += 1, + (None, ')' | ']') => depth -= 1, + (None, ',') if depth == 0 => { + items.push(current.trim().to_owned()); + current.clear(); + continue; + } + (None, _) => {} + } + current.push(character); + } + items.push(current.trim().to_owned()); + items +} + fn check_table_settings(config: &ConfigView<'_, N>) -> Check { let Some(value) = config.get(TABLE_SETTINGS_KEY) else { return Ok(()); diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs index e03994b1c2..06366bc6d4 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs @@ -91,6 +91,44 @@ fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_a query_sql: "select 1", expected_outcome: "model 'orders_daily': primary_key must be a prefix of order_by", }, + ValidateTestCase { + description: "a primary key prefix of a single sorting tuple string", + config: table(vec![ + ("order_by", Value::Str("customer_id, ordered_at")), + ("primary_key", Value::Str("customer_id")), + ]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a parenthesized sorting tuple with nested calls and quotes", + config: table(vec![ + ( + "order_by", + Value::Str( + "(customer_id, toStartOfInterval(ordered_at, INTERVAL 1 day), 'a,b')", + ), + ), + ( + "primary_key", + Value::Str("(customer_id, toStartOfInterval(ordered_at, INTERVAL 1 day))"), + ), + ]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a primary key string that is not a sorting prefix", + config: table(vec![ + ("order_by", Value::Str("(customer_id, ordered_at)")), + ("primary_key", Value::Str("ordered_at")), + ]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': primary_key must be a prefix of order_by", + }, ValidateTestCase { description: "a setting with a list value", config: table(vec![( From 59e98a4f9310c853877737663b13209f6778bde1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:35:42 +0100 Subject: [PATCH 22/34] fix: refuse ClickHouse connection settings that override required ones User connection settings were applied over the settings SQLBuild's generated SQL depends on, so for example join_use_nulls = 0 silently broke snapshot inserts. Required settings are now applied last, and a user value that differs from one fails to connect with A710 naming the setting. --- .../docs/concepts/adapters/clickhouse.md | 2 +- .../clickhouse/classes/clickhouse_adapter.py | 34 +++++++++++++- src/sqlbuild/adapters/clickhouse/constants.py | 1 + .../adapters/clickhouse/_test_types.py | 8 ++++ .../clickhouse/test_session_settings.py | 46 +++++++++++++++++++ .../docs/concepts/adapters/clickhouse.mdx | 2 +- 6 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/test_session_settings.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index a86f5b174d..cb67acc258 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -159,7 +159,7 @@ Every query runs with these settings so SQL behaves the same as on other adapter | `mutations_sync`, `lightweight_deletes_sync` | `2` | Statements finish before the next one starts | | `date_time_input_format`, `cast_string_to_date_time_mode` | `best_effort` | ISO 8601 timestamps with `T` and offsets parse | -Values under `settings` in the connection override these. +The connection's `settings` may repeat these values but not change them: a different value fails to connect with `A710`, because SQLBuild's generated SQL depends on them. ClickHouse returns a type's default value (`0`, the epoch, an empty string) from `lag` and `lead` past the edge of a window frame unless the argument is `Nullable`. SQLBuild's generated snapshot SQL wraps those arguments in `toNullable` so they return `NULL` as on other adapters. Write the same in your own models when you rely on `NULL` there. diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 7d3996db13..5b13e0269a 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -117,6 +117,7 @@ RELATION_NOT_FOUND_CODES, RELATION_SEPARATOR, SESSION_SETTINGS, + SETTING_CONFLICT_CODE, SNAPSHOT_REBUILD_INFIX, SNAPSHOT_WORK_INFIX, STATE_TABLE_ENGINE_KEYWORD, @@ -180,8 +181,7 @@ def connect(self, config: dict[str, Any]) -> _ClickHouseConnection: help="Supported options: " + ", ".join(sorted(CONNECTION_OPTION_KEYS)), ) secure: bool = bool(config.get("secure", False)) - settings: dict[str, object] = dict(SESSION_SETTINGS) - settings.update(dict(config.get("settings") or {})) + settings: dict[str, object] = _session_settings(dict(config.get("settings") or {})) client_options: dict[str, Any] = { "host": str(config.get("host", "localhost")), "port": int(config.get("port", DEFAULT_HTTPS_PORT if secure else DEFAULT_HTTP_PORT)), @@ -2645,3 +2645,33 @@ def _prune_history_sql( f"PARTITION BY {partition} ORDER BY {order}) AS __sqlbuild_history_rank FROM {table}) " f"WHERE __sqlbuild_history_rank > {retain_versions}) SETTINGS transform_null_in = 1" ) + + +def _session_settings(user_settings: dict[str, object]) -> dict[str, object]: + """Merge user settings under the required ones, refusing a user value that differs.""" + + conflicts: tuple[str, ...] = tuple( + sorted( + name + for name, value in user_settings.items() + if name in SESSION_SETTINGS + and _setting_text(value) != _setting_text(SESSION_SETTINGS[name]) + ) + ) + if conflicts: + required: str = ", ".join(f"{name} = {SESSION_SETTINGS[name]!r}" for name in conflicts) + raise AdapterUserError( + message=( + "ClickHouse connection settings override settings SQLBuild requires: " + + ", ".join(conflicts) + ), + code=SETTING_CONFLICT_CODE, + help=f"Remove them from [connection.settings]; SQLBuild runs with {required}.", + ) + return {**user_settings, **SESSION_SETTINGS} + + +def _setting_text(value: object) -> str: + if isinstance(value, bool): + return "1" if value else "0" + return str(value).strip().lower() diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index 6f0d5574a3..d472527ba4 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -18,6 +18,7 @@ "date_time_input_format": "best_effort", "cast_string_to_date_time_mode": "best_effort", } +SETTING_CONFLICT_CODE: str = "A710" CONNECTION_OPTION_KEYS: frozenset[str] = frozenset( { "host", diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index 5da6716d68..654f8cdb15 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -64,3 +64,11 @@ class TableLayoutTestCase: description: str layout: dict[str, object] expected_clauses: str + + +@dataclass(frozen=True) +class SessionSettingConflictTestCase: + description: str + settings: dict[str, object] + expected_code: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_session_settings.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_session_settings.py new file mode 100644 index 0000000000..0788ebc8fb --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_session_settings.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import ( + SessionSettingConflictTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SessionSettingConflictTestCase( + description="join_use_nulls turned off", + settings={"join_use_nulls": 0, "max_threads": 4}, + expected_code="A710", + expected_message=( + "ClickHouse connection settings override settings SQLBuild requires: join_use_nulls" + ), + ), + SessionSettingConflictTestCase( + description="two conflicting settings", + settings={"union_default_mode": "ALL", "mutations_sync": False}, + expected_code="A710", + expected_message=( + "ClickHouse connection settings override settings SQLBuild requires: " + "mutations_sync, union_default_mode" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_conflicting_user_settings_when_connecting_then_coded_error_names_them( + test_case: SessionSettingConflictTestCase, +) -> None: + with pytest.raises(AdapterUserError) as error: + ClickHouseAdapter().connect( + {"host": "127.0.0.1", "port": 1, "connect_timeout": 1, "settings": test_case.settings} + ) + + assert (error.value.code, error.value.message) == ( + test_case.expected_code, + test_case.expected_message, + ) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 6ca48f7897..628aa9d72a 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -148,7 +148,7 @@ Every query runs with these settings so SQL behaves the same as on other adapter | `mutations_sync`, `lightweight_deletes_sync` | `2` | Statements finish before the next one starts | | `date_time_input_format`, `cast_string_to_date_time_mode` | `best_effort` | ISO 8601 timestamps with `T` and offsets parse | -Values under `settings` in the connection override these. +The connection's `settings` may repeat these values but not change them: a different value fails to connect with `A710`, because SQLBuild's generated SQL depends on them. ClickHouse returns a type's default value (`0`, the epoch, an empty string) from `lag` and `lead` past the edge of a window frame unless the argument is `Nullable`. SQLBuild's generated snapshot SQL wraps those arguments in `toNullable` so they return `NULL` as on other adapters. Write the same in your own models when you rely on `NULL` there. From c19f81d2d18141ceddd180b506f925f5f08cfbf4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:40:14 +0100 Subject: [PATCH 23/34] fix: approve the ClickHouse connection's raw query boundary The ClickHouse connection runs client queries inside its statement lifecycle, like the other adapters' connection wrappers, but was missing from the raw statement audit's approved boundaries. --- scripts/fensu_policy/constants.py | 1 + src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 738556e895..f7d6084c08 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -201,6 +201,7 @@ "src/sqlbuild/adapter/contract/classes/observed_connection.py", "src/sqlbuild/adapter/contract/classes/observed_cursor.py", "src/sqlbuild/adapters/bigquery/classes/bigquery_connection.py", + "src/sqlbuild/adapters/clickhouse/classes/clickhouse_connection.py", "src/sqlbuild/adapters/databricks/classes/databricks_connection.py", "src/sqlbuild/adapters/postgres/classes/postgres_connection.py", "src/sqlbuild/adapters/snowflake/classes/snowflake_connection.py", diff --git a/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py b/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py index a8c18616ac..9e4f896e16 100644 --- a/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py +++ b/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py @@ -24,6 +24,9 @@ "adapters/bigquery/classes/bigquery_connection.py": frozenset( {("_BigQueryConnection.query_job", "query")} ), + "adapters/clickhouse/classes/clickhouse_connection.py": frozenset( + {("_ClickHouseConnection._run", "query")} + ), "adapters/databricks/classes/databricks_connection.py": frozenset( {("_DatabricksConnection._run_setting", "execute")} ), From 64d4e694e4fce826db8bb2884fe1b7fbbdfd809d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:43:10 +0100 Subject: [PATCH 24/34] fix: store NULL as NULL when a ClickHouse model starts producing it ClickHouse tables built from a query take each column's nullability from it, and inserting NULL into a non-nullable column stored the type's default ('', 0, the epoch) under on_schema_change ignore. Incremental and microbatch runs now widen such target columns to Nullable before the schema comparison through a new widen_target_nullability adapter method (a no-op elsewhere), insert_null_as_default is pinned to 0 so any other mismatch fails, and diff_schema reports nullability differences. --- .../docs/concepts/adapters/clickhouse.md | 6 + .../adapter/contract/classes/base_adapter.py | 14 ++ .../contract/classes/duckdb_backed_adapter.py | 14 ++ .../contract/classes/strict_adapter.py | 13 ++ .../bigquery/classes/bigquery_adapter.py | 14 ++ .../clickhouse/classes/clickhouse_adapter.py | 42 +++++- src/sqlbuild/adapters/clickhouse/constants.py | 1 + .../databricks/classes/databricks_adapter.py | 14 ++ .../postgres/classes/postgres_adapter.py | 14 ++ .../snowflake/classes/snowflake_adapter.py | 14 ++ .../sqlserver/classes/sqlserver_adapter.py | 14 ++ .../_helpers/materializations/incremental.py | 16 +- .../_helpers/materializations/microbatch.py | 7 + .../commands/main/clickhouse/_test_types.py | 9 ++ .../clickhouse/test_clickhouse_nullability.py | 138 ++++++++++++++++++ .../adapters/clickhouse/_test_types.py | 8 + .../adapters/clickhouse/test_client.py | 50 ++++++- .../docs/concepts/adapters/clickhouse.mdx | 5 + 18 files changed, 385 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index cb67acc258..b9a6d1b760 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/clickhouse/ - Reading ReplacingMergeTree sources - Functions - Clones +- Nullable columns - Settings SQLBuild pins - Limits @@ -146,6 +147,10 @@ SQLBuild does not deploy Python functions (`A707`) or SQL table functions (`A708 `sqb clone` creates the clone with `CREATE TABLE ... AS` and attaches every part of the origin with `ALTER TABLE ... ATTACH PARTITION ALL FROM`, which hard-links the data instead of copying it. The clone keeps the origin's engine, keys and settings and stays valid after the origin is dropped. +## Nullable columns + +ClickHouse columns hold `NULL` only when their type is `Nullable`, and a table built from a query takes each column's nullability from the query. When an incremental model later produces `NULL` in a column its table created as non-nullable, SQLBuild changes that column to `Nullable` with `ALTER TABLE ... MODIFY COLUMN` before writing, so the `NULL` is stored as `NULL`. A column in the sorting or partitioning key cannot become `Nullable` unless the table sets `allow_nullable_key`, so that write fails instead. + ## Settings SQLBuild pins Every query runs with these settings so SQL behaves the same as on other adapters: @@ -158,6 +163,7 @@ Every query runs with these settings so SQL behaves the same as on other adapter | `default_table_engine` | `MergeTree` | Tables created without an engine clause are MergeTree tables | | `mutations_sync`, `lightweight_deletes_sync` | `2` | Statements finish before the next one starts | | `date_time_input_format`, `cast_string_to_date_time_mode` | `best_effort` | ISO 8601 timestamps with `T` and offsets parse | +| `insert_null_as_default` | `0` | Inserting `NULL` into a column that cannot hold it fails instead of storing `0`, `''` or the epoch | The connection's `settings` may repeat these values but not change them: a different value fails to connect with `A710`, because SQLBuild's generated SQL depends on them. diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 58df5e5cc2..23c722f41b 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -1465,6 +1465,20 @@ def merge( ) -> int | None: raise AdapterUserError(message="merge requires an engine-specific implementation") + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Let target columns admit NULL where incoming ones do; return the target columns.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def add_columns( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index bb1ad30829..30b52b2c6b 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -2042,6 +2042,20 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Return the target columns unchanged; this warehouse never stores NULL as a default.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index b9733a11f8..aa24bad368 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -47,6 +47,19 @@ class StrictAdapter( adapter_name: ClassVar[str] + @abstractmethod + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Let target columns admit NULL where incoming ones do; return the target columns.""" + ... + @abstractmethod def supports_zero_copy_clone(self) -> bool: """Return whether clone operations can use zero-copy semantics.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index fa50a792f6..e4b1b6b8d0 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -841,6 +841,20 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Return the target columns unchanged; this warehouse never stores NULL as a default.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 5b13e0269a..5ea31ff3ab 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -97,6 +97,7 @@ from sqlbuild.adapters.clickhouse._helpers.table_layout import render_table_layout from sqlbuild.adapters.clickhouse._helpers.type_text import ( canonical_type, + is_nullable_type, nullable_type, quote_string, ) @@ -1478,8 +1479,13 @@ def diff_schema(self, *, connection: Any, left: str, right: str) -> SchemaDiffRe (ColumnInfo(name=name, type=left_map[name]), ColumnInfo(name=name, type=type_sql)) for name, type_sql in right_map.items() if name in left_map - and not types_equal( - left=canonical_type(left_map[name]), right=canonical_type(type_sql), dialect=None + and ( + not types_equal( + left=canonical_type(left_map[name]), + right=canonical_type(type_sql), + dialect=None, + ) + or is_nullable_type(left_map[name]) != is_nullable_type(type_sql) ) ) return SchemaDiffResult( @@ -2007,6 +2013,38 @@ def add_columns( for statement in statements: self.execute(connection=connection, sql=statement) + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Make target columns ``Nullable`` where the incoming column of the same type is.""" + + incoming: dict[str, str] = {column.name.lower(): column.type for column in incoming_columns} + widened: tuple[ColumnInfo, ...] = tuple( + ColumnInfo(name=column.name, type=nullable_type(column.type)) + for column in target_columns + if column.name.lower() in incoming + and is_nullable_type(incoming[column.name.lower()]) + and not is_nullable_type(column.type) + and nullable_type(column.type) != column.type + and canonical_type(incoming[column.name.lower()]) == canonical_type(column.type) + ) + if not widened: + return target_columns + self.alter_column_types( + connection=connection, + destination=destination, + columns=widened, + statement_recorder=statement_recorder, + ) + by_name: dict[str, ColumnInfo] = {column.name: column for column in widened} + return tuple(by_name.get(column.name, column) for column in target_columns) + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index d472527ba4..3ba3597d12 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -17,6 +17,7 @@ "lightweight_deletes_sync": 2, "date_time_input_format": "best_effort", "cast_string_to_date_time_mode": "best_effort", + "insert_null_as_default": 0, } SETTING_CONFLICT_CODE: str = "A710" CONNECTION_OPTION_KEYS: frozenset[str] = frozenset( diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 9006a13ce2..ad6baf39ba 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -2507,6 +2507,20 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) "the table protocol, or full-refresh the model" ) + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Return the target columns unchanged; this warehouse never stores NULL as a default.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 8a8e9af2d6..7f5e14cf43 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -2271,6 +2271,20 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Return the target columns unchanged; this warehouse never stores NULL as a default.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 0cb29d1848..1ea747e1c7 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3274,6 +3274,20 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Return the target columns unchanged; this warehouse never stores NULL as a default.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index ba50791eba..a89f7d79df 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -1046,6 +1046,20 @@ def add_columns( ) -> None: raise AdapterUserError(message="add_columns requires an engine-specific implementation") + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Return the target columns unchanged; this warehouse never stores NULL as a default.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def alter_column_types( self, *, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 5e2ffcfc01..b3297ebcad 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -200,12 +200,18 @@ def execute_incremental_entry( schema=target_schema, name=delta_table, ) - target_columns: tuple[ColumnInfo, ...] = inspect_runtime_relation_schema( - adapter=adapter, + target_columns: tuple[ColumnInfo, ...] = adapter.widen_target_nullability( connection=connection, - database=target_database, - schema=target_schema, - name=target_table, + destination=target_qualified, + target_columns=inspect_runtime_relation_schema( + adapter=adapter, + connection=connection, + database=target_database, + schema=target_schema, + name=target_table, + ), + incoming_columns=initial_delta_columns, + statement_recorder=statement_recorder, ) delta_columns, type_warnings = _prepare_incremental_delta_types( adapter=adapter, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index d42b325d1f..ccdad343b9 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -2861,6 +2861,13 @@ def _apply_microbatch_schema_change( schema=targets.target_schema, name=targets.target_table, ) + target_columns = context.adapter.widen_target_nullability( + connection=context.connection, + destination=targets.target_qualified, + target_columns=target_columns, + incoming_columns=delta_columns, + statement_recorder=state.statement_recorder, + ) schema_warnings: tuple[str, ...] = _apply_schema_change( adapter=context.adapter, connection=context.connection, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index a0cbdc7e59..2eef65d97c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -67,3 +67,12 @@ class ClickHouseLayoutChangeE2ETestCase: expected_warning_codes: tuple[str, ...] expected_full_refresh_warning_codes: tuple[str, ...] expected_rebuilt_fragment: str + + +@dataclass(frozen=True) +class ClickHouseNullabilityE2ETestCase: + description: str + model_sql: str + edited_select: str + expected_rows: tuple[tuple[object, ...], ...] + expected_types: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py new file mode 100644 index 0000000000..0bfe0fbc5c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseNullabilityE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + build_clickhouse_project_toml, + build_unique_schema_name, + cleanup_clickhouse_schema, + create_clickhouse_schema, + execute_clickhouse_sql, + fetch_clickhouse_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, + stringify_warehouse_rows, +) + +_FIRST_SELECT: str = ( + "SELECT o.order_id AS order_id, o.amount AS amount, toString(o.order_id) AS note\n" + 'FROM __source("raw_orders") AS o\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseNullabilityE2ETestCase( + description="nullIf on a merge model", + model_sql=( + "MODEL (description 'One row per order.',\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [order_id],\n" + ");\n\n" + ), + edited_select=( + "SELECT o.order_id AS order_id, nullIf(o.amount, 20) AS amount, " + "nullIf(toString(o.order_id), '3') AS note\n" + 'FROM __source("raw_orders") AS o\n' + ), + expected_rows=(("1", "10", "1"), ("2", None, "2"), ("3", "30", None)), + expected_types=(("Nullable(Int64)", "Nullable(String)"),), + ), + ClickHouseNullabilityE2ETestCase( + description="left join on an append model that ignores schema changes", + model_sql=( + "MODEL (description 'One row per order load.',\n" + " materialized incremental,\n" + " incremental_strategy append,\n" + " on_schema_change ignore,\n" + ");\n\n" + ), + edited_select=( + "SELECT o.order_id AS order_id, o.amount AS amount, r.note AS note\n" + 'FROM __source("raw_orders") AS o\n' + "LEFT JOIN (SELECT toInt64(1) AS order_id, 'one' AS note) AS r " + "ON o.order_id = r.order_id\n" + ), + expected_rows=( + ("1", "10", "1"), + ("1", "10", "one"), + ("2", "20", "2"), + ("2", "20", None), + ("3", "30", "3"), + ("3", "30", None), + ), + expected_types=(("Int64", "Nullable(String)"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_non_nullable_target_when_model_starts_producing_nulls_then_nulls_are_stored( + tmp_path: Path, + test_case: ClickHouseNullabilityE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_nulls") + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_nulls", + repo_files={ + "sqlbuild_project.toml": build_clickhouse_project_toml( + project_name="clickhouse_nulls", schema_name=schema_name, config=config + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n" + " description: Synthetic orders.\n" + f" schema: {schema_name}\n" + " table: raw_orders\n" + ), + "models/orders.sql": test_case.model_sql + _FIRST_SELECT, + }, + ) + create_clickhouse_schema(config=config, schema_name=schema_name) + execute_clickhouse_sql( + config=config, + statements=( + f"CREATE TABLE {schema_name}.raw_orders (order_id Int64, amount Int64) " + "ENGINE = MergeTree ORDER BY order_id", + f"INSERT INTO {schema_name}.raw_orders VALUES (1, 10), (2, 20), (3, 30)", + ), + ) + try: + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first.returncode == 0, first.stdout + first.stderr + (project_dir / "models" / "orders.sql").write_text( + test_case.model_sql + test_case.edited_select + ) + + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert second.returncode == 0, second.stdout + second.stderr + assert stringify_warehouse_rows( + fetch_clickhouse_rows( + config=config, + sql=f"SELECT order_id, amount, note FROM {schema_name}.orders " + "ORDER BY order_id, note NULLS LAST", + ) + ) == (test_case.expected_rows) + assert fetch_clickhouse_rows( + config=config, + sql=f"SELECT toTypeName(amount), toTypeName(note) FROM {schema_name}.orders LIMIT 1", + ) == (test_case.expected_types) + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py index ce7910fd63..ec2120a123 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -139,3 +139,11 @@ class ClickHouseStateTimestampTestCase: description: str created_at: datetime expected_created_at: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseSchemaDiffTestCase: + description: str + left_ddl_columns: str + right_ddl_columns: str + expected_type_changes: tuple[tuple[str, str, str], ...] diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py index 5c532b3db3..a343b17563 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py @@ -10,7 +10,13 @@ from sqlbuild._native import build_microbatch_insert_many_sql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.models import ColumnInfo, CursorValue, QueryResult, RelationInfo +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + CursorValue, + QueryResult, + RelationInfo, + SchemaDiffResult, +) from sqlbuild.adapter.contract.types import RelationReadStatus from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter from sqlbuild.compiler.compile.types import FunctionLanguage @@ -27,6 +33,7 @@ ClickHouseRefusedFunctionTestCase, ClickHouseRewriteFaultTestCase, ClickHouseRewriteTestCase, + ClickHouseSchemaDiffTestCase, ClickHouseSeedTestCase, ClickHouseStateTableTestCase, ClickHouseStateTimestampTestCase, @@ -894,3 +901,44 @@ def test_given_microbatch_event_when_publishing_state_then_clickhouse_keeps_micr connection=connection, sql=f"SELECT created_at FROM {clickhouse_schema}._sqlbuild_microbatches", ) == (test_case.expected_created_at) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseSchemaDiffTestCase( + description="nullability differences are type changes", + left_ddl_columns="id Int64, amount Int64, note Nullable(String), code String", + right_ddl_columns=( + "id Int64, amount Nullable(Int64), note String, code LowCardinality(String)" + ), + expected_type_changes=( + ("amount", "Int64", "Nullable(Int64)"), + ("note", "Nullable(String)", "String"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_relations_differing_in_nullability_when_diffing_schema_then_changes_are_reported( + test_case: ClickHouseSchemaDiffTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + left: str = f"{clickhouse_schema}.left_orders" + right: str = f"{clickhouse_schema}.right_orders" + adapter.execute( + connection=connection, + sql=f"CREATE TABLE {left} ({test_case.left_ddl_columns}) ENGINE = MergeTree ORDER BY id", + ) + adapter.execute( + connection=connection, + sql=f"CREATE TABLE {right} ({test_case.right_ddl_columns}) ENGINE = MergeTree ORDER BY id", + ) + + result: SchemaDiffResult = adapter.diff_schema(connection=connection, left=left, right=right) + + assert tuple( + (before.name, before.type, after.type) for before, after in result.type_changed_columns + ) == (test_case.expected_type_changes) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 628aa9d72a..18b5bf718b 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -135,6 +135,10 @@ SQLBuild does not deploy Python functions (`A707`) or SQL table functions (`A708 `sqb clone` creates the clone with `CREATE TABLE ... AS` and attaches every part of the origin with `ALTER TABLE ... ATTACH PARTITION ALL FROM`, which hard-links the data instead of copying it. The clone keeps the origin's engine, keys and settings and stays valid after the origin is dropped. +## Nullable columns + +ClickHouse columns hold `NULL` only when their type is `Nullable`, and a table built from a query takes each column's nullability from the query. When an incremental model later produces `NULL` in a column its table created as non-nullable, SQLBuild changes that column to `Nullable` with `ALTER TABLE ... MODIFY COLUMN` before writing, so the `NULL` is stored as `NULL`. A column in the sorting or partitioning key cannot become `Nullable` unless the table sets `allow_nullable_key`, so that write fails instead. + ## Settings SQLBuild pins Every query runs with these settings so SQL behaves the same as on other adapters: @@ -147,6 +151,7 @@ Every query runs with these settings so SQL behaves the same as on other adapter | `default_table_engine` | `MergeTree` | Tables created without an engine clause are MergeTree tables | | `mutations_sync`, `lightweight_deletes_sync` | `2` | Statements finish before the next one starts | | `date_time_input_format`, `cast_string_to_date_time_mode` | `best_effort` | ISO 8601 timestamps with `T` and offsets parse | +| `insert_null_as_default` | `0` | Inserting `NULL` into a column that cannot hold it fails instead of storing `0`, `''` or the epoch | The connection's `settings` may repeat these values but not change them: a different value fails to connect with `A710`, because SQLBuild's generated SQL depends on them. From 08248003d3b0a490448e2bae4203926a933d4377 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:50:28 +0100 Subject: [PATCH 25/34] fix: publish multi-partition ClickHouse rewrites with one atomic exchange A rewrite that touched several partitions swapped them in one at a time. An interruption between swaps could leave a newer partition published and an older one not, and the next run's cursor, read from the table's maximum, then skipped the unpublished rows for good. Several partitions are now rebuilt in a copy of the table that hard-links every kept part and are published with one EXCHANGE TABLES, under an in-process exclusive lock on the table; single-partition swaps keep REPLACE PARTITION under a shared table lock and their partition locks. The copy carries the table's generation in its comment, so microbatch state keeps its scope across the exchange. A fault test interrupts every statement of the rewrite and checks the next cursor run against the DuckDB oracle. --- .../docs/concepts/adapters/clickhouse.md | 6 +- .../clickhouse/_helpers/partition_rewrite.py | 116 ++++++++++--- .../clickhouse/classes/clickhouse_adapter.py | 28 +++- .../classes/relation_write_locks.py | 48 ++++++ src/sqlbuild/adapters/clickhouse/constants.py | 1 + src/sqlbuild/adapters/clickhouse/models.py | 15 +- .../adapters/clickhouse/_test_types.py | 10 +- .../sqlbuild/adapters/clickhouse/helpers.py | 28 ++++ .../adapters/clickhouse/test_client.py | 154 +++++++++++++----- .../docs/concepts/adapters/clickhouse.mdx | 6 +- 10 files changed, 336 insertions(+), 76 deletions(-) create mode 100644 src/sqlbuild/adapters/clickhouse/classes/relation_write_locks.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index b9a6d1b760..8267f988d4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -75,12 +75,12 @@ SQLBuild never relies on background merges for correctness, and it never updates - **Tables** build in a staging table and replace the live table with `EXCHANGE TABLES`, which is atomic. This needs databases with the `Atomic` engine, the default; other engines fail with `A705`. - **Views** use `CREATE OR REPLACE VIEW`. - **Incremental `append`** inserts the new rows. -- **Incremental `merge`, `delete_insert` and cursor windows** rewrite whole partitions. SQLBuild stages the incoming rows, finds the partitions they touch and the partitions holding the rows they replace, rebuilds those partitions in a staging table, and swaps each one in with `ALTER TABLE ... REPLACE PARTITION`. Each swap is atomic. If a build stops before the swaps the table is untouched; if it stops between swaps, some partitions are already new, and rerunning the build recomputes the rest and converges. -- **Microbatches** use the same partition rewrite for each batch. Concurrent batches (`batch_concurrency`) rewrite different partitions in parallel; batches of one `sqb` process that share a partition take turns. +- **Incremental `merge`, `delete_insert` and cursor windows** rewrite whole partitions. SQLBuild stages the incoming rows and finds the partitions they touch and the partitions holding the rows they replace. One partition is rebuilt in a staging table and swapped in with `ALTER TABLE ... REPLACE PARTITION`. Several partitions are published together: SQLBuild copies the table by hard-linking its parts (no data is rewritten), rebuilds the touched partitions in the copy and swaps the copy in with one `EXCHANGE TABLES`. Either way the swap is atomic, so a build interrupted at any point leaves the table as it was before the run or with every partition of the run, never part of them, and the next run's cursor window cannot skip rows. +- **Microbatches** use the same partition rewrite for each batch, so each batch is published atomically. Concurrent batches (`batch_concurrency`) swap different single partitions in parallel; batches of one `sqb` process that share a partition, or that publish several partitions with an exchange, take turns. ### One writer per model -A partition rewrite reads the partition, rebuilds it and swaps it in. ClickHouse has no transaction that spans those steps, and SQLBuild only serializes rewrites of the same partition inside one `sqb` process. If two `sqb` processes write the same model at the same time and touch the same partition, the later swap wins and the rows the other process wrote to that partition are lost. +A partition rewrite reads the table, rebuilds partitions and swaps them in. ClickHouse has no transaction that spans those steps, and SQLBuild only serializes rewrites of one table inside one `sqb` process. If two `sqb` processes write the same model at the same time, the later swap wins and rows the other process wrote can be lost. Run each model from one process at a time: one scheduled job per model or per project, with no overlapping runs. Concurrency within a run (`--concurrency`, `batch_concurrency`) is safe. - **Snapshots** apply their changes to a private copy of the snapshot table and exchange it in. Closing a version rebuilds the copy rather than updating rows, so the live snapshot changes only at the final exchange. diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py b/src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py index 5fff1d9a70..0f78e56503 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/partition_rewrite.py @@ -3,22 +3,25 @@ from __future__ import annotations import hashlib +from collections.abc import Callable from typing import Any +from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.clickhouse._helpers.type_text import quote_string from sqlbuild.adapters.clickhouse.constants import ( + GENERATION_COMMENT_PREFIX, MAX_IDENTIFIER_LENGTH, REWRITE_DELTA_INFIX, REWRITE_STAGING_INFIX, ) -from sqlbuild.adapters.clickhouse.models import PartitionRewrite, RewriteExecutor +from sqlbuild.adapters.clickhouse.models import PartitionRewrite, RewriteExecutor, RewriteTables from sqlbuild.runtime.execution_limits.main.execution_deadlines_suspended import ( execution_deadlines_suspended, ) def rewrite_partitions(*, request: PartitionRewrite, executor: RewriteExecutor) -> int: - """Apply one rewrite and return the number of incoming rows.""" + """Apply one rewrite atomically and return the number of incoming rows.""" digest: str = hashlib.sha256( f"{request.destination}\0{request.delta_insert_sql}\0{request.displaced_predicate}".encode() @@ -26,10 +29,14 @@ def rewrite_partitions(*, request: PartitionRewrite, executor: RewriteExecutor) delta: str = artifact_relation( relation=request.destination, infix=REWRITE_DELTA_INFIX, digest=digest ) - staging: str = artifact_relation( - relation=request.destination, infix=REWRITE_STAGING_INFIX, digest=digest + tables: RewriteTables = RewriteTables( + destination=request.destination, + delta=delta, + staging=artifact_relation( + relation=request.destination, infix=REWRITE_STAGING_INFIX, digest=digest + ), + displaced_predicate=request.displaced_predicate.replace("{delta}", delta), ) - predicate: str = request.displaced_predicate.replace("{delta}", delta) def run(sql: str) -> Any: executor.statement_recorder.record(sql) @@ -46,36 +53,93 @@ def run(sql: str) -> Any: for row in run( f"SELECT DISTINCT _partition_id FROM {delta} UNION DISTINCT " f"SELECT DISTINCT _partition_id FROM {request.destination} " - f"WHERE ifNull({predicate}, 0)" + f"WHERE ifNull({tables.displaced_predicate}, 0)" ).fetchall() ) ) - if not partitions: - return incoming - partition_list: str = ", ".join(quote_string(partition) for partition in partitions) - with executor.partition_locks.locked( - destination=request.destination, partitions=partitions - ): - run(f"DROP TABLE IF EXISTS {staging}") - run(f"CREATE TABLE {staging} AS {request.destination}") - run( - f"INSERT INTO {staging} SELECT * FROM {request.destination} " - f"WHERE _partition_id IN ({partition_list}) AND NOT ifNull({predicate}, 0)" - ) - run(f"INSERT INTO {staging} SELECT * FROM {delta}") - partition: str - for partition in partitions: - run( - f"ALTER TABLE {request.destination} REPLACE PARTITION ID " - f"{quote_string(partition)} FROM {staging}" - ) + if len(partitions) > 1: + _exchange_replacement(run=run, tables=tables, partitions=partitions, executor=executor) + else: + _replace_partition(run=run, tables=tables, partitions=partitions, executor=executor) return incoming finally: with execution_deadlines_suspended(): - run(f"DROP TABLE IF EXISTS {staging}") + run(f"DROP TABLE IF EXISTS {tables.staging}") run(f"DROP TABLE IF EXISTS {delta}") +def _replace_partition( + *, + run: Callable[[str], Any], + tables: RewriteTables, + partitions: tuple[str, ...], + executor: RewriteExecutor, +) -> None: + """Swap the only rewritten partition in place; nothing to do without one.""" + + if not partitions: + return + with ( + executor.relation_locks.shared(destination=tables.destination), + executor.partition_locks.locked(destination=tables.destination, partitions=partitions), + ): + run(f"DROP TABLE IF EXISTS {tables.staging}") + run(f"CREATE TABLE {tables.staging} AS {tables.destination}") + _insert_rewritten_rows(run=run, tables=tables, partitions=partitions) + run( + f"ALTER TABLE {tables.destination} REPLACE PARTITION ID " + f"{quote_string(partitions[0])} FROM {tables.staging}" + ) + + +def _exchange_replacement( + *, + run: Callable[[str], Any], + tables: RewriteTables, + partitions: tuple[str, ...], + executor: RewriteExecutor, +) -> None: + """Exchange the table for a copy that hard-links the partitions it keeps.""" + + drops: str = ", ".join( + f"DROP PARTITION ID {quote_string(partition)}" for partition in partitions + ) + with executor.relation_locks.exclusive(destination=tables.destination): + generation: str = generation_comment(run(executor.generation_query).fetchone()) + run(f"DROP TABLE IF EXISTS {tables.staging}") + run(f"CREATE TABLE {tables.staging} AS {tables.destination}") + run(f"ALTER TABLE {tables.staging} ATTACH PARTITION ALL FROM {tables.destination}") + run(f"ALTER TABLE {tables.staging} {drops}") + _insert_rewritten_rows(run=run, tables=tables, partitions=partitions) + run(f"ALTER TABLE {tables.staging} MODIFY COMMENT {quote_string(generation)}") + run(f"EXCHANGE TABLES {tables.staging} AND {tables.destination}") + + +def _insert_rewritten_rows( + *, run: Callable[[str], Any], tables: RewriteTables, partitions: tuple[str, ...] +) -> None: + """Fill the staging table with the kept rows of ``partitions`` and every delta row.""" + + partition_list: str = ", ".join(quote_string(partition) for partition in partitions) + run( + f"INSERT INTO {tables.staging} SELECT * FROM {tables.destination} " + f"WHERE _partition_id IN ({partition_list}) " + f"AND NOT ifNull({tables.displaced_predicate}, 0)" + ) + run(f"INSERT INTO {tables.staging} SELECT * FROM {tables.delta}") + + +def generation_comment(row: tuple[object, ...] | None) -> str: + """The comment naming a table's generation: its own, else one made from its UUID.""" + + if row is None: + raise AdapterUserError(message="the partition rewrite target no longer exists") + comment: str = str(row[1] or "") + if comment.startswith(GENERATION_COMMENT_PREFIX): + return comment + return f"{GENERATION_COMMENT_PREFIX}{row[0]}" + + def artifact_relation(*, relation: str, infix: str, digest: str) -> str: """Return a sibling relation name bounded by the identifier limit.""" diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 5ea31ff3ab..7d1bb6f546 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -88,6 +88,7 @@ from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.adapters.clickhouse._helpers.partition_rewrite import ( artifact_relation, + generation_comment, rewrite_partitions, ) from sqlbuild.adapters.clickhouse._helpers.snapshot_rewrite import ( @@ -103,6 +104,7 @@ ) from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection from sqlbuild.adapters.clickhouse.classes.partition_locks import PartitionLocks +from sqlbuild.adapters.clickhouse.classes.relation_write_locks import RelationWriteLocks from sqlbuild.adapters.clickhouse.constants import ( CONNECTION_OPTION_KEYS, DEFAULT_COLUMN_DEFAULT_KIND, @@ -111,6 +113,7 @@ DEFAULT_HTTPS_PORT, DSN_OPTION, EXCHANGE_UNSUPPORTED_CODES, + GENERATION_COMMENT_PREFIX, MAX_IDENTIFIER_LENGTH, MAX_QUERY_SIZE_BYTES, MINIMUM_SERVER_VERSION, @@ -164,6 +167,7 @@ class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): historical_insert=HistoricalSnapshotInsertStyle.INSERT_WITH, ) _partition_locks: ClassVar[PartitionLocks] = PartitionLocks() + _relation_locks: ClassVar[RelationWriteLocks] = RelationWriteLocks() def connect(self, config: dict[str, Any]) -> _ClickHouseConnection: try: @@ -464,16 +468,16 @@ def probe_relation_read(self, *, connection: Any, relation: str) -> RelationRead def physical_relation_generation( self, *, connection: Any, database: str | None, schema: str | None, name: str ) -> str | None: - """The table UUID, which changes whenever the table is recreated or exchanged.""" + """The table UUID, kept in the comment of copies a partition rewrite exchanges in.""" del database row: Any = self.execute( connection=connection, - sql="SELECT toString(uuid) FROM system.tables WHERE " - + _database_filter(schema) - + f" AND name = {quote_string(name)}", + sql=_table_generation_query(schema=schema, name=name), ).fetchone() - return None if row is None else str(row[0]) + if row is None: + return None + return generation_comment(tuple(row)).removeprefix(GENERATION_COMMENT_PREFIX) def estimate_relation_row_count( self, *, connection: Any, database: str | None, schema: str | None, name: str @@ -1062,6 +1066,8 @@ def _rewrite( execute=lambda sql: self.execute(connection=connection, sql=sql), statement_recorder=statement_recorder, partition_locks=self._partition_locks, + relation_locks=self._relation_locks, + generation_query=_generation_query(request.destination), ), ) @@ -2636,6 +2642,18 @@ def _split_relation(relation: str) -> tuple[str | None, str]: return parts[0].strip().strip('"`'), name +def _generation_query(relation: str) -> str: + schema, name = _split_relation(relation) + return _table_generation_query(schema=schema, name=name) + + +def _table_generation_query(*, schema: str | None, name: str) -> str: + return ( + "SELECT toString(uuid), comment FROM system.tables WHERE " + f"{_database_filter(schema)} AND name = {quote_string(name)}" + ) + + def _qualify_like(*, destination: str, origin: str) -> str: if RELATION_SEPARATOR in destination or RELATION_SEPARATOR not in origin: return destination diff --git a/src/sqlbuild/adapters/clickhouse/classes/relation_write_locks.py b/src/sqlbuild/adapters/clickhouse/classes/relation_write_locks.py new file mode 100644 index 0000000000..6f4e81ce60 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/relation_write_locks.py @@ -0,0 +1,48 @@ +"""In-process shared and exclusive locks on whole ClickHouse relations.""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager + + +class RelationWriteLocks: + """Let partition swaps share a relation while a whole-table exchange holds it alone.""" + + def __init__(self) -> None: + self._condition: threading.Condition = threading.Condition() + self._sharers: dict[str, int] = {} + self._exclusive: set[str] = set() + + @contextmanager + def shared(self, *, destination: str) -> Iterator[None]: + """Hold ``destination`` alongside other sharers while no exchange holds it.""" + + with self._condition: + self._condition.wait_for(lambda: destination not in self._exclusive) + self._sharers[destination] = self._sharers.get(destination, 0) + 1 + try: + yield + finally: + with self._condition: + self._sharers[destination] -= 1 + self._condition.notify_all() + + @contextmanager + def exclusive(self, *, destination: str) -> Iterator[None]: + """Hold ``destination`` alone, after current sharers and exchanges finish.""" + + with self._condition: + self._condition.wait_for( + lambda: ( + destination not in self._exclusive and self._sharers.get(destination, 0) == 0 + ) + ) + self._exclusive.add(destination) + try: + yield + finally: + with self._condition: + self._exclusive.discard(destination) + self._condition.notify_all() diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index 3ba3597d12..ba181eae54 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -99,3 +99,4 @@ PHYSICAL_STORAGE_CONFIG_KEY: str = "physical_storage" DEFAULT_TABLE_ENGINE: str = "MergeTree" EMPTY_SORTING_KEY: str = "tuple()" +GENERATION_COMMENT_PREFIX: str = "sqlbuild generation " diff --git a/src/sqlbuild/adapters/clickhouse/models.py b/src/sqlbuild/adapters/clickhouse/models.py index d9800b5bfd..09423fbefa 100644 --- a/src/sqlbuild/adapters/clickhouse/models.py +++ b/src/sqlbuild/adapters/clickhouse/models.py @@ -8,6 +8,7 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.clickhouse.classes.partition_locks import PartitionLocks +from sqlbuild.adapters.clickhouse.classes.relation_write_locks import RelationWriteLocks @dataclass(frozen=True) @@ -19,13 +20,25 @@ class PartitionRewrite: displaced_predicate: str +@dataclass(frozen=True) +class RewriteTables: + """The live table, the staged delta and staging relations, and the displaced rows.""" + + destination: str + delta: str + staging: str + displaced_predicate: str + + @dataclass(frozen=True) class RewriteExecutor: - """The adapter operations and process-wide partition locks a rewrite needs.""" + """The adapter operations and process-wide locks a rewrite needs.""" execute: Callable[[str], Any] statement_recorder: StatementRecorder partition_locks: PartitionLocks + relation_locks: RelationWriteLocks + generation_query: str @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py index ec2120a123..6087ce4c35 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -45,7 +45,8 @@ class ClickHouseRewriteTestCase: class ClickHouseRewriteFaultTestCase: description: str fail_statement_number: int - expected_target_unchanged: bool + expected_states_after_crash: tuple[tuple[tuple[object, ...], ...], ...] + expected_rows_after_rerun: tuple[tuple[object, ...], ...] @dataclass(frozen=True) @@ -147,3 +148,10 @@ class ClickHouseSchemaDiffTestCase: left_ddl_columns: str right_ddl_columns: str expected_type_changes: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class ClickHouseGenerationTestCase: + description: str + incoming_rows: tuple[tuple[int, str, int, str], ...] + expected_generation_kept: bool diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py index 324870a3ed..ad1aad40de 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py @@ -175,6 +175,34 @@ def apply_write( ) +def cursor_delta_sql( + *, adapter: ClickHouseAdapter, connection: Any, relation: str, source: str +) -> str: + """The source rows past the target's highest amount, as a cursor-driven run reads them.""" + + cursor: object = fetch_rows( + adapter=adapter, connection=connection, sql=f"SELECT max(amount) FROM {relation}" + )[0][0] + return f"SELECT order_id, customer, amount, order_date FROM {source} WHERE amount > {cursor}" + + +def incoming_rows_sql(rows: tuple[tuple[int, str, int, str], ...]) -> str: + return _incoming_sql(rows) + + +def write_cursor_delta( + *, adapter: ClickHouseAdapter, connection: Any, relation: str, sql: str +) -> None: + adapter.delete_insert( + connection=connection, + destination=relation, + sql=sql, + unique_key=("order_id",), + columns=ORDER_COLUMN_NAMES, + statement_recorder=StatementRecorder(), + ) + + def run_concurrent_cursor_windows( *, adapter: ClickHouseAdapter, diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py index a343b17563..e775759e5f 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py @@ -26,6 +26,7 @@ ClickHouseConcurrentWindowTestCase, ClickHouseCursorFilterTestCase, ClickHouseFunctionTestCase, + ClickHouseGenerationTestCase, ClickHouseInspectionTestCase, ClickHouseInterruptTestCase, ClickHouseQualifiedNameTestCase, @@ -45,28 +46,51 @@ InjectedFailure, apply_write, create_orders, + cursor_delta_sql, execute_all, fetch_rows, + incoming_rows_sql, interrupt_running_query, microbatch_completion, oracle_rows, order_rows_as_dates, recorded_statements, run_concurrent_cursor_windows, + write_cursor_delta, ) _PARTITIONED: str = "toYYYYMM(order_date)" _UNPARTITIONED: str = "tuple()" -_FAULT_WRITE: ClickHouseRewriteTestCase = ClickHouseRewriteTestCase( - description="three partitions", +_CURSOR_SKIP_WRITE: ClickHouseRewriteTestCase = ClickHouseRewriteTestCase( + description="an old partition's update outruns a new partition's insert", strategy="delete_insert", partition_by=_PARTITIONED, initial_rows=INITIAL_ORDERS, - incoming_rows=( - (1, "ada", 11, "2026-01-05"), - (3, "cy", 31, "2026-02-03"), - (5, "ed", 51, "2026-03-01"), - ), + incoming_rows=((1, "ada", 61, "2026-01-05"), (6, "fa", 55, "2026-03-09")), +) +_CURSOR_SKIP_SOURCE: tuple[tuple[int, str, int, str], ...] = ( + *INITIAL_ORDERS[1:], + *_CURSOR_SKIP_WRITE.incoming_rows, +) + + +_EXCHANGE_REWRITE_STEPS: tuple[str, ...] = ( + "drop stale delta", + "create delta", + "stage delta rows", + "count delta rows", + "find touched partitions", + "read the table generation", + "drop stale replacement", + "create replacement", + "attach every partition", + "drop rewritten partitions", + "insert kept rows", + "insert delta rows", + "carry the generation", + "exchange tables", + "drop replacement", + "drop delta", ) @@ -492,29 +516,19 @@ def test_given_incremental_write_when_rewriting_partitions_then_rows_match_duckd "test_case", [ ClickHouseRewriteFaultTestCase( - description="crash while staging the delta", - fail_statement_number=3, - expected_target_unchanged=True, - ), - ClickHouseRewriteFaultTestCase( - description="crash while building the partition staging", - fail_statement_number=9, - expected_target_unchanged=True, - ), - ClickHouseRewriteFaultTestCase( - description="crash before the first partition swap", - fail_statement_number=10, - expected_target_unchanged=True, - ), - ClickHouseRewriteFaultTestCase( - description="crash between partition swaps", - fail_statement_number=11, - expected_target_unchanged=False, - ), + description=step, + fail_statement_number=number, + expected_states_after_crash=( + order_rows_as_dates(_CURSOR_SKIP_WRITE.initial_rows), + oracle_rows(_CURSOR_SKIP_WRITE), + ), + expected_rows_after_rerun=oracle_rows(_CURSOR_SKIP_WRITE), + ) + for number, step in enumerate(_EXCHANGE_REWRITE_STEPS, start=1) ], ids=lambda case: case.description, ) -def test_given_failure_at_rewrite_step_when_rerunning_then_target_converges( +def test_given_interruption_at_any_rewrite_step_when_next_cursor_run_then_no_row_is_skipped( test_case: ClickHouseRewriteFaultTestCase, adapter: ClickHouseAdapter, connection: Any, @@ -522,12 +536,23 @@ def test_given_failure_at_rewrite_step_when_rerunning_then_target_converges( monkeypatch: pytest.MonkeyPatch, ) -> None: relation: str = f"{clickhouse_schema}.orders" + source: str = f"{clickhouse_schema}.source_orders" create_orders( adapter=adapter, connection=connection, relation=relation, - partition_by=_FAULT_WRITE.partition_by, - rows=_FAULT_WRITE.initial_rows, + partition_by=_CURSOR_SKIP_WRITE.partition_by, + rows=_CURSOR_SKIP_WRITE.initial_rows, + ) + create_orders( + adapter=adapter, + connection=connection, + relation=source, + partition_by=_UNPARTITIONED, + rows=_CURSOR_SKIP_SOURCE, + ) + first_delta: str = cursor_delta_sql( + adapter=adapter, connection=connection, relation=relation, source=source ) monkeypatch.setattr( adapter, @@ -537,22 +562,30 @@ def test_given_failure_at_rewrite_step_when_rerunning_then_target_converges( ), ) with pytest.raises(InjectedFailure): - apply_write( - adapter=adapter, connection=connection, test_case=_FAULT_WRITE, relation=relation + write_cursor_delta( + adapter=adapter, connection=connection, relation=relation, sql=first_delta ) + monkeypatch.undo() select_sql: str = ( - f"SELECT order_id, customer, amount, order_date FROM {relation} ORDER BY order_id" + f"SELECT order_id, customer, amount, order_date FROM {relation} " + "ORDER BY order_id, customer, amount" ) after_crash: tuple[tuple[object, ...], ...] = fetch_rows( adapter=adapter, connection=connection, sql=select_sql ) - initial: tuple[tuple[object, ...], ...] = order_rows_as_dates(_FAULT_WRITE.initial_rows) - assert (after_crash == initial) is test_case.expected_target_unchanged - apply_write(adapter=adapter, connection=connection, test_case=_FAULT_WRITE, relation=relation) + write_cursor_delta( + adapter=adapter, + connection=connection, + relation=relation, + sql=cursor_delta_sql( + adapter=adapter, connection=connection, relation=relation, source=source + ), + ) - assert fetch_rows(adapter=adapter, connection=connection, sql=select_sql) == oracle_rows( - _FAULT_WRITE + assert after_crash in test_case.expected_states_after_crash + assert fetch_rows(adapter=adapter, connection=connection, sql=select_sql) == ( + test_case.expected_rows_after_rerun ) @@ -942,3 +975,50 @@ def test_given_relations_differing_in_nullability_when_diffing_schema_then_chang assert tuple( (before.name, before.type, after.type) for before, after in result.type_changed_columns ) == (test_case.expected_type_changes) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseGenerationTestCase( + description="one partition is replaced in place", + incoming_rows=((1, "ada", 11, "2026-01-05"),), + expected_generation_kept=True, + ), + ClickHouseGenerationTestCase( + description="an exchanged copy keeps the generation", + incoming_rows=((1, "ada", 11, "2026-01-05"), (6, "fa", 60, "2026-03-09")), + expected_generation_kept=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_partition_rewrite_when_reading_generation_then_microbatch_scope_is_kept( + test_case: ClickHouseGenerationTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + relation: str = f"{clickhouse_schema}.orders" + create_orders( + adapter=adapter, + connection=connection, + relation=relation, + partition_by=_PARTITIONED, + rows=INITIAL_ORDERS, + ) + before: str | None = adapter.physical_relation_generation( + connection=connection, database=None, schema=clickhouse_schema, name="orders" + ) + + write_cursor_delta( + adapter=adapter, + connection=connection, + relation=relation, + sql=incoming_rows_sql(test_case.incoming_rows), + ) + + after: str | None = adapter.physical_relation_generation( + connection=connection, database=None, schema=clickhouse_schema, name="orders" + ) + assert (before is not None and before == after) is test_case.expected_generation_kept diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 18b5bf718b..95ef7bc0a1 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -63,12 +63,12 @@ SQLBuild never relies on background merges for correctness, and it never updates - **Tables** build in a staging table and replace the live table with `EXCHANGE TABLES`, which is atomic. This needs databases with the `Atomic` engine, the default; other engines fail with `A705`. - **Views** use `CREATE OR REPLACE VIEW`. - **Incremental `append`** inserts the new rows. -- **Incremental `merge`, `delete_insert` and cursor windows** rewrite whole partitions. SQLBuild stages the incoming rows, finds the partitions they touch and the partitions holding the rows they replace, rebuilds those partitions in a staging table, and swaps each one in with `ALTER TABLE ... REPLACE PARTITION`. Each swap is atomic. If a build stops before the swaps the table is untouched; if it stops between swaps, some partitions are already new, and rerunning the build recomputes the rest and converges. -- **Microbatches** use the same partition rewrite for each batch. Concurrent batches (`batch_concurrency`) rewrite different partitions in parallel; batches of one `sqb` process that share a partition take turns. +- **Incremental `merge`, `delete_insert` and cursor windows** rewrite whole partitions. SQLBuild stages the incoming rows and finds the partitions they touch and the partitions holding the rows they replace. One partition is rebuilt in a staging table and swapped in with `ALTER TABLE ... REPLACE PARTITION`. Several partitions are published together: SQLBuild copies the table by hard-linking its parts (no data is rewritten), rebuilds the touched partitions in the copy and swaps the copy in with one `EXCHANGE TABLES`. Either way the swap is atomic, so a build interrupted at any point leaves the table as it was before the run or with every partition of the run, never part of them, and the next run's cursor window cannot skip rows. +- **Microbatches** use the same partition rewrite for each batch, so each batch is published atomically. Concurrent batches (`batch_concurrency`) swap different single partitions in parallel; batches of one `sqb` process that share a partition, or that publish several partitions with an exchange, take turns. ### One writer per model -A partition rewrite reads the partition, rebuilds it and swaps it in. ClickHouse has no transaction that spans those steps, and SQLBuild only serializes rewrites of the same partition inside one `sqb` process. If two `sqb` processes write the same model at the same time and touch the same partition, the later swap wins and the rows the other process wrote to that partition are lost. +A partition rewrite reads the table, rebuilds partitions and swaps them in. ClickHouse has no transaction that spans those steps, and SQLBuild only serializes rewrites of one table inside one `sqb` process. If two `sqb` processes write the same model at the same time, the later swap wins and rows the other process wrote can be lost. Run each model from one process at a time: one scheduled job per model or per project, with no overlapping runs. Concurrency within a run (`--concurrency`, `batch_concurrency`) is safe. - **Snapshots** apply their changes to a private copy of the snapshot table and exchange it in. Closing a version rebuilds the copy rather than updating rows, so the live snapshot changes only at the final exchange. From 2ada8f66857ef917898d3f2a4f801847c709d1b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:07:12 +0100 Subject: [PATCH 26/34] fix: resolve UDF calls in SQL tests whose model SQL parses as a call chain On ClickHouse a UDF marker call such as __udf("name")(args) parses as a parametric function, so SQL-test planning took the parsed path, which only replaced relation markers and left the UDF marker in the real model's SQL. The parsed path now resolves UDF and table-function calls the same way the textual path does. --- .../compiler/_helpers/sql_tests/planning.rs | 15 ++++++++- .../sqlbuild-analysis/src/compiler/tests.rs | 2 ++ .../src/compiler/tests/helpers.rs | 2 +- .../src/compiler/tests/planning/helpers.rs | 31 +++++++++++++++++ .../compiler/tests/test_sql_test_planning.rs | 33 ++++++++++++++++++- .../src/compiler/tests/test_types.rs | 6 ++++ 6 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/compiler/tests/planning/helpers.rs diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index d5db202b72..cb2d4599a6 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -1455,8 +1455,21 @@ fn analyze_and_resolve_sql( replacements.insert((function_name.as_str(), referenced_name.as_str()), target); } } + let relations_resolved = + replace_relation_markers(request.query_sql, &replacements, request.patterns); + let functions_resolved = resolve_function_calls( + &resolve_function_calls( + &relations_resolved, + request.functions, + false, + request.patterns, + )?, + request.functions, + true, + request.patterns, + )?; let cte_body_sql = strip_statement_terminators( - &replace_relation_markers(request.query_sql, &replacements, request.patterns), + &functions_resolved, SliceDialect::new(Some(request.dialect_name)), ) .to_string(); diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index 2ba8066077..7e1eda4178 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/planning/helpers.rs"] +mod planning_helpers; #[path = "tests/test_scenario_expected_columns.rs"] mod scenario_expected_columns; #[path = "tests/test_scenario_helper_ctes.rs"] diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index 90616732f5..305d2fa2e7 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -1559,7 +1559,7 @@ pub(super) fn snowflake_function_synonyms_stay_as_authored() -> bool { && !sql.contains("SUBSTRING") } -fn generic_lexical_syntax() -> Value { +pub(crate) fn generic_lexical_syntax() -> Value { json!({ "backslashEscapeQuotes": [], "escapeStringPrefix": false, diff --git a/crates/sqlbuild-analysis/src/compiler/tests/planning/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/planning/helpers.rs new file mode 100644 index 0000000000..cfd6b634e4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/planning/helpers.rs @@ -0,0 +1,31 @@ +use serde_json::{Value, json}; + +use crate::compiler::tests::helpers::{generic_lexical_syntax, plan_json}; + +/// The rendered SQL of a model test whose real model calls a UDF in its projection. +pub(crate) fn projected_udf_call_sql(dialect: &str) -> String { + let request = json!({ + "lexicalSyntax": generic_lexical_syntax(), + "models": [ + {"name": "orders", "querySql": "SELECT o.order_id, __udf(\"is_open\")(o.status) AS is_open FROM __source(\"raw_orders\") AS o"} + ], + "functions": [{"name": "is_open", "udfPrefix": "analytics__is_open", "udfSuffix": ""}], + "tests": [{ + "name": "orders_case", "fileLabel": "tests/orders.sql", + "payload": { + "kind": "model", + "authoredCtes": [{"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 'open' AS status"}], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id, TRUE AS is_open"}], + "expectedModelNames": ["orders"] + } + }], + "sqlAnalysisEnabled": true, "sqlAnalysisDialect": dialect + }); + let response: Value = + serde_json::from_str(&plan_json(&request.to_string()).expect("planning succeeds")) + .expect("valid JSON"); + response["artifacts"][0]["sql"] + .as_str() + .expect("rendered SQL") + .to_owned() +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs index f7e25c954a..43e3268d98 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs @@ -17,9 +17,10 @@ use crate::compiler::tests::helpers::{ unicode_cte_after_leading_with_preserves_identifier, unresolved_reference_fast_rejection_preserves_warning, upstream_fallback_resolves, }; +use crate::compiler::tests::planning_helpers::projected_udf_call_sql; use crate::compiler::tests::test_types::{ GlueErrorProjectionTestCase, GluePlanParityTestCase, GluePlannerErrorTestCase, - SqlTestPlanningTestCase, + ProjectedUdfTestCase, SqlTestPlanningTestCase, }; /// A mocked model test, a test missing its source mock and a UDF test over two models. @@ -324,3 +325,33 @@ fn given_plan_warnings_when_projecting_errors_then_each_distinct_error_message_i ); } } + +#[test] +fn given_real_model_calling_udf_in_projection_when_planning_then_call_targets_the_function() { + let test_cases = [ + ProjectedUdfTestCase { + description: "DuckDB falls back to textual resolution", + dialect: "duckdb", + expected_call: "analytics__is_open(o.status)", + }, + ProjectedUdfTestCase { + description: "ClickHouse parses the marker as a parametric call", + dialect: "clickhouse", + expected_call: "analytics__is_open(o.status)", + }, + ]; + + for test_case in test_cases { + let sql = projected_udf_call_sql(test_case.dialect); + + assert_eq!( + ( + sql.contains(test_case.expected_call), + sql.contains("__udf(") + ), + (true, false), + "{}: {sql}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 62cb4dd15f..bf32b0365e 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -227,3 +227,9 @@ pub(crate) struct ScenarioHelperShadowTestCase { pub(crate) check_sql: &'static str, pub(crate) expected_rejected: bool, } + +pub(super) struct ProjectedUdfTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static str, + pub(super) expected_call: &'static str, +} From 5377a6e61e4166dd002173259be32bf244da62c6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:08:47 +0100 Subject: [PATCH 27/34] test: build, test and rebuild the waffle shop on ClickHouse The bundled waffle shop runs through the real CLI on ClickHouse: a full build of seeds, sources, models and audits, sqb test, and a second build over the incremental and microbatch paths, checking exit codes and row counts. The helper drops the SQL table function and the Python UDF (neither deploys to ClickHouse), aliases qualified columns in multi-join models, and writes CURRENT_TIMESTAMP() with parentheses; the adapter page documents both SQL differences. --- .../docs/concepts/adapters/clickhouse.md | 6 ++ .../commands/main/clickhouse/_test_types.py | 8 ++ .../cli/commands/main/clickhouse/helpers.py | 65 ++++++++++++++ .../clickhouse/test_clickhouse_waffle_shop.py | 88 +++++++++++++++++++ .../docs/concepts/adapters/clickhouse.mdx | 5 ++ 5 files changed, 172 insertions(+) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_waffle_shop.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index 8267f988d4..77dc78fa4a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -17,6 +17,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/clickhouse/ - Clones - Nullable columns - Settings SQLBuild pins +- SQL differences to watch - Limits The ClickHouse adapter is in beta and supports a single server. Clusters (`ON CLUSTER`, replicated and distributed tables) are not supported yet. Please [report issues](https://github.com/chio-labs/sqlbuild/issues). @@ -169,6 +170,11 @@ The connection's `settings` may repeat these values but not change them: a diffe ClickHouse returns a type's default value (`0`, the epoch, an empty string) from `lag` and `lead` past the edge of a window frame unless the argument is `Nullable`. SQLBuild's generated snapshot SQL wraps those arguments in `toNullable` so they return `NULL` as on other adapters. Write the same in your own models when you rely on `NULL` there. +## SQL differences to watch + +- In a query that joins three or more tables, ClickHouse names a selected column such as `c.customer_id` literally `c.customer_id` when another joined table also has a `customer_id`. Alias such columns (`c.customer_id AS customer_id`) so the table gets the column name other warehouses give it. +- `CURRENT_TIMESTAMP` needs parentheses: write `CURRENT_TIMESTAMP()` or `now()`. + ## Limits - Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 2eef65d97c..214ca43891 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -76,3 +76,11 @@ class ClickHouseNullabilityE2ETestCase: edited_select: str expected_rows: tuple[tuple[object, ...], ...] expected_types: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseWaffleShopE2ETestCase: + description: str + expected_return_codes: tuple[int, int, int] + expected_summary_fragments: tuple[str, str, str] + expected_row_counts: dict[str, int] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index a04079ad6a..b74d2dfc8c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -2,11 +2,14 @@ from __future__ import annotations +import re import uuid +from pathlib import Path from textwrap import dedent from typing import Any, cast from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_waffle_shop CLICKHOUSE_IMAGE: str = "clickhouse/clickhouse-server:26.1" @@ -246,3 +249,65 @@ def active_parts_by_partition( ), ) return {str(row[0]): tuple(map(str, cast(list[object], row[1]))) for row in rows} + + +def prepare_clickhouse_waffle_shop( + *, tmp_path: Path, config: dict[str, object] +) -> tuple[Path, str]: + """Copy waffle shop to tmp dir and wire it to a unique ClickHouse database.""" + + schema_name: str = build_unique_schema_name(prefix="sqb_waffle") + project_dir: Path = prepare_waffle_shop(tmp_path) + (project_dir / "functions" / "sql" / "customer_orders.sql").unlink() + (project_dir / "tests" / "unit" / "test_customer_orders_table_fn.sql").unlink() + (project_dir / "functions" / "python" / "is_completed_order_py.py").unlink() + fact_orders_path: Path = project_dir / "models" / "marts" / "fact_orders.sql" + fact_orders_path.write_text( + fact_orders_path.read_text(encoding="utf-8").replace( + '__udf("is_completed_order_py")(o.status) AS is_completed_order_py,', + '__udf("is_completed_order")(o.status) AS is_completed_order_py,', + ), + encoding="utf-8", + ) + for model_name in ("dim_customers", "fact_orders", "scenario_order_prices"): + _alias_qualified_columns(project_dir / "models" / "marts" / f"{model_name}.sql") + materialization_path: Path = project_dir / "materializations" / "partition_tracked.py" + materialization_path.write_text( + materialization_path.read_text(encoding="utf-8").replace( + "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP()" + ), + encoding="utf-8", + ) + time_macros_path: Path = project_dir / "models" / "marts" / "_macros" / "time.py" + time_macros_path.write_text( + time_macros_path.read_text(encoding="utf-8").replace( + "return f\"DATE_TRUNC('{grain}', {expression})\"", + "return f\"CAST(DATE_TRUNC('{grain}', {expression}) AS TIMESTAMP)\"", + ), + encoding="utf-8", + ) + (project_dir / "sqlbuild_project.toml").write_text( + build_clickhouse_project_toml( + project_name="waffle_shop", + schema_name=schema_name, + config=config, + extra='\n[settings]\ndefault_audit_severity = "warn"\n\n' + '[path_defaults.staging]\nmaterialized = "view"\n', + ), + encoding="utf-8", + ) + return project_dir, schema_name + + +def _alias_qualified_columns(model_path: Path) -> None: + """Alias bare qualified columns; ClickHouse names them ``alias.column`` in multi-joins.""" + + model_path.write_text( + re.sub( + r"^(\s+)(\w+)\.(\w+),$", + r"\1\2.\3 AS \3,", + model_path.read_text(encoding="utf-8"), + flags=re.MULTILINE, + ), + encoding="utf-8", + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_waffle_shop.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_waffle_shop.py new file mode 100644 index 0000000000..aaef19ec9b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_waffle_shop.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseWaffleShopE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + cleanup_clickhouse_schema, + create_clickhouse_schema, + fetch_clickhouse_rows, + prepare_clickhouse_waffle_shop, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseWaffleShopE2ETestCase( + description="waffle shop full build succeeds on clickhouse", + expected_return_codes=(0, 0, 0), + expected_summary_fragments=( + "Completed successfully", + "FAIL=0", + "Completed successfully", + ), + expected_row_counts={ + "customer_status_snapshot": 5, + "daily_activity_rollup": 3, + "daily_order_partitioned": 4, + "daily_revenue": 3, + "dim_customers": 5, + "fact_orders": 10, + "hourly_activity_with_daily_context": 0, + "hourly_order_activity": 8, + "order_status_index": 10, + "scenario_order_prices": 10, + "stg_customers": 5, + "stg_orders": 10, + "waffle_types": 6, + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_waffle_shop_when_building_testing_and_rebuilding_on_clickhouse_then_tables_match( + tmp_path: Path, + test_case: ClickHouseWaffleShopE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path + schema_name: str + project_dir, schema_name = prepare_clickhouse_waffle_shop(tmp_path=tmp_path, config=config) + create_clickhouse_schema(config=config, schema_name=schema_name) + try: + results: tuple[subprocess.CompletedProcess[str], ...] = tuple( + run_sqb(command=command, project_dir=project_dir) + for command in ( + ("--no-color", "build", "--concurrency", "4"), + ("--no-color", "test"), + ("--no-color", "build", "--concurrency", "4"), + ) + ) + + assert tuple(result.returncode for result in results) == ( + test_case.expected_return_codes + ), "\n".join(result.stdout + result.stderr for result in results) + assert [ + fragment in result.stdout + for fragment, result in zip(test_case.expected_summary_fragments, results, strict=True) + ] == [True, True, True], "\n".join(result.stdout for result in results) + assert { + name: int( + str( + fetch_clickhouse_rows( + config=config, sql=f"SELECT count() FROM {schema_name}.{name}" + )[0][0] + ) + ) + for name in test_case.expected_row_counts + } == (test_case.expected_row_counts) + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 95ef7bc0a1..7cd6dcf29f 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -157,6 +157,11 @@ The connection's `settings` may repeat these values but not change them: a diffe ClickHouse returns a type's default value (`0`, the epoch, an empty string) from `lag` and `lead` past the edge of a window frame unless the argument is `Nullable`. SQLBuild's generated snapshot SQL wraps those arguments in `toNullable` so they return `NULL` as on other adapters. Write the same in your own models when you rely on `NULL` there. +## SQL differences to watch + +- In a query that joins three or more tables, ClickHouse names a selected column such as `c.customer_id` literally `c.customer_id` when another joined table also has a `customer_id`. Alias such columns (`c.customer_id AS customer_id`) so the table gets the column name other warehouses give it. +- `CURRENT_TIMESTAMP` needs parentheses: write `CURRENT_TIMESTAMP()` or `now()`. + ## Limits - Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. From 1ac814d2920d5481da57684d6b6998cc8fad99a4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:38:20 +0100 Subject: [PATCH 28/34] fix: fail ClickHouse builds whose columns are named after a table alias In a query joining three or more tables, ClickHouse names a selected alias.column after the alias when another joined table has the same column, so a table could silently get a column such as 'o.order_id' where every other warehouse and SQL analysis say 'order_id'. Plan entries now carry the output columns SQL analysis inferred, and after creating a model's table or delta the ClickHouse adapter drops it and fails with A711 when a column is named alias.column for an expected column, naming the alias to add. Models without an inferred column list are not checked. --- .../_helpers/planner_records/constants.rs | 1 + .../planner_strategy/model_entries.rs | 15 +++ .../docs/concepts/adapters/clickhouse.md | 2 +- src/sqlbuild/_native.pyi | 2 + .../clickhouse/_helpers/output_columns.py | 19 +++ .../clickhouse/classes/clickhouse_adapter.py | 45 +++++++- src/sqlbuild/adapters/clickhouse/constants.py | 4 + .../run/_helpers/execution/staging.py | 8 +- .../run/_helpers/execution/table_config.py | 26 +++++ .../_helpers/materializations/incremental.py | 4 +- .../_helpers/materializations/microbatch.py | 4 +- .../run/_helpers/materializations/snapshot.py | 4 +- src/sqlbuild/executor/run/main/_execute.py | 8 +- .../commands/main/clickhouse/_test_types.py | 9 ++ .../test_clickhouse_output_columns.py | 109 ++++++++++++++++++ .../adapters/clickhouse/_test_types.py | 8 ++ .../clickhouse/test_output_columns.py | 45 ++++++++ .../docs/concepts/adapters/clickhouse.mdx | 2 +- 18 files changed, 294 insertions(+), 21 deletions(-) create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py create mode 100644 src/sqlbuild/executor/run/_helpers/execution/table_config.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_output_columns.py create mode 100644 tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 669ab180a8..523d1e4eb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -629,6 +629,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("snapshot_schema_change", FieldDefault::None), FieldSpec::new("table_type", FieldDefault::Text("transient")), FieldSpec::new("physical_storage", FieldDefault::EmptyDict), + FieldSpec::new("expected_output_columns", FieldDefault::None), FieldSpec::new("on_schema_change", FieldDefault::None), FieldSpec::new("type_enforcement", FieldDefault::False), FieldSpec::new("declared_columns", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 37aacdcee8..a775cca9c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -308,6 +308,20 @@ fn logical_ddl<'py>( Ok(statements.join(DDL_SEPARATOR)) } +/// The output column names SQL analysis inferred for the model, when it inferred them. +fn expected_output_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let inferred: Bound<'py, PyAny> = native_field(model, "inferred_columns")?; + if inferred.is_none() { + return Ok(none(py)); + } + let mut names: Vec> = Vec::new(); + for column in inferred.try_iter()? { + names.push(native_field(&column?, "name")?); + } + Ok(PyTuple::new(py, names)?.into_any()) +} + /// The model's physical table layout, refused on adapters whose tables have none. fn storage_fields<'py>( request: &ModelEntryDecisions<'py>, @@ -668,6 +682,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( entry.set_item("merge_exclude_columns", merge_exclude)?; entry.set_item("table_type", table_type)?; storage_fields(request, &values, &entry)?; + entry.set_item("expected_output_columns", expected_output_columns(model)?)?; entry.set_item("on_schema_change", policy)?; entry.set_item("type_enforcement", enforcement)?; contract_fields(model, &values, &entry)?; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index 77dc78fa4a..78d12d73e0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -172,7 +172,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## SQL differences to watch -- In a query that joins three or more tables, ClickHouse names a selected column such as `c.customer_id` literally `c.customer_id` when another joined table also has a `customer_id`. Alias such columns (`c.customer_id AS customer_id`) so the table gets the column name other warehouses give it. +- In a query that joins three or more tables, ClickHouse names a selected column such as `c.customer_id` literally `c.customer_id` when another joined table also has a `customer_id`. Alias such columns (`c.customer_id AS customer_id`) so the table gets the column name other warehouses give it. When SQL analysis knows the model's output columns, a build that produces such a name fails with `A711` naming the column and the alias to add. - `CURRENT_TIMESTAMP` needs parentheses: write `CURRENT_TIMESTAMP()` or `now()`. ## Limits diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4736fbce1d..bddfa169bb 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -15711,6 +15711,7 @@ class ModelPlanEntry(NativeRecord): snapshot_schema_change: str | None table_type: str physical_storage: dict[str, object] + expected_output_columns: tuple[str, ...] | None on_schema_change: OnSchemaChange | None type_enforcement: bool declared_columns: tuple[ColumnInfo, ...] @@ -15796,6 +15797,7 @@ class ModelPlanEntry(NativeRecord): snapshot_schema_change: str | None = ..., table_type: str = ..., physical_storage: dict[str, object] = ..., + expected_output_columns: tuple[str, ...] | None = ..., on_schema_change: OnSchemaChange | None = ..., type_enforcement: bool = ..., declared_columns: tuple[ColumnInfo, ...] = ..., diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py b/src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py new file mode 100644 index 0000000000..ef459f31fa --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py @@ -0,0 +1,19 @@ +"""Detect output columns ClickHouse named after their table alias.""" + +from __future__ import annotations + +from sqlbuild.adapters.clickhouse.constants import QUALIFIER_SEPARATOR + + +def alias_qualified_columns( + *, actual: tuple[str, ...], expected: tuple[str, ...] +) -> tuple[tuple[str, str], ...]: + """Pairs of an ``alias.column`` name ClickHouse produced and the ``column`` analysis expects.""" + + wanted: frozenset[str] = frozenset(expected) - frozenset(actual) + pairs: tuple[tuple[str, str], ...] = tuple( + (name, name.rpartition(QUALIFIER_SEPARATOR)[2]) for name in actual + ) + return tuple( + (produced, column) for produced, column in pairs if produced != column and column in wanted + ) diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 7d1bb6f546..924ae40e95 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -12,7 +12,7 @@ from datetime import date, datetime from decimal import Decimal from pathlib import Path -from typing import Any, ClassVar +from typing import Any, ClassVar, cast from sqlbuild._native import ( SeedCsvSettings, @@ -86,6 +86,7 @@ ) from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.clickhouse._helpers.output_columns import alias_qualified_columns from sqlbuild.adapters.clickhouse._helpers.partition_rewrite import ( artifact_relation, generation_comment, @@ -113,10 +114,13 @@ DEFAULT_HTTPS_PORT, DSN_OPTION, EXCHANGE_UNSUPPORTED_CODES, + EXPECTED_OUTPUT_COLUMNS_CONFIG_KEY, GENERATION_COMMENT_PREFIX, MAX_IDENTIFIER_LENGTH, MAX_QUERY_SIZE_BYTES, MINIMUM_SERVER_VERSION, + MODEL_NAME_CONFIG_KEY, + OUTPUT_COLUMN_NAME_CODE, PHYSICAL_STORAGE_CONFIG_KEY, RELATION_NOT_FOUND_CODES, RELATION_SEPARATOR, @@ -2155,6 +2159,45 @@ def create_table_as( stmt: str for stmt in statements: self.execute(connection=connection, sql=stmt) + expected: object = (config or {}).get(EXPECTED_OUTPUT_COLUMNS_CONFIG_KEY) + if expected: + self._check_output_column_names( + connection=connection, + destination=destination, + model_name=str((config or {}).get(MODEL_NAME_CONFIG_KEY) or destination), + expected=tuple(str(name) for name in cast(tuple[object, ...], expected)), + ) + + def _check_output_column_names( + self, *, connection: Any, destination: str, model_name: str, expected: tuple[str, ...] + ) -> None: + """Refuse a table whose columns ClickHouse named ``alias.column`` instead of ``column``.""" + + renamed: tuple[tuple[str, str], ...] = alias_qualified_columns( + actual=tuple( + column.name + for column in self.describe_relation(connection=connection, relation=destination) + ), + expected=expected, + ) + if not renamed: + return + self.execute(connection=connection, sql=f"DROP TABLE IF EXISTS {destination}") + pairs: str = ", ".join( + f"'{produced}' instead of '{wanted}'" for produced, wanted in renamed + ) + aliases: str = ", ".join(f"{produced} AS {wanted}" for produced, wanted in renamed) + raise AdapterUserError( + message=( + f"model '{model_name}' produced columns named after their table alias on " + f"ClickHouse: {pairs}" + ), + code=OUTPUT_COLUMN_NAME_CODE, + help=( + "ClickHouse names a selected alias.column after the alias when another joined " + f"table has the same column; alias it in the model: {aliases}" + ), + ) def create_view_as( self, diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index ba181eae54..bebad7d7f3 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -100,3 +100,7 @@ DEFAULT_TABLE_ENGINE: str = "MergeTree" EMPTY_SORTING_KEY: str = "tuple()" GENERATION_COMMENT_PREFIX: str = "sqlbuild generation " +EXPECTED_OUTPUT_COLUMNS_CONFIG_KEY: str = "expected_output_columns" +MODEL_NAME_CONFIG_KEY: str = "model_name" +OUTPUT_COLUMN_NAME_CODE: str = "A711" +QUALIFIER_SEPARATOR: str = "." diff --git a/src/sqlbuild/executor/run/_helpers/execution/staging.py b/src/sqlbuild/executor/run/_helpers/execution/staging.py index 605aa1fc38..379040407a 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/staging.py +++ b/src/sqlbuild/executor/run/_helpers/execution/staging.py @@ -7,7 +7,7 @@ from sqlbuild._native import Fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types +from sqlbuild.executor.run._helpers.execution.table_config import model_table_config from sqlbuild.executor.run.models import ModelMaterializationContext from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, @@ -48,11 +48,7 @@ def create_staging_relation( connection=connection, destination=staging_qualified, sql=resolved_sql, - config={ - "table_type": context.entry.table_type, - "inferred_column_types": inferred_column_types(context.entry), - "physical_storage": context.entry.physical_storage, - }, + config=model_table_config(context.entry), statement_recorder=statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_config.py b/src/sqlbuild/executor/run/_helpers/execution/table_config.py new file mode 100644 index 0000000000..7f71b3f645 --- /dev/null +++ b/src/sqlbuild/executor/run/_helpers/execution/table_config.py @@ -0,0 +1,26 @@ +"""Adapter config for the tables a model's query is materialized into.""" + +from __future__ import annotations + +from sqlbuild._native import ModelPlanEntry +from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types + + +def model_output_config(entry: ModelPlanEntry) -> dict[str, object]: + """The model name, its inferred column types and the output columns analysis expects.""" + + return { + "model_name": entry.name, + "expected_output_columns": entry.expected_output_columns, + "inferred_column_types": inferred_column_types(entry), + } + + +def model_table_config(entry: ModelPlanEntry) -> dict[str, object]: + """The model's table type, physical layout and expected output columns.""" + + return { + "table_type": entry.table_type, + "physical_storage": entry.physical_storage, + **model_output_config(entry), + } diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index b3297ebcad..075e086b73 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -37,12 +37,12 @@ run_post_hook_phase, run_pre_hook_phase, ) -from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, build_skipped_result, ) from sqlbuild.executor.run._helpers.execution.schema import inspect_runtime_relation_schema +from sqlbuild.executor.run._helpers.execution.table_config import model_output_config from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from sqlbuild.executor.run._helpers.validation.contracts import validate_runtime_contract from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( @@ -517,7 +517,7 @@ def _prepare_delta_relation( connection=connection, destination=delta_qualified, sql=preparation.resolved_sql, - config={"inferred_column_types": inferred_column_types(context.entry)}, + config=model_output_config(context.entry), statement_recorder=statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index ccdad343b9..7fc9b5fca7 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -78,12 +78,12 @@ run_post_hook_phase, run_pre_hook_phase, ) -from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, build_skipped_result, ) from sqlbuild.executor.run._helpers.execution.schema import inspect_runtime_relation_schema +from sqlbuild.executor.run._helpers.execution.table_config import model_output_config from sqlbuild.executor.run._helpers.materializations.full_refresh import ( promote_full_refresh_rebuild, relation_exists, @@ -2812,7 +2812,7 @@ def _stage_microbatch_delta( connection=context.connection, destination=targets.delta_qualified, sql=batch_sql, - config={"inferred_column_types": inferred_column_types(context.entry)}, + config=model_output_config(context.entry), statement_recorder=state.statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 88e075a15d..44964715d8 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -44,12 +44,12 @@ run_post_hook_phase, run_pre_hook_phase, ) -from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, build_skipped_result, ) from sqlbuild.executor.run._helpers.execution.schema import inspect_runtime_relation_schema +from sqlbuild.executor.run._helpers.execution.table_config import model_output_config from sqlbuild.executor.run._helpers.materializations.full_refresh import ( promote_full_refresh_rebuild, relation_exists, @@ -192,7 +192,7 @@ def execute_snapshot_entry( # noqa: PLR0915 connection=connection, destination=delta_qualified, sql=entry.resolved_sql, - config={"inferred_column_types": inferred_column_types(entry)}, + config=model_output_config(entry), statement_recorder=statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index df4d2f90ef..2e1bce918c 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -26,7 +26,6 @@ run_post_hook_phase, run_pre_hook_phase, ) -from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.promotion import promote_relation_to_destination from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, @@ -37,6 +36,7 @@ ) from sqlbuild.executor.run._helpers.execution.schema import inspect_runtime_relation_schema from sqlbuild.executor.run._helpers.execution.staging import create_staging_relation +from sqlbuild.executor.run._helpers.execution.table_config import model_table_config from sqlbuild.executor.run._helpers.execution.table_targets import resolve_table_targets from sqlbuild.executor.run._helpers.materializations.custom import ( execute_custom_entry as execute_custom_entry, @@ -423,11 +423,7 @@ def _immediate_lifecycle( connection=connection, destination=target_qualified, sql=state.resolved_sql, - config={ - "table_type": entry.table_type, - "inferred_column_types": inferred_column_types(entry), - "physical_storage": entry.physical_storage, - }, + config=model_table_config(entry), statement_recorder=statement_recorder, ) except Exception as exc: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 214ca43891..d21d7d8e96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -84,3 +84,12 @@ class ClickHouseWaffleShopE2ETestCase: expected_return_codes: tuple[int, int, int] expected_summary_fragments: tuple[str, str, str] expected_row_counts: dict[str, int] + + +@dataclass(frozen=True) +class ClickHouseOutputColumnNameE2ETestCase: + description: str + selected_order_id: str + expected_return_code: int + expected_output_fragments: tuple[str, ...] + expected_columns: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_output_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_output_columns.py new file mode 100644 index 0000000000..2a61861e46 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_output_columns.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseOutputColumnNameE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + build_clickhouse_project_toml, + build_unique_schema_name, + cleanup_clickhouse_schema, + create_clickhouse_schema, + execute_clickhouse_sql, + fetch_clickhouse_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseOutputColumnNameE2ETestCase( + description="an unaliased joined column fails with A711", + selected_order_id="o.order_id", + expected_return_code=1, + expected_output_fragments=( + "A711", + "'o.order_id' instead of 'order_id'", + "o.order_id AS order_id", + ), + expected_columns=(), + ), + ClickHouseOutputColumnNameE2ETestCase( + description="an aliased joined column builds", + selected_order_id="o.order_id AS order_id", + expected_return_code=0, + expected_output_fragments=("Completed successfully",), + expected_columns=(("order_id",), ("quantity",), ("payment_cents",)), + ), + ], + ids=lambda case: case.description, +) +def test_given_three_table_join_when_building_on_clickhouse_then_column_names_match_analysis( + tmp_path: Path, + test_case: ClickHouseOutputColumnNameE2ETestCase, + clickhouse_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_names") + config: dict[str, object] = clickhouse_e2e_config + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="clickhouse_names", + repo_files={ + "sqlbuild_project.toml": build_clickhouse_project_toml( + project_name="clickhouse_names", schema_name=schema_name, config=config + ), + "sources/raw.yml": ( + "sources:\n" + + "".join( + f" - name: {name}\n description: Synthetic {name}.\n" + f" schema: {schema_name}\n table: {name}\n" + for name in ("orders", "products", "payments") + ) + ), + "models/order_lines.sql": ( + "MODEL (description 'One row per order.');\n\n" + f"SELECT {test_case.selected_order_id}, o.quantity AS quantity, " + "p.amount AS payment_cents\n" + 'FROM __source("orders") AS o\n' + 'JOIN __source("products") AS w ON o.product_id = w.product_id\n' + 'LEFT JOIN __source("payments") AS p ON o.order_id = p.order_id\n' + ), + }, + ) + create_clickhouse_schema(config=config, schema_name=schema_name) + execute_clickhouse_sql( + config=config, + statements=( + f"CREATE TABLE {schema_name}.orders (order_id Int64, product_id Int64, " + "quantity Int64) ENGINE = MergeTree ORDER BY order_id", + f"CREATE TABLE {schema_name}.products (product_id Int64, order_id Int64) " + "ENGINE = MergeTree ORDER BY product_id", + f"CREATE TABLE {schema_name}.payments (order_id Int64, amount Int64) " + "ENGINE = MergeTree ORDER BY order_id", + f"INSERT INTO {schema_name}.orders VALUES (1, 7, 2)", + f"INSERT INTO {schema_name}.products VALUES (7, 0)", + f"INSERT INTO {schema_name}.payments VALUES (1, 900)", + ), + ) + try: + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr + assert [ + fragment in result.stdout + result.stderr + for fragment in test_case.expected_output_fragments + ] == [True] * len(test_case.expected_output_fragments), result.stdout + result.stderr + assert fetch_clickhouse_rows( + config=config, + sql=f"SELECT name FROM system.columns WHERE database = '{schema_name}' " + "AND table = 'order_lines' ORDER BY position", + ) == (test_case.expected_columns) + finally: + cleanup_clickhouse_schema(config=config, schema_name=schema_name) diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index 654f8cdb15..4f3217da2e 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -72,3 +72,11 @@ class SessionSettingConflictTestCase: settings: dict[str, object] expected_code: str expected_message: str + + +@dataclass(frozen=True) +class AliasQualifiedColumnsTestCase: + description: str + actual: tuple[str, ...] + expected: tuple[str, ...] + expected_pairs: tuple[tuple[str, str], ...] diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py new file mode 100644 index 0000000000..926580859d --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.clickhouse._helpers.output_columns import alias_qualified_columns +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import AliasQualifiedColumnsTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + AliasQualifiedColumnsTestCase( + description="a joined column named after its alias", + actual=("o.order_id", "quantity", "payment_amount_cents"), + expected=("order_id", "quantity", "payment_amount_cents"), + expected_pairs=(("o.order_id", "order_id"),), + ), + AliasQualifiedColumnsTestCase( + description="matching names", + actual=("order_id", "quantity"), + expected=("order_id", "quantity"), + expected_pairs=(), + ), + AliasQualifiedColumnsTestCase( + description="a deliberate dotted name the analysis also expects", + actual=("order.id", "quantity"), + expected=("order.id", "quantity"), + expected_pairs=(), + ), + AliasQualifiedColumnsTestCase( + description="a dotted name whose suffix the query also produces", + actual=("o.order_id", "order_id"), + expected=("order_id",), + expected_pairs=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_output_columns_when_comparing_with_analysis_then_alias_named_columns_are_found( + test_case: AliasQualifiedColumnsTestCase, +) -> None: + assert ( + alias_qualified_columns(actual=test_case.actual, expected=test_case.expected) + == test_case.expected_pairs + ) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 7cd6dcf29f..f405329b98 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -159,7 +159,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## SQL differences to watch -- In a query that joins three or more tables, ClickHouse names a selected column such as `c.customer_id` literally `c.customer_id` when another joined table also has a `customer_id`. Alias such columns (`c.customer_id AS customer_id`) so the table gets the column name other warehouses give it. +- In a query that joins three or more tables, ClickHouse names a selected column such as `c.customer_id` literally `c.customer_id` when another joined table also has a `customer_id`. Alias such columns (`c.customer_id AS customer_id`) so the table gets the column name other warehouses give it. When SQL analysis knows the model's output columns, a build that produces such a name fails with `A711` naming the column and the alias to add. - `CURRENT_TIMESTAMP` needs parentheses: write `CURRENT_TIMESTAMP()` or `now()`. ## Limits From 200e75ea5df4827a1accfde500bf9aaafb97f87c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:45:53 +0100 Subject: [PATCH 29/34] fix: widen ClickHouse columns to Nullable only when on_schema_change allows it Nullability widening ran before the on_schema_change check, so fail and ignore models still had their columns altered. Widening now happens only under append_new_columns and sync_all_columns; fail reports the type change through its usual path, and ignore leaves the table alone so a NULL insert fails with A713 explaining the fix. A key column that would need widening fails with A712 explaining allow_nullable_key and coalesce() instead of a raw server error. --- .../docs/concepts/adapters/clickhouse.md | 6 +- .../clickhouse/_helpers/output_columns.py | 11 +++- .../clickhouse/classes/clickhouse_adapter.py | 66 ++++++++++++++++++- src/sqlbuild/adapters/clickhouse/constants.py | 8 +++ .../_helpers/materializations/incremental.py | 18 ++++- .../_helpers/materializations/microbatch.py | 6 +- .../commands/main/clickhouse/_test_types.py | 2 + .../clickhouse/test_clickhouse_nullability.py | 62 +++++++++++++---- .../adapters/clickhouse/_test_types.py | 7 ++ .../clickhouse/test_output_columns.py | 38 ++++++++++- .../docs/concepts/adapters/clickhouse.mdx | 6 +- 11 files changed, 208 insertions(+), 22 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index 78d12d73e0..b8bdd8f42d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -150,7 +150,11 @@ SQLBuild does not deploy Python functions (`A707`) or SQL table functions (`A708 ## Nullable columns -ClickHouse columns hold `NULL` only when their type is `Nullable`, and a table built from a query takes each column's nullability from the query. When an incremental model later produces `NULL` in a column its table created as non-nullable, SQLBuild changes that column to `Nullable` with `ALTER TABLE ... MODIFY COLUMN` before writing, so the `NULL` is stored as `NULL`. A column in the sorting or partitioning key cannot become `Nullable` unless the table sets `allow_nullable_key`, so that write fails instead. +ClickHouse columns hold `NULL` only when their type is `Nullable`, and a table built from a query takes each column's nullability from the query. When an incremental model later produces `NULL` in a column its table created as non-nullable, what happens follows `on_schema_change`: + +- `append_new_columns` (the default) and `sync_all_columns`: SQLBuild changes the column to `Nullable` with `ALTER TABLE ... MODIFY COLUMN` before writing, so the `NULL` is stored as `NULL`. A column in the sorting, partitioning or primary key cannot become `Nullable` unless the table sets `allow_nullable_key`, so that run fails with `A712`; keep the key non-null with `coalesce()` or `assumeNotNull()`, or set `table_settings (allow_nullable_key true)` and rebuild with `--full-refresh`. +- `fail`: the run fails as a schema change and the table is untouched. +- `ignore`: the table is not changed, and writing a `NULL` into the non-nullable column fails with `A713` instead of storing `0`, `''` or the epoch. ## Settings SQLBuild pins diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py b/src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py index ef459f31fa..699b8a1afe 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/output_columns.py @@ -1,8 +1,8 @@ -"""Detect output columns ClickHouse named after their table alias.""" +"""Column names in ClickHouse query outputs and table key expressions.""" from __future__ import annotations -from sqlbuild.adapters.clickhouse.constants import QUALIFIER_SEPARATOR +from sqlbuild.adapters.clickhouse.constants import KEY_IDENTIFIER_PATTERN, QUALIFIER_SEPARATOR def alias_qualified_columns( @@ -17,3 +17,10 @@ def alias_qualified_columns( return tuple( (produced, column) for produced, column in pairs if produced != column and column in wanted ) + + +def key_identifiers(key_expressions: tuple[str, ...]) -> frozenset[str]: + """The column names that appear in a table's key expressions.""" + + matches: list[tuple[str, str]] = KEY_IDENTIFIER_PATTERN.findall(" ".join(key_expressions)) + return frozenset(quoted or bare for quoted, bare in matches) diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 924ae40e95..93235c3832 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -86,7 +86,10 @@ ) from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.adapters.clickhouse._helpers.output_columns import alias_qualified_columns +from sqlbuild.adapters.clickhouse._helpers.output_columns import ( + alias_qualified_columns, + key_identifiers, +) from sqlbuild.adapters.clickhouse._helpers.partition_rewrite import ( artifact_relation, generation_comment, @@ -120,6 +123,11 @@ MAX_QUERY_SIZE_BYTES, MINIMUM_SERVER_VERSION, MODEL_NAME_CONFIG_KEY, + NULL_IN_ORDINARY_COLUMN_CODE, + NULL_IN_ORDINARY_COLUMN_SERVER_CODE, + NULL_TARGET_COLUMN_PATTERN, + NULLABLE_KEY_CODE, + NULLABLE_KEY_SETTING, OUTPUT_COLUMN_NAME_CODE, PHYSICAL_STORAGE_CONFIG_KEY, RELATION_NOT_FOUND_CODES, @@ -226,7 +234,25 @@ def create_client() -> Any: def _execute(self, *, connection: _ClickHouseConnection, sql: str) -> Any: log_sql(logger=logging.getLogger("sqlbuild.adapter.clickhouse"), sql=sql) - return connection.execute(sql) + try: + return connection.execute(sql) + except Exception as error: + if _error_code(error) != NULL_IN_ORDINARY_COLUMN_SERVER_CODE: + raise + column: re.Match[str] | None = NULL_TARGET_COLUMN_PATTERN.search(str(error)) + raise AdapterUserError( + message=( + "ClickHouse cannot store NULL in non-Nullable column " + f"'{column.group(1) if column else '?'}'" + ), + code=NULL_IN_ORDINARY_COLUMN_CODE, + help=( + "The model produces NULL where the table's column is not Nullable. With " + "on_schema_change append_new_columns (the default) or sync_all_columns " + "SQLBuild makes the column Nullable; otherwise wrap the expression in " + "coalesce() or rebuild the table with --full-refresh." + ), + ) from error def close(self, connection: _ClickHouseConnection) -> None: connection.close() @@ -2046,6 +2072,11 @@ def widen_target_nullability( ) if not widened: return target_columns + self._refuse_nullable_key_columns( + connection=connection, + destination=destination, + columns=tuple(column.name for column in widened), + ) self.alter_column_types( connection=connection, destination=destination, @@ -2055,6 +2086,37 @@ def widen_target_nullability( by_name: dict[str, ColumnInfo] = {column.name: column for column in widened} return tuple(by_name.get(column.name, column) for column in target_columns) + def _refuse_nullable_key_columns( + self, *, connection: Any, destination: str, columns: tuple[str, ...] + ) -> None: + """Explain a key column that cannot become ``Nullable`` before ClickHouse refuses it.""" + + schema, name = _split_relation(destination) + row: Any = self.execute( + connection=connection, + sql="SELECT sorting_key, partition_key, primary_key, create_table_query " + f"FROM system.tables WHERE {_database_filter(schema)} AND name = {quote_string(name)}", + ).fetchone() + if row is None or NULLABLE_KEY_SETTING in str(row[3]): + return + keyed: tuple[str, ...] = tuple( + column for column in columns if column in key_identifiers(tuple(map(str, row[:3]))) + ) + if not keyed: + return + raise AdapterUserError( + message=( + f"{destination}: the model now produces NULL in key column(s) " + f"{', '.join(keyed)}, which ClickHouse cannot make Nullable" + ), + code=NULLABLE_KEY_CODE, + help=( + "Keep the key non-null in the model, for example coalesce(column, default) or " + "assumeNotNull(column), or set table_settings (allow_nullable_key true) and " + "rebuild the table with --full-refresh." + ), + ) + def alter_column_types( self, *, diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index bebad7d7f3..d4c415feb4 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -1,5 +1,7 @@ """ClickHouse adapter constants.""" +import re + DEFAULT_DATABASE: str = "default" DEFAULT_HTTP_PORT: int = 8123 DEFAULT_HTTPS_PORT: int = 8443 @@ -104,3 +106,9 @@ MODEL_NAME_CONFIG_KEY: str = "model_name" OUTPUT_COLUMN_NAME_CODE: str = "A711" QUALIFIER_SEPARATOR: str = "." +NULLABLE_KEY_CODE: str = "A712" +NULLABLE_KEY_SETTING: str = "allow_nullable_key = 1" +KEY_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"`([^`]+)`|([A-Za-z_][A-Za-z0-9_]*)") +NULL_IN_ORDINARY_COLUMN_SERVER_CODE: int = 349 +NULL_IN_ORDINARY_COLUMN_CODE: str = "A713" +NULL_TARGET_COLUMN_PATTERN: re.Pattern[str] = re.compile(r"to destination column (\S+?):") diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 075e086b73..380f3272a4 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -75,6 +75,9 @@ ) _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS +_SCHEMA_CHANGING_POLICIES: frozenset[OnSchemaChange] = frozenset( + {OnSchemaChange.APPEND_NEW_COLUMNS, OnSchemaChange.SYNC_ALL_COLUMNS} +) @dataclass(frozen=True) @@ -210,7 +213,10 @@ def execute_incremental_entry( schema=target_schema, name=target_table, ), - incoming_columns=initial_delta_columns, + incoming_columns=_widenable_columns( + on_schema_change=entry.on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE, + incoming_columns=initial_delta_columns, + ), statement_recorder=statement_recorder, ) delta_columns, type_warnings = _prepare_incremental_delta_types( @@ -706,6 +712,16 @@ def _prepare_incremental_delta_types( return prepared_columns, warnings +def _widenable_columns( + *, on_schema_change: OnSchemaChange, incoming_columns: tuple[ColumnInfo, ...] +) -> tuple[ColumnInfo, ...]: + """The incoming columns target nullability may widen to, under a policy that changes schema.""" + + if on_schema_change in _SCHEMA_CHANGING_POLICIES: + return incoming_columns + return () + + def _apply_schema_change( *, adapter: BaseAdapter, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 7fc9b5fca7..422e775eda 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -92,6 +92,7 @@ from sqlbuild.executor.run._helpers.materializations.incremental import ( _apply_schema_change, _execute_dml, + _widenable_columns, ) from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( @@ -2865,7 +2866,10 @@ def _apply_microbatch_schema_change( connection=context.connection, destination=targets.target_qualified, target_columns=target_columns, - incoming_columns=delta_columns, + incoming_columns=_widenable_columns( + on_schema_change=context.entry.on_schema_change or _DEFAULT_ON_SCHEMA_CHANGE, + incoming_columns=delta_columns, + ), statement_recorder=state.statement_recorder, ) schema_warnings: tuple[str, ...] = _apply_schema_change( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index d21d7d8e96..f46daea1cb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -74,6 +74,8 @@ class ClickHouseNullabilityE2ETestCase: description: str model_sql: str edited_select: str + expected_return_code: int + expected_output_fragment: str expected_rows: tuple[tuple[object, ...], ...] expected_types: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py index 0bfe0fbc5c..f94a53c2d5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py @@ -45,11 +45,53 @@ "nullIf(toString(o.order_id), '3') AS note\n" 'FROM __source("raw_orders") AS o\n' ), + expected_return_code=0, + expected_output_fragment="Completed successfully", expected_rows=(("1", "10", "1"), ("2", None, "2"), ("3", "30", None)), expected_types=(("Nullable(Int64)", "Nullable(String)"),), ), ClickHouseNullabilityE2ETestCase( - description="left join on an append model that ignores schema changes", + description="nullIf on a merge model that fails on schema changes", + model_sql=( + "MODEL (description 'One row per order.',\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [order_id],\n" + " on_schema_change fail,\n" + ");\n\n" + ), + edited_select=( + "SELECT o.order_id AS order_id, nullIf(o.amount, 20) AS amount, " + "toString(o.order_id) AS note\n" + 'FROM __source("raw_orders") AS o\n' + ), + expected_return_code=1, + expected_output_fragment="type changes: amount", + expected_rows=(("1", "10", "1"), ("2", "20", "2"), ("3", "30", "3")), + expected_types=(("Int64", "String"),), + ), + ClickHouseNullabilityE2ETestCase( + description="nullIf on a sorting key column", + model_sql=( + "MODEL (description 'One row per order.',\n" + " materialized incremental,\n" + " incremental_strategy merge,\n" + " unique_key [order_id],\n" + " order_by [note],\n" + ");\n\n" + ), + edited_select=( + "SELECT o.order_id AS order_id, o.amount AS amount, " + "nullIf(toString(o.order_id), '3') AS note\n" + 'FROM __source("raw_orders") AS o\n' + ), + expected_return_code=1, + expected_output_fragment="A712", + expected_rows=(("1", "10", "1"), ("2", "20", "2"), ("3", "30", "3")), + expected_types=(("Int64", "String"),), + ), + ClickHouseNullabilityE2ETestCase( + description="left join on an append model that ignores schema changes fails loudly", model_sql=( "MODEL (description 'One row per order load.',\n" " materialized incremental,\n" @@ -63,20 +105,15 @@ "LEFT JOIN (SELECT toInt64(1) AS order_id, 'one' AS note) AS r " "ON o.order_id = r.order_id\n" ), - expected_rows=( - ("1", "10", "1"), - ("1", "10", "one"), - ("2", "20", "2"), - ("2", "20", None), - ("3", "30", "3"), - ("3", "30", None), - ), - expected_types=(("Int64", "Nullable(String)"),), + expected_return_code=1, + expected_output_fragment="A713", + expected_rows=(("1", "10", "1"), ("2", "20", "2"), ("3", "30", "3")), + expected_types=(("Int64", "String"),), ), ], ids=lambda case: case.description, ) -def test_given_non_nullable_target_when_model_starts_producing_nulls_then_nulls_are_stored( +def test_given_non_nullable_target_when_model_starts_producing_nulls_then_policy_decides( tmp_path: Path, test_case: ClickHouseNullabilityE2ETestCase, clickhouse_e2e_config: dict[str, object], @@ -122,7 +159,8 @@ def test_given_non_nullable_target_when_model_starts_producing_nulls_then_nulls_ command=("--no-color", "build"), project_dir=project_dir ) - assert second.returncode == 0, second.stdout + second.stderr + assert second.returncode == test_case.expected_return_code, second.stdout + second.stderr + assert test_case.expected_output_fragment in second.stdout + second.stderr assert stringify_warehouse_rows( fetch_clickhouse_rows( config=config, diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py index 4f3217da2e..2f6d7bf201 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -80,3 +80,10 @@ class AliasQualifiedColumnsTestCase: actual: tuple[str, ...] expected: tuple[str, ...] expected_pairs: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class KeyIdentifiersTestCase: + description: str + key_expressions: tuple[str, ...] + expected_identifiers: frozenset[str] diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py index 926580859d..71cdbaeb58 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_output_columns.py @@ -2,8 +2,14 @@ import pytest -from sqlbuild.adapters.clickhouse._helpers.output_columns import alias_qualified_columns -from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import AliasQualifiedColumnsTestCase +from sqlbuild.adapters.clickhouse._helpers.output_columns import ( + alias_qualified_columns, + key_identifiers, +) +from tests.unit.src.sqlbuild.adapters.clickhouse._test_types import ( + AliasQualifiedColumnsTestCase, + KeyIdentifiersTestCase, +) @pytest.mark.parametrize( @@ -43,3 +49,31 @@ def test_given_output_columns_when_comparing_with_analysis_then_alias_named_colu alias_qualified_columns(actual=test_case.actual, expected=test_case.expected) == test_case.expected_pairs ) + + +@pytest.mark.parametrize( + "test_case", + [ + KeyIdentifiersTestCase( + description="sorting, partitioning and quoted keys", + key_expressions=( + "customer_id, toDate(ordered_at)", + "toYYYYMM(ordered_at)", + "`order id`", + ), + expected_identifiers=frozenset( + {"customer_id", "toDate", "ordered_at", "toYYYYMM", "order id"} + ), + ), + KeyIdentifiersTestCase( + description="an unsorted table", + key_expressions=("", "", ""), + expected_identifiers=frozenset(), + ), + ], + ids=lambda case: case.description, +) +def test_given_key_expressions_when_reading_then_column_names_are_found( + test_case: KeyIdentifiersTestCase, +) -> None: + assert key_identifiers(test_case.key_expressions) == test_case.expected_identifiers diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index f405329b98..5aa6becff3 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -137,7 +137,11 @@ SQLBuild does not deploy Python functions (`A707`) or SQL table functions (`A708 ## Nullable columns -ClickHouse columns hold `NULL` only when their type is `Nullable`, and a table built from a query takes each column's nullability from the query. When an incremental model later produces `NULL` in a column its table created as non-nullable, SQLBuild changes that column to `Nullable` with `ALTER TABLE ... MODIFY COLUMN` before writing, so the `NULL` is stored as `NULL`. A column in the sorting or partitioning key cannot become `Nullable` unless the table sets `allow_nullable_key`, so that write fails instead. +ClickHouse columns hold `NULL` only when their type is `Nullable`, and a table built from a query takes each column's nullability from the query. When an incremental model later produces `NULL` in a column its table created as non-nullable, what happens follows `on_schema_change`: + +- `append_new_columns` (the default) and `sync_all_columns`: SQLBuild changes the column to `Nullable` with `ALTER TABLE ... MODIFY COLUMN` before writing, so the `NULL` is stored as `NULL`. A column in the sorting, partitioning or primary key cannot become `Nullable` unless the table sets `allow_nullable_key`, so that run fails with `A712`; keep the key non-null with `coalesce()` or `assumeNotNull()`, or set `table_settings (allow_nullable_key true)` and rebuild with `--full-refresh`. +- `fail`: the run fails as a schema change and the table is untouched. +- `ignore`: the table is not changed, and writing a `NULL` into the non-nullable column fails with `A713` instead of storing `0`, `''` or the epoch. ## Settings SQLBuild pins From d6b1d56bd7ab42f83e92f5eb5cf969f88ccb08c2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:48:01 +0100 Subject: [PATCH 30/34] fix: give ClickHouse table copies their own generation Copies made with CREATE TABLE ... AS inherited the source's generation comment, so a clone, hard copy, migration stage or snapshot working copy shared the original table's microbatch generation. Every such copy now clears the comment and takes its own UUID as its generation; only the partition-rewrite exchange carries the generation over. --- .../clickhouse/_helpers/snapshot_rewrite.py | 5 +- .../clickhouse/classes/clickhouse_adapter.py | 7 +-- src/sqlbuild/adapters/clickhouse/constants.py | 1 + .../adapters/clickhouse/_test_types.py | 7 +++ .../sqlbuild/adapters/clickhouse/helpers.py | 28 +++++++++++ .../adapters/clickhouse/test_client.py | 50 +++++++++++++++++++ .../adapters/clickhouse/test_helpers.py | 2 +- 7 files changed, 94 insertions(+), 6 deletions(-) diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py b/src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py index 4499a06766..c51d08dc43 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/snapshot_rewrite.py @@ -5,6 +5,7 @@ import re from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.clickhouse.constants import FRESH_GENERATION_CLAUSE from sqlbuild.adapters.clickhouse.models import SnapshotWorkspace _UPDATE_PREFIX: str = "UPDATE " @@ -34,7 +35,7 @@ def rewrite_snapshot_statements( rewritten: list[str] = [ f"DROP TABLE IF EXISTS {workspace.work}", f"DROP TABLE IF EXISTS {workspace.rebuild}", - f"CREATE TABLE {workspace.work} AS {workspace.destination}", + f"CREATE TABLE {workspace.work} AS {workspace.destination}{FRESH_GENERATION_CLAUSE}", f"INSERT INTO {workspace.work} SELECT * FROM {workspace.destination}", ] statement: str @@ -77,7 +78,7 @@ def _close_statements(*, workspace: SnapshotWorkspace, statement: str) -> tuple[ "GROUP BY __sqb_row" ) return ( - f"CREATE TABLE {workspace.rebuild} AS {workspace.work}", + f"CREATE TABLE {workspace.rebuild} AS {workspace.work}{FRESH_GENERATION_CLAUSE}", f"INSERT INTO {workspace.rebuild} SELECT __target.* REPLACE (" f"if(__closed.__sqb_closed = 1 AND __target.{valid_to} IS NULL, " f"CAST(__closed.__sqb_valid_to, toTypeName(__target.{valid_to})), " diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 93235c3832..bcf53d1924 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -118,6 +118,7 @@ DSN_OPTION, EXCHANGE_UNSUPPORTED_CODES, EXPECTED_OUTPUT_COLUMNS_CONFIG_KEY, + FRESH_GENERATION_CLAUSE, GENERATION_COMMENT_PREFIX, MAX_IDENTIFIER_LENGTH, MAX_QUERY_SIZE_BYTES, @@ -799,7 +800,7 @@ def render_replace_table_from_relation( ) -> tuple[str, ...]: return ( f"DROP TABLE IF EXISTS {destination}", - f"CREATE TABLE {destination} AS {origin}", + f"CREATE TABLE {destination} AS {origin}{FRESH_GENERATION_CLAUSE}", f"INSERT INTO {destination} SELECT * FROM {origin}", ) @@ -818,7 +819,7 @@ def render_clone( return self.render_replace_table_from_relation(destination=destination, origin=origin) return ( f"DROP TABLE IF EXISTS {destination}", - f"CREATE TABLE {destination} AS {origin}", + f"CREATE TABLE {destination} AS {origin}{FRESH_GENERATION_CLAUSE}", f"ALTER TABLE {destination} ATTACH PARTITION ALL FROM {origin}", ) @@ -841,7 +842,7 @@ def render_migration_stage( return MigrationStagePlan( transfer=MigrationTransfer.COPY, statements=( - f"CREATE TABLE {stage} AS {origin}", + f"CREATE TABLE {stage} AS {origin}{FRESH_GENERATION_CLAUSE}", f"INSERT INTO {stage} SELECT * FROM {origin}", ), ) diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index d4c415feb4..7dce598a92 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -112,3 +112,4 @@ NULL_IN_ORDINARY_COLUMN_SERVER_CODE: int = 349 NULL_IN_ORDINARY_COLUMN_CODE: str = "A713" NULL_TARGET_COLUMN_PATTERN: re.Pattern[str] = re.compile(r"to destination column (\S+?):") +FRESH_GENERATION_CLAUSE: str = " COMMENT ''" diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py index 6087ce4c35..f637bf8f58 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/_test_types.py @@ -155,3 +155,10 @@ class ClickHouseGenerationTestCase: description: str incoming_rows: tuple[tuple[int, str, int, str], ...] expected_generation_kept: bool + + +@dataclass(frozen=True) +class ClickHouseCopyGenerationTestCase: + description: str + copy_kind: str + expected_copy_shares_generation: bool diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py index ad1aad40de..93802fa307 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py @@ -422,3 +422,31 @@ def _write_merge( "cursor": _write_cursor, "merge": _write_merge, } + + +def copy_relation( + *, adapter: ClickHouseAdapter, connection: Any, kind: str, origin: str, destination: str +) -> None: + """Copy ``origin`` to ``destination`` through one of the adapter's copy statements.""" + + execute_all( + adapter=adapter, + connection=connection, + statements=_COPY_STATEMENTS[kind](adapter, origin, destination), + ) + + +_COPY_STATEMENTS: dict[str, Callable[[ClickHouseAdapter, str, str], tuple[str, ...]]] = { + "clone": lambda adapter, origin, destination: adapter.render_clone( + origin=origin, destination=destination + ), + "hard copy": lambda adapter, origin, destination: adapter.render_clone( + origin=origin, destination=destination, hard_copy=True + ), + "replace from relation": lambda adapter, origin, destination: ( + adapter.render_replace_table_from_relation(destination=destination, origin=origin) + ), + "migration stage": lambda adapter, origin, destination: ( + adapter.render_migration_stage(origin=origin, stage=destination).statements + ), +} diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py index e775759e5f..5e4337fc3b 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py @@ -24,6 +24,7 @@ ClickHouseCloneTestCase, ClickHouseColumnMigrationTestCase, ClickHouseConcurrentWindowTestCase, + ClickHouseCopyGenerationTestCase, ClickHouseCursorFilterTestCase, ClickHouseFunctionTestCase, ClickHouseGenerationTestCase, @@ -45,6 +46,7 @@ FailingExecute, InjectedFailure, apply_write, + copy_relation, create_orders, cursor_delta_sql, execute_all, @@ -1022,3 +1024,51 @@ def test_given_partition_rewrite_when_reading_generation_then_microbatch_scope_i connection=connection, database=None, schema=clickhouse_schema, name="orders" ) assert (before is not None and before == after) is test_case.expected_generation_kept + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseCopyGenerationTestCase( + description=kind, copy_kind=kind, expected_copy_shares_generation=False + ) + for kind in ("clone", "hard copy", "replace from relation", "migration stage") + ], + ids=lambda case: case.description, +) +def test_given_exchanged_table_when_copying_then_copy_gets_its_own_generation( + test_case: ClickHouseCopyGenerationTestCase, + adapter: ClickHouseAdapter, + connection: Any, + clickhouse_schema: str, +) -> None: + relation: str = f"{clickhouse_schema}.orders" + create_orders( + adapter=adapter, + connection=connection, + relation=relation, + partition_by=_PARTITIONED, + rows=INITIAL_ORDERS, + ) + write_cursor_delta( + adapter=adapter, + connection=connection, + relation=relation, + sql=incoming_rows_sql(((1, "ada", 11, "2026-01-05"), (6, "fa", 60, "2026-03-09"))), + ) + origin_generation: str | None = adapter.physical_relation_generation( + connection=connection, database=None, schema=clickhouse_schema, name="orders" + ) + + copy_relation( + adapter=adapter, + connection=connection, + kind=test_case.copy_kind, + origin=relation, + destination=f"{clickhouse_schema}.orders_copy", + ) + + copy_generation: str | None = adapter.physical_relation_generation( + connection=connection, database=None, schema=clickhouse_schema, name="orders_copy" + ) + assert (copy_generation == origin_generation) is test_case.expected_copy_shares_generation diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py index 3a76620376..16d7731e36 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_helpers.py @@ -158,7 +158,7 @@ def test_given_snapshot_close_when_rewriting_then_working_copy_is_rebuilt_and_ex assert statements[:5] == ( "DROP TABLE IF EXISTS sales.customers__w", "DROP TABLE IF EXISTS sales.customers__r", - "CREATE TABLE sales.customers__w AS sales.customers", + "CREATE TABLE sales.customers__w AS sales.customers COMMENT ''", "INSERT INTO sales.customers__w SELECT * FROM sales.customers", "INSERT INTO sales.customers__w SELECT 1", ) From bbec1d183252d35f2d50deb1943b928177ff7cfb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:14:05 +0100 Subject: [PATCH 31/34] fix: size the lifecycle adapter list for both new adapters --- crates/sqlbuild-core/src/lifecycle_events/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index fed6366c4a..cb337572fc 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -258,7 +258,7 @@ pub const OPERATION_STRATEGIES: [&str; 10] = [ "sql", "sync_all_columns", ]; -pub const OPERATION_ADAPTERS: [&str; 9] = [ +pub const OPERATION_ADAPTERS: [&str; 10] = [ "bigquery", "clickhouse", "custom", From 926e24aa20ff14ec6978f9bfa10034647d114bd4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:22:55 +0100 Subject: [PATCH 32/34] fix: define nullability widening on the SQLite adapter --- .../adapters/sqlite/classes/sqlite_adapter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 330cb97958..21b1f790f6 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -1394,6 +1394,20 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def widen_target_nullability( + self, + *, + connection: Any, + destination: str, + target_columns: tuple[ColumnInfo, ...], + incoming_columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> tuple[ColumnInfo, ...]: + """Return the target columns unchanged; this warehouse never stores NULL as a default.""" + + del connection, destination, incoming_columns, statement_recorder + return target_columns + def alter_column_types( self, *, From a298397988a4cd57ec6fd3662a7c6d494fb16f3f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:43:59 +0100 Subject: [PATCH 33/34] test: record the source final flag in the native assembly goldens Assembled sources now carry the ClickHouse final read option, which defaults to false, so the two assembly golden digests that include sources change. --- ...ct_assembly_error_and_python_value_edges.json | 2 +- ...eeds_and_targets_hooks_audits_a_function.json | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json b/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json index c8a3d30096..6cd965767f 100644 --- a/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json +++ b/tests/goldens/python_oracle/project_assembly_error_and_python_value_edges.json @@ -117,6 +117,6 @@ ], [ "project_assembly", -"cdd9d32ea79d9111e6be4551" +"77a21099e4cf0ed8378806e7" ] ] diff --git a/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json b/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json index e03d4f59aa..c51fc0ba0c 100644 --- a/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json +++ b/tests/goldens/python_oracle/project_assembly_managed_sources_templated_seeds_and_targets_hooks_audits_a_function.json @@ -1,34 +1,34 @@ [ [ "project_assembly", -"c624532f45ee2f49f32bb014" +"c0cc684fdc0838367c4aa5a5" ], [ "project_assembly", -"e8f758602c1eff42b55ae130" +"89e88c0fa0a676eefb1838f8" ], [ "project_assembly", -"c864759e0faad475584471c2" +"9bcbea02e4f8292c20f34a7e" ], [ "project_assembly", -"bef8fb45b929dfd76002a363" +"3b563ba79ba36067df383ee6" ], [ "project_assembly", -"72eebcec28c52bab2cfaff77" +"b9e8a53734face2e3949dd37" ], [ "project_assembly", -"545af810ba171c3734719656" +"f934f6542c039b80ea0ee187" ], [ "project_assembly", -"69e7548205912c60d17e51ba" +"5186cd735cf200339781e260" ], [ "project_assembly", -"2d3e75fa9a354fbf256f6bcb" +"9a072d9d5401f86627a5830b" ] ] From 9fa3cc48eaff8ac4b869740f10f47f6efd9a02de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:56:34 +0100 Subject: [PATCH 34/34] docs: list ClickHouse and SQLite in the temporary test function table --- .../skills/sqlbuild/references/docs/concepts/testing.md | 3 ++- website/src/content/docs/docs/concepts/testing.mdx | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 6b6a0b0fa1..5efe46e116 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -605,7 +605,8 @@ when the test connections close. | PostgreSQL | SQL function in the session's `pg_temp` schema, called as `pg_temp.` | | Snowflake | `CREATE TEMPORARY FUNCTION __sqb_test__` in the session's current schema, for SQL and Python UDFs and table functions | | Databricks | `CREATE TEMPORARY FUNCTION __sqb_test__` for SQL functions | -| BigQuery, SQL Server | Not supported; tests call the built function | +| BigQuery, SQL Server, ClickHouse | Not supported; tests call the built function (ClickHouse functions are server-wide) | +| SQLite | Not applicable; SQLite has no SQL functions | When an adapter cannot create a copy (a Databricks Python UDF, for example), or the warehouse refuses to create it (the role lacks permission, or the Snowflake session has no current schema), diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index ef3f447579..999c0fcae3 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -580,7 +580,8 @@ when the test connections close. | PostgreSQL | SQL function in the session's `pg_temp` schema, called as `pg_temp.` | | Snowflake | `CREATE TEMPORARY FUNCTION __sqb_test__` in the session's current schema, for SQL and Python UDFs and table functions | | Databricks | `CREATE TEMPORARY FUNCTION __sqb_test__` for SQL functions | -| BigQuery, SQL Server | Not supported; tests call the built function | +| BigQuery, SQL Server, ClickHouse | Not supported; tests call the built function (ClickHouse functions are server-wide) | +| SQLite | Not applicable; SQLite has no SQL functions | When an adapter cannot create a copy (a Databricks Python UDF, for example), or the warehouse refuses to create it (the role lacks permission, or the Snowflake session has no current schema),