From 8010267ded1503520b973ec89a721da84fb1e59b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 02:57:34 +0100 Subject: [PATCH 01/32] fix(sql-lint): honour backtick identifiers for the project dialect SQL lint's interpolation-site lexer treated backticks as ordinary code, so in BigQuery and Databricks projects a close paren inside a backtick identifier ended a macro call early, and an apostrophe inside a backtick identifier hid every later macro site. `sqb format` then rewrote the hidden macro (for example `@label(...)` became `@LABEL(...)`), breaking compile. Lint now derives backtick quoting from the configured lint dialect using the same policy rules use, in both the native preparation pass and the Python fallback used for formatting and non-ASCII bodies. Postgres and SQL Server keep treating backticks as code; backslash escapes are unchanged. --- .../src/bindings/_helpers/functions.rs | 29 +++++-- .../src/engine/tests/test_sql_scanning.rs | 59 ++++++++++++- .../src/engine/tests/test_types.rs | 7 ++ .../src/rules/main/mod.rs | 1 + .../src/rules/main/quote_policy.rs | 4 + .../src/sql_lint/_helpers/preparation.rs | 42 ++++++--- .../src/sql_lint/main/backtick_identifiers.rs | 4 + .../src/sql_lint/main/mod.rs | 1 + .../src/sql_lint/main/preparation.rs | 8 +- .../src/sql_scan/models.rs | 10 ++- src/sqlbuild/_native.pyi | 11 ++- src/sqlbuild/lint/_helpers/expansion.py | 16 +++- src/sqlbuild/lint/_helpers/fixes.py | 4 +- src/sqlbuild/lint/_helpers/native_format.py | 2 +- src/sqlbuild/lint/_helpers/native_sql.py | 14 ++- src/sqlbuild/lint/_helpers/sqlbuild_tokens.py | 85 ++++++++++++++----- src/sqlbuild/lint/constants.py | 1 + src/sqlbuild/lint/main/run_lint.py | 6 +- .../sqlbuild/cli/commands/main/_test_types.py | 9 ++ .../sqlbuild/cli/commands/main/test_format.py | 48 +++++++++++ .../src/sqlbuild/lint/_helpers/_test_types.py | 11 +++ .../lint/_helpers/test_sqlbuild_tokens.py | 77 +++++++++++++++-- .../src/sqlbuild/lint/test_expanded_lint.py | 4 +- 23 files changed, 388 insertions(+), 65 deletions(-) create mode 100644 crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 6ecc7baca1..2e4535d748 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{ Bound, IntoPyObject, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use crate::configuration::main::load; use crate::constants::API_VERSION; @@ -38,19 +38,37 @@ fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult, + dialect: String, +} + #[pyfunction] fn prepare_lint_sql( py: Python<'_>, - expanded: &str, - before_expansion: &str, - prior_sites: Vec, + request: LintPreparationRequest, ) -> PyResult> { py.detach(|| { - crate::sql_lint::main::preparation::prepare(expanded, before_expansion, &prior_sites) + crate::sql_lint::main::preparation::prepare( + &request.expanded, + &request.before_expansion, + &request.prior_sites, + &request.dialect, + ) }) .map_err(value_error) } +#[pyfunction] +fn lint_backtick_identifiers(dialect: &str) -> bool { + crate::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect) +} + #[pyfunction] fn lint_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { py.detach(|| crate::sql_lint::main::batch_engine::lint_batch_json(request_json)) @@ -325,6 +343,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; + module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_batch_json, module)?)?; diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs index baea4d051b..f2e5c1b7b4 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers::sql_references::extraction::extract; -use crate::engine::tests::test_types::SqlScannerTestCase; +use crate::engine::tests::test_types::{LintDialectSiteTestCase, SqlScannerTestCase}; use crate::rules::_helpers::evaluation::{normalize_rules_sql, rules_quote_policy}; use crate::rules::_helpers::numeric_decisions::compact_sql; use crate::sql_lint::_helpers::preparation::prepare; @@ -282,7 +282,7 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann test_case.description ); let lint_sql = format!("SELECT @m({fragment} x) AS y"); - let lint_site = prepare(&lint_sql, &lint_sql, &[])?.map(|prepared| { + let lint_site = prepare(&lint_sql, &lint_sql, &[], "postgres")?.map(|prepared| { prepared .1 .first() @@ -304,3 +304,58 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann } Ok(()) } + +#[test] +fn given_lint_dialect_when_preparing_macro_sites_then_backticks_follow_rules_quoting() +-> Result<(), String> { + let test_cases = [ + LintDialectSiteTestCase { + description: "databricks backtick containing close paren", + dialect: "databricks", + fragment: "`a)b`", + expected_lint_site: "@m(`a)b` x)", + }, + LintDialectSiteTestCase { + description: "bigquery doubled backtick", + dialect: "bigquery", + fragment: "`a``)b`", + expected_lint_site: "@m(`a``)b` x)", + }, + LintDialectSiteTestCase { + description: "databricks single quote inside backticks", + dialect: "databricks", + fragment: "`it's)`", + expected_lint_site: "@m(`it's)` x)", + }, + LintDialectSiteTestCase { + description: "databricks backslash in single quote still escapes", + dialect: "databricks", + fragment: "'a\\')'", + expected_lint_site: "@m('a\\')' x)", + }, + LintDialectSiteTestCase { + description: "postgres backtick remains code", + dialect: "postgres", + fragment: "`a)b`", + expected_lint_site: "@m(`a)", + }, + LintDialectSiteTestCase { + description: "sqlserver backtick remains code", + dialect: "tsql", + fragment: "`a)b`", + expected_lint_site: "@m(`a)", + }, + ]; + for test_case in test_cases { + let lint_sql = format!("SELECT @m({} x) AS y", test_case.fragment); + let lint_site = prepare(&lint_sql, &lint_sql, &[], test_case.dialect)? + .and_then(|prepared| prepared.1.first().map(|site| site.5.clone())) + .unwrap_or_default(); + assert_eq!( + lint_site, test_case.expected_lint_site, + "SQL lint macro site: {}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs index 9c319a4f50..ab954a9371 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs @@ -169,6 +169,13 @@ pub(crate) struct EmptyInputMinimumTestsTestCase { } /// One SQL fragment scanned by every native quote- and comment-aware scanner. +pub(crate) struct LintDialectSiteTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) fragment: &'static str, + pub(crate) expected_lint_site: &'static str, +} + pub(crate) struct SqlScannerTestCase { pub(crate) description: &'static str, pub(crate) fragment: &'static str, diff --git a/crates/sqlbuild-rules-native/src/rules/main/mod.rs b/crates/sqlbuild-rules-native/src/rules/main/mod.rs index 3ecc50b539..605292c0ac 100644 --- a/crates/sqlbuild-rules-native/src/rules/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/rules/main/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod catalogue; pub(crate) mod evaluate; pub(crate) mod evaluate_project; pub(crate) mod fingerprint; +pub(crate) mod quote_policy; pub(crate) mod resolve_threshold_overrides; pub(crate) mod select; pub(crate) mod selected_codes; diff --git a/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs b/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs new file mode 100644 index 0000000000..a33a47e140 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs @@ -0,0 +1,4 @@ +/// Return the quoting policy rules use when scanning SQL for `dialect_name`. +pub(crate) fn quote_policy(dialect_name: &str) -> crate::sql_scan::models::QuotePolicy { + crate::rules::_helpers::evaluation::rules_quote_policy(dialect_name) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs index 872d157226..e0d6f65181 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs @@ -5,14 +5,22 @@ use std::sync::LazyLock; use regex::Regex; +use crate::rules::main::quote_policy::quote_policy; use crate::sql_lint::types::{InterpolationSite, PreparedSql}; use crate::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; use crate::sql_scan::models::QuotePolicy; -static SITES: LazyLock> = LazyLock::new(|| { - Regex::new( - r#"--[^\n]*(?:\n|\z)|/\*[\s\S]*?(?:\*/|\z)|'(?:\\.|''|[^'\\])*(?:'|\z)|"(?:\\.|""|[^"\\])*(?:"|\z)|(?P@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()"#, - ) +const SITE_PATTERN: &str = + r#"(?P@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()"#; +const NON_CODE_PATTERN: &str = r#"--[^\n]*(?:\n|\z)|/\*[\s\S]*?(?:\*/|\z)|'(?:\\.|''|[^'\\])*(?:'|\z)|"(?:\\.|""|[^"\\])*(?:"|\z)"#; +const BACKTICK_PATTERN: &str = r#"`(?:``|[^`])*(?:`|\z)"#; + +static SITES: LazyLock> = + LazyLock::new(|| Regex::new(&format!("{NON_CODE_PATTERN}|{SITE_PATTERN}"))); +static BACKTICK_SITES: LazyLock> = LazyLock::new(|| { + Regex::new(&format!( + "{NON_CODE_PATTERN}|{BACKTICK_PATTERN}|{SITE_PATTERN}" + )) }); static CTES: LazyLock> = LazyLock::new(|| { Regex::new( @@ -24,15 +32,27 @@ static WORDS: LazyLock> = static OPAQUE_CTE: LazyLock> = LazyLock::new(|| Regex::new(r"(?i)\b[A-Za-z_][A-Za-z0-9_]*\s+AS\s*\(\s*(?:--[^\n]*\n\s*)?$")); +/// Return whether SQL lint treats backticks as identifier quotes for `dialect`, as rules do. +pub(crate) fn backtick_identifiers(dialect: &str) -> bool { + quote_policy(dialect).backtick_identifiers +} + pub(crate) fn prepare( expanded: &str, before_expansion: &str, prior_sites: &[usize], + dialect: &str, ) -> Result, String> { if !expanded.is_ascii() || !before_expansion.is_ascii() { return Ok(None); } - let site_pattern = SITES.as_ref().map_err(|error| error.to_string())?; + let policy = QuotePolicy::SQL_LINT.with_backtick_identifiers(backtick_identifiers(dialect)); + let sites_regex = if policy.backtick_identifiers { + &BACKTICK_SITES + } else { + &SITES + }; + let site_pattern = sites_regex.as_ref().map_err(|error| error.to_string())?; let cte_pattern = CTES.as_ref().map_err(|error| error.to_string())?; let word_pattern = WORDS.as_ref().map_err(|error| error.to_string())?; let opaque_pattern = OPAQUE_CTE.as_ref().map_err(|error| error.to_string())?; @@ -47,7 +67,7 @@ pub(crate) fn prepare( if start < copied_to { continue; } - let Some(end) = site_end(expanded, start) else { + let Some(end) = site_end(expanded, start, policy) else { continue; }; text.push_str(&expanded[copied_to..start]); @@ -96,7 +116,7 @@ pub(crate) fn prepare( Ok(Some((text, sites, referenced))) } -fn site_end(text: &str, start: usize) -> Option { +fn site_end(text: &str, start: usize, policy: QuotePolicy) -> Option { let bytes = text.as_bytes(); match bytes[start] { b'@' if bytes.get(start + 1) == Some(&b'@') => Some(scan_name(bytes, start + 2, true)), @@ -110,7 +130,7 @@ fn site_end(text: &str, start: usize) -> Option { } let end = scan_name(bytes, start + 1, false); if bytes.get(end) == Some(&b'(') { - matching_paren(bytes, end) + matching_paren(bytes, end, policy) } else { Some(end) } @@ -119,7 +139,7 @@ fn site_end(text: &str, start: usize) -> Option { b'_' => { let end = scan_name(bytes, start, false); (bytes.get(end) == Some(&b'(')) - .then(|| matching_paren(bytes, end)) + .then(|| matching_paren(bytes, end, policy)) .flatten() } _ => None, @@ -137,8 +157,8 @@ fn scan_name(bytes: &[u8], mut index: usize, interpolation: bool) -> usize { index } -fn matching_paren(bytes: &[u8], open: usize) -> Option { - match scan_matching_paren(bytes, open, QuotePolicy::SQL_LINT) { +fn matching_paren(bytes: &[u8], open: usize, policy: QuotePolicy) -> Option { + match scan_matching_paren(bytes, open, policy) { Ok(close) => Some(close + 1), Err(_) => None, } diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs b/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs new file mode 100644 index 0000000000..f0cc7a2a75 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs @@ -0,0 +1,4 @@ +/// Return whether SQL lint treats backticks as identifier quotes for `dialect`. +pub(crate) fn backtick_identifiers(dialect: &str) -> bool { + crate::sql_lint::_helpers::preparation::backtick_identifiers(dialect) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs index 506a01d172..f5c37dd789 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod backtick_identifiers; pub(crate) mod batch_engine; pub(crate) mod batch_formatter; pub(crate) mod catalogue; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs b/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs index 6a3aa9e3ec..a1b2710ffd 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs @@ -6,6 +6,12 @@ pub(crate) fn prepare( expanded: &str, before_expansion: &str, prior_sites: &[usize], + dialect: &str, ) -> Result, String> { - crate::sql_lint::_helpers::preparation::prepare(expanded, before_expansion, prior_sites) + crate::sql_lint::_helpers::preparation::prepare( + expanded, + before_expansion, + prior_sites, + dialect, + ) } diff --git a/crates/sqlbuild-rules-native/src/sql_scan/models.rs b/crates/sqlbuild-rules-native/src/sql_scan/models.rs index 74e0f5a908..4f508b195c 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/models.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/models.rs @@ -27,13 +27,21 @@ impl QuotePolicy { double_quote_backslash_escapes: false, }; - /// SQL lint policy, mirroring its site lexer: backslashes escape, backticks are code. + /// SQL lint base policy: backslashes escape; lint enables backticks per dialect. pub(crate) const SQL_LINT: Self = Self { backtick_identifiers: false, single_quote_backslash_escapes: true, double_quote_backslash_escapes: true, }; + /// Return this policy with backtick-delimited identifiers enabled or disabled. + pub(crate) const fn with_backtick_identifiers(self, backtick_identifiers: bool) -> Self { + Self { + backtick_identifiers, + ..self + } + } + /// Return whether `byte` opens quoted text under this policy. pub(crate) fn is_quote(self, byte: u8) -> bool { match byte { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 264de8d6fb..f4a8b177bd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,5 +1,13 @@ """Private SQLBuild native engine bindings.""" +from typing import TypedDict + +class LintPreparationRequest(TypedDict): + expanded: str + before_expansion: str + prior_sites: list[int] + dialect: str + def catalogue_json() -> str: ... def evaluate_json(request_json: str) -> str: ... def finalize_rule_findings_json(request_json: str) -> str: ... @@ -8,8 +16,9 @@ def format_sql_batch_json(request_json: str) -> str: ... def lint_sql_json(request_json: str) -> str: ... def lint_sql_batch_json(request_json: str) -> str: ... def prepare_lint_sql( - expanded: str, before_expansion: str, prior_sites: list[int] + request: LintPreparationRequest, ) -> tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None: ... +def lint_backtick_identifiers(dialect: str) -> bool: ... def load_config_json(project_dir: str) -> str: ... def selected_codes_json(request_json: str) -> str: ... def render_owned_skill(content: str, input_fingerprint: str) -> str: ... diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 72068bc867..3ad24a1da7 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -135,9 +135,9 @@ def prepare_lint_body( project_dir: Path, file_path: Path, contents: str, - body_start: int, - body_end: int, + body_range: tuple[int, int], context: SqlExpansionContext, + dialect: str, external_identifiers: tuple[str, ...] = (), allows_ceremonial_select: bool = False, allows_dynamic_output_star: bool = False, @@ -145,6 +145,9 @@ def prepare_lint_body( ) -> LintBody: """Expand one authored body and neutralize whatever interpolation remains.""" + body_start: int + body_end: int + body_start, body_end = body_range authored_body: str = contents[body_start:body_end] pre_expansion_sites: tuple[InterpolationSite, ...] = () expansion_input: str = authored_body @@ -184,11 +187,16 @@ def prepare_lint_body( sites: tuple[InterpolationSite, ...] native_prepared: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( _native.prepare_lint_sql( - expanded, expansion_input, [site.neutralized_start for site in pre_expansion_sites] + { + "expanded": expanded, + "before_expansion": expansion_input, + "prior_sites": [site.neutralized_start for site in pre_expansion_sites], + "dialect": dialect, + } ) ) if native_prepared is None: - neutralized, sites = neutralize_interpolation(body=expanded) + neutralized, sites = neutralize_interpolation(body=expanded, dialect=dialect) externally_referenced_ctes: tuple[str, ...] = _externally_referenced_ctes( expanded=expanded, interpolation_sites=sites, diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 93eb37b407..f97f085b54 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -89,9 +89,9 @@ def lint_contents( project_dir=project_dir, file_path=file_path, contents=contents, - body_start=body_start, - body_end=body_end, + body_range=(body_start, body_end), context=context, + dialect=config.dialect, ) ) if bodies: diff --git a/src/sqlbuild/lint/_helpers/native_format.py b/src/sqlbuild/lint/_helpers/native_format.py index 0dc110b5de..2f55ea9ff1 100644 --- a/src/sqlbuild/lint/_helpers/native_format.py +++ b/src/sqlbuild/lint/_helpers/native_format.py @@ -66,7 +66,7 @@ def format_native_sql_bodies( body: str = contents[body_start:body_end] trailing: str = body[len(body.rstrip()) :] core: str = body[: len(body) - len(trailing)] if trailing else body - neutralized, sites = neutralize_interpolation(body=core) + neutralized, sites = neutralize_interpolation(body=core, dialect=config.dialect) cache_key: tuple[str, str] = (neutralized, config.dialect) requests_by_key.setdefault( cache_key, diff --git a/src/sqlbuild/lint/_helpers/native_sql.py b/src/sqlbuild/lint/_helpers/native_sql.py index a18b081d4e..ab02e7f638 100644 --- a/src/sqlbuild/lint/_helpers/native_sql.py +++ b/src/sqlbuild/lint/_helpers/native_sql.py @@ -132,6 +132,7 @@ def run_native_sql_lint( raw_diagnostic=raw_diagnostic, body=body, contents=contents_by_path[body.file_path], + dialect=config.dialect, ) if violation is not None: violations_by_file.setdefault(body.file_path, []).append(violation) @@ -220,7 +221,7 @@ def _parse_failure_violation( def _authored_violation( - *, raw_diagnostic: object, body: LintBody, contents: str + *, raw_diagnostic: object, body: LintBody, contents: str, dialect: str ) -> LintViolation | None: if not isinstance(raw_diagnostic, dict): raise NativeLintError("native lint diagnostic must be an object") @@ -263,6 +264,7 @@ def _authored_violation( body=body, contents=contents, line_starts=starts, + dialect=dialect, ) fix: LintEdit | None = _authored_fix( raw_fix=raw_fix, @@ -282,6 +284,7 @@ def _authored_violation( mapped=mapped, contents=contents, absolute_offset=absolute_offset, + dialect=dialect, ), severity=VIOLATION_SEVERITY_WARNING, engine=LINT_ENGINE_NATIVE, @@ -341,11 +344,14 @@ def _authored_end_position( body: LintBody, contents: str, line_starts: tuple[int, ...], + dialect: str, ) -> tuple[int, int] | None: """Map a native exclusive end offset only when authored continuity is provable.""" if mapped_start.generated: - token: str | None = interpolation_text_at(body=contents, start=absolute_start) + token: str | None = interpolation_text_at( + body=contents, start=absolute_start, dialect=dialect + ) if token is None: return None return _offset_position(offset=absolute_start + len(token), line_starts=line_starts) @@ -359,11 +365,11 @@ def _authored_end_position( def _violation_message( - *, message: str, mapped: MappedOffset, contents: str, absolute_offset: int + *, message: str, mapped: MappedOffset, contents: str, absolute_offset: int, dialect: str ) -> str: if not mapped.generated: return message - token: str | None = interpolation_text_at(body=contents, start=absolute_offset) + token: str | None = interpolation_text_at(body=contents, start=absolute_offset, dialect=dialect) if token is None: return f"{message} {GENERATED_SQL_MESSAGE_SUFFIX}" return f"{message} {GENERATED_SQL_MESSAGE_TEMPLATE.format(token=token)}" diff --git a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py index 3519ed0970..bea7500304 100644 --- a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py +++ b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py @@ -3,7 +3,9 @@ from __future__ import annotations import re +from functools import cache +import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import ( MACRO_TOKEN, SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN, @@ -12,6 +14,7 @@ ) from sqlbuild.compiler.compile.models import ExpansionSpan from sqlbuild.lint.constants import ( + BACKTICK_CHARACTER, CLOSING_PAREN_CHARACTER, EXPECTED_SENTINEL_OCCURRENCES, IDENTIFIER_EXTRA_CHARACTER, @@ -43,15 +46,29 @@ _SQLBUILD_FUNCTION_PATTERN: str = "|".join( re.escape(name) for name in sorted(_SQLBUILD_FUNCTION_NAMES) ) +_NON_CODE_SCAN_PATTERN: str = ( + r"--[^\n]*(?:\n|\Z)" + r"|/\*[\s\S]*?(?:\*/|\Z)" + r"|'(?:\\.|''|[^'\\])*(?:'|\Z)" + r'|"(?:\\.|""|[^"\\])*(?:"|\Z)' +) +_BACKTICK_SCAN_PATTERN: str = r"|`(?:``|[^`])*(?:`|\Z)" +_SITE_SCAN_PATTERN: str = rf"|(?P@@|\$\{{|@|(?:{_SQLBUILD_FUNCTION_PATTERN})\s*\()" _INTERPOLATION_SCAN_PATTERN: re.Pattern[str] = re.compile( - rf"--[^\n]*(?:\n|\Z)" - rf"|/\*[\s\S]*?(?:\*/|\Z)" - rf"|'(?:\\.|''|[^'\\])*(?:'|\Z)" - rf'|"(?:\\.|""|[^"\\])*(?:"|\Z)' - rf"|(?P@@|\$\{{|@|(?:{_SQLBUILD_FUNCTION_PATTERN})\s*\()" + _NON_CODE_SCAN_PATTERN + _SITE_SCAN_PATTERN +) +_BACKTICK_INTERPOLATION_SCAN_PATTERN: re.Pattern[str] = re.compile( + _NON_CODE_SCAN_PATTERN + _BACKTICK_SCAN_PATTERN + _SITE_SCAN_PATTERN ) +@cache +def _backtick_identifiers(dialect: str) -> bool: + """Return whether lint treats backticks as identifier quotes, matching the native lexer.""" + + return _native.lint_backtick_identifiers(dialect) + + def neutralize_context_interpolation(*, body: str) -> tuple[str, tuple[InterpolationSite, ...]]: """Replace runtime CTX interpolation with parseable generated sentinels.""" @@ -64,9 +81,7 @@ def neutralize_context_interpolation(*, body: str) -> tuple[str, tuple[Interpola start: int = body.find(token, copied_to) if start < 0: break - end: int | None = _interpolation_site_end(body=body, start=start) - if end is None: - break + end: int = _interpolation_name_end(body=body, start=start + len(SQL_INTERPOLATION_TOKEN)) literal: str = body[copied_to:start] sentinel: str = _CONTEXT_SENTINEL_TEMPLATE.format(index=len(sites)) pieces.extend((literal, sentinel)) @@ -123,21 +138,31 @@ def neutralize_generic_audit_parameters(*, body: str) -> tuple[str, tuple[Interp return "".join(pieces), tuple(sites) -def neutralize_interpolation(*, body: str) -> tuple[str, tuple[InterpolationSite, ...]]: +def neutralize_interpolation( + *, body: str, dialect: str +) -> tuple[str, tuple[InterpolationSite, ...]]: """Replace every interpolation site with a unique sentinel identifier.""" if not _contains_interpolation_candidate(body=body): return body, () + backtick_identifiers: bool = _backtick_identifiers(dialect) + scan_pattern: re.Pattern[str] = ( + _BACKTICK_INTERPOLATION_SCAN_PATTERN + if backtick_identifiers + else _INTERPOLATION_SCAN_PATTERN + ) sites: list[InterpolationSite] = [] pieces: list[str] = [] neutralized_length: int = 0 copied_to: int = 0 match: re.Match[str] - for match in _INTERPOLATION_SCAN_PATTERN.finditer(body): + for match in scan_pattern.finditer(body): if match.group("site") is None or match.start() < copied_to: continue site_start: int = match.start() - site_end: int | None = _interpolation_site_end(body=body, start=site_start) + site_end: int | None = _interpolation_site_end( + body=body, start=site_start, backtick_identifiers=backtick_identifiers + ) if site_end is None: continue literal: str = body[copied_to:site_start] @@ -190,10 +215,12 @@ def restore_interpolation(*, fixed: str, sites: tuple[InterpolationSite, ...]) - return restored -def interpolation_text_at(*, body: str, start: int) -> str | None: +def interpolation_text_at(*, body: str, start: int, dialect: str) -> str | None: """Return the interpolation token beginning at an offset, if there is one.""" - site_end: int | None = _interpolation_site_end(body=body, start=start) + site_end: int | None = _interpolation_site_end( + body=body, start=start, backtick_identifiers=_backtick_identifiers(dialect) + ) if site_end is None: return None return body[start:site_end] @@ -232,24 +259,28 @@ def map_neutralized_offset(*, offset: int, sites: tuple[InterpolationSite, ...]) return mapped -def _interpolation_site_end(*, body: str, start: int) -> int | None: +def _interpolation_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: character: str = body[start] if character == MACRO_TOKEN: if body.startswith(SQL_INTERPOLATION_TOKEN, start): return _interpolation_name_end(body=body, start=start + len(SQL_INTERPOLATION_TOKEN)) - return _macro_site_end(body=body, start=start) + return _macro_site_end(body=body, start=start, backtick_identifiers=backtick_identifiers) if character == TEMPLATE_INTERPOLATION_START[0]: return _template_site_end(body=body, start=start) if character == IDENTIFIER_EXTRA_CHARACTER and body.startswith(_SQLBUILD_FUNCTION_NAMES, start): - return _sqlbuild_function_site_end(body=body, start=start) + return _sqlbuild_function_site_end( + body=body, start=start, backtick_identifiers=backtick_identifiers + ) return None -def _sqlbuild_function_site_end(*, body: str, start: int) -> int | None: +def _sqlbuild_function_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: name_end: int = _identifier_end(body=body, start=start) if name_end >= len(body) or body[name_end] != OPENING_PAREN_CHARACTER: return None - call_end: int | None = _matching_paren_end(body=body, opening_index=name_end) + call_end: int | None = _matching_paren_end( + body=body, opening_index=name_end, backtick_identifiers=backtick_identifiers + ) return call_end @@ -262,7 +293,7 @@ def _template_site_end(*, body: str, start: int) -> int | None: return end_index + len(TEMPLATE_INTERPOLATION_END) -def _macro_site_end(*, body: str, start: int) -> int | None: +def _macro_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: name_start: int = start + len(MACRO_TOKEN) if name_start >= len(body): return None @@ -276,10 +307,12 @@ def _macro_site_end(*, body: str, start: int) -> int | None: name_end: int = _identifier_end(body=body, start=name_start) if name_end >= len(body) or body[name_end] != OPENING_PAREN_CHARACTER: return name_end - return _matching_paren_end(body=body, opening_index=name_end) + return _matching_paren_end( + body=body, opening_index=name_end, backtick_identifiers=backtick_identifiers + ) -def _matching_paren_end(*, body: str, opening_index: int) -> int | None: +def _matching_paren_end(*, body: str, opening_index: int, backtick_identifiers: bool) -> int | None: depth: int = 0 index: int = opening_index length: int = len(body) @@ -295,7 +328,11 @@ def _matching_paren_end(*, body: str, opening_index: int) -> int | None: continue character: str = body[index] if quote_character is not None: - if character == SQL_ESCAPE_CHARACTER and index + 1 < length: + if ( + character == SQL_ESCAPE_CHARACTER + and quote_character in SQL_QUOTE_CHARACTERS + and index + 1 < length + ): index += 2 continue if character == quote_character: @@ -305,7 +342,9 @@ def _matching_paren_end(*, body: str, opening_index: int) -> int | None: quote_character = None index += 1 continue - if character in SQL_QUOTE_CHARACTERS: + if character in SQL_QUOTE_CHARACTERS or ( + backtick_identifiers and character == BACKTICK_CHARACTER + ): quote_character = character index += 1 continue diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index e39076ac2e..cf2dcf769b 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -78,6 +78,7 @@ ADAPTER_CONFIG_KEY: str = "adapter" SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) +BACKTICK_CHARACTER: str = "`" SQL_ESCAPE_CHARACTER: str = "\\" SINGLE_QUOTE_CHARACTER: str = "'" OPENING_PAREN_CHARACTER: str = "(" diff --git a/src/sqlbuild/lint/main/run_lint.py b/src/sqlbuild/lint/main/run_lint.py index 522387c993..8c177abae2 100644 --- a/src/sqlbuild/lint/main/run_lint.py +++ b/src/sqlbuild/lint/main/run_lint.py @@ -78,6 +78,7 @@ def run_lint( contents=contents, headers=headers, context=context, + dialect=config.dialect, project_dir=project_dir, allows_dynamic_output_star=file_path.resolve() in dynamic_output_paths, compiled_expansions=compiled_expansions, @@ -124,6 +125,7 @@ def _prepared_bodies( contents: str, headers: tuple[HeaderSpan, ...], context: SqlExpansionContext, + dialect: str, project_dir: Path, allows_dynamic_output_star: bool, compiled_expansions: dict[Path, CompiledSqlExpansion] | None, @@ -151,9 +153,9 @@ def _prepared_bodies( project_dir=project_dir, file_path=file_path, contents=contents, - body_start=body_start, - body_end=body_end, + body_range=(body_start, body_end), context=context, + dialect=dialect, external_identifiers=external_identifiers, allows_ceremonial_select=allows_ceremonial_select, allows_dynamic_output_star=allows_dynamic_output_star, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 33339a443e..26614e5391 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -149,6 +149,15 @@ class FormatCompileIntegrationTestCase: expected_literal: str +@dataclass(frozen=True) +class BacktickDialectFormatIntegrationTestCase: + """One backtick-identifier adapter whose macro call must survive formatting.""" + + description: str + adapter: str + expected_literal: str + + @dataclass(frozen=True) class DescriptionFormatIntegrationTestCase: """One description wrapping expectation through the real CLI and compiler.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index bad46249d0..961dcb0c07 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -10,6 +10,7 @@ from sqlbuild.cli.commands.main.entrypoint.entry import main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + BacktickDialectFormatIntegrationTestCase, CanonicalFixtureFormatIntegrationTestCase, DescriptionFormatIntegrationTestCase, FormatCompileIntegrationTestCase, @@ -926,3 +927,50 @@ def test_given_leading_cte_comment_when_formatting_then_comment_rule_still_passe assert test_case.expected_fragment in formatted_sql assert model_path.read_text(encoding="utf-8") == formatted_sql assert "SQBRSQL033" not in output.out + output.err + + +@pytest.mark.parametrize( + "test_case", + [ + BacktickDialectFormatIntegrationTestCase( + description="databricks apostrophe inside a backtick identifier keeps the macro", + adapter="databricks", + expected_literal='@label("order_id") AS order_id', + ), + BacktickDialectFormatIntegrationTestCase( + description="bigquery apostrophe inside a backtick identifier keeps the macro", + adapter="bigquery", + expected_literal='@label("order_id") AS order_id', + ), + ], + ids=lambda case: case.description, +) +def test_given_backtick_identifier_apostrophe_when_formatting_then_macro_call_still_compiles( + test_case: BacktickDialectFormatIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.adapter}"\n', encoding="utf-8" + ) + macros: Path = tmp_path / "models" / "_macros" + macros.mkdir(parents=True) + (macros / "labels.py").write_text( + 'def label(expression: str) -> str:\n """Return one column expression."""\n' + " return expression\n", + encoding="utf-8", + ) + model_path: Path = tmp_path / "models" / "orders.sql" + model_path.write_text( + 'MODEL (description "Orders", database warehouse, schema analytics);\n\n' + 'select `customer\'s id` as customer_id, @label("order_id") as order_id ' + "from raw_orders\n", + encoding="utf-8", + ) + + format_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "format"]) + compile_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "compile"]) + output: CaptureResult[str] = capsys.readouterr() + + assert (format_exit, compile_exit) == (0, 0), output.out + output.err + assert test_case.expected_literal in model_path.read_text(encoding="utf-8") diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index b22f40f168..ff9a6dba16 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -146,6 +146,17 @@ class NeutralizeInterpolationTestCase: expected_original_texts: tuple[str, ...] +@dataclass(frozen=True) +class DialectNeutralizeInterpolationTestCase: + """Test case for interpolation neutralization under one lint dialect.""" + + description: str + dialect: str + body: str + expected_neutralized: str + expected_original_texts: tuple[str, ...] + + @dataclass(frozen=True) class MapOffsetTestCase: """Test case for neutralized-to-original offset mapping.""" diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index f3a00e4407..3ac410fd38 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -13,6 +13,7 @@ from sqlbuild.lint.exceptions import InterpolationRestorationError from sqlbuild.lint.models import InterpolationSite from tests.unit.src.sqlbuild.lint._helpers._test_types import ( + DialectNeutralizeInterpolationTestCase, MapOffsetTestCase, NeutralizeInterpolationTestCase, RestoreFailureTestCase, @@ -147,11 +148,75 @@ def test_given_body_when_neutralizing_then_sentinels_replace_interpolation( ) -> None: neutralized: str sites: tuple[InterpolationSite, ...] - neutralized, sites = neutralize_interpolation(body=test_case.body) + neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") assert neutralized == test_case.expected_neutralized assert tuple(site.original_text for site in sites) == test_case.expected_original_texts native: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( - _native.prepare_lint_sql(test_case.body, test_case.body, []) + _native.prepare_lint_sql( + { + "expanded": test_case.body, + "before_expansion": test_case.body, + "prior_sites": [], + "dialect": "generic", + } + ) + ) + assert native is not None + assert native[0] == test_case.expected_neutralized + assert tuple(InterpolationSite(*site) for site in native[1]) == sites + + +@pytest.mark.parametrize( + "test_case", + [ + DialectNeutralizeInterpolationTestCase( + description="databricks backtick keeps a close paren inside a macro call", + dialect="databricks", + body="SELECT @m(`a)b` x) AS y", + expected_neutralized="SELECT __sqb_lint_0__ AS y", + expected_original_texts=("@m(`a)b` x)",), + ), + DialectNeutralizeInterpolationTestCase( + description="bigquery backtick apostrophe does not hide a later macro", + dialect="bigquery", + body='SELECT `customer\'s id` AS customer_id, @label("order_id") AS order_id', + expected_neutralized="SELECT `customer's id` AS customer_id, __sqb_lint_0__ AS order_id", + expected_original_texts=('@label("order_id")',), + ), + DialectNeutralizeInterpolationTestCase( + description="postgres backtick remains ordinary code", + dialect="postgres", + body="SELECT @m(`a)b` x) AS y", + expected_neutralized="SELECT __sqb_lint_0__b` x) AS y", + expected_original_texts=("@m(`a)",), + ), + DialectNeutralizeInterpolationTestCase( + description="sqlserver backtick remains ordinary code", + dialect="tsql", + body="SELECT @m(`a)b` x) AS y", + expected_neutralized="SELECT __sqb_lint_0__b` x) AS y", + expected_original_texts=("@m(`a)",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_quoting_when_neutralizing_then_python_and_native_sites_agree( + test_case: DialectNeutralizeInterpolationTestCase, +) -> None: + neutralized: str + sites: tuple[InterpolationSite, ...] + neutralized, sites = neutralize_interpolation(body=test_case.body, dialect=test_case.dialect) + assert neutralized == test_case.expected_neutralized + assert tuple(site.original_text for site in sites) == test_case.expected_original_texts + native: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( + _native.prepare_lint_sql( + { + "expanded": test_case.body, + "before_expansion": test_case.body, + "prior_sites": [], + "dialect": test_case.dialect, + } + ) ) assert native is not None assert native[0] == test_case.expected_neutralized @@ -177,7 +242,7 @@ def test_given_many_sites_when_neutralizing_then_every_sentinel_is_distinct( ) -> None: neutralized: str sites: tuple[InterpolationSite, ...] - neutralized, sites = neutralize_interpolation(body=test_case.body) + neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") assert neutralized == test_case.expected_neutralized sentinels: tuple[str, ...] = tuple(site.sentinel for site in sites) assert len(set(sentinels)) == len(test_case.expected_original_texts) @@ -223,7 +288,7 @@ def test_given_neutralized_offset_when_mapping_then_authored_offset_matches( test_case: MapOffsetTestCase, ) -> None: sites: tuple[InterpolationSite, ...] - _neutralized, sites = neutralize_interpolation(body=test_case.body) + _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") mapped: int = map_neutralized_offset(offset=test_case.neutralized_offset, sites=sites) assert mapped == test_case.expected_original_offset @@ -283,7 +348,7 @@ def test_given_fixed_text_when_restoring_then_original_interpolation_returns( test_case: RestoreInterpolationTestCase, ) -> None: sites: tuple[InterpolationSite, ...] - _neutralized, sites = neutralize_interpolation(body=test_case.body) + _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") restored: str = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) assert restored == test_case.expected_restored @@ -310,7 +375,7 @@ def test_given_mangled_sentinels_when_restoring_then_error_is_raised( test_case: RestoreFailureTestCase, ) -> None: sites: tuple[InterpolationSite, ...] - _neutralized, sites = neutralize_interpolation(body=test_case.body) + _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") with pytest.raises(InterpolationRestorationError) as error: _ = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) assert test_case.expected_message_fragment in str(error.value) diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 6f9b2ffb29..d4c8801470 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -271,9 +271,9 @@ def test_given_generic_audit_argument_when_mapping_then_generated_region_is_not_ project_dir=tmp_path, file_path=audit, contents=contents, - body_start=10, - body_end=len(contents), + body_range=(10, len(contents)), context=context, + dialect="duckdb", ) sentinel_offset: int = body.lint_text.index("__sqlbuild_audit_parameter_0__") mapped: MappedOffset = map_expanded_offset(offset=sentinel_offset, passes=body.passes) From a73bb1b5141e1dd5ef711657fb518b44e2615b96 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:01:57 +0100 Subject: [PATCH 02/32] fix(rules): skip comments in the generic parser fallback normalisation When a model's dialect parser rejected it, the generic fallback rewrote `:` paths and `->` lambdas while skipping only quoted text. An apostrophe in a `--` or `/* */` comment (for example `-- don't`) opened a phantom quote that never closed, so normalisation stopped and the fallback parse failed. The fallback now skips comments and quotes through the shared scanner. --- .../src/engine/tests/test_evaluation.rs | 42 +++++++++++++++++++ .../src/rules/_helpers/evaluation.rs | 14 +++---- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs index 03d28d174c..7b904bff6f 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs @@ -479,6 +479,48 @@ fn given_dialect_quoting_around_table_functions_when_evaluating_rules_then_model Ok(()) } +#[test] +fn given_comment_apostrophes_when_generic_fallback_normalizes_rules_sql_then_model_parses() +-> Result<(), String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let test_cases = [ + test_types::DialectEvaluationTestCase { + description: "line comment apostrophe before a colon path", + dialect: "postgres", + query_sql: "-- don't drop the customer path\nSELECT payload:customer_id AS customer_id FROM orders", + expected_code: "SQBRCONTRACT101", + }, + test_types::DialectEvaluationTestCase { + description: "block comment apostrophe before a colon path", + dialect: "postgres", + query_sql: "/* customer's path */ SELECT payload:customer_id AS customer_id FROM orders", + expected_code: "SQBRCONTRACT101", + }, + test_types::DialectEvaluationTestCase { + description: "colon inside a line comment stays commented", + dialect: "postgres", + query_sql: "SELECT payload:customer_id AS customer_id -- note: it's nested\nFROM orders", + expected_code: "SQBRCONTRACT101", + }, + ]; + + for test_case in test_cases { + let config = json!({"select": ["SQBRCONTRACT101"], "cache": {"enabled": false}}); + let mut request: Value = serde_json::from_str(&helpers::request(&project_dir, &config)) + .map_err(|error| error.to_string())?; + request["dialect"] = json!(test_case.dialect); + request["models"][0]["query_sql"] = json!(test_case.query_sql); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + assert_eq!( + result["faults"][0]["code"], test_case.expected_code, + "{}: {result}", + test_case.description + ); + } + Ok(()) +} + #[test] fn given_enforced_contract_outputs_when_evaluating_explicit_type_rule_then_returns_expected_faults() -> Result<(), String> { diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs index 6732ebc948..01407def13 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs @@ -13,7 +13,6 @@ use crate::rules::models::{ }; use crate::sql_scan::main::matching_paren::matching_paren; use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::main::quote_end::quote_end; use crate::sql_scan::models::QuotePolicy; use globset::{Glob, GlobSetBuilder}; use sqlparser::ast::{ @@ -273,12 +272,13 @@ fn normalize_generic_fallback(sql: &str) -> String { while index < bytes.len() { let byte = bytes[index]; let next = bytes.get(index + 1).copied(); - if policy.is_quote(byte) { - let Ok(end) = quote_end(&bytes, index, policy) else { - break; - }; - index = end; - continue; + match non_code_end(&bytes, index, policy) { + Ok(Some(end)) => { + index = end; + continue; + } + Ok(None) => {} + Err(_) => break, } if byte == b'-' && next == Some(b'>') { bytes[index] = b','; From 7d3b9c71697a08fee3b2aa1f1debc8d7b2cae281 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:12:07 +0100 Subject: [PATCH 03/32] fix(sql-tests): share the expected-CTE UNION splitter with the Polyglot fallback When Polyglot could not parse an expected CTE, the fallback split set operations with its own scanner: it ignored comments (an apostrophe in `-- don't` hid every later branch, and `UNION` inside a comment split), left `DISTINCT` at the start of a `UNION DISTINCT` branch so the branch parse raised a raw parser error, and split on INTERSECT/EXCEPT even though the parsed path rejects them. Both the fallback and the textual expected-projection path now use one shared `split_union_branches` over the comment- and quote-aware `iter_code_positions`, handling `UNION [ALL|DISTINCT]`. The fallback rejects top-level INTERSECT/EXCEPT with the parsed path's error, and defers to the textual path when a split branch still cannot be parsed. --- .../compile/_helpers/analysis/tests.py | 121 ++++++------------ .../compile/_helpers/sql_tests/core.py | 32 +---- src/sqlbuild/compiler/compile/constants.py | 3 - .../sql_analysis/_helpers/set_operations.py | 86 +++++++++++++ .../main/_contains_top_level_keyword.py | 13 ++ .../main/_split_union_branches.py | 9 ++ .../compiler/compile/_helpers/_test_types.py | 2 +- .../test_expected_projection_scanning.py | 5 +- .../_helpers/test_sql_analysis_tests.py | 40 ++++++ 9 files changed, 191 insertions(+), 120 deletions(-) create mode 100644 src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py create mode 100644 src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py create mode 100644 src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py index 43294b01a3..391bca4cd7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py @@ -9,19 +9,18 @@ POLYGLOT_SELECT_EXPRESSION_NAME, POLYGLOT_UNION_EXPRESSION_NAME, POLYGLOT_WRAPPER_EXPRESSION_NAMES, - SQL_CLOSE_PAREN_TOKEN, - SQL_IDENTIFIER_EXTRA_TOKEN, - SQL_OPEN_PAREN_TOKEN, - SQL_QUOTE_TOKENS, - SQL_SET_OPERATION_KEYWORDS, - SQL_UNION_ALL_KEYWORD, - SQL_UNION_KEYWORD, ) from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.sql_analysis.main._contains_top_level_keyword import ( + contains_top_level_keyword, +) +from sqlbuild.compiler.sql_analysis.main._split_union_branches import split_union_branches from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _POLYGLOT_VALUES_EXPRESSION_NAME: str = "Values" _POLYGLOT_VALUES_SET_ALIAS: str = "_values" +_SCAN_CONTEXT: str = "SQL test" +_UNSUPPORTED_SET_OPERATION_KEYWORDS: tuple[str, ...] = ("INTERSECT", "EXCEPT") def extract_expected_branch_column_names_with_sql_analysis( @@ -34,20 +33,39 @@ def extract_expected_branch_column_names_with_sql_analysis( try: parsed_expression: Any = polyglot_module.parse_one(sql, dialect="generic") except polyglot_module.PolyglotError: - branches: tuple[str, ...] = _split_set_operation_branches(sql) - if len(branches) > 1: - return tuple( - _extract_branch_names( - expression=polyglot_module.parse_one(branch_sql, dialect="generic"), - file_label=file_label, - label=label, - )[0] - for branch_sql in branches - ) - return None + return _extract_unparsed_branch_names( + sql=sql, polyglot_module=polyglot_module, file_label=file_label, label=label + ) return _extract_branch_names(expression=parsed_expression, file_label=file_label, label=label) +def _extract_unparsed_branch_names( + *, sql: str, polyglot_module: Any, file_label: str, label: str +) -> tuple[tuple[str, ...], ...] | None: + """Parse UNION branches separately, rejecting set operations the parsed path rejects.""" + + if contains_top_level_keyword( + sql=sql, keywords=_UNSUPPORTED_SET_OPERATION_KEYWORDS, context=_SCAN_CONTEXT + ): + raise CompileInputError( + f"SQL test '{file_label}' must define each {label} set-operation branch as a " + "SELECT query" + ) + branches: tuple[str, ...] = split_union_branches(sql=sql, context=_SCAN_CONTEXT) + if len(branches) <= 1: + return None + try: + parsed_branches: tuple[Any, ...] = tuple( + polyglot_module.parse_one(branch_sql, dialect="generic") for branch_sql in branches + ) + except polyglot_module.PolyglotError: + return None + return tuple( + _extract_branch_names(expression=expression, file_label=file_label, label=label)[0] + for expression in parsed_branches + ) + + def _extract_branch_names( *, expression: Any, file_label: str, label: str ) -> tuple[tuple[str, ...], ...]: @@ -113,70 +131,3 @@ def _extract_select_names(*, expression: Any, file_label: str, label: str) -> tu f"SQL test '{file_label}' must project at least one column in {label}" ) return tuple(names) - - -def _split_set_operation_branches(sql: str) -> tuple[str, ...]: - branches: list[str] = [] - branch_start: int = 0 - depth: int = 0 - quote: str | None = None - index: int = 0 - while index < len(sql): - character: str = sql[index] - if quote is not None: - if character == quote: - quote = None - index += 1 - continue - if character in SQL_QUOTE_TOKENS: - quote = character - index += 1 - continue - if character == SQL_OPEN_PAREN_TOKEN: - depth += 1 - index += 1 - continue - if character == SQL_CLOSE_PAREN_TOKEN: - depth = max(0, depth - 1) - index += 1 - continue - operation_end: int | None = _consume_set_operation(sql=sql, start=index) - if depth == 0 and operation_end is not None: - branch_sql: str = sql[branch_start:index].strip() - if branch_sql: - branches.append(branch_sql) - branch_start = operation_end - index = operation_end - continue - index += 1 - final_branch_sql: str = sql[branch_start:].strip() - if final_branch_sql: - branches.append(final_branch_sql) - return tuple(branches) - - -def _consume_set_operation(*, sql: str, start: int) -> int | None: - keyword: str - for keyword in SQL_SET_OPERATION_KEYWORDS: - end: int = start + len(keyword) - if sql[start:end].upper() != keyword: - continue - if start > 0 and (sql[start - 1].isalnum() or sql[start - 1] == SQL_IDENTIFIER_EXTRA_TOKEN): - continue - if end < len(sql) and (sql[end].isalnum() or sql[end] == SQL_IDENTIFIER_EXTRA_TOKEN): - continue - index: int = _skip_whitespace(sql=sql, start=end) - if keyword == SQL_UNION_KEYWORD and sql[index : index + 3].upper() == SQL_UNION_ALL_KEYWORD: - all_end: int = index + 3 - if all_end == len(sql) or not ( - sql[all_end].isalnum() or sql[all_end] == SQL_IDENTIFIER_EXTRA_TOKEN - ): - index = _skip_whitespace(sql=sql, start=all_end) - return index - return None - - -def _skip_whitespace(*, sql: str, start: int) -> int: - while start < len(sql) and sql[start].isspace(): - start += 1 - return start diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index 9887de4642..aa49baaf8d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -63,6 +63,7 @@ from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import ( skip_quoted_text, ) +from sqlbuild.compiler.sql_analysis.main._split_union_branches import split_union_branches from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _CONTEXT: str = "SQL test" @@ -702,42 +703,13 @@ def _extract_expected_branch_column_names( ) if sql_analysis_column_names is not None: return sql_analysis_column_names - branches: tuple[str, ...] = _split_set_operation_branches(sql) + branches: tuple[str, ...] = split_union_branches(sql=sql, context=_CONTEXT) return tuple( _extract_expected_select_column_names(branch_sql=branch, file_label=file_label, label=label) for branch in branches ) -def _split_set_operation_branches(sql: str) -> tuple[str, ...]: - branches: list[str] = [] - branch_start: int = 0 - resume: int = 0 - index: int - depth: int - for index, depth in iter_code_positions(sql=sql, context=_CONTEXT): - if index < resume or depth != 0: - continue - union_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="UNION") - if union_end is None: - continue - branch_sql: str = sql[branch_start:index].strip() - if branch_sql: - branches.append(branch_sql) - resume = _skip_ignorable(sql=sql, start=union_end) - quantifier_end: int | None = _try_consume_keyword( - sql=sql, start=resume, keyword="ALL" - ) or _try_consume_keyword(sql=sql, start=resume, keyword="DISTINCT") - if quantifier_end is not None: - resume = _skip_ignorable(sql=sql, start=quantifier_end) - branch_start = resume - - final_branch_sql: str = sql[branch_start:].strip() - if final_branch_sql: - branches.append(final_branch_sql) - return tuple(branches) - - def _extract_expected_select_column_names( *, branch_sql: str, file_label: str, label: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 494ef088f9..4765f97827 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -47,9 +47,6 @@ SQL_QUALIFIER_SEPARATOR_TOKEN: str = "." SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'}) SQL_IDENTIFIER_EXTRA_TOKEN: str = "_" -SQL_UNION_KEYWORD: str = "UNION" -SQL_UNION_ALL_KEYWORD: str = "ALL" -SQL_SET_OPERATION_KEYWORDS: tuple[str, ...] = ("UNION", "INTERSECT", "EXCEPT") SQL_WITH_KEYWORD: str = "WITH" SQL_CEREMONIAL_SELECT_VALUE: str = "1" UNKNOWN_SQL_TYPE_NAME: str = "UNKNOWN" diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py b/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py new file mode 100644 index 0000000000..c90cf46415 --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py @@ -0,0 +1,86 @@ +"""Top-level SQL set-operation scanning over the shared code-position iterator.""" + +from __future__ import annotations + +from sqlbuild.compiler.sql_analysis._helpers.scanning import ( + is_identifier_character_impl, + iter_code_positions_impl, + skip_block_comment_impl, + skip_line_comment_impl, +) + +_UNION_KEYWORD: str = "UNION" +_UNION_ALL_KEYWORD: str = "ALL" +_UNION_DISTINCT_KEYWORD: str = "DISTINCT" + + +def split_union_branches_impl(*, sql: str, context: str) -> tuple[str, ...]: + """Split SQL on top-level `UNION [ALL | DISTINCT]`, ignoring quotes and comments.""" + + branches: list[str] = [] + branch_start: int = 0 + resume: int = 0 + index: int + depth: int + for index, depth in iter_code_positions_impl(sql=sql, context=context): + if index < resume or depth != 0: + continue + union_end: int | None = _keyword_end(sql=sql, start=index, keyword=_UNION_KEYWORD) + if union_end is None: + continue + branch_sql: str = sql[branch_start:index].strip() + if branch_sql: + branches.append(branch_sql) + resume = _skip_ignorable(sql=sql, start=union_end, context=context) + quantifier_end: int | None = _keyword_end( + sql=sql, start=resume, keyword=_UNION_ALL_KEYWORD + ) or _keyword_end(sql=sql, start=resume, keyword=_UNION_DISTINCT_KEYWORD) + if quantifier_end is not None: + resume = _skip_ignorable(sql=sql, start=quantifier_end, context=context) + branch_start = resume + final_branch_sql: str = sql[branch_start:].strip() + if final_branch_sql: + branches.append(final_branch_sql) + return tuple(branches) + + +def contains_top_level_keyword_impl(*, sql: str, keywords: tuple[str, ...], context: str) -> bool: + """Return whether any keyword appears as top-level code outside quotes and comments.""" + + index: int + depth: int + for index, depth in iter_code_positions_impl(sql=sql, context=context): + if depth != 0: + continue + keyword: str + for keyword in keywords: + if _keyword_end(sql=sql, start=index, keyword=keyword) is not None: + return True + return False + + +def _keyword_end(*, sql: str, start: int, keyword: str) -> int | None: + keyword_end: int = start + len(keyword) + if sql[start:keyword_end].upper() != keyword: + return None + if keyword_end < len(sql) and is_identifier_character_impl(sql[keyword_end]): + return None + if start > 0 and is_identifier_character_impl(sql[start - 1]): + return None + return keyword_end + + +def _skip_ignorable(*, sql: str, start: int, context: str) -> int: + index: int = start + while index < len(sql): + if sql[index].isspace(): + index += 1 + continue + if sql.startswith("--", index): + index = skip_line_comment_impl(sql=sql, start=index) + continue + if sql.startswith("/*", index): + index = skip_block_comment_impl(sql=sql, start=index, context=context) + continue + return index + return index diff --git a/src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py b/src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py new file mode 100644 index 0000000000..a47c7a3a5a --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py @@ -0,0 +1,13 @@ +"""Top-level SQL keyword detection entrypoint.""" + +from sqlbuild.compiler.sql_analysis._helpers.set_operations import ( + contains_top_level_keyword_impl, +) + + +def contains_top_level_keyword( + *, sql: str, keywords: tuple[str, ...], context: str = "SQL" +) -> bool: + """Return whether any keyword appears as top-level code outside quotes and comments.""" + + return contains_top_level_keyword_impl(sql=sql, keywords=keywords, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py b/src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py new file mode 100644 index 0000000000..8e00c5c25b --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py @@ -0,0 +1,9 @@ +"""Top-level SQL UNION branch splitting entrypoint.""" + +from sqlbuild.compiler.sql_analysis._helpers.set_operations import split_union_branches_impl + + +def split_union_branches(*, sql: str, context: str = "SQL") -> tuple[str, ...]: + """Split SQL on top-level `UNION [ALL | DISTINCT]`, ignoring quotes and comments.""" + + return split_union_branches_impl(sql=sql, context=context) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 2981975de0..2ccee5fb81 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -453,7 +453,7 @@ class BuildScenarioInputsErrorTestCase: class ExtractSqlAnalysisExpectedBranchesTestCase: description: str sql: str - expected_branch_column_names: tuple[tuple[str, ...], ...] + expected_branch_column_names: tuple[tuple[str, ...], ...] | None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py index af71f97607..1d79fde89c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py @@ -5,6 +5,7 @@ import pytest from sqlbuild.compiler.compile._helpers.sql_tests import core as sql_test_core +from sqlbuild.compiler.sql_analysis.main._split_union_branches import split_union_branches from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedProjectionScanTestCase, ) @@ -82,7 +83,9 @@ def test_given_expected_cte_sql_when_scanning_top_level_then_results_are_characterised( test_case: ExpectedProjectionScanTestCase, ) -> None: - assert sql_test_core._split_set_operation_branches(test_case.sql) == test_case.expected_branches + assert ( + split_union_branches(sql=test_case.sql, context="SQL test") == test_case.expected_branches + ) assert ( sql_test_core._find_select_list_end(sql=test_case.sql, start=0) == test_case.expected_select_list_end diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py index ede8156d7c..9bb38ed4ff 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py @@ -33,6 +33,31 @@ sql="SELECT order_id, status FROM expected_rows", expected_branch_column_names=(("order_id", "status"),), ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback ignores an apostrophe in a line comment", + sql="SELECT 1 AS order_id; -- don't mix\nUNION ALL SELECT 2 AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback ignores a set operation inside a block comment", + sql="SELECT 1 AS order_id; /* UNION ALL */ UNION ALL SELECT 2 AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback splits union distinct", + sql="SELECT 1 AS order_id; UNION DISTINCT SELECT 2 AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback keeps doubled quotes inside one string", + sql="SELECT 1 AS order_id; UNION SELECT 'it''s UNION' AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback defers to textual analysis when a branch cannot parse", + sql='SELECT __udf("net_amount")(1) AS amount UNION ALL SELECT 2 AS amount', + expected_branch_column_names=None, + ), ], ids=lambda case: case.description, ) @@ -62,6 +87,21 @@ def test_given_sql_analysis_available_when_extracting_expected_branches_then_it_ sql="SELECT 1 AS order_id UNION ALL VALUES (2)", expected_error_fragment="set-operation branch as a SELECT query", ), + ExtractSqlAnalysisExpectedBranchesErrorTestCase( + description="parsed intersect is not a supported expected branch", + sql="SELECT 1 AS order_id INTERSECT SELECT 2 AS order_id", + expected_error_fragment="set-operation branch as a SELECT query", + ), + ExtractSqlAnalysisExpectedBranchesErrorTestCase( + description="fallback rejects intersect like the parsed path", + sql="SELECT 1 AS order_id; INTERSECT SELECT 2 AS order_id", + expected_error_fragment="set-operation branch as a SELECT query", + ), + ExtractSqlAnalysisExpectedBranchesErrorTestCase( + description="fallback rejects except like the parsed path", + sql="SELECT 1 AS order_id; EXCEPT SELECT 2 AS order_id", + expected_error_fragment="set-operation branch as a SELECT query", + ), ExtractSqlAnalysisExpectedBranchesErrorTestCase( description="raises when expected select uses star projection", sql="SELECT * FROM expected_rows", From f0d48b72a2ff0c1d9ad637d222834755f52aa28c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:15:51 +0100 Subject: [PATCH 04/32] fix(sql-tests): merge textual assertion CTEs into a leading WITH The textual assertion fallback prefixed lifted CTEs as `WITH ... {body}` even when the assertion body began with its own `WITH`, producing `WITH a AS (...) WITH b AS (...) SELECT ...`. That resolved SQL is what the renderer emits when an assertion's lifted CTEs conflict with earlier steps. The fallback now uses the shared `with_leading_ctes`, merging lifted CTEs into the body's leading WITH clause like every other planning path. --- .../compiler/_helpers/sql_tests/planning.rs | 10 +-- .../src/compiler/tests/helpers.rs | 64 +++++++++++++++++++ .../compiler/tests/test_sql_test_planning.rs | 7 +- 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs index 58f157ebad..058724762e 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs @@ -1056,15 +1056,7 @@ fn resolve_assertion_textual_sql( ); let body_sql = resolution.sql; let reached = resolution.reached; - let resolved_sql = if lifted_ctes.is_empty() { - body_sql.clone() - } else { - let cte_parts: Vec = lifted_ctes - .iter() - .map(|(name, body)| cte_definition_sql(name, body)) - .collect(); - format!("WITH {} {body_sql}", cte_parts.join(", ")) - }; + let resolved_sql = with_leading_ctes(&lifted_ctes, &body_sql); Ok(( TextualStep { resolved_sql, diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs index 422ac14668..a54f05ebf5 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs @@ -657,6 +657,70 @@ pub(crate) fn plan_without_rendering_returns_executable_steps() -> bool { true } +pub(crate) fn textual_assertion_with_clause_merges_lifted_ctes() -> bool { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_planning::plan_and_render_json( + &json!({ + "models": [ + { + "name": "stg_orders", + "querySql": "SELECT * FROM __source(\"raw_orders\")", + "modelDependencies": [] + }, + { + "name": "orders", + "querySql": "SELECT * FROM __ref(\"stg_orders\")", + "modelDependencies": ["stg_orders"] + } + ], + "tests": [{ + "name": "orders_case", + "fileLabel": "tests/orders.sql", + "payload": { + "kind": "model", + "authoredCtes": [{ + "name": "__source__raw_orders", + "sqlBody": "SELECT 1 AS order_id" + }], + "expectedCtes": [], + "expectedModelNames": [], + "assertionCtes": [{ + "name": "__assert__no_negative_orders", + "sqlBody": "WITH negative_orders AS (SELECT * FROM __ref(\"orders\") WHERE order_id < 0) SELECT * FROM negative_orders" + }] + } + }], + "sqlAnalysisEnabled": false, + "sqlAnalysisDialect": "duckdb", + "renderSql": false + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + + let resolved_sql = response["artifacts"][0]["assertions"][0]["resolvedSql"] + .as_str() + .expect("resolved assertion SQL"); + assert_eq!( + resolved_sql, + concat!( + "WITH __ref__stg_orders AS (SELECT * FROM (SELECT 1 AS order_id)), ", + "__ref__orders AS (SELECT * FROM __ref__stg_orders), ", + "negative_orders AS (SELECT * FROM __ref__orders WHERE order_id < 0) ", + "SELECT * FROM negative_orders" + ) + ); + assert!( + polyglot_sql::Dialect::get(polyglot_sql::DialectType::DuckDB) + .parse(resolved_sql) + .is_ok(), + "{resolved_sql}" + ); + true +} + pub(crate) fn upstream_fallback_resolves() -> bool { let request = json!({ "models": [ diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs index b343728391..eb6f3a9359 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs @@ -5,7 +5,7 @@ use crate::compiler::tests::helpers::{ mock_read_through_helper_brings_its_mock_dependencies_into_scope, model_test_batch_returns_ordered_artifact, plan_without_rendering_returns_executable_steps, scoped_helper_named_like_generated_cte_is_rejected, - shared_textual_chain_renders_each_model_once, + shared_textual_chain_renders_each_model_once, textual_assertion_with_clause_merges_lifted_ctes, unicode_cte_after_leading_with_preserves_identifier, unresolved_reference_fast_rejection_preserves_warning, upstream_fallback_resolves, }; @@ -91,6 +91,11 @@ fn given_sql_test_planning_cases_when_exercising_native_planner_then_expected_be run: scoped_helper_named_like_generated_cte_is_rejected, expected_success: true, }, + SqlTestPlanningTestCase { + description: "textual assertion WITH clause merges lifted CTEs", + run: textual_assertion_with_clause_merges_lifted_ctes, + expected_success: true, + }, ]; for test_case in test_cases { From e7dac37f7d5840096c19d5f21b3bd743b5a7ed1f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:22:30 +0100 Subject: [PATCH 05/32] refactor(native): share one top-level code scan in the SQL-test extractor `split_unions`, `split_top_level` and `find_top_level_keyword` each repeated the same loop around the shared scanner: skip comments and quotes, track parenthesis depth, then act on top-level code. They now run on a single `scan_code` walker whose visitor advances, resumes after a consumed keyword, or stops with a result. Characterisation tests pin union splits, comma splits, keyword offsets, multibyte text, unbalanced parentheses and unclosed-input errors; results are unchanged. --- .../compiler/_helpers/sql_tests/extraction.rs | 135 +++++++++--------- .../tests/test_sql_test_extraction.rs | 103 ++++++++++++- .../src/compiler/tests/test_types.rs | 8 ++ 3 files changed, 175 insertions(+), 71 deletions(-) diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs index ebcdd035c2..c2c43537dd 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs @@ -844,10 +844,20 @@ fn contains_select_star(sql: &str) -> Result { Ok(false) } -fn split_unions(sql: &str) -> Result, String> { - let mut values: Vec<&str> = Vec::new(); - let mut start = 0; - let mut index = 0; +/// What a top-level scan does after visiting one code offset. +enum CodeStep { + Advance, + Resume(usize), + Stop(T), +} + +/// Visit each code offset outside quotes and comments with its depth after that byte's parenthesis. +fn scan_code( + sql: &str, + start: usize, + mut visit: impl FnMut(usize, isize) -> Result, String>, +) -> Result, String> { + let mut index = start; let mut depth = 0; while index < sql.len() { let next = skip_non_code(sql, index)?; @@ -860,83 +870,68 @@ fn split_unions(sql: &str) -> Result, String> { Some(b')') => depth -= 1, _ => {} } - if depth == 0 - && let Some(end) = consume_keyword(sql, index, "UNION") - { - let value = sql[start..index].trim(); - if !value.is_empty() { - values.push(value); - } - index = skip_ignorable(sql, end)?; - if let Some(quantifier_end) = consume_keyword(sql, index, "ALL") - .or_else(|| consume_keyword(sql, index, "DISTINCT")) - { - index = skip_ignorable(sql, quantifier_end)?; - } - start = index; - continue; + match visit(index, depth)? { + CodeStep::Advance => index += char_len(sql, index), + CodeStep::Resume(resume) => index = resume, + CodeStep::Stop(value) => return Ok(Some(value)), } - index += char_len(sql, index); } - let value = sql[start..].trim(); - if !value.is_empty() { - values.push(value); - } - Ok(values) + Ok(None) } -fn split_top_level(sql: &str, separator: u8) -> Result, String> { +fn non_empty_trimmed(value: &str) -> Option<&str> { + Some(value.trim()).filter(|trimmed| !trimmed.is_empty()) +} + +pub(crate) fn split_unions(sql: &str) -> Result, String> { let mut values: Vec<&str> = Vec::new(); let mut start = 0; - let mut index = 0; - let mut depth = 0; - while index < sql.len() { - let next = skip_non_code(sql, index)?; - if next != index { - index = next; - continue; - } - match byte_at(sql, index) { - Some(b'(') => depth += 1, - Some(b')') => depth -= 1, - Some(value) if value == separator && depth == 0 => { - let item = sql[start..index].trim(); - if !item.is_empty() { - values.push(item); - } - start = index + 1; - } - _ => {} + scan_code::<()>(sql, 0, |index, depth| { + let Some(end) = consume_keyword(sql, index, "UNION").filter(|_| depth == 0) else { + return Ok(CodeStep::Advance); + }; + values.extend(non_empty_trimmed(&sql[start..index])); + let mut resume = skip_ignorable(sql, end)?; + if let Some(quantifier_end) = + consume_keyword(sql, resume, "ALL").or_else(|| consume_keyword(sql, resume, "DISTINCT")) + { + resume = skip_ignorable(sql, quantifier_end)?; } - index += char_len(sql, index); - } - let item = sql[start..].trim(); - if !item.is_empty() { - values.push(item); - } + start = resume; + Ok(CodeStep::Resume(resume)) + })?; + values.extend(non_empty_trimmed(&sql[start..])); Ok(values) } -fn find_top_level_keyword(sql: &str, start: usize, keyword: &str) -> Result, String> { - let mut index = start; - let mut depth = 0; - while index < sql.len() { - let next = skip_non_code(sql, index)?; - if next != index { - index = next; - continue; - } - match byte_at(sql, index) { - Some(b'(') => depth += 1, - Some(b')') => depth -= 1, - _ => {} - } - if depth == 0 && consume_keyword(sql, index, keyword).is_some() { - return Ok(Some(index)); +pub(crate) fn split_top_level(sql: &str, separator: u8) -> Result, String> { + let mut values: Vec<&str> = Vec::new(); + let mut start = 0; + scan_code::<()>(sql, 0, |index, depth| { + if depth == 0 && byte_at(sql, index) == Some(separator) { + values.extend(non_empty_trimmed(&sql[start..index])); + start = index + 1; } - index += char_len(sql, index); - } - Ok(None) + Ok(CodeStep::Advance) + })?; + values.extend(non_empty_trimmed(&sql[start..])); + Ok(values) +} + +pub(crate) fn find_top_level_keyword( + sql: &str, + start: usize, + keyword: &str, +) -> Result, String> { + scan_code(sql, start, |index, depth| { + Ok( + if depth == 0 && consume_keyword(sql, index, keyword).is_some() { + CodeStep::Stop(index) + } else { + CodeStep::Advance + }, + ) + }) } fn find_top_level_clause_keyword( diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs index 671caec8be..7be75f53cb 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs @@ -1,3 +1,6 @@ +use crate::compiler::_helpers::sql_tests::extraction::{ + find_top_level_keyword, split_top_level, split_unions, +}; use crate::compiler::tests::helpers::{ dependent_assertion_returns_authoritative_error, empty_model_fixture_marker_preserves_direct_mode_validation, @@ -5,7 +8,7 @@ use crate::compiler::tests::helpers::{ mixed_expanded_tests_preserve_order_and_payloads, quoted_ctes_and_implicit_alias_preserve_payload, }; -use crate::compiler::tests::test_types::SqlTestExtractionTestCase; +use crate::compiler::tests::test_types::{SqlTestExtractionTestCase, TopLevelScanTestCase}; #[test] fn given_sql_test_cases_when_extracting_native_payloads_then_expected_behavior_holds() { @@ -46,3 +49,101 @@ fn given_sql_test_cases_when_extracting_native_payloads_then_expected_behavior_h ); } } + +#[test] +fn given_expected_cte_sql_when_scanning_top_level_then_unions_commas_and_keywords_are_pinned() { + let unclosed_quote = "SQL test contains an unclosed quoted string".to_owned(); + let unclosed_comment = "SQL test contains an unclosed block comment".to_owned(); + let test_cases = [ + TopLevelScanTestCase { + description: "plain select", + sql: "SELECT a, b AS c FROM orders", + expected_unions: Ok(vec!["SELECT a, b AS c FROM orders"]), + expected_commas: Ok(vec!["SELECT a", "b AS c FROM orders"]), + expected_from: Ok(Some(17)), + }, + TopLevelScanTestCase { + description: "union quantifiers and comments between keywords", + sql: "SELECT 1 AS a UNION ALL SELECT 2 AS a UNION /* x */ DISTINCT -- y\n SELECT 3 AS a UNION SELECT 4 AS a", + expected_unions: Ok(vec![ + "SELECT 1 AS a", + "SELECT 2 AS a", + "SELECT 3 AS a", + "SELECT 4 AS a", + ]), + expected_commas: Ok(vec![ + "SELECT 1 AS a UNION ALL SELECT 2 AS a UNION /* x */ DISTINCT -- y\n SELECT 3 AS a UNION SELECT 4 AS a", + ]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "quotes comments and parentheses hide separators", + sql: "SELECT 'x, UNION (' AS a, /* UNION , FROM */ \"b)\", `c,d`, f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t", + expected_unions: Ok(vec![ + "SELECT 'x, UNION (' AS a, /* UNION , FROM */ \"b)\", `c,d`, f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t", + ]), + expected_commas: Ok(vec![ + "SELECT 'x, UNION (' AS a", + "/* UNION , FROM */ \"b)\"", + "`c,d`", + "f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t", + ]), + expected_from: Ok(Some(107)), + }, + TopLevelScanTestCase { + description: "keyword boundaries and empty pieces", + sql: ",SELECT reunion, union_id,, x FROMAGE UNION", + expected_unions: Ok(vec![",SELECT reunion, union_id,, x FROMAGE"]), + expected_commas: Ok(vec!["SELECT reunion", "union_id", "x FROMAGE UNION"]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "unbalanced close paren goes negative", + sql: "SELECT a) UNION SELECT b, c FROM t", + expected_unions: Ok(vec!["SELECT a) UNION SELECT b, c FROM t"]), + expected_commas: Ok(vec!["SELECT a) UNION SELECT b, c FROM t"]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "multibyte text", + sql: "SELECT 'é' AS ñ, ü FROM t UNION SELECT 1, 2", + expected_unions: Ok(vec!["SELECT 'é' AS ñ, ü FROM t", "SELECT 1, 2"]), + expected_commas: Ok(vec!["SELECT 'é' AS ñ", "ü FROM t UNION SELECT 1", "2"]), + expected_from: Ok(Some(22)), + }, + TopLevelScanTestCase { + description: "unclosed quote", + sql: "SELECT 'a, b FROM t", + expected_unions: Err(unclosed_quote.clone()), + expected_commas: Err(unclosed_quote.clone()), + expected_from: Err(unclosed_quote), + }, + TopLevelScanTestCase { + description: "unclosed block comment after union", + sql: "SELECT 1 UNION /* never closed", + expected_unions: Err(unclosed_comment.clone()), + expected_commas: Err(unclosed_comment.clone()), + expected_from: Err(unclosed_comment), + }, + ]; + for test_case in test_cases { + assert_eq!( + split_unions(test_case.sql), + test_case.expected_unions, + "unions: {}", + test_case.description + ); + assert_eq!( + split_top_level(test_case.sql, b','), + test_case.expected_commas, + "commas: {}", + test_case.description + ); + assert_eq!( + find_top_level_keyword(test_case.sql, 0, "FROM"), + test_case.expected_from, + "FROM: {}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs index ee2a9052a0..895efdcd05 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs @@ -34,3 +34,11 @@ pub(crate) struct ExpectedColumnsTestCase { pub(crate) sql: &'static str, pub(crate) expected_columns: Option<&'static [&'static str]>, } + +pub(crate) struct TopLevelScanTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_unions: Result, String>, + pub(crate) expected_commas: Result, String>, + pub(crate) expected_from: Result, String>, +} From 825574b627fbfff416b1ffa68dcbbc017a49ce58 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:25:54 +0100 Subject: [PATCH 06/32] fix(identity): include declared loader columns in version identity Loader version identity hashed the destination, write strategy, cursor, unique key and contract but not the declared `columns`, so editing a loader's column declarations (for example through a module-level column list) left its recorded version unchanged. Loaders now include `columns` in their identity decorator config, matching assets. Retry and connection mode remain excluded for every kind. Upgrading gives one round of new versions for loaders with declared columns: their first run after the upgrade records a new version hash. --- .../python_nodes/_helpers/inventory.py | 1 + .../cli/commands/main/build/_test_types.py | 12 ++ .../main/build/test_python_node_identity.py | 149 ++++++++++++++++++ .../python_nodes/_helpers/test_inventory.py | 1 + 4 files changed, 163 insertions(+) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py index fbe89b7bd4..5070249fad 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py @@ -216,6 +216,7 @@ def _build_loader_node( discovered=loader, provider_by_name=provider_by_name, decorator_config={ + "columns": loader.columns, "contract": loader.contract, "cursor_column": loader.cursor_column, "destination": loader.destination, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 5ae694aeed..6766f3f217 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1109,3 +1109,15 @@ class SqlHookIdentityBuildE2ETestCase: added_hook_path: str added_hook_contents: str expected_reason: str + + +@dataclass(frozen=True) +class PythonNodeIdentityBuildE2ETestCase: + """Test case for Python node version identity after one decorator-input edit.""" + + description: str + edited_path: str + original_text: str + edited_text: str + expected_asset_identity_status: str + expected_loader_version_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py new file mode 100644 index 0000000000..f67b193c43 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -0,0 +1,149 @@ +"""E2E tests for Python node version identity across decorator-input edits.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + PythonNodeIdentityBuildE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_LOADERS_PATH: str = "loaders/orders.py" +_ASSETS_PATH: str = "assets/exports.py" +_ORIGINAL_COLUMNS: str = 'LoaderColumnSpec(name="order_id", type="INTEGER")' +_ORIGINAL_RETRY: str = "RetryPolicy(max_attempts=2, retry_on=RuntimeError)" + + +@pytest.mark.parametrize( + "test_case", + [ + PythonNodeIdentityBuildE2ETestCase( + description="loader declared column edit records a new loader version", + edited_path=_LOADERS_PATH, + original_text=_ORIGINAL_COLUMNS, + edited_text=( + 'LoaderColumnSpec(name="order_id", type="INTEGER", ' + 'description="Staged order identifier")' + ), + expected_asset_identity_status="unchanged", + expected_loader_version_count=2, + ), + PythonNodeIdentityBuildE2ETestCase( + description="asset retry edit keeps every version unchanged", + edited_path=_ASSETS_PATH, + original_text=_ORIGINAL_RETRY, + edited_text="RetryPolicy(max_attempts=5, retry_on=RuntimeError)", + expected_asset_identity_status="unchanged", + expected_loader_version_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_built_python_nodes_when_editing_decorator_inputs_then_identity_tracks_columns( + test_case: PythonNodeIdentityBuildE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_node_identity_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "python_node_identity_project" + adapter = "duckdb" + + [connection] + database = "warehouse.duckdb" + """ + ).strip() + + "\n", + _LOADERS_PATH: dedent( + f""" + from sqlbuild.loaders import LoaderColumnSpec, loader + + STAGED_ORDER_COLUMNS = [{_ORIGINAL_COLUMNS}] + + + @loader(write_strategy="table", columns=STAGED_ORDER_COLUMNS) + def staged_orders(ctx): + return [{{"order_id": 1}}] + + + @loader(depends_on=[staged_orders]) + def raw_orders(ctx): + staged = ctx.loader(staged_orders).destination + rows = ctx.query(f"SELECT order_id FROM {{staged}}") + return [{{"order_id": row[0]}} for row in rows] + """ + ).lstrip(), + "sources/raw.yml": dedent( + """ + sources: + - name: raw_orders + managed: true + write_strategy: table + columns: + - name: order_id + type: INTEGER + """ + ).lstrip(), + _ASSETS_PATH: dedent( + f""" + from sqlbuild.assets import asset + from sqlbuild.retries import RetryPolicy + + EXPORT_RETRY = {_ORIGINAL_RETRY} + + + @asset(retry=EXPORT_RETRY) + def export_orders(ctx): + return None + """ + ).lstrip(), + }, + ) + first_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first_build.returncode == 0, first_build.stdout + first_build.stderr + + edited: Path = project_dir / test_case.edited_path + edited.write_text( + edited.read_text(encoding="utf-8").replace(test_case.original_text, test_case.edited_text), + encoding="utf-8", + ) + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("plan", "--json"), project_dir=project_dir + ) + second_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert second_build.returncode == 0, second_build.stdout + second_build.stderr + payload: dict[str, object] = json.loads(plan.stdout) + statuses_by_name: dict[str, str] = { + str(entry["name"]): str(entry["identity_status"]) for entry in payload["python_nodes"] + } + assert statuses_by_name["export_orders"] == test_case.expected_asset_identity_status + assert query_duckdb( + db_path=project_dir / "warehouse.duckdb", + sql=( + "SELECT COUNT(DISTINCT version_hash) FROM main._sqlbuild_fingerprints " + "WHERE node_type = 'loader' AND node_name = 'staged_orders'" + ), + ) == [(test_case.expected_loader_version_count,)] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index b6eea07a48..5970d9c0ec 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -237,6 +237,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node description="identity decorator config keeps each kind's authored field set", expected_decorator_config_keys_by_selector={ "loader:load_events": ( + "columns", "contract", "cursor_column", "destination", From b1bdf28d904eac1617ad2e8bf64846e45385bc3d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:43:07 +0100 Subject: [PATCH 07/32] fix(sql-tests): support INTERSECT and EXCEPT in expected CTEs and check every branch Expected CTEs now accept UNION, INTERSECT and EXCEPT, each with an optional ALL or DISTINCT, in model, macro, UDF and table-function tests, and every branch's projection names and count must match the first branch. - Native extractor: `split_unions` becomes `split_set_operations`, so an INTERSECT or EXCEPT branch is validated instead of being folded into the first branch; `* EXCEPT (...)` star modifiers are not treated as operators. - Polyglot path: Intersect and Except nodes are flattened like Union instead of being rejected as non-SELECT branches. - Shared Python fallback: `split_union_branches` becomes `split_set_operation_branches` and no longer rejects INTERSECT/EXCEPT. - Model-test expected-CTE errors now name the actual CTE (for example `__expected__orders`) instead of `__expected__`, matching direct-logic tests; the message wording is otherwise unchanged. --- .../compiler/_helpers/sql_tests/extraction.rs | 26 +++- .../src/compiler/tests/helpers.rs | 88 ++++++++++- .../tests/test_sql_test_extraction.rs | 41 ++++- .../compile/_helpers/analysis/tests.py | 21 +-- .../compile/_helpers/sql_tests/core.py | 7 +- src/sqlbuild/compiler/compile/constants.py | 4 +- .../sql_analysis/_helpers/set_operations.py | 57 ++++--- .../main/_contains_top_level_keyword.py | 13 -- .../main/_split_set_operation_branches.py | 11 ++ .../main/_split_union_branches.py | 9 -- .../cli/commands/main/test/_test_types.py | 10 ++ .../main/test/test_set_operation_expected.py | 136 +++++++++++++++++ .../compiler/compile/_helpers/_test_types.py | 22 +++ .../test_expected_projection_scanning.py | 29 +++- .../test_set_operation_expected_ctes.py | 141 ++++++++++++++++++ .../_helpers/test_sql_analysis_tests.py | 35 +++-- .../compiler/compile/_helpers/test_tests.py | 6 +- 17 files changed, 567 insertions(+), 89 deletions(-) delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py create mode 100644 src/sqlbuild/compiler/sql_analysis/main/_split_set_operation_branches.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs index c2c43537dd..7e7688c379 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs @@ -202,7 +202,8 @@ fn classify_model(ctes: Vec, file: &str) -> Result { authored.push(cte); } else if let Some(value) = name.strip_prefix("__expected__") { expected_models.push(required(value, "__expected__", file)?); - validate_expected(&cte, file, "__expected__", true)?; + let label = cte.0.clone(); + validate_expected(&cte, file, &label, true)?; expected.push(cte); } else if let Some(value) = name.strip_prefix("__assert__") { assertion_names.push(required(value, "__assert__", file)?); @@ -431,7 +432,7 @@ fn validate_expected( "SQL test '{file}' must not use SELECT * in {label} CTEs" )); } - let branches = split_unions(&cte.1)?; + let branches = split_set_operations(&cte.1)?; let mut names: Vec> = Vec::new(); for branch in branches { names.push(projection_names(branch, file, label)?); @@ -883,11 +884,15 @@ fn non_empty_trimmed(value: &str) -> Option<&str> { Some(value.trim()).filter(|trimmed| !trimmed.is_empty()) } -pub(crate) fn split_unions(sql: &str) -> Result, String> { +/// Split on top-level `UNION`, `INTERSECT` and `EXCEPT`, each with an optional `ALL`/`DISTINCT`. +pub(crate) fn split_set_operations(sql: &str) -> Result, String> { let mut values: Vec<&str> = Vec::new(); let mut start = 0; scan_code::<()>(sql, 0, |index, depth| { - let Some(end) = consume_keyword(sql, index, "UNION").filter(|_| depth == 0) else { + if depth != 0 { + return Ok(CodeStep::Advance); + } + let Some(end) = set_operator_end(sql, index)? else { return Ok(CodeStep::Advance); }; values.extend(non_empty_trimmed(&sql[start..index])); @@ -904,6 +909,19 @@ pub(crate) fn split_unions(sql: &str) -> Result, String> { Ok(values) } +/// Return the end of a set operator at `index`; `* EXCEPT (...)` star modifiers are not operators. +fn set_operator_end(sql: &str, index: usize) -> Result, String> { + if let Some(end) = + consume_keyword(sql, index, "UNION").or_else(|| consume_keyword(sql, index, "INTERSECT")) + { + return Ok(Some(end)); + } + match consume_keyword(sql, index, "EXCEPT") { + Some(end) if previous_code_byte(sql, index)? != Some(b'*') => Ok(Some(end)), + _ => Ok(None), + } +} + pub(crate) fn split_top_level(sql: &str, separator: u8) -> Result, String> { let mut values: Vec<&str> = Vec::new(); let mut start = 0; diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs index a54f05ebf5..d72e8bf7ac 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs @@ -212,7 +212,7 @@ pub(crate) fn expected_projection_errors_name_the_expected_cte() -> bool { ), ( r#"{"tests":[{"sql":"WITH __source__raw_orders AS (SELECT 1 AS id), __expected__orders AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/orders.sql","mode":"model"}]}"#, - "SQL test 'tests/orders.sql' must alias every non-trivial __expected__ projection", + "SQL test 'tests/orders.sql' must alias every non-trivial __expected__orders projection", ), ]; for (request, expected_error) in cases { @@ -222,6 +222,92 @@ pub(crate) fn expected_projection_errors_name_the_expected_cte() -> bool { true } +fn extract_expected( + mode: &str, + actual: &str, + expected_name: &str, + expected_sql: &str, +) -> Result { + let sql = format!("WITH {actual}, {expected_name} AS ({expected_sql}) SELECT 1"); + extract_batch_json( + &json!({"tests": [{"sql": sql, "fileLabel": "tests/orders.sql", "mode": mode}]}) + .to_string(), + ) +} + +pub(crate) fn set_operation_expected_ctes_validate_every_branch() -> bool { + let kinds = [ + ( + "model", + "__source__raw_orders AS (SELECT 1 AS order_id)", + "__expected__orders", + ), + ( + "macro", + "__macro_actual__ AS (SELECT 1 AS order_id)", + "__macro_expected__", + ), + ( + "udf", + "__udf_actual__ AS (SELECT 1 AS order_id)", + "__udf_expected__", + ), + ( + "table_fn", + "__table_fn_actual__ AS (SELECT 1 AS order_id)", + "__table_fn_expected__", + ), + ]; + let accepted = [ + "SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + "SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", + "SELECT 1 AS order_id INTERSECT /* EXCEPT */ -- UNION\n SELECT 1 AS order_id", + ]; + for (mode, actual, expected_name) in kinds { + for expected_sql in accepted { + assert!( + extract_expected(mode, actual, expected_name, expected_sql).is_ok(), + "{mode}: {expected_sql}" + ); + } + let mismatch = format!( + "SQL test 'tests/orders.sql' must use the same {expected_name} projection names and order in every set-operation branch; branch 2 does not match branch 1" + ); + let third_mismatch = mismatch.replace("branch 2", "branch 3"); + let not_select = format!( + "SQL test 'tests/orders.sql' must define each {expected_name} set-operation branch as a SELECT query" + ); + let rejected = [ + ( + "SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", + &mismatch, + ), + ( + "SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + &mismatch, + ), + ( + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + &third_mismatch, + ), + ("SELECT 1 AS order_id EXCEPT VALUES (1)", ¬_select), + ]; + for (expected_sql, expected_error) in rejected { + assert_eq!( + extract_expected(mode, actual, expected_name, expected_sql).as_ref(), + Err(expected_error), + "{mode}: {expected_sql}" + ); + } + } + true +} + pub(crate) fn concurrent_requests_initialize_shared_template_once() -> bool { let templates = Arc::new(Mutex::new(HashMap::new())); let starts = Arc::new(Barrier::new(4)); diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs index 7be75f53cb..e5b777fb29 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers::sql_tests::extraction::{ - find_top_level_keyword, split_top_level, split_unions, + find_top_level_keyword, split_set_operations, split_top_level, }; use crate::compiler::tests::helpers::{ dependent_assertion_returns_authoritative_error, @@ -7,6 +7,7 @@ use crate::compiler::tests::helpers::{ expected_projection_errors_name_the_expected_cte, mixed_expanded_tests_preserve_order_and_payloads, quoted_ctes_and_implicit_alias_preserve_payload, + set_operation_expected_ctes_validate_every_branch, }; use crate::compiler::tests::test_types::{SqlTestExtractionTestCase, TopLevelScanTestCase}; @@ -38,6 +39,11 @@ fn given_sql_test_cases_when_extracting_native_payloads_then_expected_behavior_h run: expected_projection_errors_name_the_expected_cte, expected_success: true, }, + SqlTestExtractionTestCase { + description: "set-operation expected CTEs validate every branch", + run: set_operation_expected_ctes_validate_every_branch, + expected_success: true, + }, ]; for test_case in test_cases { @@ -51,7 +57,8 @@ fn given_sql_test_cases_when_extracting_native_payloads_then_expected_behavior_h } #[test] -fn given_expected_cte_sql_when_scanning_top_level_then_unions_commas_and_keywords_are_pinned() { +fn given_expected_cte_sql_when_scanning_top_level_then_set_operations_commas_and_keywords_are_pinned() + { let unclosed_quote = "SQL test contains an unclosed quoted string".to_owned(); let unclosed_comment = "SQL test contains an unclosed block comment".to_owned(); let test_cases = [ @@ -111,6 +118,34 @@ fn given_expected_cte_sql_when_scanning_top_level_then_unions_commas_and_keyword expected_commas: Ok(vec!["SELECT 'é' AS ñ", "ü FROM t UNION SELECT 1", "2"]), expected_from: Ok(Some(22)), }, + TopLevelScanTestCase { + description: "intersect and except quantifiers mixed with union", + sql: "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a UNION SELECT 4 AS a EXCEPT SELECT 5 AS a", + expected_unions: Ok(vec![ + "SELECT 1 AS a", + "SELECT 2 AS a", + "SELECT 3 AS a", + "SELECT 4 AS a", + "SELECT 5 AS a", + ]), + expected_commas: Ok(vec![ + "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a UNION SELECT 4 AS a EXCEPT SELECT 5 AS a", + ]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "star except modifier and identifier boundaries are not set operations", + sql: "SELECT t.* EXCEPT (status), exceptional, intersection FROM t", + expected_unions: Ok(vec![ + "SELECT t.* EXCEPT (status), exceptional, intersection FROM t", + ]), + expected_commas: Ok(vec![ + "SELECT t.* EXCEPT (status)", + "exceptional", + "intersection FROM t", + ]), + expected_from: Ok(Some(54)), + }, TopLevelScanTestCase { description: "unclosed quote", sql: "SELECT 'a, b FROM t", @@ -128,7 +163,7 @@ fn given_expected_cte_sql_when_scanning_top_level_then_unions_commas_and_keyword ]; for test_case in test_cases { assert_eq!( - split_unions(test_case.sql), + split_set_operations(test_case.sql), test_case.expected_unions, "unions: {}", test_case.description diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py index 391bca4cd7..c56f1ce44b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py @@ -7,20 +7,18 @@ from sqlbuild.compiler.compile.constants import ( POLYGLOT_COLUMN_EXPRESSION_NAME, POLYGLOT_SELECT_EXPRESSION_NAME, - POLYGLOT_UNION_EXPRESSION_NAME, + POLYGLOT_SET_OPERATION_EXPRESSION_NAMES, POLYGLOT_WRAPPER_EXPRESSION_NAMES, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis.main._contains_top_level_keyword import ( - contains_top_level_keyword, +from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( + split_set_operation_branches, ) -from sqlbuild.compiler.sql_analysis.main._split_union_branches import split_union_branches from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _POLYGLOT_VALUES_EXPRESSION_NAME: str = "Values" _POLYGLOT_VALUES_SET_ALIAS: str = "_values" _SCAN_CONTEXT: str = "SQL test" -_UNSUPPORTED_SET_OPERATION_KEYWORDS: tuple[str, ...] = ("INTERSECT", "EXCEPT") def extract_expected_branch_column_names_with_sql_analysis( @@ -42,16 +40,9 @@ def extract_expected_branch_column_names_with_sql_analysis( def _extract_unparsed_branch_names( *, sql: str, polyglot_module: Any, file_label: str, label: str ) -> tuple[tuple[str, ...], ...] | None: - """Parse UNION branches separately, rejecting set operations the parsed path rejects.""" + """Parse each top-level set-operation branch separately when the whole query cannot parse.""" - if contains_top_level_keyword( - sql=sql, keywords=_UNSUPPORTED_SET_OPERATION_KEYWORDS, context=_SCAN_CONTEXT - ): - raise CompileInputError( - f"SQL test '{file_label}' must define each {label} set-operation branch as a " - "SELECT query" - ) - branches: tuple[str, ...] = split_union_branches(sql=sql, context=_SCAN_CONTEXT) + branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_SCAN_CONTEXT) if len(branches) <= 1: return None try: @@ -70,7 +61,7 @@ def _extract_branch_names( *, expression: Any, file_label: str, label: str ) -> tuple[tuple[str, ...], ...]: expression = _unwrap_expression(expression=expression) - if expression.__class__.__name__ == POLYGLOT_UNION_EXPRESSION_NAME: + if expression.__class__.__name__ in POLYGLOT_SET_OPERATION_EXPRESSION_NAMES: left_expression: Any = expression.args["left"] right_expression: Any = expression.args["right"] return ( diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index aa49baaf8d..3b49726ee3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -63,7 +63,9 @@ from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import ( skip_quoted_text, ) -from sqlbuild.compiler.sql_analysis.main._split_union_branches import split_union_branches +from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( + split_set_operation_branches, +) from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _CONTEXT: str = "SQL test" @@ -480,6 +482,7 @@ def _classify_model_sql_test_ctes( _validate_expected_cte_query( cte=cte, file_label=file_label, + label=cte.name, allow_empty_fixture=True, ) expected_ctes.append(cte) @@ -703,7 +706,7 @@ def _extract_expected_branch_column_names( ) if sql_analysis_column_names is not None: return sql_analysis_column_names - branches: tuple[str, ...] = split_union_branches(sql=sql, context=_CONTEXT) + branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_CONTEXT) return tuple( _extract_expected_select_column_names(branch_sql=branch, file_label=file_label, label=label) for branch in branches diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 4765f97827..06b89ae639 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -56,7 +56,9 @@ RIGHT_JOIN_SIDE: str = "RIGHT" FULL_JOIN_SIDE: str = "FULL" -POLYGLOT_UNION_EXPRESSION_NAME: str = "Union" +POLYGLOT_SET_OPERATION_EXPRESSION_NAMES: frozenset[str] = frozenset( + {"Union", "Intersect", "Except"} +) POLYGLOT_SELECT_EXPRESSION_NAME: str = "Select" POLYGLOT_COLUMN_EXPRESSION_NAME: str = "Column" POLYGLOT_WRAPPER_EXPRESSION_NAMES: frozenset[str] = frozenset({"Subquery", "Paren"}) diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py b/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py index c90cf46415..f42cb54430 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py @@ -9,32 +9,43 @@ skip_line_comment_impl, ) -_UNION_KEYWORD: str = "UNION" -_UNION_ALL_KEYWORD: str = "ALL" -_UNION_DISTINCT_KEYWORD: str = "DISTINCT" +_SET_OPERATOR_KEYWORDS: tuple[str, ...] = ("UNION", "INTERSECT") +_EXCEPT_KEYWORD: str = "EXCEPT" +_ALL_KEYWORD: str = "ALL" +_DISTINCT_KEYWORD: str = "DISTINCT" +_STAR_CHARACTER: str = "*" -def split_union_branches_impl(*, sql: str, context: str) -> tuple[str, ...]: - """Split SQL on top-level `UNION [ALL | DISTINCT]`, ignoring quotes and comments.""" +def split_set_operation_branches_impl(*, sql: str, context: str) -> tuple[str, ...]: + """Split SQL on top-level `UNION`, `INTERSECT` and `EXCEPT` with optional quantifiers.""" branches: list[str] = [] branch_start: int = 0 resume: int = 0 + previous_code: str | None = None + previous_depth: int = 0 index: int depth: int for index, depth in iter_code_positions_impl(sql=sql, context=context): - if index < resume or depth != 0: - continue - union_end: int | None = _keyword_end(sql=sql, start=index, keyword=_UNION_KEYWORD) - if union_end is None: + if depth != previous_depth: + previous_code = None + previous_depth = depth + operator_end: int | None = ( + None + if index < resume or depth != 0 + else _set_operator_end(sql=sql, start=index, previous_code=previous_code) + ) + if not sql[index].isspace(): + previous_code = sql[index] + if operator_end is None: continue branch_sql: str = sql[branch_start:index].strip() if branch_sql: branches.append(branch_sql) - resume = _skip_ignorable(sql=sql, start=union_end, context=context) + resume = _skip_ignorable(sql=sql, start=operator_end, context=context) quantifier_end: int | None = _keyword_end( - sql=sql, start=resume, keyword=_UNION_ALL_KEYWORD - ) or _keyword_end(sql=sql, start=resume, keyword=_UNION_DISTINCT_KEYWORD) + sql=sql, start=resume, keyword=_ALL_KEYWORD + ) or _keyword_end(sql=sql, start=resume, keyword=_DISTINCT_KEYWORD) if quantifier_end is not None: resume = _skip_ignorable(sql=sql, start=quantifier_end, context=context) branch_start = resume @@ -44,19 +55,17 @@ def split_union_branches_impl(*, sql: str, context: str) -> tuple[str, ...]: return tuple(branches) -def contains_top_level_keyword_impl(*, sql: str, keywords: tuple[str, ...], context: str) -> bool: - """Return whether any keyword appears as top-level code outside quotes and comments.""" +def _set_operator_end(*, sql: str, start: int, previous_code: str | None) -> int | None: + """Return a set operator's end; `* EXCEPT (...)` star modifiers are not operators.""" - index: int - depth: int - for index, depth in iter_code_positions_impl(sql=sql, context=context): - if depth != 0: - continue - keyword: str - for keyword in keywords: - if _keyword_end(sql=sql, start=index, keyword=keyword) is not None: - return True - return False + keyword: str + for keyword in _SET_OPERATOR_KEYWORDS: + keyword_end: int | None = _keyword_end(sql=sql, start=start, keyword=keyword) + if keyword_end is not None: + return keyword_end + if previous_code == _STAR_CHARACTER: + return None + return _keyword_end(sql=sql, start=start, keyword=_EXCEPT_KEYWORD) def _keyword_end(*, sql: str, start: int, keyword: str) -> int | None: diff --git a/src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py b/src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py deleted file mode 100644 index a47c7a3a5a..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_contains_top_level_keyword.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Top-level SQL keyword detection entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.set_operations import ( - contains_top_level_keyword_impl, -) - - -def contains_top_level_keyword( - *, sql: str, keywords: tuple[str, ...], context: str = "SQL" -) -> bool: - """Return whether any keyword appears as top-level code outside quotes and comments.""" - - return contains_top_level_keyword_impl(sql=sql, keywords=keywords, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_split_set_operation_branches.py b/src/sqlbuild/compiler/sql_analysis/main/_split_set_operation_branches.py new file mode 100644 index 0000000000..b0d1ad4714 --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_split_set_operation_branches.py @@ -0,0 +1,11 @@ +"""Top-level SQL set-operation branch splitting entrypoint.""" + +from sqlbuild.compiler.sql_analysis._helpers.set_operations import ( + split_set_operation_branches_impl, +) + + +def split_set_operation_branches(*, sql: str, context: str = "SQL") -> tuple[str, ...]: + """Split SQL on top-level `UNION`, `INTERSECT` and `EXCEPT`, ignoring quotes and comments.""" + + return split_set_operation_branches_impl(sql=sql, context=context) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py b/src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py deleted file mode 100644 index 8e00c5c25b..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_split_union_branches.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Top-level SQL UNION branch splitting entrypoint.""" - -from sqlbuild.compiler.sql_analysis._helpers.set_operations import split_union_branches_impl - - -def split_union_branches(*, sql: str, context: str = "SQL") -> tuple[str, ...]: - """Split SQL on top-level `UNION [ALL | DISTINCT]`, ignoring quotes and comments.""" - - return split_union_branches_impl(sql=sql, context=context) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index d75c81bf04..b78f987df6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -176,3 +176,13 @@ class CursorWindowE2ETestCase: expected_exit_code: int expected_output_fragments: tuple[str, ...] expected_compiled_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SetOperationExpectedE2ETestCase: + """SQL tests whose expected CTEs combine branches with set operations.""" + + description: str + test_files: dict[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py new file mode 100644 index 0000000000..32378e33a7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py @@ -0,0 +1,136 @@ +"""E2E coverage for INTERSECT and EXCEPT in SQL-test expected CTEs.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + SetOperationExpectedE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + run_sqb, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "set_operation_expected"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "set_operation_expected.duckdb"\n\n' + '[defaults]\nmaterialized = "table"\n' + ), + "macros/status.py": ( + 'def normalize_status(value: str) -> str:\n return f"LOWER(TRIM({value}))"\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: status\n type: VARCHAR\n" + ), + "models/orders.sql": ( + "MODEL ();\n\n" + 'SELECT order_id, @normalize_status("status") AS status FROM __source("raw_orders")\n' + ), +} +_SOURCE_MOCK: str = ( + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, ' PAID ' AS status\n" + " UNION ALL SELECT 2 AS order_id, 'void' AS status\n" + "),\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SetOperationExpectedE2ETestCase( + description="model and macro tests pass with intersect and except", + test_files={ + "tests/unit/test_orders_except.sql": ( + 'TEST (name "orders_except");\n\nWITH\n' + + _SOURCE_MOCK + + "__expected__orders AS (\n" + " SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id\n" + " UNION ALL SELECT 3 AS order_id\n" + " EXCEPT SELECT 3 AS order_id\n" + ")\nSELECT 1\n" + ), + "tests/unit/test_orders_intersect.sql": ( + 'TEST (name "orders_intersect");\n\nWITH\n' + + _SOURCE_MOCK + + "__expected__orders AS (\n" + " SELECT order_id FROM (VALUES (1), (2), (3)) AS candidates(order_id)\n" + " INTERSECT ALL\n" + " SELECT order_id FROM (VALUES (1), (2)) AS loaded(order_id)\n" + ")\nSELECT 1\n" + ), + "tests/unit/test_normalize_status.sql": ( + 'TEST (mode macro, name "normalizes_status");\n\nWITH\n' + "input_values AS (SELECT ' PAID ' AS raw_status),\n" + "__macro_actual__ AS (\n" + ' SELECT @normalize_status("raw_status") AS status FROM input_values\n' + "),\n" + "__macro_expected__ AS (\n" + " SELECT 'paid' AS status\n" + " INTERSECT DISTINCT SELECT 'paid' AS status\n" + " EXCEPT SELECT 'void' AS status\n" + ")\nSELECT 1\n" + ), + }, + expected_exit_code=0, + expected_output_fragments=("PASS=3", "FAIL=0"), + ), + SetOperationExpectedE2ETestCase( + description="model test with a mismatched intersect branch fails clearly", + test_files={ + "tests/unit/test_orders_mismatch.sql": ( + 'TEST (name "orders_mismatch");\n\nWITH\n' + + _SOURCE_MOCK + + "__expected__orders AS (\n" + " SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id\n" + ")\nSELECT 1\n" + ), + }, + expected_exit_code=1, + expected_output_fragments=( + "SQL test 'tests/unit/test_orders_mismatch.sql' must use the same " + "__expected__orders projection names and order in every set-operation " + "branch; branch 2 does not match branch 1", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_set_operation_expected_ctes_when_testing_then_every_branch_is_checked( + test_case: SetOperationExpectedE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="set_operation_expected", + repo_files={**_PROJECT_FILES, **test_case.test_files}, + ) + execute_duckdb( + db_path=project_dir / "set_operation_expected.duckdb", + sql="CREATE TABLE raw_orders AS SELECT 1 AS order_id, 'paid' AS status", + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + + output: str = tested.stdout + tested.stderr + assert tested.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 2ccee5fb81..d5f642b92d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -910,3 +910,25 @@ class ExpectedProjectionScanTestCase: expected_commas: tuple[str, ...] expected_alias: str | None expected_contains_select_star: bool + + +@dataclass(frozen=True) +class SetOperationExpectedKind: + """One SQL-test kind whose expected CTE accepts set operations.""" + + description: str + mode: SqlTestMode + support_ctes: tuple[tuple[str, str], ...] + expected_cte_name: str + payload_type: str + + +@dataclass(frozen=True) +class SetOperationExpectedTestCase: + """One expected-CTE set operation in one SQL-test kind.""" + + description: str + kind: SetOperationExpectedKind + sql: str + expected_payload_type: str = "" + expected_error_template: str = "" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py index 1d79fde89c..1377c29cfb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py @@ -5,7 +5,9 @@ import pytest from sqlbuild.compiler.compile._helpers.sql_tests import core as sql_test_core -from sqlbuild.compiler.sql_analysis.main._split_union_branches import split_union_branches +from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( + split_set_operation_branches, +) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedProjectionScanTestCase, ) @@ -59,6 +61,28 @@ expected_alias="a", expected_contains_select_star=False, ), + ExpectedProjectionScanTestCase( + description="intersect and except split branches with quantifiers", + sql="SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a", + expected_branches=("SELECT 1 AS a", "SELECT 2 AS a", "SELECT 3 AS a"), + expected_select_list_end=len( + "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a" + ), + expected_commas=( + "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a", + ), + expected_alias="a", + expected_contains_select_star=False, + ), + ExpectedProjectionScanTestCase( + description="star except modifier is not a set operation", + sql="SELECT t.* EXCEPT (status), exceptional FROM t", + expected_branches=("SELECT t.* EXCEPT (status), exceptional FROM t",), + expected_select_list_end=len("SELECT t.* EXCEPT (status), exceptional "), + expected_commas=("SELECT t.* EXCEPT (status)", "exceptional FROM t"), + expected_alias=None, + expected_contains_select_star=False, + ), ExpectedProjectionScanTestCase( description="backtick identifiers are quoted", sql="SELECT `a, b` AS `c`, * FROM t", @@ -84,7 +108,8 @@ def test_given_expected_cte_sql_when_scanning_top_level_then_results_are_charact test_case: ExpectedProjectionScanTestCase, ) -> None: assert ( - split_union_branches(sql=test_case.sql, context="SQL test") == test_case.expected_branches + split_set_operation_branches(sql=test_case.sql, context="SQL test") + == test_case.expected_branches ) assert ( sql_test_core._find_select_list_end(sql=test_case.sql, start=0) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py new file mode 100644 index 0000000000..123f8a652d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py @@ -0,0 +1,141 @@ +"""Set-operation expected CTEs across every SQL-test kind validate every branch.""" + +from __future__ import annotations + +from itertools import product + +import pytest + +from sqlbuild.compiler.compile._helpers.sql_tests.core import classify_sql_test_ctes +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileSqlTestCte, CompileSqlTestCtes +from sqlbuild.compiler.compile.types import SqlTestMode +from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + SetOperationExpectedKind, + SetOperationExpectedTestCase, +) + +_FILE: str = "tests/unit/orders.sql" +_MISMATCH: str = ( + "SQL test '{file}' must use the same {cte} projection names and order in every " + "set-operation branch; branch {branch} does not match branch 1" +) +_NOT_SELECT: str = "SQL test '{file}' must define each {cte} set-operation branch as a SELECT query" +_KINDS: tuple[SetOperationExpectedKind, ...] = ( + SetOperationExpectedKind( + description="model", + mode=SqlTestMode.MODEL, + support_ctes=(("__source__raw_orders", "SELECT 1 AS order_id"),), + expected_cte_name="__expected__orders", + payload_type="CompileModelSqlTestCtes", + ), + SetOperationExpectedKind( + description="macro", + mode=SqlTestMode.MACRO, + support_ctes=(("__macro_actual__", "SELECT 1 AS order_id"),), + expected_cte_name="__macro_expected__", + payload_type="CompileDirectLogicSqlTestCtes", + ), + SetOperationExpectedKind( + description="udf", + mode=SqlTestMode.UDF, + support_ctes=(("__udf_actual__", "SELECT 1 AS order_id"),), + expected_cte_name="__udf_expected__", + payload_type="CompileDirectLogicSqlTestCtes", + ), + SetOperationExpectedKind( + description="table_fn", + mode=SqlTestMode.TABLE_FN, + support_ctes=(("__table_fn_actual__", "SELECT 1 AS order_id"),), + expected_cte_name="__table_fn_expected__", + payload_type="CompileDirectLogicSqlTestCtes", + ), +) +_ACCEPTED_SQL: tuple[str, ...] = ( + "SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + "SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", +) +_REJECTED_SQL: tuple[tuple[str, str], ...] = ( + ("SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", _MISMATCH.replace("{branch}", "2")), + ( + "SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + _MISMATCH.replace("{branch}", "2"), + ), + ( + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + _MISMATCH.replace("{branch}", "3"), + ), + ("SELECT 1 AS order_id EXCEPT VALUES (1)", _NOT_SELECT), +) + + +@pytest.mark.parametrize( + "test_case", + [ + SetOperationExpectedTestCase( + description=f"{kind.description}: {sql}", + kind=kind, + sql=sql, + expected_payload_type=kind.payload_type, + ) + for kind, sql in product(_KINDS, _ACCEPTED_SQL) + ], + ids=lambda case: case.description, +) +def test_given_set_operation_expected_cte_when_classifying_then_it_is_accepted( + test_case: SetOperationExpectedTestCase, +) -> None: + ctes: tuple[CompileSqlTestCte, ...] = ( + *( + CompileSqlTestCte(name=name, sql_body=body) + for name, body in test_case.kind.support_ctes + ), + CompileSqlTestCte(name=test_case.kind.expected_cte_name, sql_body=test_case.sql), + ) + + classified: CompileSqlTestCtes = classify_sql_test_ctes( + ctes=ctes, file_label=_FILE, mode=test_case.kind.mode + ) + + assert type(classified.payload).__name__ == test_case.expected_payload_type + + +@pytest.mark.parametrize( + "test_case", + [ + SetOperationExpectedTestCase( + description=f"{kind.description}: {rejected[0]}", + kind=kind, + sql=rejected[0], + expected_error_template=rejected[1], + ) + for kind, rejected in product(_KINDS, _REJECTED_SQL) + ], + ids=lambda case: case.description, +) +def test_given_invalid_set_operation_branch_when_classifying_then_it_names_the_expected_cte( + test_case: SetOperationExpectedTestCase, +) -> None: + ctes: tuple[CompileSqlTestCte, ...] = ( + *( + CompileSqlTestCte(name=name, sql_body=body) + for name, body in test_case.kind.support_ctes + ), + CompileSqlTestCte(name=test_case.kind.expected_cte_name, sql_body=test_case.sql), + ) + + with pytest.raises(CompileInputError) as error_info: + _ = classify_sql_test_ctes(ctes=ctes, file_label=_FILE, mode=test_case.kind.mode) + + assert str(error_info.value) == test_case.expected_error_template.format( + file=_FILE, cte=test_case.kind.expected_cte_name + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py index 9bb38ed4ff..236430d065 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py @@ -53,6 +53,27 @@ sql="SELECT 1 AS order_id; UNION SELECT 'it''s UNION' AS order_id", expected_branch_column_names=(("order_id",), ("order_id",)), ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="parsed intersect and except return every branch in order", + sql=( + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT ALL " + "SELECT 3 AS other_id INTERSECT DISTINCT SELECT 4 AS extra_id" + ), + expected_branch_column_names=( + ("order_id",), + ("order_id",), + ("other_id",), + ("extra_id",), + ), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback splits intersect and except with quantifiers", + sql=( + "SELECT 1 AS order_id; INTERSECT ALL SELECT 2 AS order_id " + "EXCEPT DISTINCT SELECT 3 AS other_id" + ), + expected_branch_column_names=(("order_id",), ("order_id",), ("other_id",)), + ), ExtractSqlAnalysisExpectedBranchesTestCase( description="fallback defers to textual analysis when a branch cannot parse", sql='SELECT __udf("net_amount")(1) AS amount UNION ALL SELECT 2 AS amount', @@ -88,18 +109,8 @@ def test_given_sql_analysis_available_when_extracting_expected_branches_then_it_ expected_error_fragment="set-operation branch as a SELECT query", ), ExtractSqlAnalysisExpectedBranchesErrorTestCase( - description="parsed intersect is not a supported expected branch", - sql="SELECT 1 AS order_id INTERSECT SELECT 2 AS order_id", - expected_error_fragment="set-operation branch as a SELECT query", - ), - ExtractSqlAnalysisExpectedBranchesErrorTestCase( - description="fallback rejects intersect like the parsed path", - sql="SELECT 1 AS order_id; INTERSECT SELECT 2 AS order_id", - expected_error_fragment="set-operation branch as a SELECT query", - ), - ExtractSqlAnalysisExpectedBranchesErrorTestCase( - description="fallback rejects except like the parsed path", - sql="SELECT 1 AS order_id; EXCEPT SELECT 2 AS order_id", + description="fallback rejects a values branch after except", + sql="SELECT 1 AS order_id; EXCEPT VALUES (2)", expected_error_fragment="set-operation branch as a SELECT query", ), ExtractSqlAnalysisExpectedBranchesErrorTestCase( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py index 0476e95425..4f2e2d17ba 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py @@ -648,7 +648,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return __expected__orders AS (SELECT * FROM __source__raw_orders) SELECT 1 """.strip(), - expected_error_fragment=r"must not use SELECT \* in __expected__ CTEs", + expected_error_fragment=r"must not use SELECT \* in __expected__orders CTEs", ), ExtractSqlTestCtesErrorTestCase( description="rejects empty fixture marker for direct logic expected output", @@ -684,7 +684,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return ) SELECT 1 """.strip(), - expected_error_fragment="must alias every non-trivial __expected__ projection", + expected_error_fragment="must alias every non-trivial __expected__orders projection", ), ExtractSqlTestCtesErrorTestCase( description="raises when non union projection alias is missing", @@ -695,7 +695,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return ) SELECT 1 """.strip(), - expected_error_fragment="must alias every non-trivial __expected__ projection", + expected_error_fragment="must alias every non-trivial __expected__orders projection", ), ExtractSqlTestCtesErrorTestCase( description="raises when expected branch is not a select query", From ac5f62fee5a7ec5359aaf823f01c1479ecd597ac Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:16:13 +0100 Subject: [PATCH 08/32] refactor(compiler): remove unused code --- .../compile/_helpers/analysis/columns.py | 247 ------------------ .../compile/_helpers/analysis/compact.py | 26 -- .../compile/_helpers/attachment/audits.py | 38 --- .../compile/_helpers/attachment/core.py | 27 -- .../_helpers/attachment/model_config.py | 23 -- .../compile/_helpers/render/declarations.py | 28 -- .../compile/_helpers/render/sql_vars.py | 45 ---- src/sqlbuild/compiler/compile/constants.py | 6 - .../compiler/compile/main/_cursor_roles.py | 1 - src/sqlbuild/compiler/compile/models.py | 1 - src/sqlbuild/compiler/contracts/models.py | 6 - .../discovery/_helpers/filesystem/core.py | 41 --- .../discovery/_helpers/sql/model_files.py | 50 ---- src/sqlbuild/compiler/discovery/models.py | 1 - .../compiler/fingerprints/constants.py | 18 -- .../compiler/lineage/_helpers/fast_columns.py | 33 --- src/sqlbuild/compiler/lineage/models.py | 87 ------ .../command_planning/local_node_planning.py | 35 --- .../planner/_helpers/identity/functions.py | 20 -- .../_helpers/output/cursor_type_check.py | 17 -- .../planner/_helpers/output/plan_entry.py | 4 - .../planner/_helpers/output/strategy.py | 98 ------- src/sqlbuild/compiler/planner/constants.py | 1 - src/sqlbuild/compiler/planner/models.py | 22 -- src/sqlbuild/compiler/planner/types.py | 22 -- .../compiler/scopes/_helpers/lookup.py | 10 - .../scopes/_helpers/report_operations.py | 6 - .../scopes/_helpers/report_projection.py | 22 -- src/sqlbuild/compiler/scopes/constants.py | 1 - src/sqlbuild/compiler/scopes/exceptions.py | 6 - .../scopes/main/_normalize_scope_path.py | 13 - .../scopes/main/_parse_qualified_identity.py | 12 - .../main/_path_is_equal_or_descendant.py | 13 - .../scopes/main/_qualified_identity_text.py | 12 - .../scopes/main/_query_scope_target.py | 23 -- .../scopes/main/_scope_is_path_visible.py | 30 --- src/sqlbuild/compiler/scopes/models.py | 2 - .../_helpers/test_filesystem_loaders.py | 8 +- .../_helpers/test_filesystem_python_nodes.py | 18 +- .../discovery/_helpers/test_sql_models.py | 1 - .../sqlbuild/compiler/lineage/_test_types.py | 3 - .../sqlbuild/compiler/lineage/test_columns.py | 28 +- .../compiler/planner/_helpers/_test_types.py | 14 - .../planner/_helpers/test_strategy_ddl.py | 200 -------------- .../compiler/planner/main/_test_types.py | 12 - .../planner/main/test_local_node_planning.py | 95 ------- .../scopes/test_builder_and_visibility.py | 6 +- .../compiler/scopes/test_identities.py | 9 +- .../sqlbuild/compiler/scopes/test_paths.py | 64 +---- 49 files changed, 28 insertions(+), 1477 deletions(-) delete mode 100644 src/sqlbuild/compiler/planner/_helpers/command_planning/local_node_planning.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_normalize_scope_path.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_parse_qualified_identity.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_path_is_equal_or_descendant.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_qualified_identity_text.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_query_scope_target.py delete mode 100644 src/sqlbuild/compiler/scopes/main/_scope_is_path_visible.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_ddl.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/main/test_local_node_planning.py diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py index 36d1ccf721..0cb65058e7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py @@ -20,9 +20,6 @@ ) from sqlbuild.compiler.compile.constants import ( DECIMAL_SQL_TYPE_NAME, - FULL_JOIN_SIDE, - LEFT_JOIN_SIDE, - RIGHT_JOIN_SIDE, SQL_QUALIFIER_SEPARATOR_TOKEN, SQL_WILDCARD_TOKEN, ) @@ -447,61 +444,6 @@ def _analyze_columns_and_lineage_from_polyglot_ast( return tuple(columns), tuple(lineage_columns), has_star -def _extract_polyglot_lineage_facts( - *, - parsed: Any, - references: tuple[CompileSqlReference, ...], -) -> tuple[tuple[CompiledLineageColumnFact, ...], bool]: - if str(getattr(parsed, "kind", "")) != _POLYGLOT_KIND_SELECT: - return (), False - alias_map: dict[str, tuple[CompiledResourceType, str]] = _polyglot_reference_alias_map( - parsed=parsed, - references=references, - ) - unqualified_resource: tuple[CompiledResourceType, str] | None = _single_alias_resource( - alias_map - ) - lineage_columns: list[CompiledLineageColumnFact] = [] - has_star: bool = False - projection: Any - for projection in getattr(parsed, "expressions", ()): - projection = _unwrap_polyglot_annotations(projection) - if bool(getattr(projection, "is_star", False)): - has_star = True - continue - inner: Any = ( - projection.this - if str(getattr(projection, "kind", "")) == _POLYGLOT_KIND_ALIAS - else projection - ) - if bool(getattr(inner, "is_star", False)): - has_star = True - continue - output_column: str = str(getattr(projection, "output_name", "") or "") - if not output_column or output_column == SQL_WILDCARD_TOKEN: - continue - upstream_columns, confidence = _polyglot_lineage_upstream_columns( - projection=projection, - alias_map=alias_map, - unqualified_resource=unqualified_resource, - ) - transform_kind: ColumnTransformKind = _polyglot_lineage_transform_kind( - expression=inner, - has_upstream=bool(upstream_columns), - ) - lineage_columns.append( - CompiledLineageColumnFact( - output_column=output_column, - upstream_columns=upstream_columns, - transform_kind=transform_kind, - confidence=confidence - if upstream_columns or transform_kind == ColumnTransformKind.CONSTANT - else ColumnLineageConfidence.UNKNOWN, - ) - ) - return tuple(lineage_columns), has_star - - def _polyglot_reference_alias_map( *, parsed: Any, references: tuple[CompileSqlReference, ...] ) -> dict[str, tuple[CompiledResourceType, str]]: @@ -893,27 +835,6 @@ def _infer_polyglot_column_nullability( return InferredNullability.UNKNOWN -def _polyglot_columns_in_expression(expression: Any) -> tuple[Any, ...]: - if str(getattr(expression, "kind", "")) == _POLYGLOT_KIND_COLUMN: - return (expression,) - columns: list[Any] = [] - seen: set[int] = set() - - def visit(*, node: Any, visited: set[int], found: list[Any]) -> tuple[set[int], list[Any]]: - node_id: int = id(node) - if node_id in visited: - return visited, found - visited = visited | {node_id} - if str(getattr(node, "kind", "")) == _POLYGLOT_KIND_COLUMN: - return visited, [*found, node] - for child in _polyglot_child_expressions(node): - visited, found = visit(node=child, visited=visited, found=found) - return visited, found - - seen, columns = visit(node=expression, visited=seen, found=columns) - return tuple(columns) - - def _polyglot_column_table_name(column: Any) -> str: payload: object = column.to_dict().get(_POLYGLOT_PAYLOAD_COLUMN, {}) if not isinstance(payload, dict): @@ -925,18 +846,6 @@ def _polyglot_column_table_name(column: Any) -> str: return raw_name if isinstance(raw_name, str) else "" -def _polyglot_child_expressions(expression: Any) -> tuple[Any, ...]: - children: list[Any] = [] - for attr_name in ("this", "expression", "left", "right"): - child: Any | None = getattr(expression, attr_name, None) - if child is not None and str(getattr(child, "kind", "")): - children.append(child) - for child in getattr(expression, "expressions", ()) or (): - if str(getattr(child, "kind", "")): - children.append(child) - return tuple(children) - - def _polyglot_expression_args(expression: Any) -> tuple[Any, ...]: args: list[Any] = [] primary_arg: Any | None = getattr(expression, "this", None) @@ -1012,13 +921,6 @@ def _polyglot_alias_nullability_from_select( return alias_nullability -def _polyglot_table_alias_or_name(table: Any) -> str: - alias_or_name: str = str(getattr(table, "alias_or_name", "") or "") - if alias_or_name: - return alias_or_name - return str(getattr(table, "name", "") or "") - - def _polyglot_table_payload_alias_and_name(table_payload: dict[str, object]) -> tuple[str, str]: table_name: str = _polyglot_name_payload_value(table_payload.get(_POLYGLOT_PAYLOAD_NAME)) alias: str = ( @@ -1111,102 +1013,6 @@ def _replace_table_function_calls_with_stubs( return "".join(parts) -def _find_outermost_select(*, parsed: Any, expressions_module: Any) -> Any | None: - """Find the outermost SELECT statement from a parsed expression.""" - - union_type: type[Any] = expressions_module.Union - select_type: type[Any] = expressions_module.Select - intersect_type: type[Any] = expressions_module.Intersect - except_type: type[Any] = expressions_module.Except - - if isinstance(parsed, (union_type, intersect_type, except_type)): - return parsed.find(select_type) - if isinstance(parsed, select_type): - return parsed - - body: Any | None = getattr(parsed, "this", None) - if body is None: - return None - if isinstance(body, (union_type, intersect_type, except_type)): - return body.find(select_type) - if isinstance(body, select_type): - return body - return parsed.find(select_type) - - -def _is_set_operation(*, parsed: Any, expressions_module: Any) -> bool: - union_type: type[Any] = expressions_module.Union - intersect_type: type[Any] = expressions_module.Intersect - except_type: type[Any] = expressions_module.Except - if isinstance(parsed, (union_type, intersect_type, except_type)): - return True - body: Any | None = getattr(parsed, "this", None) - return isinstance(body, (union_type, intersect_type, except_type)) - - -def _extract_columns_from_select( - *, - select: Any, - expressions_module: Any, - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - infer_nullability: bool, - inference_profile: ExpressionInferenceProfile, -) -> tuple[InferredColumn, ...]: - """Extract output column names and types from a SELECT's projection list.""" - - column_nullability_by_table = dict(column_nullability_by_table) - star_type: type[Any] = expressions_module.Star - annotated_type: type[Any] = expressions_module.Annotated - alias_type: type[Any] = expressions_module.Alias - column_type: type[Any] = expressions_module.Column - cast_type: type[Any] = expressions_module.Cast - try_cast_type: type[Any] = expressions_module.TryCast - alias_nullability: dict[str, InferredNullability] = _alias_nullability_from_select( - select=select, - expressions_module=expressions_module, - column_nullability_by_table=column_nullability_by_table, - ) - - projection_list: list[Any] = select.args.get("expressions", []) - columns: list[InferredColumn] = [] - - expression: Any - for expression in projection_list: - while isinstance(expression, annotated_type): - expression = expression.this - if isinstance(expression, star_type): - continue - - name: str - inner: Any - if isinstance(expression, alias_type): - name = expression.alias - inner = expression.this - elif isinstance(expression, column_type): - name = expression.name - inner = expression - else: - continue - - col_type: str | None = None - if isinstance(inner, (cast_type, try_cast_type)): - col_type = inner.to.sql() - - nullability: InferredNullability = InferredNullability.UNKNOWN - if infer_nullability: - nullability = _infer_expression_nullability( - expression=inner, - expressions_module=expressions_module, - alias_nullability=alias_nullability, - column_nullability_by_table=column_nullability_by_table, - inference_profile=inference_profile, - ) - - columns.append(InferredColumn(name=name, type=col_type, nullability=nullability)) - - return tuple(columns) - - def _infer_expression_nullability( *, expression: Any, @@ -1346,59 +1152,6 @@ def _expression_function_args(expression: Any) -> tuple[Any, ...]: return tuple(args) -def _alias_nullability_from_select( - *, - select: Any, - expressions_module: Any, - column_nullability_by_table: dict[str, dict[str, InferredNullability]], -) -> dict[str, InferredNullability]: - table_type: type[Any] = expressions_module.Table - alias_nullability: dict[str, InferredNullability] = {} - current_aliases: set[str] = set() - - from_expression: Any | None = select.args.get("from_") - from_table: Any | None = getattr(from_expression, "this", None) - if isinstance(from_table, table_type): - alias: str = _table_alias_or_name(from_table) - current_aliases.add(alias) - alias_nullability[alias] = InferredNullability.UNKNOWN - _copy_table_facts_to_alias( - alias=alias, - table_name=from_table.name, - column_nullability_by_table=column_nullability_by_table, - ) - - for join in select.args.get("joins") or []: - joined_table: Any | None = join.this - if not isinstance(joined_table, table_type): - continue - joined_alias: str = _table_alias_or_name(joined_table) - side: str = str(join.args.get("side") or "").upper() - if side == LEFT_JOIN_SIDE: - alias_nullability[joined_alias] = InferredNullability.NULLABLE - elif side == RIGHT_JOIN_SIDE: - for alias in current_aliases: - alias_nullability[alias] = InferredNullability.NULLABLE - alias_nullability[joined_alias] = InferredNullability.UNKNOWN - elif side == FULL_JOIN_SIDE: - for alias in current_aliases: - alias_nullability[alias] = InferredNullability.NULLABLE - alias_nullability[joined_alias] = InferredNullability.NULLABLE - else: - alias_nullability[joined_alias] = InferredNullability.UNKNOWN - current_aliases.add(joined_alias) - _copy_table_facts_to_alias( - alias=joined_alias, - table_name=joined_table.name, - column_nullability_by_table=column_nullability_by_table, - ) - return alias_nullability - - -def _table_alias_or_name(table: Any) -> str: - return str(table.alias_or_name or table.name) - - def _copy_table_facts_to_alias( *, alias: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py index 569ca031a9..bf65ecd3bc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py @@ -221,32 +221,6 @@ def infer_columns_with_sql_analysis( return None -def analyze_columns_with_polyglot( - *, - query_sql: str, - placeholders: dict[str, str] | None = None, - column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, - inference_profile: ExpressionInferenceProfile | None = None, -) -> tuple[InferredColumn, ...] | None | bool: - """Infer columns with one Polyglot parse, returning False when unavailable.""" - - profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() - cleaned_sql: str = _replace_refs_with_stubs( - query_sql=query_sql, - dialect=profile.sql_analysis_dialect, - ) - if placeholders: - cleaned_sql = substitute_placeholder_defaults( - query_sql=cleaned_sql, placeholders=placeholders - ) - return _infer_columns_with_polyglot( - cleaned_sql=cleaned_sql, - dialect=profile.sql_analysis_dialect, - column_nullability_by_table=column_nullability_by_table or {}, - inference_profile=profile, - ) - - def analyze_columns_and_lineage_with_polyglot( *, query_sql: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 95cea9d4b5..339fb45626 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -24,7 +24,6 @@ from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile._helpers.render.arguments import ( render_parameterized_sql, - render_sql_argument_value, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion @@ -676,43 +675,6 @@ def render_generic_audit_sql( ) -def render_generic_audit_argument( - *, - argument_name: str, - arguments: dict[str, object], - owner_file: Path, - definition_name: str, - quoted: bool, -) -> str: - """Render one generic attached-audit parameter value into SQL text.""" - - return render_parameterized_sql( - sql=f"@'{argument_name}'" if quoted else f"@{argument_name}", - arguments=arguments, - owner_label=str(owner_file), - definition_label=f"generic audit '{definition_name}'", - ) - - -def render_generic_audit_argument_value( - *, - argument_value: object, - owner_file: Path, - definition_name: str, - argument_name: str, - quoted: bool, -) -> str: - """Render one generic attached-audit argument value using raw or literal SQL rules.""" - - return render_sql_argument_value( - argument_value=argument_value, - owner_label=str(owner_file), - definition_label=f"audit '{definition_name}'", - argument_name=argument_name, - quoted=quoted, - ) - - def resolve_audit_severity( *, instance_severity: str | AuditSeverity | None, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 30b56e3067..1371ed1ff6 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -1113,33 +1113,6 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi ) -def expand_model_hook_macros( - *, - values: dict[str, object], - file_path: Path, - effective_vars: dict[str, object], - context_values: dict[str, str | None], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - declaration_expansion: DeclarationExpansionContext, - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] | None = None, - consumer: ResourceIdentity | None = None, -) -> dict[str, object]: - """Expand SQL interpolation and macros within executable hook SQL strings.""" - - return expand_model_hook_macros_result( - values=values, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=declaration_expansion, - sql_hook_definitions=sql_hook_definitions, - consumer=consumer, - ).values - - def expand_model_hook_macros_result( *, values: dict[str, object], diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index ea7c56e8c0..863629bf07 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -521,29 +521,6 @@ def _resolve_model_schema( return declaration -def _merge_schema_tags( - *, config: CompileModelConfig, schema_entry: SchemaModelEntry -) -> CompileModelConfig: - """Union schema.yml tags into model config values.""" - - if not schema_entry.tags: - return config - merged_values: dict[str, object] = _merged_with_tag_union( - base=dict(config.values), - overlay={"tags": list(schema_entry.tags)}, - ) - return CompileModelConfig( - values=merged_values, - model_header_keys=config.model_header_keys, - matched_path_default=config.matched_path_default, - logical_schema=config.logical_schema, - layer_schema=config.layer_schema, - logical_database=config.logical_database, - time_travel_retention=config.time_travel_retention, - table_type=config.table_type, - ) - - def find_matching_path_default( *, model_file: DiscoveredSqlModelFile, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index f150d64e1c..d847acff0c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -474,34 +474,6 @@ def _validate_no_inherited_column_overrides( ) -def expand_declaration_references( - *, - sql: str, - file_path: Path, - enums: dict[str, EnumDeclaration], - constants: dict[str, ConstantDeclaration], - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - inaccessible_enums: dict[str, DeclarationRecord] | None = None, - inaccessible_constants: dict[str, DeclarationRecord] | None = None, -) -> str: - """Resolve enum-member and constant references to SQL scalar literals.""" - - result: DeclarationExpansionResult = expand_declaration_references_result( - sql=sql, - file_path=file_path, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - declarations=DeclarationResolutionContext( - enums=enums, - constants=constants, - inaccessible_enums=inaccessible_enums or {}, - inaccessible_constants=inaccessible_constants or {}, - ), - ) - return result.sql - - def expand_declaration_references_with_spans( *, sql: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 561c891295..2d4dd10672 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -55,51 +55,6 @@ _NATIVE_RESULT_LENGTH: int = 2 -def validate_var_macro_collision( - *, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], -) -> None: - """Raise if any project var name collides with a macro name.""" - - collisions: set[str] = set(effective_vars) & set(loaded_macros) - if collisions: - names: str = ", ".join(sorted(collisions)) - raise CompileInputError( - f"project variable names collide with macro names: {names}. " - f"Rename the variable or macro to avoid ambiguity." - ) - - -def expand_authored_sql( # noqa: PLR0913 - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - context_values: Mapping[str, str | None] | None = None, - declarations: DeclarationResolutionContext | None = None, - declaration_resolver: DeclarationScopeResolver | None = None, -) -> str: - """Apply SQL interpolation and macro expansion to authored SQL text.""" - - return expand_authored_sql_result( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=macro_context, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - context_values=context_values, - declarations=declarations, - declaration_resolver=declaration_resolver, - ).sql - - def expand_authored_sql_result( # noqa: PLR0913 *, sql: str, diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 06b89ae639..724111951c 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -17,10 +17,7 @@ MODEL_AUDIT_OVERRIDE_KEYS: frozenset[str] = frozenset({"by_type", "by_column"}) MODEL_FULL_REFRESH_CONFIG_KEY: str = "full_refresh" CURSOR_INPUTS_CONFIG_KEY: str = "cursor_inputs" -CURSOR_FILTER_INPUTS_CONFIG_KEY: str = "cursor_filter_inputs" -CURSOR_WATERMARK_INPUTS_CONFIG_KEY: str = "cursor_watermark_inputs" MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" -MICROBATCH_LIMIT_CONFIG_KEY: str = "microbatch_limit" MICROBATCH_LIMIT_MAX_BATCHES_KEY: str = "max_batches" MICROBATCH_LIMIT_ACTION_KEY: str = "action" WATERMARK_CURSOR_INPUT_BLOCK_KEYS: frozenset[str] = frozenset({"column", "roles"}) @@ -52,9 +49,6 @@ UNKNOWN_SQL_TYPE_NAME: str = "UNKNOWN" DECIMAL_SQL_TYPE_NAME: str = "DECIMAL" RESOLVED_SOURCE_CONFIDENCE: str = "resolved" -LEFT_JOIN_SIDE: str = "LEFT" -RIGHT_JOIN_SIDE: str = "RIGHT" -FULL_JOIN_SIDE: str = "FULL" POLYGLOT_SET_OPERATION_EXPRESSION_NAMES: frozenset[str] = frozenset( {"Union", "Intersect", "Except"} diff --git a/src/sqlbuild/compiler/compile/main/_cursor_roles.py b/src/sqlbuild/compiler/compile/main/_cursor_roles.py index 06762b8d57..fc5b437af6 100644 --- a/src/sqlbuild/compiler/compile/main/_cursor_roles.py +++ b/src/sqlbuild/compiler/compile/main/_cursor_roles.py @@ -47,7 +47,6 @@ def resolve_cursor_input_roles(*, model: CompiledModel) -> CursorInputRoles: watermark_inputs=watermark_inputs, filter_field=CURSOR_INPUTS_CONFIG_KEY, watermark_field=CURSOR_INPUTS_CONFIG_KEY, - uses_legacy_alias=False, ) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index f8fd4c18cd..d54e22ac83 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -220,7 +220,6 @@ class CursorInputRoles: watermark_inputs: dict[str, str] filter_field: str watermark_field: str - uses_legacy_alias: bool @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/contracts/models.py b/src/sqlbuild/compiler/contracts/models.py index 4040f2076d..70d9863a9d 100644 --- a/src/sqlbuild/compiler/contracts/models.py +++ b/src/sqlbuild/compiler/contracts/models.py @@ -12,9 +12,3 @@ class ContractValidationResult: """Diagnostics emitted by model contract validation.""" diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - - @property - def has_errors(self) -> bool: - """Return whether contract validation found errors.""" - - return any(diagnostic.is_error for diagnostic in self.diagnostics) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index ebc6b82114..d02b078ab4 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -450,7 +450,6 @@ def _discover_model_file( model_name=file_path.stem, relative_path=relative_path, ), - output_column_locations_extracted=extract_output_column_locations, extract_implicit_alias_columns=extract_implicit_alias_columns, ) @@ -894,46 +893,6 @@ def discover_materialization_files( return tuple(discovered_files) -def discover_loader_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredLoaderFunction, ...]: - """Discover decorated source loader functions under loaders/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).loaders - ) - - -def discover_task_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredTaskFunction, ...]: - """Discover decorated task functions under tasks/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).tasks - ) - - -def discover_asset_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredAssetFunction, ...]: - """Discover decorated asset functions under assets/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).assets - ) - - -def discover_check_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredCheckFunction, ...]: - """Discover decorated check functions under checks/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).checks - ) - - def discover_python_node_functions( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () ) -> DiscoveredPythonNodeFunctions: diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index 3cad384065..37579a7b5f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -436,24 +436,6 @@ def _location_for_projection_range( ) -def _location_for_header_token( - *, - contents: str, - header_start: int, - token: _ModelHeaderToken, - relative_path: Path, - line_starts: tuple[int, ...], -) -> SourceLocation: - absolute_position: int = header_start + token.position - return _location_for_absolute_span( - contents=contents, - start=absolute_position, - end=absolute_position + len(token.value), - relative_path=relative_path, - line_starts=line_starts, - ) - - def _location_for_absolute_span( *, contents: str, start: int, end: int, relative_path: Path, line_starts: tuple[int, ...] ) -> SourceLocation: @@ -485,38 +467,6 @@ def _line_starts(contents: str) -> tuple[int, ...]: return tuple(starts) -def _find_top_level_keyword(*, sql: str, keyword: str, start: int) -> int | None: - depth: int = 0 - index: int = start - in_quote: str | None = None - while index < len(sql): - character: str = sql[index] - if in_quote is not None: - if character == _MODEL_HEADER_ESCAPE_CHARACTER: - index += 2 - continue - if character == in_quote: - in_quote = None - index += 1 - continue - if character in _MODEL_HEADER_QUOTE_NAMES: - in_quote = character - index += 1 - continue - if character == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - index += 1 - continue - if character == _MODEL_HEADER_CLOSE_PAREN: - depth = max(0, depth - 1) - index += 1 - continue - if depth == 0 and _keyword_at(sql=sql, keyword=keyword, index=index): - return index - index += 1 - return None - - def _split_top_level_select_items(*, sql: str, start: int, end: int) -> tuple[tuple[int, int], ...]: items: list[tuple[int, int]] = [] depth: int = 0 diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 530fd3a6be..b2c4ea0166 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -77,7 +77,6 @@ class DiscoveredSqlModelFile: query_sql: str enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) - output_column_locations_extracted: bool = True extract_implicit_alias_columns: bool = True diff --git a/src/sqlbuild/compiler/fingerprints/constants.py b/src/sqlbuild/compiler/fingerprints/constants.py index 2add2ac5c5..7ca63166e8 100644 --- a/src/sqlbuild/compiler/fingerprints/constants.py +++ b/src/sqlbuild/compiler/fingerprints/constants.py @@ -27,25 +27,7 @@ NODE_TYPE_TABLE_FN: str = "table_fn" FUNCTION_NODE_TYPES: tuple[str, str] = (NODE_TYPE_UDF, NODE_TYPE_TABLE_FN) NODE_TYPE_SEED: str = "seed" -NODE_TYPE_LOADER: str = "loader" -NODE_TYPE_TASK: str = "task" -NODE_TYPE_ASSET: str = "asset" -NODE_TYPE_CHECK: str = "check" NODE_TYPE_HOOK: str = "hook" NODE_TYPE_QUERY_DIFF_ARTIFACT: str = "query_diff_artifact" -FINGERPRINT_COLUMNS: tuple[str, ...] = ( - COLUMN_NODE_TYPE, - COLUMN_NODE_NAME, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_SCHEMA, - COLUMN_TARGET_NAME, - COLUMN_RUN_ID, - COLUMN_DEFINITION_HASH, - COLUMN_VERSION_HASH, - COLUMN_SCHEMA_FINGERPRINT, - COLUMN_DEFINITION_B64, - COLUMN_METADATA_JSON_B64, - COLUMN_TIMESTAMP, -) AUDIT_GATE_METADATA_KEY: str = "audit_gate" diff --git a/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py b/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py index ed09b462e4..63b5c8e5e2 100644 --- a/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py +++ b/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py @@ -518,39 +518,6 @@ def _polyglot_column_table_name(column: Any) -> str: return raw_name if isinstance(raw_name, str) else "" -def _polyglot_columns_in_expression(expression: Any) -> tuple[Any, ...]: - if str(getattr(expression, "kind", "")) == POLYGLOT_KIND_COLUMN: - return (expression,) - columns: list[Any] = [] - seen: set[int] = set() - - def visit(*, node: Any, visited: set[int], found: list[Any]) -> tuple[set[int], list[Any]]: - node_id: int = id(node) - if node_id in visited: - return visited, found - visited = visited | {node_id} - if str(getattr(node, "kind", "")) == POLYGLOT_KIND_COLUMN: - return visited, [*found, node] - for child in _polyglot_child_expressions(node): - visited, found = visit(node=child, visited=visited, found=found) - return visited, found - - seen, columns = visit(node=expression, visited=seen, found=columns) - return tuple(columns) - - -def _polyglot_child_expressions(expression: Any) -> tuple[Any, ...]: - children: list[Any] = [] - for attr_name in ("this", "expression", "left", "right"): - child: Any | None = getattr(expression, attr_name, None) - if child is not None and str(getattr(child, "kind", "")): - children.append(child) - for child in getattr(expression, "expressions", ()) or (): - if str(getattr(child, "kind", "")): - children.append(child) - return tuple(children) - - def _polyglot_classify_transform( *, expression: Any, diff --git a/src/sqlbuild/compiler/lineage/models.py b/src/sqlbuild/compiler/lineage/models.py index 9c812a48e9..23b65bd473 100644 --- a/src/sqlbuild/compiler/lineage/models.py +++ b/src/sqlbuild/compiler/lineage/models.py @@ -61,31 +61,6 @@ def as_qualified_column(self) -> QualifiedLineageColumn: ) -@dataclass(frozen=True) -class ColumnLineageNode: - """One internal SQL analysis lineage graph node.""" - - id: str - name: str - expression_sql: str | None = None - source_sql: str | None = None - resource_type: CompiledResourceType | str | None = None - resource_name: str | None = None - scope_name: str | None = None - - def __post_init__(self) -> None: - if self.resource_type is not None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - -@dataclass(frozen=True) -class InternalColumnLineageEdge: - """One edge in the internal SQL analysis lineage graph.""" - - upstream_node_id: str - downstream_node_id: str - - @dataclass(frozen=True) class ColumnLineageEdge: """One collapsed SQLBuild lineage graph edge.""" @@ -118,8 +93,6 @@ class ColumnLineage: expression_sql: str | None upstream_columns: tuple[ColumnLineageSource, ...] nullability: InferredNullability = InferredNullability.UNKNOWN - nodes: tuple[ColumnLineageNode, ...] = field(default_factory=tuple) - edges: tuple[InternalColumnLineageEdge, ...] = field(default_factory=tuple) confidence: ColumnLineageConfidence = ColumnLineageConfidence.UNKNOWN @@ -406,18 +379,6 @@ def edges_targeting(self, model_name: str) -> tuple[ColumnLineageEdge, ...]: for record in self._edge_records_by_target_model.get(model_name, ()) ) - def producing_edge( - self, - *, - model_name: str, - column_name: str, - ) -> ColumnLineageEdge | None: - """Return the first edge that produces `model_name.column_name`.""" - - self._ensure_indexes() - record: object | None = self._edge_record_by_target_column.get((model_name, column_name)) - return self._materialize_edge(record) if record is not None else None - def edges_sourced_from(self, resource_name: str) -> tuple[ColumnLineageEdge, ...]: """Return edges whose source resource is `resource_name`.""" @@ -441,54 +402,6 @@ def column_consumers( if edge.source.column_name == column_name ) - def trace_column( - self, - *, - model_name: str, - column_name: str, - ) -> tuple[ColumnLineageEdge, ...]: - """Trace a model column upstream through project lineage edges.""" - - result: list[ColumnLineageEdge] = [] - stack: list[tuple[str, str]] = [(model_name, column_name)] - visited: set[tuple[str, str]] = set() - - while stack: - current_model, current_column = stack.pop() - if (current_model, current_column) in visited: - continue - visited.add((current_model, current_column)) - for edge in self.edges_targeting(current_model): - if edge.target.column_name == current_column: - result.append(edge) - stack.append((edge.source.resource_name, edge.source.column_name)) - - return tuple(result) - - def trace_column_downstream( - self, - *, - resource_name: str, - column_name: str, - ) -> tuple[ColumnLineageEdge, ...]: - """Trace a resource column downstream through project lineage edges.""" - - result: list[ColumnLineageEdge] = [] - stack: list[tuple[str, str]] = [(resource_name, column_name)] - visited: set[tuple[str, str]] = set() - - while stack: - current_resource, current_column = stack.pop() - if (current_resource, current_column) in visited: - continue - visited.add((current_resource, current_column)) - for edge in self.edges_sourced_from(current_resource): - if edge.source.column_name == current_column: - result.append(edge) - stack.append((edge.target.resource_name, edge.target.column_name)) - - return tuple(result) - @staticmethod def _edge_record_identity(record: object) -> tuple[str, str, str]: if isinstance(record, ColumnLineageEdge): diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/local_node_planning.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/local_node_planning.py deleted file mode 100644 index b073ba1f21..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/local_node_planning.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Public local node planning classification entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import LocalNodePlanInput, LocalNodePlanOutcome -from sqlbuild.compiler.planner.types import LocalNodePlanAction, LocalNodePlanReason - - -def classify_local_node_plan(input: LocalNodePlanInput) -> LocalNodePlanOutcome: - """Classify local node state using planner semantics shared by adapters.""" - - if input.full_refresh: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.FULL_REFRESH, - ) - if not input.fingerprint_exists: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.FIRST_RUN, - ) - if not input.relation_exists: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.RELATION_MISSING, - ) - if input.local_hash is not None and input.previous_hash != input.local_hash: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.LOCAL_CHANGED, - ) - return LocalNodePlanOutcome( - action=LocalNodePlanAction.CURRENT, - reason=LocalNodePlanReason.NO_CHANGE, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py index 6f22ca9682..29ca640fb0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py @@ -29,26 +29,6 @@ def build_compiled_function_fingerprint_sql(function: CompiledFunction) -> str: ) -def detect_function_backfill( - *, - function: CompiledFunction, - fingerprint_sql: str, - snapshot: WarehouseSnapshot, - query_change_tracking: bool, - full_refresh: bool, -) -> BackfillResult: - """Resolve a function definition change into a cascadeable backfill.""" - - result: tuple[PlanReason, BackfillResult] = detect_function_change( - function=function, - fingerprint_sql=fingerprint_sql, - snapshot=snapshot, - query_change_tracking=query_change_tracking, - full_refresh=full_refresh, - ) - return result[1] - - def detect_function_change( *, function: CompiledFunction, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py b/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py index e79d773568..96a6981e17 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py @@ -190,23 +190,6 @@ def _check_with_heuristic( return None -def _classify_type_heuristic(warehouse_type: str) -> CursorType | None: - """Classify a warehouse type string as timestamp or integer via substrings.""" - - upper: str = warehouse_type.upper() - - substring: str - for substring in _TIMESTAMP_SUBSTRINGS: - if substring in upper: - return CursorType.TIMESTAMP - - for substring in _INTEGER_SUBSTRINGS: - if substring in upper: - return CursorType.INTEGER - - return None - - def classify_cursor_sql_type(sql_type: str) -> CursorType | None: """Classify a supported cross-adapter SQL cursor type.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 84778d33f8..985719ee71 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -851,10 +851,6 @@ def plan_model_from_change( if plan_config.incremental_mode == IncrementalMode.MICROBATCH else get_config_str(values=model.config.values, key="lookback") ), - lookback_is_default=( - plan_config.incremental_mode == IncrementalMode.MICROBATCH - and get_config_str(values=model.config.values, key="lookback") is None - ), cursor_bounds=cursor_bounds, cursor_input_relations=cursor_input_relations, batch_size=plan_config.batch_size, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index d9c5ab26f3..c124677f32 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -2,10 +2,8 @@ from __future__ import annotations -from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import ( CompiledModel, - CompiledRelationLocation, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -129,47 +127,6 @@ def resolve_schema_actions( return tuple(actions) -def build_logical_ddl( - *, - action: PlanAction, - resolved_sql: str, - target: CompiledRelationLocation, - unique_key: tuple[str, ...], - warehouse_columns: tuple[ColumnInfo, ...], - merge_exclude_columns: tuple[str, ...] = (), -) -> str: - """Generate the logical DDL string for a model plan entry.""" - - qualified_name: str = target.qualified_name or target.name - - if action == PlanAction.CREATE_VIEW: - return f"CREATE OR REPLACE VIEW {qualified_name} AS (\n{resolved_sql}\n)" - - if action == PlanAction.CREATE_TABLE: - return f"CREATE TABLE {qualified_name} AS (\n{resolved_sql}\n)" - - if action == PlanAction.INCREMENTAL_APPEND: - return f"INSERT INTO {qualified_name}\n{resolved_sql}" - - if action == PlanAction.INCREMENTAL_DELETE_INSERT: - return _build_delete_insert_ddl( - qualified_name=qualified_name, - resolved_sql=resolved_sql, - unique_key=unique_key, - ) - - if action == PlanAction.INCREMENTAL_MERGE: - return _build_merge_ddl( - qualified_name=qualified_name, - resolved_sql=resolved_sql, - unique_key=unique_key, - warehouse_columns=warehouse_columns, - merge_exclude_columns=merge_exclude_columns, - ) - - return "" - - def build_model_warnings( *, model_name: str, @@ -388,61 +345,6 @@ def _incremental_action( return action, reason -def _build_delete_insert_ddl( - *, - qualified_name: str, - resolved_sql: str, - unique_key: tuple[str, ...], -) -> str: - """Build logical DDL for delete+insert strategy.""" - - key_list: str = ", ".join(unique_key) - source_key_list: str = ", ".join(unique_key) - delete_stmt: str = ( - f"DELETE FROM {qualified_name}\n" - f"WHERE ({key_list}) IN (SELECT {source_key_list} FROM (\n{resolved_sql}\n))" - ) - insert_stmt: str = f"INSERT INTO {qualified_name}\n{resolved_sql}" - return f"{delete_stmt};\n\n{insert_stmt}" - - -def _build_merge_ddl( - *, - qualified_name: str, - resolved_sql: str, - unique_key: tuple[str, ...], - warehouse_columns: tuple[ColumnInfo, ...], - merge_exclude_columns: tuple[str, ...], -) -> str: - """Build logical DDL for merge/upsert strategy.""" - - on_clause: str = " AND ".join( - f"{_DDL_MERGE_TARGET_ALIAS}.{k} = {_DDL_MERGE_SOURCE_ALIAS}.{k}" for k in unique_key - ) - - immutable_columns: frozenset[str] = frozenset( - column.lower() for column in (*unique_key, *merge_exclude_columns) - ) - non_key_columns: list[str] = [ - col.name for col in warehouse_columns if col.name.lower() not in immutable_columns - ] - all_columns: list[str] = [col.name for col in warehouse_columns] - - update_clause: str = ", ".join( - f"{col} = {_DDL_MERGE_SOURCE_ALIAS}.{col}" for col in non_key_columns - ) - insert_columns: str = ", ".join(all_columns) - insert_values: str = ", ".join(f"{_DDL_MERGE_SOURCE_ALIAS}.{col}" for col in all_columns) - - return ( - f"MERGE INTO {qualified_name} AS {_DDL_MERGE_TARGET_ALIAS}\n" - f"USING (\n{resolved_sql}\n) AS {_DDL_MERGE_SOURCE_ALIAS}\n" - f"ON {on_clause}\n" - f"WHEN MATCHED THEN UPDATE SET {update_clause}\n" - f"WHEN NOT MATCHED THEN INSERT ({insert_columns}) VALUES ({insert_values})" - ) - - def _describe_finding(finding: SchemaFinding) -> str: """Build a human-readable description for one schema finding.""" diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index dc7fa0bb6a..f546ba1de9 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -37,7 +37,6 @@ MODEL_PLACEHOLDERS_CONFIG_KEY: str = "placeholders" MODEL_PRE_HOOKS_CONFIG_KEY: str = "pre_hooks" MODEL_POST_HOOKS_CONFIG_KEY: str = "post_hooks" -MODEL_CONTRACT_CONFIG_KEY: str = "contract" MICROBATCH_START_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.START) MICROBATCH_END_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.END) METADATA_NAME_FILTER_LIMIT: int = 250 diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 58ff77e55e..a28995307a 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -38,8 +38,6 @@ CursorWatermarkMode, FixtureKey, GraphResourceKind, - LocalNodePlanAction, - LocalNodePlanReason, MaterializationType, OnSchemaChange, PlanAction, @@ -157,25 +155,6 @@ class SelectorExpansion: downstream: bool = False -@dataclass(frozen=True) -class LocalNodePlanInput: - """Local state used to classify one planner graph node.""" - - fingerprint_exists: bool - relation_exists: bool - full_refresh: bool = False - local_hash: str | None = None - previous_hash: str | None = None - - -@dataclass(frozen=True) -class LocalNodePlanOutcome: - """Local planner action and reason for one graph node.""" - - action: LocalNodePlanAction - reason: LocalNodePlanReason - - @dataclass(frozen=True) class ParsedScenarioArtifactName: """Parsed physical name for one planner-owned scenario artifact.""" @@ -1001,7 +980,6 @@ class ModelPlanEntry: cursor_start: str | None = None cursor_end: str | None = None lookback: str | None = None - lookback_is_default: bool = False cursor_bounds: CursorBounds | None = None cursor_input_relations: tuple[CursorInputRelation, ...] = field(default_factory=tuple) batch_size: str | None = None diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index f7ba982083..8cf32feb3f 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -28,23 +28,6 @@ class CursorInputRole(StrEnum): WATERMARK = "watermark" -class LocalNodePlanAction(StrEnum): - """Action for one locally classified planner graph node.""" - - RUN = "run" - CURRENT = "current" - - -class LocalNodePlanReason(StrEnum): - """Reason for one locally classified planner graph node.""" - - FIRST_RUN = "first_run" - FULL_REFRESH = "full_refresh" - RELATION_MISSING = "relation_missing" - LOCAL_CHANGED = "local_changed" - NO_CHANGE = "no_change" - - class RelationMarkerTargetResolver(Protocol): def __call__(self, *, function_name: str, referenced_name: str) -> str | None: ... @@ -106,11 +89,6 @@ class SchemaChangeKind(StrEnum): COLUMN_TYPE_CHANGED = "column_type_changed" -class SchemaChangeBackfillKey(StrEnum): - ADD_COLUMN = "add_column" - TYPE_CHANGE = "type_change" - - class SchemaColumnSource(StrEnum): YML = "yml" SQLGLOT = "sql_analysis" diff --git a/src/sqlbuild/compiler/scopes/_helpers/lookup.py b/src/sqlbuild/compiler/scopes/_helpers/lookup.py index 59eccbe38b..f9d2453e32 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/lookup.py +++ b/src/sqlbuild/compiler/scopes/_helpers/lookup.py @@ -116,16 +116,6 @@ def build_lookup(*, index: ScopeIndex) -> ScopeLookup: key=lambda item: item.resource, sort_key=grant_sort_key, ), - visibility_by_resource=group_records( - records=canonical_index.visibility, - key=lambda item: item.resource, - sort_key=visibility_sort_key, - ), - inaccessible_by_resource=group_records( - records=canonical_index.inaccessible, - key=lambda item: item.resource, - sort_key=inaccessible_sort_key, - ), ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_operations.py b/src/sqlbuild/compiler/scopes/_helpers/report_operations.py index 6cf4c31436..c85de9bc2c 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/report_operations.py +++ b/src/sqlbuild/compiler/scopes/_helpers/report_operations.py @@ -390,12 +390,6 @@ def serialize_result(*, result: ScopeReport | ScopeBrowseResult | ScopeListResul return json.dumps(payload, sort_keys=True, ensure_ascii=True, separators=(",", ":")) + "\n" -def serialize_report(*, report: ScopeReport) -> str: - """Serialize the schema-version-one report payload as deterministic ASCII JSON.""" - - return serialize_result(result=report) - - def _reports( *, lookup: ScopeLookup, diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py b/src/sqlbuild/compiler/scopes/_helpers/report_projection.py index eacc9fa534..0cd2f4b5b5 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py +++ b/src/sqlbuild/compiler/scopes/_helpers/report_projection.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Iterable from typing import cast from sqlbuild.compiler.scopes._helpers.lookup import identity_key @@ -378,27 +377,6 @@ def diagnostic_projection(*, diagnostic: ScopeDiagnostic) -> dict[str, object]: } -def records_for_identities( - *, lookup: ScopeLookup, identities: Iterable[DeclarationIdentity] -) -> tuple[DeclarationRecord, ...]: - """Return every known duplicate match in stable identity/location order.""" - - records: list[DeclarationRecord] = [] - for identity in identities: - records.extend(lookup.declarations.get(identity, ())) - return tuple( - sorted( - records, - key=lambda item: ( - identity_key(item.identity), - item.path, - item.line, - item.column, - ), - ) - ) - - def safe_scope_path(*, path: str) -> str: """Return a normalized relative path without leaking invalid absolute input.""" diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index 1f4f6dbd65..0979ca1350 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -38,7 +38,6 @@ SCOPE_ENUM_MEMBER_FIELDS: frozenset[str] = frozenset({"name"}) DEFAULT_ENUM_MEMBER_PREVIEW: int = 20 QUALIFIED_IDENTITY_SEPARATOR: str = ":" -PRIVATE_IDENTITY_SEPARATOR: str = "." PATH_SEPARATOR: str = "/" WINDOWS_PATH_SEPARATOR: str = "\\" CURRENT_PATH_COMPONENT: str = "." diff --git a/src/sqlbuild/compiler/scopes/exceptions.py b/src/sqlbuild/compiler/scopes/exceptions.py index c79d597432..e368b911e8 100644 --- a/src/sqlbuild/compiler/scopes/exceptions.py +++ b/src/sqlbuild/compiler/scopes/exceptions.py @@ -28,12 +28,6 @@ class InvalidQualifiedIdentityError(ScopeError): code: str = "S002" -class DuplicateScopeIdentityError(ScopeError): - """Raised when records duplicate a canonical identity.""" - - code: str = "S003" - - class ScopeCacheDecodeError(ScopeError): """Raised when a persistent scope-index payload is malformed.""" diff --git a/src/sqlbuild/compiler/scopes/main/_normalize_scope_path.py b/src/sqlbuild/compiler/scopes/main/_normalize_scope_path.py deleted file mode 100644 index 256eb057a9..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_normalize_scope_path.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Normalize one project-relative scope path.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.paths import normalize_path - - -def normalize_scope_path(*, path: str | PurePath) -> str: - """Normalize POSIX or Windows input to a project-relative POSIX path.""" - - return normalize_path(path=path) diff --git a/src/sqlbuild/compiler/scopes/main/_parse_qualified_identity.py b/src/sqlbuild/compiler/scopes/main/_parse_qualified_identity.py deleted file mode 100644 index 302e22be0d..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_parse_qualified_identity.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Parse one qualified scope identity.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes._helpers.identities import parse_identity -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity - - -def parse_qualified_identity(*, value: str) -> ResourceIdentity | DeclarationIdentity: - """Parse only explicit kind-qualified identities; bare names are invalid.""" - - return parse_identity(value=value) diff --git a/src/sqlbuild/compiler/scopes/main/_path_is_equal_or_descendant.py b/src/sqlbuild/compiler/scopes/main/_path_is_equal_or_descendant.py deleted file mode 100644 index b0f301a499..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_path_is_equal_or_descendant.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Compare normalized project-relative path components.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant - - -def path_is_equal_or_descendant(*, path: str | PurePath, ancestor: str | PurePath) -> bool: - """Compare normalized path components rather than textual prefixes.""" - - return is_equal_or_descendant(path=path, ancestor=ancestor) diff --git a/src/sqlbuild/compiler/scopes/main/_qualified_identity_text.py b/src/sqlbuild/compiler/scopes/main/_qualified_identity_text.py deleted file mode 100644 index a09bdf00c2..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_qualified_identity_text.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Format one qualified scope identity.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes._helpers.identities import format_identity -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity - - -def format_qualified_identity(*, identity: ResourceIdentity | DeclarationIdentity) -> str: - """Format a resource, public declaration, or owner-qualified private declaration.""" - - return format_identity(identity=identity) diff --git a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py b/src/sqlbuild/compiler/scopes/main/_query_scope_target.py deleted file mode 100644 index ba2f9e5702..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Query a scope resource by qualified identity or normalized authored path.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.visibility import query_target -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeLookup, - ScopeTargetQuery, -) - - -def query_scope_target( - *, - lookup: ScopeLookup, - target: ResourceIdentity | DeclarationIdentity | str | PurePath, -) -> ScopeTargetQuery: - """Return matching resource or declaration records without conflating absence.""" - - return query_target(lookup=lookup, target=target) diff --git a/src/sqlbuild/compiler/scopes/main/_scope_is_path_visible.py b/src/sqlbuild/compiler/scopes/main/_scope_is_path_visible.py deleted file mode 100644 index 6b6658df2d..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_scope_is_path_visible.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Resolve path-derived lexical scope visibility.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant, normalize_path -from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, PATH_SEPARATOR -from sqlbuild.compiler.scopes.types import ScopeKind - - -def scope_is_path_visible( - *, scope: ScopeKind, owning_path: str | PurePath, resource_path: str | PurePath -) -> bool: - """Resolve path-derived visibility without project discovery or I/O.""" - - if scope is ScopeKind.GLOBAL: - return True - owner: str = normalize_path(path=owning_path) - resource: str = normalize_path(path=resource_path) - resource_parent: str = ( - resource.rsplit(PATH_SEPARATOR, maxsplit=1)[0] - if PATH_SEPARATOR in resource - else CURRENT_PATH_COMPONENT - ) - if scope is ScopeKind.INHERITED: - return is_equal_or_descendant(path=resource_parent, ancestor=owner) - if scope is ScopeKind.LOCAL: - return resource_parent == owner - return False diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py index 10ce5c6632..6f46e02a7e 100644 --- a/src/sqlbuild/compiler/scopes/models.py +++ b/src/sqlbuild/compiler/scopes/models.py @@ -219,8 +219,6 @@ class ScopeLookup: usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] - visibility_by_resource: Mapping[ResourceIdentity, tuple[VisibilityRecord, ...]] - inaccessible_by_resource: Mapping[ResourceIdentity, tuple[InaccessibleRecord, ...]] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index d228860e25..d700029f9c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_loader_functions +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_python_node_functions from sqlbuild.compiler.discovery.exceptions import LoaderDiscoveryError from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( @@ -170,7 +170,9 @@ def test_given_project_dir_when_discovering_loaders_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - result: tuple[DiscoveredLoaderFunction, ...] = discover_loader_functions(project_dir=tmp_path) + result: tuple[DiscoveredLoaderFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).loaders + ) assert tuple(loader.name for loader in result) == test_case.expected_names assert tuple(loader.destination for loader in result) == test_case.expected_targets @@ -224,4 +226,4 @@ def test_given_loader_import_error_when_discovering_loaders_then_raises_clear_er file_path.write_text(contents, encoding="utf-8") with pytest.raises(LoaderDiscoveryError, match=test_case.expected_error_fragment): - discover_loader_functions(project_dir=tmp_path) + discover_python_node_functions(project_dir=tmp_path) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 0f81f83eab..c84d689d24 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -7,9 +7,7 @@ import pytest from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_asset_functions, - discover_check_functions, - discover_task_functions, + discover_python_node_functions, ) from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError from sqlbuild.compiler.discovery.models import ( @@ -93,8 +91,12 @@ def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - tasks: tuple[DiscoveredTaskFunction, ...] = discover_task_functions(project_dir=tmp_path) - assets: tuple[DiscoveredAssetFunction, ...] = discover_asset_functions(project_dir=tmp_path) + tasks: tuple[DiscoveredTaskFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).tasks + ) + assets: tuple[DiscoveredAssetFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).assets + ) assert tuple(task.name for task in tasks) == test_case.expected_task_names assert ( @@ -148,7 +150,7 @@ def test_given_python_node_import_error_when_discovering_then_raises_clear_error file_path.write_text(contents, encoding="utf-8") with pytest.raises(PythonNodeDiscoveryError, match=test_case.expected_error_fragment): - discover_task_functions(project_dir=tmp_path) + discover_python_node_functions(project_dir=tmp_path) @pytest.mark.parametrize( @@ -211,7 +213,9 @@ def test_given_project_dir_when_discovering_checks_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - checks: tuple[DiscoveredCheckFunction, ...] = discover_check_functions(project_dir=tmp_path) + checks: tuple[DiscoveredCheckFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).checks + ) assert tuple(check.name for check in checks) == test_case.expected_check_names assert tuple(len(check.depends_on) for check in checks) == ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 1a9ff291a1..9f3ede17d7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -204,7 +204,6 @@ def test_given_deferred_output_locations_when_discovering_models_then_projection )[0] assert model_file.output_column_locations == {} - assert model_file.output_column_locations_extracted is False assert ( model_file.extract_implicit_alias_columns is test_case.expected_extract_implicit_alias_columns diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index 3484661282..11d8c51746 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -16,15 +16,12 @@ class ColumnLineageAnalyzerTestCase: expected_column: str expected_upstream_columns: tuple[str, ...] expected_transform_kind: ColumnTransformKind - expected_internal_scope_names: tuple[str, ...] = () @dataclass(frozen=True) class ProjectLineageGraphTestCase: description: str - expected_trace: tuple[str, ...] expected_consumers: tuple[str, ...] - expected_downstream_trace: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index e01d20d06d..d628fb85c7 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -186,8 +186,6 @@ def test_given_compiled_project_when_building_column_lineage_then_infers_expecte ) assert upstream_columns == tuple(sorted(test_case.expected_upstream_columns)) assert column_lineage.transform_kind == test_case.expected_transform_kind - for expected_scope_name in test_case.expected_internal_scope_names: - assert expected_scope_name in {node.scope_name for node in column_lineage.nodes} @pytest.mark.parametrize( @@ -378,15 +376,13 @@ def test_given_select_star_when_building_column_lineage_then_expands_known_schem "test_case", [ ProjectLineageGraphTestCase( - description="traces upstream and downstream through linear project graph", - expected_trace=("b.id->c.id", "a.id->b.id"), + description="finds column consumers through linear project graph", expected_consumers=("a.id->b.id",), - expected_downstream_trace=("a.id->b.id", "b.id->c.id"), ) ], ids=lambda case: case.description, ) -def test_given_linear_project_when_tracing_column_lineage_then_returns_expected_edges( +def test_given_linear_project_when_listing_column_consumers_then_returns_expected_edges( test_case: ProjectLineageGraphTestCase, ) -> None: project: CompiledProject = make_compiled_project( @@ -404,15 +400,6 @@ def test_given_linear_project_when_tracing_column_lineage_then_returns_expected_ result: ProjectColumnLineage | None = build_project_column_lineage(project=project) assert result is not None - trace: tuple[str, ...] = tuple( - edge_label( - edge.source.resource_name, - edge.source.column_name, - edge.target.resource_name, - edge.target.column_name, - ) - for edge in result.trace_column(model_name="c", column_name="id") - ) consumers: tuple[str, ...] = tuple( edge_label( edge.source.resource_name, @@ -422,18 +409,7 @@ def test_given_linear_project_when_tracing_column_lineage_then_returns_expected_ ) for edge in result.column_consumers(resource_name="a", column_name="id") ) - downstream_trace: tuple[str, ...] = tuple( - edge_label( - edge.source.resource_name, - edge.source.column_name, - edge.target.resource_name, - edge.target.column_name, - ) - for edge in result.trace_column_downstream(resource_name="a", column_name="id") - ) - assert trace == test_case.expected_trace assert consumers == test_case.expected_consumers - assert downstream_trace == test_case.expected_downstream_trace @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 2882afb428..bd20f33618 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -523,19 +522,6 @@ class ResolveSchemaActionsTestCase: expected_actions: tuple[SchemaAction, ...] -@dataclass(frozen=True) -class BuildLogicalDdlTestCase: - description: str - action: PlanAction - resolved_sql: str - qualified_name: str | None - unique_key: tuple[str, ...] - warehouse_columns: tuple[ColumnInfo, ...] - expected_ddl_fragment: str - merge_exclude_columns: tuple[str, ...] = () - unexpected_ddl_fragments: tuple[str, ...] = () - - @dataclass(frozen=True) class BuildModelWarningsTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_ddl.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_ddl.py deleted file mode 100644 index ebfdaead50..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_ddl.py +++ /dev/null @@ -1,200 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner._helpers.output.strategy import build_logical_ddl -from sqlbuild.compiler.planner.types import PlanAction -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - BuildLogicalDdlTestCase, -) - -_SIMPLE_SQL: str = "SELECT id, name FROM raw_orders" - -_WAREHOUSE_COLUMNS: tuple[ColumnInfo, ...] = ( - ColumnInfo(name="order_id", type="INTEGER"), - ColumnInfo(name="name", type="VARCHAR"), - ColumnInfo(name="amount", type="DECIMAL"), -) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildLogicalDdlTestCase( - description="create view wraps sql in create or replace view", - action=PlanAction.CREATE_VIEW, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="CREATE OR REPLACE VIEW staging.orders AS", - ), - BuildLogicalDdlTestCase( - description="create table wraps sql in create table as", - action=PlanAction.CREATE_TABLE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="CREATE TABLE staging.orders AS", - ), - BuildLogicalDdlTestCase( - description="incremental append wraps sql in insert into", - action=PlanAction.INCREMENTAL_APPEND, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="INSERT INTO staging.orders", - ), - BuildLogicalDdlTestCase( - description="delete_insert produces delete and insert statements", - action=PlanAction.INCREMENTAL_DELETE_INSERT, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=(), - expected_ddl_fragment="DELETE FROM staging.orders", - ), - BuildLogicalDdlTestCase( - description="delete_insert includes unique key in where clause", - action=PlanAction.INCREMENTAL_DELETE_INSERT, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=(), - expected_ddl_fragment="WHERE (order_id) IN", - ), - BuildLogicalDdlTestCase( - description="merge produces merge into with on clause", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment="MERGE INTO staging.orders AS target", - ), - BuildLogicalDdlTestCase( - description="merge includes update set for non-key columns", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment=( - "WHEN MATCHED THEN UPDATE SET name = __source.name, amount = __source.amount" - ), - ), - BuildLogicalDdlTestCase( - description="merge includes insert clause with all columns", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment="WHEN NOT MATCHED THEN INSERT (order_id, name, amount)", - ), - BuildLogicalDdlTestCase( - description="merge exclusion only omits matched update column", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - merge_exclude_columns=("NAME",), - expected_ddl_fragment="WHEN MATCHED THEN UPDATE SET amount = __source.amount", - unexpected_ddl_fragments=("UPDATE SET name = __source.name",), - ), - BuildLogicalDdlTestCase( - description="merge on clause uses unique key", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment="ON target.order_id = __source.order_id", - ), - BuildLogicalDdlTestCase( - description="skip action produces empty ddl", - action=PlanAction.SKIP, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="", - ), - BuildLogicalDdlTestCase( - description="create view includes resolved sql in body", - action=PlanAction.CREATE_VIEW, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment=_SIMPLE_SQL, - ), - BuildLogicalDdlTestCase( - description="delete_insert with composite key uses both columns", - action=PlanAction.INCREMENTAL_DELETE_INSERT, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id", "line_id"), - warehouse_columns=(), - expected_ddl_fragment="WHERE (order_id, line_id) IN", - ), - BuildLogicalDdlTestCase( - description="load_seed action produces empty ddl", - action=PlanAction.LOAD_SEED, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="", - ), - BuildLogicalDdlTestCase( - description="merge with composite key joins on both columns", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id", "line_id"), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment=( - "ON target.order_id = __source.order_id AND target.line_id = __source.line_id" - ), - ), - BuildLogicalDdlTestCase( - description=("falls back to target name when qualified name is none"), - action=PlanAction.CREATE_TABLE, - resolved_sql=_SIMPLE_SQL, - qualified_name=None, - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="CREATE TABLE orders AS", - ), - ], - ids=lambda case: case.description, -) -def test_given_action_and_sql_when_building_ddl_then_contains_expected_fragment( - test_case: BuildLogicalDdlTestCase, -) -> None: - target: CompiledRelationLocation = CompiledRelationLocation( - database=None, - schema="staging", - name="orders", - qualified_name=test_case.qualified_name, - ) - - result: str = build_logical_ddl( - action=test_case.action, - resolved_sql=test_case.resolved_sql, - target=target, - unique_key=test_case.unique_key, - warehouse_columns=test_case.warehouse_columns, - merge_exclude_columns=test_case.merge_exclude_columns, - ) - - assert test_case.expected_ddl_fragment in result - unexpected_fragment: str - for unexpected_fragment in test_case.unexpected_ddl_fragments: - assert unexpected_fragment not in result diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index cf5dd77382..25ddd6b3a7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -97,18 +97,6 @@ class DirectInputBaselineTestCase: unexpected_baseline_names: tuple[str, ...] -@dataclass(frozen=True) -class LocalNodePlanningTestCase: - description: str - fingerprint_exists: bool - relation_exists: bool - full_refresh: bool - local_hash: str | None - previous_hash: str | None - expected_action: str - expected_reason: str - - @dataclass(frozen=True) class SelectorExpansionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_local_node_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_local_node_planning.py deleted file mode 100644 index 8e30470fe1..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_local_node_planning.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.planner._helpers.command_planning.local_node_planning import ( - classify_local_node_plan, -) -from sqlbuild.compiler.planner.models import LocalNodePlanInput, LocalNodePlanOutcome -from sqlbuild.compiler.planner.types import LocalNodePlanAction, LocalNodePlanReason -from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( - LocalNodePlanningTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - LocalNodePlanningTestCase( - description="full refresh takes precedence over all local state", - fingerprint_exists=False, - relation_exists=False, - full_refresh=True, - local_hash="current", - previous_hash="previous", - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.FULL_REFRESH, - ), - LocalNodePlanningTestCase( - description="missing fingerprint is first run", - fingerprint_exists=False, - relation_exists=True, - full_refresh=False, - local_hash="current", - previous_hash=None, - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.FIRST_RUN, - ), - LocalNodePlanningTestCase( - description="missing relation with fingerprint reruns relation", - fingerprint_exists=True, - relation_exists=False, - full_refresh=False, - local_hash="current", - previous_hash="current", - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.RELATION_MISSING, - ), - LocalNodePlanningTestCase( - description="changed local hash reruns relation", - fingerprint_exists=True, - relation_exists=True, - full_refresh=False, - local_hash="current", - previous_hash="previous", - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.LOCAL_CHANGED, - ), - LocalNodePlanningTestCase( - description="matching local hash is current", - fingerprint_exists=True, - relation_exists=True, - full_refresh=False, - local_hash="current", - previous_hash="current", - expected_action=LocalNodePlanAction.CURRENT, - expected_reason=LocalNodePlanReason.NO_CHANGE, - ), - LocalNodePlanningTestCase( - description="missing local hash skips hash comparison", - fingerprint_exists=True, - relation_exists=True, - full_refresh=False, - local_hash=None, - previous_hash="previous", - expected_action=LocalNodePlanAction.CURRENT, - expected_reason=LocalNodePlanReason.NO_CHANGE, - ), - ], - ids=lambda case: case.description, -) -def test_given_local_node_state_when_classifying_then_returns_expected_outcome( - test_case: LocalNodePlanningTestCase, -) -> None: - outcome: LocalNodePlanOutcome = classify_local_node_plan( - LocalNodePlanInput( - fingerprint_exists=test_case.fingerprint_exists, - relation_exists=test_case.relation_exists, - full_refresh=test_case.full_refresh, - local_hash=test_case.local_hash, - previous_hash=test_case.previous_hash, - ) - ) - - assert outcome.action == test_case.expected_action - assert outcome.reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 3daf38b0f2..904a68c487 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -23,9 +23,9 @@ EnumDeclaration, EnumMember, ) +from sqlbuild.compiler.scopes._helpers.visibility import query_target from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index -from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup @@ -293,7 +293,7 @@ def test_given_unknown_qualified_target_when_querying_then_distinguishes_unknown ) -> None: lookup: ScopeLookup = build_scope_lookup(index=ScopeIndex()) - query: ScopeTargetQuery = query_scope_target(lookup=lookup, target="model:missing") + query: ScopeTargetQuery = query_target(lookup=lookup, target="model:missing") resolution: VisibilityResolution = resolve_scope_visibility( lookup=lookup, target="model:missing" ) @@ -319,7 +319,7 @@ def test_given_declaration_qualified_target_when_querying_then_returns_declarati ) lookup: ScopeLookup = build_scope_lookup(index=ScopeIndex(declarations=(declaration,))) - query: ScopeTargetQuery = query_scope_target( + query: ScopeTargetQuery = query_target( lookup=lookup, target="constant:minimum_value", ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index fbb96d762d..f9a33b4eca 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -4,9 +4,8 @@ import pytest +from sqlbuild.compiler.scopes._helpers.identities import format_identity, parse_identity from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError -from sqlbuild.compiler.scopes.main._parse_qualified_identity import parse_qualified_identity -from sqlbuild.compiler.scopes.main._qualified_identity_text import format_qualified_identity from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( @@ -43,10 +42,10 @@ def test_given_qualified_identity_when_parsing_and_formatting_then_round_trips( test_case: QualifiedIdentityCase, ) -> None: - parsed: ResourceIdentity | DeclarationIdentity = parse_qualified_identity(value=test_case.text) + parsed: ResourceIdentity | DeclarationIdentity = parse_identity(value=test_case.text) assert parsed == test_case.expected_identity - assert format_qualified_identity(identity=parsed) == test_case.text + assert format_identity(identity=parsed) == test_case.text @pytest.mark.parametrize( @@ -77,7 +76,7 @@ def test_given_bare_name_when_parsing_then_rejects_unqualified_identity( test_case: InvalidIdentityCase, ) -> None: with pytest.raises(test_case.expected_error, match="kind-qualified"): - parse_qualified_identity(value=test_case.value) + parse_identity(value=test_case.value) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py index a7a8d4cf6b..a16976601d 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py @@ -4,12 +4,7 @@ import pytest -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.main._normalize_scope_path import normalize_scope_path -from sqlbuild.compiler.scopes.main._path_is_equal_or_descendant import ( - path_is_equal_or_descendant, -) -from sqlbuild.compiler.scopes.main._scope_is_path_visible import scope_is_path_visible +from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant, normalize_path from sqlbuild.compiler.scopes.types import ScopeKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( PathNormalizationCacheCase, @@ -42,7 +37,7 @@ def test_given_posix_or_windows_path_when_normalizing_then_returns_canonical_posix_path( test_case: PathNormalizationCase, ) -> None: - assert normalize_scope_path(path=test_case.path) == test_case.expected_path + assert normalize_path(path=test_case.path) == test_case.expected_path @pytest.mark.parametrize( @@ -75,59 +70,6 @@ def test_given_repeated_path_when_normalizing_then_reuses_bounded_cache( normalize_path.cache_clear() -@pytest.mark.parametrize( - "test_case", - [ - PathVisibilityCase("global", ScopeKind.GLOBAL, "constants", "tests/orders.sql", True), - PathVisibilityCase( - "inherited_owner", ScopeKind.INHERITED, "models/staging", "models/staging/a.sql", True - ), - PathVisibilityCase( - "inherited_descendant", - ScopeKind.INHERITED, - "models/staging", - "models/staging/orders/a.sql", - True, - ), - PathVisibilityCase( - "local_owner", ScopeKind.LOCAL, "models/staging", "models/staging/a.sql", True - ), - PathVisibilityCase( - "local_descendant", - ScopeKind.LOCAL, - "models/staging", - "models/staging/orders/a.sql", - False, - ), - PathVisibilityCase( - "textual_prefix", ScopeKind.INHERITED, "models/order", "models/orders/a.sql", False - ), - PathVisibilityCase( - "descendant_hidden", - ScopeKind.INHERITED, - "models/staging/orders", - "models/staging/a.sql", - False, - ), - PathVisibilityCase( - "private", ScopeKind.PRIVATE, "models/staging", "models/staging/a.sql", False - ), - ], - ids=lambda case: case.description, -) -def test_given_scope_and_resource_path_when_resolving_then_enforces_component_visibility( - test_case: PathVisibilityCase, -) -> None: - assert ( - scope_is_path_visible( - scope=test_case.scope, - owning_path=test_case.owner, - resource_path=test_case.resource, - ) - is test_case.expected_visible - ) - - @pytest.mark.parametrize( "test_case", [ @@ -141,7 +83,7 @@ def test_given_component_prefix_only_when_comparing_descendant_then_returns_fals test_case: PathVisibilityCase, ) -> None: assert ( - path_is_equal_or_descendant(path=test_case.resource, ancestor=test_case.owner) + is_equal_or_descendant(path=test_case.resource, ancestor=test_case.owner) is test_case.expected_visible ) From 32d3dbac3b87660085e11ffdc3c6efed89612a35 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:19:08 +0100 Subject: [PATCH 09/32] refactor(cli): remove unused code --- .../cli/commands/_helpers/entry/parsing.py | 6 ----- .../cli/commands/_helpers/skills/update.py | 4 ---- .../cli/commands/classes/cli_namespace.py | 2 -- src/sqlbuild/cli/commands/constants.py | 1 - .../_helpers/execution_result_document.py | 9 -------- src/sqlbuild/cli/output/models.py | 23 ------------------- .../nested_command_progress_callbacks.py | 2 -- .../cli/commands/main/compile/_test_types.py | 1 - .../compile/test_compile_target_writer.py | 8 ++----- .../cli/commands/main/entry/test_main.py | 7 ++++-- 10 files changed, 7 insertions(+), 56 deletions(-) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py index 5fb090c963..0f7a8f1f63 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py @@ -58,12 +58,6 @@ def resolve_env_default_concurrency(explicit_concurrency: int | None) -> int | N return concurrency -def read_selector_files(paths: list[str]) -> tuple[str, ...]: - """Read newline-delimited selectors.""" - - return read_selector_file_inputs(paths).selectors - - def read_selector_file_inputs(paths: list[str]) -> SelectorInputs: """Read newline-delimited selectors while retaining file provenance.""" diff --git a/src/sqlbuild/cli/commands/_helpers/skills/update.py b/src/sqlbuild/cli/commands/_helpers/skills/update.py index 6f899e5441..2072fb1691 100644 --- a/src/sqlbuild/cli/commands/_helpers/skills/update.py +++ b/src/sqlbuild/cli/commands/_helpers/skills/update.py @@ -110,10 +110,6 @@ def normalize_skill_targets(*, targets: tuple[str, ...], source: str) -> tuple[s return tuple(normalized_targets) -def load_project_skill_targets(*, project_dir: Path) -> tuple[str, ...]: - return load_project_skill_settings(project_dir=project_dir).targets - - def load_project_skill_settings(*, project_dir: Path) -> SkillSettings: config_path: Path = project_dir / PROJECT_CONFIG_FILENAME if not config_path.exists(): diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index 38ca9e3714..9e812a66a9 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -13,7 +13,6 @@ "dbt_project_dir": None, "dbt_profiles_dir": None, "dbt_target": None, - "dbt_target_path": None, "dbt_profile": None, "sqb_output_dir": None, "dry_run": False, @@ -186,7 +185,6 @@ class CliNamespace: dbt_project_dir: str | None dbt_profiles_dir: str | None dbt_target: str | None - dbt_target_path: str | None dbt_profile: str | None sqb_output_dir: str | None dry_run: bool diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index 0d44f9df82..941a51f346 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -82,7 +82,6 @@ COST_DEFAULT_HISTORY_LIMIT: int = 10 ISO_DATE_LENGTH: int = 10 COST_NARROW_TERMINAL_WIDTH: int = 120 -COST_TABLE_COLUMN_COUNT: int = 6 COST_HISTORY_NUMERIC_COLUMNS: frozenset[int] = frozenset({4, 5}) COST_RESOURCE_NUMERIC_START_COLUMN: int = 2 BYTE_SCALE: int = 1024 diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py index e594579651..724c45bed9 100644 --- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py +++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py @@ -625,15 +625,6 @@ def _build_projection_is_partial( ) -def _projected_asset_summary(*, assets: tuple[dict[str, object], ...]) -> dict[str, int]: - return { - "success_count": sum(1 for asset in assets if asset.get("status") == _SUCCESS_STATUS), - "failure_count": sum(1 for asset in assets if asset.get("status") == _FAILED_STATUS), - "skipped_count": sum(1 for asset in assets if asset.get("status") == _SKIPPED_STATUS), - "warning_count": sum(1 for asset in assets if asset.get("status") == _WARNING_STATUS), - } - - def _format_model_assets( *, results: tuple[ModelExecutionResult, ...], plan: PlanOutput | None ) -> tuple[dict[str, object], ...]: diff --git a/src/sqlbuild/cli/output/models.py b/src/sqlbuild/cli/output/models.py index 2e895c203f..2c207f0727 100644 --- a/src/sqlbuild/cli/output/models.py +++ b/src/sqlbuild/cli/output/models.py @@ -579,29 +579,6 @@ class WrittenTarget: target_dir: Path diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - def summary_line(self) -> str: - """Build a human-readable summary line.""" - - parts: list[str] = [] - if self.model_count: - model_label: str = "model" if self.model_count == 1 else "models" - parts.append(f"{self.model_count} {model_label}") - if self.seed_count: - seed_label: str = "seed" if self.seed_count == 1 else "seeds" - parts.append(f"{self.seed_count} {seed_label}") - if self.function_count: - function_label: str = "function" if self.function_count == 1 else "functions" - parts.append(f"{self.function_count} {function_label}") - if self.audit_count: - audit_label: str = "audit" if self.audit_count == 1 else "audits" - parts.append(f"{self.audit_count} {audit_label}") - if self.test_count: - test_label: str = "test" if self.test_count == 1 else "tests" - parts.append(f"{self.test_count} {test_label}") - if not parts: - return "Compiled 0 resources" - return f"Compiled {', '.join(parts)}" - @dataclass(frozen=True) class SkillInstallTarget: diff --git a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py index 1d88465d0a..fd4b4d9365 100644 --- a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py +++ b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py @@ -53,7 +53,6 @@ def __init__( self._is_tty: bool = hasattr(stream, "isatty") and stream.isatty() self._counter: int = 0 self._current_group: str = "" - self._active_group: str = "" self._active_name: str = "" self._active_group_is_new: bool = False self._spinner_frame_index: int = 0 @@ -74,7 +73,6 @@ def on_item_start( self._projector.expect_resource_enrichment( resource_name=canonical_resource_name or item_name ) - self._active_group = group_name self._active_name = item_name self._active_group_is_new = group_name != self._current_group if self._active_group_is_new: diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py index 0d0a17d142..efc2c81fc4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -10,7 +10,6 @@ class TargetWriterTestCase: description: str expected_files: dict[str, str] = field(default_factory=dict) initial_files: dict[str, str] = field(default_factory=dict) - expected_summary_line: str = "" model_count: int = 0 expected_max_component_bytes: int = 0 diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index c5579797b0..3c75541eb4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -56,7 +56,6 @@ "SELECT order_id FROM analytics.orders WHERE order_id IS NULL\n" ), }, - expected_summary_line="Compiled 1 model, 1 seed, 2 functions, 1 audit, 1 test", ) ], ids=lambda case: case.description, @@ -74,7 +73,7 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( expected_files: dict[str, str] = dict(test_case.expected_files) expected_files["compiled/tests/_chain_/orders__stg_orders/orders_chain.sql"] = expected_test_sql - written: WrittenTarget = write_compile_target( + write_compile_target( target_dir=tmp_path / "target", adapter=DuckDbAdapter(), plan_output=plan_output, @@ -90,7 +89,6 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( manifest=manifest, ) - assert written.summary_line() == test_case.expected_summary_line assert read_target_files(tmp_path / "target", expected_files) == expected_files assert json.loads(manifest_path.read_text()) == manifest assert manifest_path.stat().st_mtime_ns == unchanged_mtime_ns @@ -194,7 +192,6 @@ def test_given_parameterized_test_entries_when_writing_target_then_case_artifact ")\n" ), }, - expected_summary_line="Compiled 1 model, 1 function", ) ], ids=lambda case: case.description, @@ -209,7 +206,7 @@ def test_given_compiled_project_when_writing_static_target_then_expected_files_a stale_path.parent.mkdir(parents=True) stale_path.write_text("SELECT 'stale'\n", encoding="utf-8") - written: WrittenTarget = write_static_compile_target( + write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), project=project, @@ -223,7 +220,6 @@ def test_given_compiled_project_when_writing_static_target_then_expected_files_a project=project, ) - assert written.summary_line() == test_case.expected_summary_line assert read_target_files(target_dir, test_case.expected_files) == test_case.expected_files assert model_path.stat().st_mtime_ns == unchanged_mtime_ns assert not stale_path.exists() diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 339cf22e89..50d4ebc625 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -8,7 +8,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild.cli.commands._helpers.entry.parsing import read_selector_files +from sqlbuild.cli.commands._helpers.entry.parsing import read_selector_file_inputs from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.main.entrypoint.entry import _main_with_dependencies, main from sqlbuild.cli.commands.models import ( @@ -2430,7 +2430,10 @@ def run_plan(request: PlanCommandRequest) -> int: assert exit_code == test_case.expected_exit_code assert received_selects == [test_case.expected_select] - assert read_selector_files([str(tmp_path / "selectors.txt")]) == ("customers", "payments") + assert read_selector_file_inputs([str(tmp_path / "selectors.txt")]).selectors == ( + "customers", + "payments", + ) @pytest.mark.parametrize( From 35119596a4868786c8a7575b2d794960624653b9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:23:49 +0100 Subject: [PATCH 10/32] refactor(executor): remove unused code --- .../executor/build/_helpers/output.py | 861 ------------------ .../executor/build/classes/build_scheduler.py | 6 - .../clone/_helpers/prephase_progress.py | 18 - src/sqlbuild/executor/clone/types.py | 6 - src/sqlbuild/executor/load/_helpers/rows.py | 6 - .../python_nodes/_helpers/lifecycle_nodes.py | 11 - .../read_side_python_execution_tracker.py | 4 - .../executor/python_nodes/main/_run.py | 23 - .../_helpers/materializations/microbatch.py | 6 - src/sqlbuild/executor/run/main/_promote.py | 34 - .../executor/build/_helpers/_test_types.py | 42 +- .../executor/build/_helpers/helpers.py | 62 -- .../build/_helpers/test_batch_summary.py | 147 --- .../executor/build/_helpers/test_output.py | 805 ---------------- .../src/sqlbuild/executor/load/test_rows.py | 6 +- .../_helpers/test_lifecycle_nodes.py | 51 +- 16 files changed, 5 insertions(+), 2083 deletions(-) delete mode 100644 src/sqlbuild/executor/build/_helpers/output.py delete mode 100644 src/sqlbuild/executor/python_nodes/main/_run.py delete mode 100644 src/sqlbuild/executor/run/main/_promote.py delete mode 100644 tests/unit/src/sqlbuild/executor/build/_helpers/test_batch_summary.py delete mode 100644 tests/unit/src/sqlbuild/executor/build/_helpers/test_output.py diff --git a/src/sqlbuild/executor/build/_helpers/output.py b/src/sqlbuild/executor/build/_helpers/output.py deleted file mode 100644 index 2ffdfdd146..0000000000 --- a/src/sqlbuild/executor/build/_helpers/output.py +++ /dev/null @@ -1,861 +0,0 @@ -"""Build execution output formatting.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope -from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display -from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end -from sqlbuild.compiler.planner.main.execution.model_execution_annotation import ( - model_execution_annotation, -) -from sqlbuild.compiler.planner.main.execution.model_resource_type import model_resource_type -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.aligned_name_value import format_aligned_name_value -from sqlbuild.presentation.main.coded_error_text import format_coded_error -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.main.resolve_name_column_width import resolve_name_column_width -from sqlbuild.presentation.main.status_cell import format_status_cell -from sqlbuild.presentation.main.summary_footer import format_summary_footer -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.types import CompletionState - - -@dataclass(frozen=True) -class _AuditDisplayEntry: - """Aggregated audit result for display.""" - - label: str - display_name: str - outcome: AuditOutcome - total_row_count: int - batch_pass: int - batch_total: int - executed_sql: str | None = None - - -def format_build_output( - *, - result: BuildExecutionResult, - plan: PlanOutput, - target: str | None = None, - concurrency: int = 1, - elapsed_seconds: float = 0.0, - use_color: bool = False, - verbose: bool = False, -) -> str: - """Format the complete build execution output.""" - - model_entry_map: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} - test_results_by_model: dict[str, SqlTestExecutionResult] = _build_test_results_by_model( - result.test_results - ) - lines: list[str] = [] - lines.append(_format_header(target=target, concurrency=concurrency, use_color=use_color)) - lines.append("") - - counter: int = 0 - total: int = _count_top_level_nodes(result) - top_level_name_width: int = _top_level_name_width(result=result, plan=plan) - sub_name_width: int = _sub_name_width(result) - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - counter += 1 - lines.append( - _format_seed_line( - seed_result=seed_result, - counter=counter, - total=total, - use_color=use_color, - name_width=top_level_name_width, - ) - ) - - model_result: ModelExecutionResult - for model_result in result.model_results: - counter += 1 - plan_entry: ModelPlanEntry | None = model_entry_map.get(model_result.model_name) - resource_type: str = _resolve_resource_type(plan_entry) - annotation: str = _resolve_annotation(plan_entry) - - lines.append( - _format_model_line( - model_result=model_result, - resource_type=resource_type, - annotation=annotation, - counter=counter, - total=total, - use_color=use_color, - name_width=top_level_name_width, - ) - ) - - batch_line: str | None = _format_batch_summary_line( - model_result=model_result, use_color=use_color - ) - if batch_line is not None: - lines.append(batch_line) - - if verbose: - event: LifeCycleEvent - for event in _resolve_verbose_events( - model_result=model_result, - plan_entry=plan_entry, - ): - if event.kind == LifeCycleEventKind.SQL: - lines.extend(_format_sql_block(event.content)) - elif event.kind == LifeCycleEventKind.LOG: - lines.extend(_format_log_block(message=event.content, use_color=use_color)) - - test_result: SqlTestExecutionResult | None = test_results_by_model.get( - model_result.model_name - ) - if test_result is not None: - lines.append( - _format_sub_line( - sub_type="test", - name=test_result.test_name, - status=_test_outcome_to_status(test_result.outcome), - use_color=use_color, - name_width=sub_name_width, - ) - ) - step_result: StepResult - for step_result in test_result.step_results: - lines.append( - _format_test_expectation_sub_line( - step_result=step_result, - use_color=use_color, - name_width=sub_name_width, - ) - ) - - audit_entries: list[_AuditDisplayEntry] = _aggregate_audit_results( - model_result.audit_results - ) - audit_entry: _AuditDisplayEntry - for audit_entry in audit_entries: - lines.append( - _format_audit_sub_line( - entry=audit_entry, - use_color=use_color, - name_width=sub_name_width, - ) - ) - if verbose and audit_entry.executed_sql is not None: - lines.extend(_format_sql_block(audit_entry.executed_sql)) - - lines.append("") - lines.append( - _format_completion_message( - status=result.status, - warning_count=result.warning_count, - use_color=use_color, - summary=_format_summary_counts(result=result, elapsed_seconds=elapsed_seconds), - ) - ) - - failure_lines: list[str] = _format_failure_details(result=result, use_color=use_color) - if failure_lines: - lines.append("") - lines.extend(failure_lines) - - warning_lines: list[str] = _format_warning_details(result) - if warning_lines: - lines.append("") - lines.extend(warning_lines) - - return "\n".join(lines) - - -def _format_header(*, target: str | None, concurrency: int, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - context_parts: list[str] = [] - if target is not None: - context_parts.append(f"target: {target}") - context_parts.append(f"concurrency: {concurrency}") - context: str = f"sqb build ({', '.join(context_parts)})" - return format_surface_header(style=style, title="Execution", context=context) - - -def _format_seed_line( - *, - seed_result: SeedExecutionResult, - counter: int, - total: int, - use_color: bool, - name_width: int, -) -> str: - counter_str: str = f"{counter}/{total}".rjust(len(str(total)) * 2 + 1) - status: str = _execution_status_to_display(seed_result.status) - style: CliStyle = CliStyle(use_color=use_color) - status_cell: str = format_status_cell(style=style, status=status) - duration: str = _format_duration(seed_result.duration_ms) - return format_aligned_name_value( - plain_name=seed_result.seed_name, - styled_name=seed_result.seed_name, - value=f"{status_cell} {style.muted(duration) if duration else ''}", - name_column_width=name_width, - prefix=f" {counter_str} {style.muted('seed ')} ", - ) - - -def _format_model_line( - *, - model_result: ModelExecutionResult, - resource_type: str, - annotation: str, - counter: int, - total: int, - use_color: bool, - name_width: int, -) -> str: - counter_str: str = f"{counter}/{total}".rjust(len(str(total)) * 2 + 1) - name_and_annotation: str = model_result.model_name - if annotation: - name_and_annotation = f"{model_result.model_name} ({annotation})" - - status: str = _execution_status_to_display(model_result.status) - style: CliStyle = CliStyle(use_color=use_color) - status_cell: str = format_status_cell(style=style, status=status) - duration: str = _format_duration(model_result.duration_ms) - detail: str = "" - if model_result.status == ExecutionStatus.FAILED and model_result.failed_phase is not None: - detail = f" {model_result.failed_phase}" - elif model_result.status == ExecutionStatus.SKIPPED: - duration = "" - detail = _model_skip_detail(model_result) - rendered_duration: str = style.muted(duration) if duration else "" - line: str = format_aligned_name_value( - plain_name=name_and_annotation, - styled_name=name_and_annotation, - value=f"{status_cell} {rendered_duration}{detail}", - name_column_width=name_width, - prefix=f" {counter_str} {style.muted(f'{resource_type:<6}')} ", - ) - return line - - -def _format_sql_block(sql: str) -> list[str]: - """Format a SQL block with minimal indent for verbose output.""" - - lines: list[str] = [""] - sql_line: str - for sql_line in _format_display_sql(sql).split("\n"): - lines.append(f" {sql_line}") - lines.append("") - return lines - - -def _format_display_sql(sql: str) -> str: - stripped: str = sql.rstrip() - if not stripped: - return sql - if stripped.endswith(";"): - return stripped - return f"{stripped};" - - -def _resolve_verbose_events( - *, model_result: ModelExecutionResult, plan_entry: ModelPlanEntry | None -) -> tuple[LifeCycleEvent, ...]: - if model_result.lifecycle_events: - return model_result.lifecycle_events - if plan_entry is not None: - return (LifeCycleEvent(kind=LifeCycleEventKind.SQL, content=plan_entry.logical_ddl),) - return () - - -def _format_log_block(*, message: str, use_color: bool) -> list[str]: - """Format a log message with indent for verbose output.""" - - line: str = f" log {message}" - style: CliStyle = CliStyle(use_color=use_color) - line = style.log_label(line) - return ["", line, ""] - - -def _format_batch_summary_line( - *, model_result: ModelExecutionResult, use_color: bool -) -> str | None: - """Return the muted microbatch summary line, if any.""" - - if model_result.batch_count is None: - return None - batch_label: str = "batch" if model_result.batch_count == 1 else "batches" - count_text: str = f"{model_result.batch_count} {batch_label}" - if model_result.batch_size is not None: - count_text = f"{count_text} ({model_result.batch_size})" - parts: list[str] = [count_text] - range_text: str | None = _format_batch_range(model_result=model_result) - if range_text is not None: - parts.append(f"range {range_text}") - if model_result.rows_affected is not None: - parts.append(_format_abbreviated_row_count(count=model_result.rows_affected)) - line: str = f" {' '.join(parts)}" - style: CliStyle = CliStyle(use_color=use_color) - return style.muted(line) - - -def _format_abbreviated_row_count(*, count: int) -> str: - """Format a row count with K/M/B abbreviation above 10,000.""" - - row_label: str = "row" if count == 1 else "rows" - _BILLION: int = 1_000_000_000 - _MILLION: int = 1_000_000 - _THOUSAND: int = 1_000 - _ABBREVIATION_THRESHOLD: int = 10_000 - if count >= _BILLION: - return f"{count / _BILLION:.1f}B {row_label}" - if count >= _MILLION: - return f"{count / _MILLION:.1f}M {row_label}" - if count >= _ABBREVIATION_THRESHOLD: - return f"{count / _THOUSAND:.1f}K {row_label}" - return f"{count:,} {row_label}" - - -def _format_batch_range(*, model_result: ModelExecutionResult) -> str | None: - """Return the inclusive cursor range for a microbatch model, if resolvable.""" - - if model_result.cursor_range_start is None or model_result.cursor_range_end is None: - return None - start: str = cursor_bound_display( - value=model_result.cursor_range_start, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - inclusive_end: str = inclusive_cursor_end( - end=model_result.cursor_range_end, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - return f"{start} \u2192 {inclusive_end}" - - -def _format_sub_line( - *, sub_type: str, name: str, status: str, use_color: bool, name_width: int -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - colored_status: str = style.status(status=status) - padding: str = " " * 10 - return format_aligned_name_value( - plain_name=name, - styled_name=name, - value=colored_status, - name_column_width=name_width, - prefix=f"{padding} {style.muted(f'{sub_type:<6}')} ", - ) - - -def _format_audit_sub_line(*, entry: _AuditDisplayEntry, use_color: bool, name_width: int) -> str: - status: str = _audit_outcome_to_display(entry.outcome) - detail: str = "" - if entry.outcome != AuditOutcome.PASS and entry.total_row_count > 0: - row_label: str = "row" if entry.total_row_count == 1 else "rows" - detail = f" {entry.total_row_count} {row_label}" - if entry.batch_total > 1: - detail = f" {entry.batch_pass}/{entry.batch_total}" + detail - return _format_sub_line( - sub_type=entry.label, - name=entry.display_name, - status=f"{status}{detail}", - use_color=use_color, - name_width=name_width, - ) - - -def _format_test_expectation_sub_line( - *, step_result: StepResult, use_color: bool, name_width: int -) -> str: - status: str = _test_outcome_to_status(step_result.outcome) - detail: str = "" - if step_result.outcome != SqlTestOutcome.PASS: - if step_result.model_name.startswith("assertion "): - row_label: str = "row" if step_result.actual_row_count == 1 else "rows" - detail = f" {step_result.actual_row_count} {row_label}" - else: - detail = f" {step_result.mismatched_row_count} mismatched" - style: CliStyle = CliStyle(use_color=use_color) - colored_status: str = style.status(status=status, text=f"{status}{detail}") - name: str = _format_test_expectation_name(step_result.model_name) - return format_aligned_name_value( - plain_name=name, - styled_name=name, - value=colored_status, - name_column_width=name_width, - prefix=f"{'':>14}{style.muted(f'{"expect":<6}')} ", - ) - - -def _format_test_expectation_name(model_name: str) -> str: - if model_name.startswith("assertion "): - return model_name - return f"expected {model_name}" - - -def _format_completion_message( - *, status: BuildStatus, warning_count: int, use_color: bool, summary: str | None = None -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - if status == BuildStatus.FAILED: - return format_completion_line( - style=style, state=CompletionState.FAIL, label="Completed with errors", summary=summary - ) - if warning_count > 0: - return format_completion_line( - style=style, - state=CompletionState.WARN, - label="Completed with warnings", - summary=summary, - ) - return format_completion_line( - style=style, state=CompletionState.OK, label="Completed successfully", summary=summary - ) - - -def _top_level_name_width(*, result: BuildExecutionResult, plan: PlanOutput) -> int: - model_entry_map: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} - names: list[str] = [seed_result.seed_name for seed_result in result.seed_results] - names.extend(function_result.function_name for function_result in result.function_results) - model_result: ModelExecutionResult - for model_result in result.model_results: - plan_entry: ModelPlanEntry | None = model_entry_map.get(model_result.model_name) - annotation: str = _resolve_annotation(plan_entry) - if annotation: - names.append(f"{model_result.model_name} ({annotation})") - else: - names.append(model_result.model_name) - return resolve_name_column_width(names=names) - - -def _sub_name_width(result: BuildExecutionResult) -> int: - names: list[str] = [] - test_result: SqlTestExecutionResult - for test_result in result.test_results: - names.append(test_result.test_name) - step_result: StepResult - for step_result in test_result.step_results: - names.append(_format_test_expectation_name(step_result.model_name)) - model_result: ModelExecutionResult - for model_result in result.model_results: - audit_entry: _AuditDisplayEntry - for audit_entry in _aggregate_audit_results(model_result.audit_results): - names.append(audit_entry.display_name) - return resolve_name_column_width(names=names) - - -def _format_summary_counts( - *, - result: BuildExecutionResult, - elapsed_seconds: float, -) -> str: - pass_count: int = 0 - warn_count: int = 0 - fail_count: int = 0 - skip_count: int = 0 - insufficient_count: int = 0 - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif model_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif model_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - audit_result: AuditExecutionResult - for audit_result in model_result.audit_results: - if audit_result.outcome == AuditOutcome.PASS: - pass_count += 1 - elif audit_result.outcome == AuditOutcome.WARN: - warn_count += 1 - elif audit_result.outcome == AuditOutcome.ERROR: - fail_count += 1 - elif audit_result.outcome == AuditOutcome.INSUFFICIENT: - insufficient_count += 1 - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif seed_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif seed_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif function_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif function_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - warn_count += len(function_result.warning_messages) - - test_result: SqlTestExecutionResult - for test_result in result.test_results: - if test_result.outcome == SqlTestOutcome.PASS: - pass_count += 1 - else: - fail_count += 1 - - end_audit: AuditExecutionResult - for end_audit in result.end_audit_results: - if end_audit.outcome == AuditOutcome.PASS: - pass_count += 1 - elif end_audit.outcome == AuditOutcome.WARN: - warn_count += 1 - elif end_audit.outcome == AuditOutcome.ERROR: - fail_count += 1 - elif end_audit.outcome == AuditOutcome.INSUFFICIENT: - insufficient_count += 1 - - source_audit: AuditExecutionResult - for source_audit in result.source_audit_results: - if source_audit.outcome == AuditOutcome.PASS: - pass_count += 1 - elif source_audit.outcome == AuditOutcome.WARN: - warn_count += 1 - elif source_audit.outcome == AuditOutcome.ERROR: - fail_count += 1 - elif source_audit.outcome == AuditOutcome.INSUFFICIENT: - insufficient_count += 1 - - total: int = pass_count + warn_count + fail_count + skip_count + insufficient_count - elapsed_str: str = f"{elapsed_seconds:.2f}s" if elapsed_seconds > 0 else "" - return format_summary_footer( - counts=( - ("PASS", pass_count), - ("WARN", warn_count), - ("FAIL", fail_count), - ("INSUFFICIENT", insufficient_count), - ("SKIP", skip_count), - ("TOTAL", total), - ), - use_color=False, - elapsed=elapsed_str or None, - ) - - -def _format_failure_details(*, result: BuildExecutionResult, use_color: bool) -> list[str]: - lines: list[str] = [] - has_failures: bool = False - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - lines.append(f" {seed_result.seed_name} (seed)") - if seed_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=seed_result.error_code, - error_message=seed_result.error_message, - error_help=seed_result.error_help, - use_color=use_color, - ) - ) - lines.append("") - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - lines.append(f" {function_result.function_name} ({function_result.function_kind})") - if function_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=function_result.error_code, - error_message=function_result.error_message, - error_help=function_result.error_help, - use_color=use_color, - ) - ) - lines.append("") - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - phase_str: str = f" ({model_result.failed_phase})" if model_result.failed_phase else "" - lines.append(f" {model_result.model_name}{phase_str}") - if model_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=model_result.error_code, - error_message=model_result.error_message, - error_help=model_result.error_help, - use_color=use_color, - ) - ) - if model_result.staging_relation is not None: - lines.append(f" {_inspection_relation_message(model_result.staging_relation)}") - lines.append("") - - test_result: SqlTestExecutionResult - for test_result in result.test_results: - if test_result.outcome == SqlTestOutcome.PASS: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - lines.append(f" {test_result.test_name} (test)") - if test_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=test_result.error_code, - error_message=test_result.error_message, - error_help=test_result.error_help, - use_color=use_color, - ) - ) - lines.append("") - - return lines - - -def _format_result_error_lines( - *, error_code: str | None, error_message: str, error_help: str | None, use_color: bool -) -> list[str]: - rendered_error: str = ( - error_message - if error_code is None - else format_coded_error( - code=error_code, - message=error_message, - help=error_help, - use_color=use_color, - ) - ) - return [f" {line}" for line in rendered_error.splitlines()] - - -def _format_warning_details(result: BuildExecutionResult) -> list[str]: - lines: list[str] = [] - has_warnings: bool = False - - model_result: ModelExecutionResult - for model_result in result.model_results: - model_warnings: list[str] = [] - audit_result: AuditExecutionResult - for audit_result in model_result.audit_results: - if audit_result.outcome == AuditOutcome.WARN: - name: str = audit_result.audit_name - if audit_result.attached_column_name: - name = f"{audit_result.audit_name} ({audit_result.attached_column_name})" - if audit_result.evaluation_mode == AuditEvaluationMode.MEASUREMENT: - model_warnings.append( - f" audit {name} measured {audit_result.measured_value}" - ) - else: - row_label: str = "row" if audit_result.row_count == 1 else "rows" - model_warnings.append( - f" audit {name} returned {audit_result.row_count} {row_label}" - ) - warning_msg: str - for warning_msg in model_result.warning_messages: - model_warnings.append(f" {warning_msg}") - if model_warnings: - if not has_warnings: - lines.append("Warnings:") - lines.append("") - has_warnings = True - lines.append(f" {model_result.model_name}") - lines.extend(model_warnings) - lines.append("") - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if not function_result.warning_messages: - continue - if not has_warnings: - lines.append("Warnings:") - lines.append("") - has_warnings = True - lines.append(f" {function_result.function_name} ({function_result.function_kind})") - warning_msg: str - for warning_msg in function_result.warning_messages: - lines.append(f" {warning_msg}") - lines.append("") - - return lines - - -def _inspection_relation_message(relation_name: str) -> str: - if relation_name.endswith("__delta"): - return f"delta table kept for inspection: {relation_name}" - return f"staging table kept for inspection: {relation_name}" - - -def _count_top_level_nodes(result: BuildExecutionResult) -> int: - return len(result.model_results) + len(result.seed_results) - - -def _build_test_results_by_model( - test_results: tuple[SqlTestExecutionResult, ...], -) -> dict[str, SqlTestExecutionResult]: - result_map: dict[str, SqlTestExecutionResult] = {} - test_result: SqlTestExecutionResult - for test_result in test_results: - step: object - for step in test_result.step_results: - if hasattr(step, "model_name"): - result_map[step.model_name] = test_result - return result_map - - -def _resolve_resource_type(plan_entry: ModelPlanEntry | None) -> str: - return model_resource_type(plan_entry) - - -def _resolve_annotation(plan_entry: ModelPlanEntry | None) -> str: - return model_execution_annotation(plan_entry) - - -def _format_duration(duration_ms: int | None) -> str: - if duration_ms is None: - return "" - seconds: float = duration_ms / 1000.0 - seconds_per_minute: int = 60 - if seconds < seconds_per_minute: - return f"{seconds:.2f}s" - minutes: int = int(seconds // 60) - remaining: float = seconds - minutes * 60 - return f"{minutes}m{remaining:.1f}s" - - -def _execution_status_to_display(status: ExecutionStatus) -> str: - if status == ExecutionStatus.SUCCESS: - return "OK" - if status == ExecutionStatus.FAILED: - return "FAIL" - if status == ExecutionStatus.SKIPPED: - return "SKIP" - return str(status) - - -def _model_skip_detail(result: ModelExecutionResult) -> str: - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _aggregate_audit_results( - audit_results: tuple[AuditExecutionResult, ...], -) -> list[_AuditDisplayEntry]: - """Aggregate per-batch delta audit results into single display entries.""" - - has_delta: bool = any(a.run_scope_phase == AuditRunScope.DELTA_AND_FINAL for a in audit_results) - - groups: dict[tuple[str, str | None, str], list[AuditExecutionResult]] = {} - audit: AuditExecutionResult - for audit in audit_results: - key: tuple[str, str | None, str] = ( - audit.audit_name, - audit.attached_column_name, - audit.run_scope_phase, - ) - groups.setdefault(key, []).append(audit) - - entries: list[_AuditDisplayEntry] = [] - results: list[AuditExecutionResult] - for (_name, _col, _phase), results in groups.items(): - display_name: str = _name - if _col is not None: - display_name = f"{_name} ({_col})" - worst: AuditOutcome = _worst_audit_outcome(results) - total_rows: int = sum(r.row_count for r in results) - pass_count: int = sum(1 for r in results if r.outcome == AuditOutcome.PASS) - label: str = _phase_label(phase=_phase, has_delta_audits=has_delta) - entries.append( - _AuditDisplayEntry( - label=label, - display_name=display_name, - outcome=worst, - total_row_count=total_rows, - batch_pass=pass_count, - batch_total=len(results), - executed_sql=results[0].executed_sql if results else None, - ) - ) - - return entries - - -def _worst_audit_outcome(results: list[AuditExecutionResult]) -> AuditOutcome: - """Return the worst outcome from a list of audit results.""" - - has_error: bool = any(r.outcome == AuditOutcome.ERROR for r in results) - if has_error: - return AuditOutcome.ERROR - has_warn: bool = any(r.outcome == AuditOutcome.WARN for r in results) - if has_warn: - return AuditOutcome.WARN - if any(r.outcome == AuditOutcome.INSUFFICIENT for r in results): - return AuditOutcome.INSUFFICIENT - return AuditOutcome.PASS - - -def _phase_label(*, phase: str, has_delta_audits: bool) -> str: - """Return the audit type label, annotated with phase when delta audits are present.""" - - if not has_delta_audits: - return "audit" - if phase == AuditRunScope.DELTA_AND_FINAL: - return "audit (d)" - return "audit (f)" - - -def _audit_outcome_to_display(outcome: AuditOutcome) -> str: - if outcome == AuditOutcome.PASS: - return "PASS" - if outcome == AuditOutcome.WARN: - return "WARN" - if outcome == AuditOutcome.ERROR: - return "FAIL" - if outcome == AuditOutcome.INSUFFICIENT: - return "INSUFFICIENT" - return str(outcome) - - -def _test_outcome_to_status(outcome: SqlTestOutcome) -> str: - if outcome == SqlTestOutcome.PASS: - return "PASS" - if outcome == SqlTestOutcome.FAIL: - return "FAIL" - if outcome == SqlTestOutcome.ERROR: - return "FAIL" - return str(outcome) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index f566f4adc2..51fd5091d0 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -11,7 +11,6 @@ from collections.abc import Callable, Mapping from concurrent.futures import FIRST_COMPLETED, CancelledError, Future, ThreadPoolExecutor, wait from contextvars import copy_context -from datetime import datetime from functools import partial from pathlib import Path from typing import Any, cast @@ -203,11 +202,6 @@ def __init__( loader_functions=customizations.loader_functions, ).loader_ref_entries ) - self._loader_is_reload: bool = runtime.loader_is_reload - self._start_cursor_ts: datetime | None = runtime.start_cursor_ts - self._end_cursor_ts: datetime | None = runtime.end_cursor_ts - self._start_cursor_int: int | None = runtime.start_cursor_int - self._end_cursor_int: int | None = runtime.end_cursor_int self._target: str = runtime.target self._effective_vars: dict[str, object] = runtime.effective_vars or {} self._warehouse_relations: dict[str, RelationInfo] = initial_state.warehouse_relations or {} diff --git a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py index fb089cd1d4..5276e7a13f 100644 --- a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py +++ b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py @@ -123,24 +123,6 @@ def write_prephase_header(*, stream: TextIO, title: str, use_color: bool) -> Non stream.flush() -def write_prephase_rows( - *, stream: TextIO, rows: tuple[PrephaseProgressRow, ...], use_color: bool -) -> None: - if not rows: - return - total: int = len(rows) - row: PrephaseProgressRow - for index, row in enumerate(rows, start=1): - write_prephase_row( - stream=stream, - row=row, - index=index, - total=total, - use_color=use_color, - ) - stream.flush() - - def write_prephase_row( *, stream: TextIO, row: PrephaseProgressRow, index: int, total: int, use_color: bool ) -> None: diff --git a/src/sqlbuild/executor/clone/types.py b/src/sqlbuild/executor/clone/types.py index 7f498ac274..9f30a97f63 100644 --- a/src/sqlbuild/executor/clone/types.py +++ b/src/sqlbuild/executor/clone/types.py @@ -23,12 +23,6 @@ class CloneFingerprintProgressReporter(CloneFingerprintProgressCallback, Protoco def write_interrupted(self) -> None: ... -class CloneStartCallback(Protocol): - def __call__( - self, *, origin_target_name: str, destination_target_name: str, total: int - ) -> None: ... - - class CloneStatus(StrEnum): SUCCESS = "success" WARNING = "warning" diff --git a/src/sqlbuild/executor/load/_helpers/rows.py b/src/sqlbuild/executor/load/_helpers/rows.py index 86dabf2db5..9df83270e5 100644 --- a/src/sqlbuild/executor/load/_helpers/rows.py +++ b/src/sqlbuild/executor/load/_helpers/rows.py @@ -14,12 +14,6 @@ from sqlbuild.spec.contracts.models import SourceColumnEntry -def normalize_loader_rows(value: object) -> tuple[dict[str, object], ...]: - """Validate and normalize a loader return value to dict rows.""" - - return tuple(iter_normalized_loader_rows(value)) - - def iter_loader_row_batches( *, value: object, batch_size: int ) -> Iterator[tuple[dict[str, object], ...]]: diff --git a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py index 64cf1ff845..9a1654c004 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py @@ -21,17 +21,6 @@ def build_ingress_lifecycle_nodes( ) -def build_read_side_python_lifecycle_nodes( - *, plan: PythonSqlRunLifecyclePlan, python_graph: PythonNodeGraph -) -> tuple[LifecycleExecutionNode, ...]: - """Build schedulable SQL-read Python nodes from a lifecycle plan.""" - - return _build_python_lifecycle_nodes( - selected_names=plan.read_side_python_node_names, - python_graph=python_graph, - ) - - def _build_python_lifecycle_nodes( *, selected_names: frozenset[str], python_graph: PythonNodeGraph ) -> tuple[LifecycleExecutionNode, ...]: diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index 85d423612f..825dcedefa 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -69,10 +69,6 @@ def __init__( def results(self) -> tuple[PythonNodeExecutionResult, ...]: return tuple(self._results_by_name.values()) - @property - def completed_python_names(self) -> frozenset[str]: - return frozenset(self._completed_python_names) - def finalize_unrun_python_nodes(self) -> tuple[PythonNodeExecutionResult, ...]: """Record selected Python nodes that could not run after SQL scheduling finished.""" diff --git a/src/sqlbuild/executor/python_nodes/main/_run.py b/src/sqlbuild/executor/python_nodes/main/_run.py deleted file mode 100644 index ffe28e6266..0000000000 --- a/src/sqlbuild/executor/python_nodes/main/_run.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public executor entrypoint for task/asset Python nodes.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.executor.python_nodes._helpers.execution import execute_python_nodes -from sqlbuild.executor.python_nodes.models import PythonNodeExecutorResult, PythonNodeRuntime -from sqlbuild.executor.python_nodes.types import ExecutablePythonNode - - -def run_python_nodes( - *, - nodes: tuple[ExecutablePythonNode, ...], - runtime: PythonNodeRuntime, - statement_recorder: StatementRecorder, -) -> PythonNodeExecutorResult: - """Execute task/asset Python nodes in dependency order.""" - - return execute_python_nodes( - nodes=nodes, - runtime=runtime, - statement_recorder=statement_recorder, - ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index f5e2e88e0d..0c221d3a3c 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -2748,12 +2748,6 @@ def _cursor_bound(*, values: list[str], cursor_type: str, maximum: bool) -> str: ) -def _cursor_result_string(value: object) -> str: - if isinstance(value, datetime | date): - return value.isoformat() - return str(value) - - def _reported_rows_affected(*, total_rows: int, row_count_known: bool) -> int | None: """Preserve a known zero row count while keeping unavailable counts as None.""" diff --git a/src/sqlbuild/executor/run/main/_promote.py b/src/sqlbuild/executor/run/main/_promote.py deleted file mode 100644 index 21170d79b0..0000000000 --- a/src/sqlbuild/executor/run/main/_promote.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public relation promotion entrypoint for run execution.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.executor.run._helpers.execution.promotion import promote_relation_to_destination - - -def promote_run_relation_to_destination( - *, - adapter: BaseAdapter, - connection: Any, - origin_relation: str, - destination_relation: str, - destination_database: str | None, - destination_schema: str | None, - destination_name: str, - statement_recorder: StatementRecorder, -) -> None: - """Promote an already-created run relation into its final destination.""" - - _ = promote_relation_to_destination( - adapter=adapter, - connection=connection, - origin_relation=origin_relation, - destination_relation=destination_relation, - destination_database=destination_database, - destination_schema=destination_schema, - destination_name=destination_name, - statement_recorder=statement_recorder, - ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index aaedf2c0f3..7507f93334 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -1,5 +1,5 @@ from collections.abc import Callable -from dataclasses import dataclass, field +from dataclasses import dataclass from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RetentionChangePhase @@ -10,11 +10,9 @@ RetentionDirection, RetentionPlanPhase, ) -from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ModelPlanOverride @dataclass(frozen=True) @@ -30,22 +28,6 @@ class LifecycleProgressTestCase: expected_event_types: tuple[str, ...] -@dataclass(frozen=True) -class BuildOutputTestCase: - """Test case for build output formatting.""" - - description: str - result: BuildExecutionResult - expected_output_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] = field(default_factory=tuple) - model_plan_overrides: tuple[ModelPlanOverride, ...] = field(default_factory=tuple) - target: str | None = None - concurrency: int = 1 - elapsed_seconds: float = 1.5 - verbose: bool = False - use_color: bool = False - - @dataclass(frozen=True) class BuildSchedulerSourceLoadTestCase: description: str @@ -151,28 +133,6 @@ class ExternalBuildSourceLoadTestCase: expected_lifecycle_message: str -@dataclass(frozen=True) -class AbbreviatedRowCountTestCase: - description: str - count: int - expected_output: str - - -@dataclass(frozen=True) -class BatchSummaryTestCase: - description: str - batch_count: int | None - rows_affected: int | None - cursor_range_start: str | None - cursor_range_end: str | None - cursor_type: str | None - cursor_grain: str | None - expected_fragments: tuple[str, ...] - batch_size: str | None = None - expected_absent_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_none: bool = False - - @dataclass(frozen=True) class MaterializationRecreatesRelationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index da4a54d744..fb4e0f42ae 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -2,8 +2,6 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import UTC, datetime from pathlib import Path from typing import Any from unittest.mock import Mock @@ -32,51 +30,11 @@ PlanAction, PlanReason, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.spec.contracts.models import SeedCsvSettings, SourceColumnEntry, SourceEntry from sqlbuild.spec.contracts.types import SourceWriteStrategy -@dataclass(frozen=True) -class ModelPlanOverride: - """Override for a model plan entry in output tests.""" - - name: str - materialization_type: MaterializationType = MaterializationType.TABLE - action: PlanAction = PlanAction.CREATE_TABLE - snapshot_strategy: str | None = None - observed_at_column: str | None = None - historical_input: str | None = None - - -def build_model_result_fields( - *, - name: str, - status: ExecutionStatus, - duration_ms: int = 100, - failed_phase: ExecutionPhase | None = None, - staging_relation: str | None = None, - promoted_relation: str | None = None, - error_message: str | None = None, - audit_results: tuple[AuditExecutionResult, ...] = (), - warning_messages: tuple[str, ...] = (), -) -> dict[str, object]: - return { - "model_name": name, - "status": status, - "duration_ms": duration_ms, - "failed_phase": failed_phase, - "staging_relation": staging_relation, - "promoted_relation": promoted_relation, - "error_message": error_message, - "audit_results": audit_results, - "warning_messages": warning_messages, - } - - def build_audit_result( *, name: str, @@ -135,26 +93,6 @@ def build_model_plan_entry( ) -def build_source_freshness_record( - *, - source_name: str, - data_hash: str, - data_version: str = "2026-06-30T12:00:00", -) -> SourceFreshnessRecord: - return SourceFreshnessRecord( - source_name=source_name, - target_database=None, - target_schema="main", - target_name=source_name, - run_id="run-1", - strategy="adapter", - value_kind="timestamp", - data_version=data_version, - data_version_hash=data_hash, - observed_at=datetime(2026, 6, 30, 12, 1, tzinfo=UTC), - ) - - def build_seed_plan_entry(*, name: str) -> SeedPlanEntry: return SeedPlanEntry( key=CompiledObjectKey(resource_type=CompiledResourceType.SEED, name=name), diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_batch_summary.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_batch_summary.py deleted file mode 100644 index 9f89537fb1..0000000000 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_batch_summary.py +++ /dev/null @@ -1,147 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.executor.build._helpers.output import ( - _format_abbreviated_row_count, - _format_batch_summary_line, -) -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( - AbbreviatedRowCountTestCase, - BatchSummaryTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - AbbreviatedRowCountTestCase( - description="exact count below abbreviation threshold", - count=438, - expected_output="438 rows", - ), - AbbreviatedRowCountTestCase( - description="single row uses singular label", - count=1, - expected_output="1 row", - ), - AbbreviatedRowCountTestCase( - description="count at threshold abbreviates to K", - count=10_000, - expected_output="10.0K rows", - ), - AbbreviatedRowCountTestCase( - description="millions abbreviate to M", - count=32_689_379, - expected_output="32.7M rows", - ), - AbbreviatedRowCountTestCase( - description="billions abbreviate to B", - count=1_847_293_612, - expected_output="1.8B rows", - ), - AbbreviatedRowCountTestCase( - description="just below threshold stays exact", - count=9_999, - expected_output="9,999 rows", - ), - ], - ids=lambda case: case.description, -) -def test_given_row_count_when_abbreviating_then_matches_expected_format( - test_case: AbbreviatedRowCountTestCase, -) -> None: - result: str = _format_abbreviated_row_count(count=test_case.count) - - assert result == test_case.expected_output - - -@pytest.mark.parametrize( - "test_case", - [ - BatchSummaryTestCase( - description="microbatch result includes batch count size range and rows", - batch_count=5, - batch_size="1mo", - rows_affected=32_689_379, - cursor_range_start="2014-01-01", - cursor_range_end="2015-01-01", - cursor_type="timestamp", - cursor_grain="day", - expected_fragments=( - "5 batches (1mo)", - "2014-01-01", - "2014-12-31", - "32.7M rows", - ), - ), - BatchSummaryTestCase( - description="microbatch result without rows omits row count", - batch_count=3, - rows_affected=None, - cursor_range_start="2014-01-01", - cursor_range_end="2014-04-01", - cursor_type="timestamp", - cursor_grain="day", - expected_fragments=("3 batches",), - expected_absent_fragments=("rows",), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_result_when_formatting_batch_summary_then_matches_contract( - test_case: BatchSummaryTestCase, -) -> None: - result: ModelExecutionResult = ModelExecutionResult( - model_name="test_model", - status=ExecutionStatus.SUCCESS, - batch_count=test_case.batch_count, - batch_size=test_case.batch_size, - rows_affected=test_case.rows_affected, - cursor_range_start=test_case.cursor_range_start, - cursor_range_end=test_case.cursor_range_end, - cursor_type=test_case.cursor_type, - cursor_grain=test_case.cursor_grain, - ) - - line: str | None = _format_batch_summary_line(model_result=result, use_color=False) - - assert line is not None - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in line - for fragment in test_case.expected_absent_fragments: - assert fragment not in line - - -@pytest.mark.parametrize( - "test_case", - [ - BatchSummaryTestCase( - description="non-microbatch result returns none", - batch_count=None, - rows_affected=None, - cursor_range_start=None, - cursor_range_end=None, - cursor_type=None, - cursor_grain=None, - expected_fragments=(), - expected_none=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_non_microbatch_result_when_formatting_batch_summary_then_returns_none( - test_case: BatchSummaryTestCase, -) -> None: - result: ModelExecutionResult = ModelExecutionResult( - model_name="test_model", - status=ExecutionStatus.SUCCESS, - batch_count=test_case.batch_count, - ) - - line: str | None = _format_batch_summary_line(model_result=result, use_color=False) - - assert (line is None) == test_case.expected_none diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_output.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_output.py deleted file mode 100644 index 787354b0b0..0000000000 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_output.py +++ /dev/null @@ -1,805 +0,0 @@ -"""Unit tests for build output formatting.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, -) -from sqlbuild.executor.build._helpers.output import format_build_output -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( - BuildOutputTestCase, - ModelPlanOverride, -) -from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ( - build_audit_result, - build_model_plan_entry, - build_plan_output, - build_seed_plan_entry, -) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildOutputTestCase( - description="microbatch model shows batch count cursor range and row count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="events", - status=ExecutionStatus.SUCCESS, - duration_ms=1000, - batch_count=5, - rows_affected=32689379, - cursor_range_start="2014-01-01", - cursor_range_end="2015-01-01", - cursor_type="timestamp", - cursor_grain="day", - ), - ), - success_count=1, - ), - expected_output_fragments=( - "events", - "5 batches", - "range 2014-01-01 \u2192 2014-12-31", - "32.7M rows", - ), - ), - BuildOutputTestCase( - description="non-microbatch model shows no batch summary line", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=150, - ), - ), - success_count=1, - ), - expected_output_fragments=("orders", "OK"), - expected_absent_fragments=("batches", "range"), - ), - BuildOutputTestCase( - description="success build shows OK status and completed successfully", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=150, - ), - ), - success_count=1, - ), - expected_output_fragments=( - "1/1", - "table", - "orders", - "OK", - "0.15s", - "\u2713 Completed successfully", - "PASS=1", - "FAIL=0", - ), - ), - BuildOutputTestCase( - description="snapshot model output shows strategy and historical shape annotation", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="customer_snapshot", - status=ExecutionStatus.SUCCESS, - duration_ms=150, - ), - ), - success_count=1, - ), - model_plan_overrides=( - ModelPlanOverride( - name="customer_snapshot", - materialization_type=MaterializationType.SNAPSHOT, - action=PlanAction.SNAPSHOT, - snapshot_strategy="check", - observed_at_column="observed_at", - historical_input="snapshot", - ), - ), - expected_output_fragments=( - "1/1", - "snapshot", - "customer_snapshot (check, historical snapshot)", - "OK", - "0.15s", - ), - expected_absent_fragments=("table",), - ), - BuildOutputTestCase( - description="failed model shows FAIL status with phase and failure detail", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.STAGING, - duration_ms=50, - error_code="R002", - error_message="Table raw_orders does not exist", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "1/1", - "FAIL", - "staging", - "\u2717 Completed with errors", - "FAIL=1", - "Failures:", - "orders (staging)", - "error[R002]: Table raw_orders does not exist", - "Table raw_orders does not exist", - ), - ), - BuildOutputTestCase( - description="colorized failed model shows styled coded error and help", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.PROMOTION, - error_code="R007", - error_help="inspect staging relation", - error_message="promotion failed", - ), - ), - failure_count=1, - ), - use_color=True, - expected_output_fragments=( - "\033[38;5;167m\033[1merror[R007]:\033[0m promotion failed", - "\033[2m= help:\033[0m inspect staging relation", - ), - ), - BuildOutputTestCase( - description="skipped model shows SKIP with no duration", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="upstream", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.STAGING, - duration_ms=10, - error_message="bad SQL", - ), - ModelExecutionResult( - model_name="downstream", - status=ExecutionStatus.SKIPPED, - ), - ), - failure_count=1, - skipped_count=1, - ), - expected_output_fragments=( - "1/2", - "upstream", - "FAIL", - "2/2", - "downstream", - "SKIP", - "SKIP=1", - ), - expected_absent_fragments=("downstream" + "0.00s",), - ), - BuildOutputTestCase( - description="warn audit shows WARN with row count in output and warnings section", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.WARN, - row_count=3, - column_name="email", - ), - ), - ), - ), - success_count=1, - warning_count=1, - ), - expected_output_fragments=( - "WARN", - "3 rows", - "\u2713 Completed with warnings", - "WARN=1", - "Warnings:", - "not_null (email) returned 3 rows", - ), - ), - BuildOutputTestCase( - description="passing audit shows PASS with no row count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - ), - ), - ), - ), - success_count=1, - ), - expected_output_fragments=( - "audit", - "not_null (id)", - "PASS", - ), - expected_absent_fragments=( - "Warnings:", - "Failures:", - ), - ), - BuildOutputTestCase( - description="seed line shows seed type and OK status", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - seed_results=( - SeedExecutionResult( - seed_name="country_codes", - status=ExecutionStatus.SUCCESS, - duration_ms=20, - ), - ), - success_count=1, - ), - expected_output_fragments=( - "1/1", - "seed", - "country_codes", - "OK", - "0.02s", - ), - ), - BuildOutputTestCase( - description="failed seed shows failure detail", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - seed_results=( - SeedExecutionResult( - seed_name="waffle_types", - status=ExecutionStatus.FAILED, - duration_ms=20, - error_message="failed to load seed CSV: invalid input syntax", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "seed", - "waffle_types", - "FAIL", - "Failures:", - "waffle_types (seed)", - "failed to load seed CSV: invalid input syntax", - ), - ), - BuildOutputTestCase( - description="failed function shows failure detail", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - function_results=( - FunctionExecutionResult( - function_name="is_completed_order", - status=ExecutionStatus.FAILED, - function_kind="udf", - error_message="function target could not be qualified", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "FAIL=1", - "Failures:", - "is_completed_order (udf)", - "function target could not be qualified", - ), - ), - BuildOutputTestCase( - description="function warning appears in summary and warnings section", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - function_results=( - FunctionExecutionResult( - function_name="is_completed_order_py", - status=ExecutionStatus.SUCCESS, - function_kind="udf", - warning_messages=("fingerprint write skipped",), - ), - ), - success_count=1, - warning_count=1, - ), - expected_output_fragments=( - "\u2713 Completed with warnings", - "PASS=1", - "WARN=1", - "Warnings:", - "is_completed_order_py (udf)", - "fingerprint write skipped", - ), - ), - BuildOutputTestCase( - description="view model shows view type", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="stg_orders", - status=ExecutionStatus.SUCCESS, - duration_ms=10, - ), - ), - success_count=1, - ), - model_plan_overrides=( - ModelPlanOverride( - name="stg_orders", - materialization_type=MaterializationType.VIEW, - action=PlanAction.CREATE_VIEW, - ), - ), - expected_output_fragments=( - "view", - "stg_orders", - "OK", - ), - ), - BuildOutputTestCase( - description="header includes target and concurrency", - result=BuildExecutionResult(status=BuildStatus.SUCCESS), - target="prod", - concurrency=4, - expected_output_fragments=( - "sqb build", - "target: prod", - "concurrency: 4", - ), - ), - BuildOutputTestCase( - description="counter counts seeds and models together", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - seed_results=( - SeedExecutionResult( - seed_name="seed_a", - status=ExecutionStatus.SUCCESS, - duration_ms=10, - ), - ), - model_results=( - ModelExecutionResult( - model_name="model_a", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ModelExecutionResult( - model_name="model_b", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - success_count=3, - ), - expected_output_fragments=( - "1/3", - "seed_a", - "2/3", - "model_a", - "3/3", - "model_b", - "TOTAL=3", - ), - ), - BuildOutputTestCase( - description="failed test shows in summary counts as FAIL", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SKIPPED, - ), - ), - test_results=( - SqlTestExecutionResult( - test_name="test_orders", - outcome=SqlTestOutcome.FAIL, - step_results=( - StepResult( - model_name="orders", - outcome=SqlTestOutcome.FAIL, - mismatched_row_count=1, - ), - ), - error_message="test 'test_orders' failed for models: orders", - ), - ), - failure_count=1, - skipped_count=1, - ), - expected_output_fragments=( - "SKIP", - "expect expected orders", - "FAIL 1 mismatched", - "\u2717 Completed with errors", - "FAIL=1", - "SKIP=1", - "Failures:", - "test_orders (test)", - ), - ), - BuildOutputTestCase( - description="unit test expectation rows show assertion names", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="fact_orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - test_results=( - SqlTestExecutionResult( - test_name="test_fact_orders", - outcome=SqlTestOutcome.PASS, - step_results=( - StepResult( - model_name="fact_orders", - outcome=SqlTestOutcome.PASS, - ), - StepResult( - model_name="assertion line_totals_are_non_negative", - outcome=SqlTestOutcome.PASS, - ), - ), - ), - ), - success_count=2, - ), - expected_output_fragments=( - "test test_fact_orders", - "expect expected fact_orders", - "expect assertion line_totals_are_non_negative PASS", - ), - ), - BuildOutputTestCase( - description="counter excludes tests from total count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - seed_results=( - SeedExecutionResult( - seed_name="seed_data", - status=ExecutionStatus.SUCCESS, - duration_ms=10, - ), - ), - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - test_results=( - SqlTestExecutionResult( - test_name="test_orders", - outcome=SqlTestOutcome.PASS, - step_results=( - StepResult( - model_name="orders", - outcome=SqlTestOutcome.PASS, - ), - ), - ), - ), - success_count=2, - ), - expected_output_fragments=( - "1/2", - "2/2", - "TOTAL=3", - ), - expected_absent_fragments=( - "3/2", - "1/3", - "2/3", - "3/3", - ), - ), - BuildOutputTestCase( - description="staging retained info shows in failure details", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.AUDIT, - duration_ms=200, - error_message=( - "final audit for 'orders' failed before replacing target table " - "with severity level: error" - ), - staging_relation="main.orders__staging", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "Failures:", - "orders (audit)", - "final audit for 'orders' failed before replacing target table", - "staging table kept for inspection: main.orders__staging", - ), - ), - BuildOutputTestCase( - description="elapsed time appears in summary", - result=BuildExecutionResult(status=BuildStatus.SUCCESS), - elapsed_seconds=2.34, - expected_output_fragments=("(2.34s)",), - ), - BuildOutputTestCase( - description="delta_and_final audits show phase labels and batch count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - ), - ), - success_count=1, - ), - expected_output_fragments=( - "audit (d)", - "audit (f)", - "not_null (id)", - "2/2", - ), - ), - BuildOutputTestCase( - description="final-only audits show plain audit label without batch count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - ), - ), - ), - ), - success_count=1, - ), - expected_output_fragments=( - "audit", - "not_null (id)", - "PASS", - ), - expected_absent_fragments=( - "audit (d)", - "audit (f)", - ), - ), - BuildOutputTestCase( - description="verbose mode shows executed lifecycle SQL and audit SQL", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - lifecycle_events=( - LifeCycleEvent( - kind=LifeCycleEventKind.SQL, - content="DROP TABLE IF EXISTS main.orders__staging", - ), - LifeCycleEvent( - kind=LifeCycleEventKind.LOG, - content="building partition 2024-01-01", - ), - LifeCycleEvent( - kind=LifeCycleEventKind.SQL, - content="CREATE OR REPLACE TABLE main.orders__staging AS SELECT 1", - ), - ), - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - ), - ), - ), - ), - success_count=1, - ), - verbose=True, - expected_output_fragments=( - "DROP TABLE IF EXISTS main.orders__staging", - "log building partition 2024-01-01", - "CREATE OR REPLACE TABLE main.orders__staging AS SELECT 1;", - "SELECT 1;", - "orders", - "OK", - ), - expected_absent_fragments=("CREATE TABLE main.orders AS SELECT 1",), - ), - BuildOutputTestCase( - description="non-verbose mode omits model DDL", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - success_count=1, - ), - verbose=False, - expected_output_fragments=( - "orders", - "OK", - ), - expected_absent_fragments=("CREATE TABLE",), - ), - BuildOutputTestCase( - description="colorized verbose mode shows lifecycle log messages dimmed", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - lifecycle_events=( - LifeCycleEvent( - kind=LifeCycleEventKind.LOG, - content="building partition 2024-01-01", - ), - ), - ), - ), - success_count=1, - ), - verbose=True, - use_color=True, - expected_output_fragments=("\033[2m log building partition 2024-01-01\033[0m",), - ), - ], - ids=lambda case: case.description, -) -def test_given_build_result_when_formatting_output_then_contains_expected_fragments( - test_case: BuildOutputTestCase, -) -> None: - override_map: dict[str, ModelPlanOverride] = {o.name: o for o in test_case.model_plan_overrides} - plan_entries: list[ModelPlanEntry] = [] - model_result: ModelExecutionResult - for model_result in test_case.result.model_results: - override: ModelPlanOverride | None = override_map.get(model_result.model_name) - mat_type: MaterializationType = getattr( - override, "materialization_type", MaterializationType.TABLE - ) - action: PlanAction = getattr(override, "action", PlanAction.CREATE_TABLE) - plan_entries.append( - build_model_plan_entry( - name=model_result.model_name, - materialization_type=mat_type, - action=action, - snapshot_strategy=getattr(override, "snapshot_strategy", None), - observed_at_column=getattr(override, "observed_at_column", None), - historical_input=getattr(override, "historical_input", None), - ) - ) - - seed_entries: tuple[SeedPlanEntry, ...] = tuple( - build_seed_plan_entry(name=sr.seed_name) for sr in test_case.result.seed_results - ) - - plan: object = build_plan_output( - model_entries=tuple(plan_entries), - seed_entries=seed_entries, - ) - - output: str = format_build_output( - result=test_case.result, - plan=plan, - target=test_case.target, - concurrency=test_case.concurrency, - elapsed_seconds=test_case.elapsed_seconds, - use_color=test_case.use_color, - verbose=test_case.verbose, - ) - - expected_fragment: str - for expected_fragment in test_case.expected_output_fragments: - assert expected_fragment in output, ( - f"Expected fragment {expected_fragment!r} not found in output:\n{output}" - ) - - absent_fragment: str - for absent_fragment in test_case.expected_absent_fragments: - assert absent_fragment not in output, ( - f"Fragment {absent_fragment!r} should not be in output:\n{output}" - ) diff --git a/tests/unit/src/sqlbuild/executor/load/test_rows.py b/tests/unit/src/sqlbuild/executor/load/test_rows.py index ee6f2c5b22..afcc0b9c14 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_rows.py +++ b/tests/unit/src/sqlbuild/executor/load/test_rows.py @@ -13,7 +13,7 @@ from sqlbuild.executor.load._helpers.rows import ( build_rows_sql, iter_loader_row_batches, - normalize_loader_rows, + iter_normalized_loader_rows, update_loader_rows_schema, ) from sqlbuild.executor.load.models import LoaderRowsSchema @@ -176,7 +176,7 @@ def test_given_reserved_loader_row_columns_when_executing_sql_then_quotes_identi def test_given_loader_return_value_when_normalizing_then_returns_expected_rows( test_case: LoaderRowsNormalizeTestCase, ) -> None: - assert normalize_loader_rows(test_case.value) == test_case.expected_rows + assert tuple(iter_normalized_loader_rows(test_case.value)) == test_case.expected_rows @pytest.mark.parametrize( @@ -209,7 +209,7 @@ def test_given_invalid_loader_return_value_when_normalizing_then_raises( test_case: LoaderRowsNormalizeErrorTestCase, ) -> None: with pytest.raises(ExecutorInputError) as exc_info: - normalize_loader_rows(test_case.value) + tuple(iter_normalized_loader_rows(test_case.value)) assert test_case.expected_error_fragment in str(exc_info.value) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py index 58155442f3..98819bdafa 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py @@ -5,10 +5,7 @@ import pytest from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan -from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import ( - build_ingress_lifecycle_nodes, - build_read_side_python_lifecycle_nodes, -) +from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import build_ingress_lifecycle_nodes from sqlbuild.executor.scheduling.models import LifecycleExecutionNode from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( PythonNodeLifecycleNodeBuildTestCase, @@ -64,49 +61,3 @@ def test_given_lifecycle_plan_when_building_ingress_nodes_then_returns_python_sc assert tuple(lifecycle_node_payload_name(node) for node in result) == ( test_case.expected_payload_names ) - - -@pytest.mark.parametrize( - "test_case", - [ - PythonNodeLifecycleNodeBuildTestCase( - description="builds read only asset scheduler node", - python_graph_case="orders", - selected_names=frozenset({"export_orders"}), - expected_names=("export_orders",), - expected_kinds=("asset",), - expected_upstream_names=((),), - expected_payload_names=("export_orders",), - ), - PythonNodeLifecycleNodeBuildTestCase( - description="builds read only task scheduler node", - python_graph_case="orders", - selected_names=frozenset({"prepare_orders"}), - expected_names=("prepare_orders",), - expected_kinds=("task",), - expected_upstream_names=((),), - expected_payload_names=("prepare_orders",), - ), - ], - ids=lambda case: case.description, -) -def test_given_lifecycle_plan_when_building_read_side_nodes_then_returns_python_scheduler_nodes( - test_case: PythonNodeLifecycleNodeBuildTestCase, -) -> None: - graph: PythonNodeGraph = python_graph_for_lifecycle_case(test_case.python_graph_case) - plan: PythonSqlRunLifecyclePlan = build_lifecycle_plan_for_selected_python_names( - graph=graph, - selected_names=test_case.selected_names, - ) - - result: tuple[LifecycleExecutionNode, ...] = build_read_side_python_lifecycle_nodes( - plan=plan, - python_graph=graph, - ) - - assert tuple(node.name for node in result) == test_case.expected_names - assert tuple(node.kind for node in result) == test_case.expected_kinds - assert tuple(node.upstream_names for node in result) == test_case.expected_upstream_names - assert tuple(lifecycle_node_payload_name(node) for node in result) == ( - test_case.expected_payload_names - ) From c4e35bdfb2996be42305e35d4967f39aaa1288cc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:25:39 +0100 Subject: [PATCH 11/32] refactor(virtual): remove unused code --- .../virtual/executor/_helpers/build.py | 1 - .../virtual/executor/_helpers/rollback.py | 18 ------ src/sqlbuild/virtual/executor/models.py | 1 - .../planner/_helpers/state_metadata.py | 43 +------------ .../state/_helpers/state_storage/locks.py | 23 ------- src/sqlbuild/virtual/state/constants.py | 2 - .../_checkpoint_physical_relations.py | 63 ------------------- .../virtual/state/_helpers/_test_types.py | 1 - .../virtual/state/_helpers/test_locks.py | 14 ----- 9 files changed, 1 insertion(+), 165 deletions(-) delete mode 100644 src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_physical_relations.py diff --git a/src/sqlbuild/virtual/executor/_helpers/build.py b/src/sqlbuild/virtual/executor/_helpers/build.py index cd893bc423..931a0e1242 100644 --- a/src/sqlbuild/virtual/executor/_helpers/build.py +++ b/src/sqlbuild/virtual/executor/_helpers/build.py @@ -374,7 +374,6 @@ def run_virtual_build( return VirtualBuildPipelineResult( project=rewritten.project, - direct_plan_output=plan.plan_output, display_plan_output=plan.plan_output, execution_plan=plan.executor_plan_output, execution_result=result, diff --git a/src/sqlbuild/virtual/executor/_helpers/rollback.py b/src/sqlbuild/virtual/executor/_helpers/rollback.py index 3c869f7a12..198331aa0c 100644 --- a/src/sqlbuild/virtual/executor/_helpers/rollback.py +++ b/src/sqlbuild/virtual/executor/_helpers/rollback.py @@ -168,24 +168,6 @@ def guard_partial_rollback_scope( return tuple(sorted({*selected_model_names, *stale_upstream_set})), final_version_hashes -def read_model_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, Any]: - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - } - - def read_physical_relations( *, backend: Any, diff --git a/src/sqlbuild/virtual/executor/models.py b/src/sqlbuild/virtual/executor/models.py index 39dc9240a9..9f88f4e69a 100644 --- a/src/sqlbuild/virtual/executor/models.py +++ b/src/sqlbuild/virtual/executor/models.py @@ -152,7 +152,6 @@ class VirtualBuildPipelineResult: """Result returned by the virtual build pipeline.""" project: CompiledProject - direct_plan_output: PlanOutput display_plan_output: PlanOutput execution_plan: PlanOutput execution_result: BuildExecutionResult diff --git a/src/sqlbuild/virtual/planner/_helpers/state_metadata.py b/src/sqlbuild/virtual/planner/_helpers/state_metadata.py index 42bd023651..c88945b3a8 100644 --- a/src/sqlbuild/virtual/planner/_helpers/state_metadata.py +++ b/src/sqlbuild/virtual/planner/_helpers/state_metadata.py @@ -2,11 +2,8 @@ from __future__ import annotations -from typing import Any - -from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.virtual.state.main.encoding._decode_state_text import decode_state_text -from sqlbuild.virtual.state.models import FunctionVersionRecord, ModelVersionRecord +from sqlbuild.virtual.state.models import ModelVersionRecord def decode_model_version_query_sqls( @@ -39,41 +36,3 @@ def decode_model_version_metadata_jsons( continue result[model_name] = metadata_json return result - - -def read_previous_function_query_sqls( - *, - backend: Any, - state_connection: Any, - schema: str, - graph: ProjectGraph, - virtual_environment_name: str, -) -> dict[str, str]: - """Read persisted virtual function fingerprint SQL by function name.""" - - function_refs: dict[str, str] = { - ref.function_name: ref.version_hash - for ref in backend.get_virtual_environment_function_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - } - function_versions: dict[str, FunctionVersionRecord | None] = { - function.name: backend.get_function_version( - connection=state_connection, - schema=schema, - function_name=function.name, - version_hash=function_refs[function.name], - ) - for function in graph.project.functions - if function.name in function_refs - } - result: dict[str, str] = {} - for function_name, function_version in function_versions.items(): - if function_version is None: - continue - query_sql: str | None = decode_state_text(function_version.definition_text_b64) - if query_sql is not None: - result[function_name] = query_sql - return result diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py b/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py index 994a954e95..4eab0e47fd 100644 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py +++ b/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py @@ -6,7 +6,6 @@ from typing import Any from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.constants import STATE_MIGRATION_LOCK_KEY from sqlbuild.virtual.state.models import StateLockLease @@ -69,28 +68,6 @@ def acquire_model_version_lock( ) -def acquire_state_migration_lock( - *, - backend: StateBackend, - connection: Any, - schema: str, - owner_id: str, - ttl: timedelta, - now: datetime | None = None, -) -> StateLockLease | None: - """Acquire the lock for state schema lifecycle operations.""" - - return acquire_state_lock( - backend=backend, - connection=connection, - schema=schema, - lock_key=STATE_MIGRATION_LOCK_KEY, - owner_id=owner_id, - ttl=ttl, - now=now, - ) - - def acquire_state_lock( *, backend: StateBackend, diff --git a/src/sqlbuild/virtual/state/constants.py b/src/sqlbuild/virtual/state/constants.py index f62be969f3..309451c7fb 100644 --- a/src/sqlbuild/virtual/state/constants.py +++ b/src/sqlbuild/virtual/state/constants.py @@ -33,7 +33,6 @@ ) VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE: str = "virtual_environment_checkpoint_seed_refs" LOCK_TABLE: str = "locks" -STATE_MIGRATION_LOCK_KEY: str = "state_migration" STATE_OPERATION_TABLE: str = "state_operations" PLAN_RUN_TABLE: str = "plan_runs" VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_TABLE: str = "virtual_environment_model_ref_events" @@ -243,7 +242,6 @@ "observed_at": StateColumnType.TIMESTAMP, "updated_at": StateColumnType.TIMESTAMP, } -SOURCE_FRESHNESS_NAIVE_UTC_WALL_CLOCK_COLUMNS: frozenset[str] = frozenset({"observed_at"}) VIRTUAL_ENVIRONMENT_CHECKPOINT_COLUMNS: dict[str, StateColumnType] = { "checkpoint_id": StateColumnType.TEXT, diff --git a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_physical_relations.py b/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_physical_relations.py deleted file mode 100644 index 295e9dec47..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_physical_relations.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Public helper for physical relations referenced by retained checkpoints.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, -) - - -def list_checkpoint_physical_relations( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str, -) -> tuple[PhysicalRelationRecord, ...]: - """List physical relations referenced by retained checkpoints for a VDE.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - backend.list_virtual_environment_checkpoints( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ) - ) - relations: dict[tuple[str | None, str | None, str], PhysicalRelationRecord] = {} - checkpoint: VirtualEnvironmentCheckpointRecord - for checkpoint in checkpoints: - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_model_refs( - connection=connection, - schema=config.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - ref: VirtualEnvironmentCheckpointModelRefRecord - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=connection, - schema=config.schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is None: - continue - relations[ - (relation.database_name, relation.schema_name, relation.relation_name) - ] = relation - return tuple(relations[key] for key in sorted(relations)) - finally: - backend.close(connection) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py index 4cdc1f1a89..68dcd3a0f0 100644 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py @@ -45,7 +45,6 @@ class StateLockServiceTestCase: ttl: timedelta expected_virtual_environment_lock_key: str expected_model_version_lock_key: str - expected_state_migration_lock_key: str @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py index df8660c332..b4ecf1ed0f 100644 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py +++ b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py @@ -6,7 +6,6 @@ from sqlbuild.virtual.state._helpers.state_storage.locks import ( acquire_model_version_lock, - acquire_state_migration_lock, acquire_virtual_environment_lock, release_state_lock, ) @@ -28,7 +27,6 @@ ttl=timedelta(minutes=15), expected_virtual_environment_lock_key="virtual_env:dev", expected_model_version_lock_key="model_version:fact_orders:abc123", - expected_state_migration_lock_key="state_migration", ) ], ids=lambda case: case.description, @@ -58,26 +56,15 @@ def test_given_state_lock_service_when_acquiring_scoped_locks_then_uses_expected ttl=test_case.ttl, now=test_case.now, ) - state_migration_lease: StateLockLease | None = acquire_state_migration_lock( - backend=backend, - connection=connection, - schema=test_case.schema, - owner_id=test_case.owner_id, - ttl=test_case.ttl, - now=test_case.now, - ) assert virtual_environment_lease is not None assert model_version_lease is not None - assert state_migration_lease is not None assert virtual_environment_lease.lock_key == test_case.expected_virtual_environment_lock_key assert model_version_lease.lock_key == test_case.expected_model_version_lock_key - assert state_migration_lease.lock_key == test_case.expected_state_migration_lock_key assert virtual_environment_lease.expires_at == test_case.now + test_case.ttl assert tuple(call[0] for call in backend.acquire_calls) == ( test_case.expected_virtual_environment_lock_key, test_case.expected_model_version_lock_key, - test_case.expected_state_migration_lock_key, ) assert release_state_lock( backend=backend, @@ -101,7 +88,6 @@ def test_given_state_lock_service_when_acquiring_scoped_locks_then_uses_expected ttl=timedelta(minutes=15), expected_virtual_environment_lock_key="virtual_env:dev", expected_model_version_lock_key="model_version:fact_orders:abc123", - expected_state_migration_lock_key="state_migration", ) ], ids=lambda case: case.description, From 04ab4a4aeb2fa6c07b64247e0717a70ceaa05fc9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:26:18 +0100 Subject: [PATCH 12/32] refactor(runtime): remove unused code --- src/sqlbuild/runtime/event_exporting/classes/dispatcher.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py b/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py index 28b877e083..077045087d 100644 --- a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py +++ b/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py @@ -155,9 +155,6 @@ def thread(self) -> threading.Thread: def notification_thread(self) -> threading.Thread | None: return self._notification_thread - @property - def health_thread(self) -> threading.Thread | None: - return self._health_thread def enqueue(self, event: LifecycleEvent) -> None: """Filter and enqueue eligible exporter attempts without waiting.""" From e5684fc6e33848b72000df1f1e92288434fb084b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:27:39 +0100 Subject: [PATCH 13/32] refactor(integrations): remove unused code --- .../integrations/dagster/constants.py | 1 - src/sqlbuild/integrations/dagster/types.py | 3 +-- .../dbt/_helpers/manifest/compile_refs.py | 20 ------------------- src/sqlbuild/integrations/dbt/constants.py | 15 -------------- .../main/manifest/_build_manifest_index.py | 14 ------------- .../manifest/_resolve_compile_model_refs.py | 17 ---------------- .../main/manifest/_resolve_manifest_model.py | 20 ------------------- src/sqlbuild/integrations/dbt/models.py | 9 --------- src/sqlbuild/integrations/dlt/constants.py | 13 ------------ .../integrations/ingestr/_helpers/output.py | 12 ----------- src/sqlbuild/integrations/rivers/types.py | 3 +-- .../event_exporting/classes/dispatcher.py | 1 - .../sqlbuild/integrations/dlt/_test_types.py | 8 -------- .../integrations/dlt/test_destination.py | 19 ------------------ 14 files changed, 2 insertions(+), 153 deletions(-) delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_build_manifest_index.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_resolve_compile_model_refs.py delete mode 100644 src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_model.py diff --git a/src/sqlbuild/integrations/dagster/constants.py b/src/sqlbuild/integrations/dagster/constants.py index f8562e2c62..03b8ad1f02 100644 --- a/src/sqlbuild/integrations/dagster/constants.py +++ b/src/sqlbuild/integrations/dagster/constants.py @@ -52,7 +52,6 @@ EVENT_OUTPUT_FLAG: str = "--event-output" LIVE_EVENT_COMMANDS: frozenset[str] = ASSET_SELECTION_COMMANDS VIRTUAL_ENV_FLAG: str = "--virtual-env" -VERBOSE_FLAGS: frozenset[str] = frozenset({"--verbose", "-v"}) EXPLICIT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "-s", "--select-file"}) JSON_OUTPUT_FLAGS: frozenset[str] = frozenset({"--json", "--json-output"}) JSON_OUTPUT_FLAG: str = "--json-output" diff --git a/src/sqlbuild/integrations/dagster/types.py b/src/sqlbuild/integrations/dagster/types.py index 0109e5c3c1..e370d93436 100644 --- a/src/sqlbuild/integrations/dagster/types.py +++ b/src/sqlbuild/integrations/dagster/types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -10,4 +10,3 @@ type DagNode = Mapping[str, Any] type DagCheck = Mapping[str, Any] type DagEdge = Mapping[str, Any] -type AssetKeyParts = Sequence[str] diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py index ebcb96f68a..f01974bac9 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py +++ b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py @@ -85,23 +85,3 @@ def validate_compile_dbt_model_reference( package_name=reference.ref_package, name=reference.ref_name, ) - - -def resolve_compile_dbt_ref_references( - *, query_sql: str, dbt_manifest: DbtManifestIndex | None -) -> str: - """Replace model __dbt_ref() calls with dbt manifest relation names.""" - - if dbt_manifest is None: - return query_sql - - def _replace_dbt_ref(match: re.Match[str]) -> str: - first_arg: str = match.group(1) - second_arg: str | None = match.group(2) - return resolve_dbt_manifest_model( - manifest=dbt_manifest, - package_name=first_arg if second_arg is not None else None, - name=second_arg if second_arg is not None else first_arg, - ).relation_name - - return _DBT_REF_PATTERN.sub(_replace_dbt_ref, query_sql) diff --git a/src/sqlbuild/integrations/dbt/constants.py b/src/sqlbuild/integrations/dbt/constants.py index b96485b59b..c7f0a71beb 100644 --- a/src/sqlbuild/integrations/dbt/constants.py +++ b/src/sqlbuild/integrations/dbt/constants.py @@ -10,24 +10,13 @@ ) ) DBT_EXECUTION_DISPLAY_FLAGS: frozenset[str] = frozenset(("--json", "--verbose", "-v")) -DBT_SUCCESSFUL_RESULT_STATUSES: frozenset[str] = frozenset({"ok", "success", "pass", "passed"}) DBT_MANIFEST_CONFIG_KEY: str = "config" -DBT_MANIFEST_INCREMENTAL_STRATEGY_KEY: str = "incremental_strategy" DBT_MANIFEST_MATERIALIZED_KEY: str = "materialized" -DBT_MANIFEST_META_KEY: str = "meta" -DBT_MANIFEST_RESOURCE_TYPE_KEY: str = "resource_type" -DBT_MANIFEST_SQLBUILD_META_KEY: str = "sqlbuild" DBT_EXECUTABLE_ENV_VAR: str = "DBT_EXECUTABLE" DEFAULT_DBT_EXECUTABLE: str = "dbt" -DBT_MATERIALIZATION_EPHEMERAL: str = "ephemeral" -DBT_MATERIALIZATION_INCREMENTAL: str = "incremental" -DBT_MATERIALIZATION_MICROBATCH: str = "microbatch" -DBT_MATERIALIZATION_SNAPSHOT: str = "snapshot" -DBT_MATERIALIZATION_TABLE: str = "table" -DBT_MATERIALIZATION_VIEW: str = "view" DBT_DEFER_FLAG: str = "--defer" DBT_EXECUTION_FAIL_STATUSES: frozenset[str] = frozenset({"error", "fail", "failed"}) @@ -36,11 +25,7 @@ {"ok", "success", "pass", "passed", "warn", "warning"} ) DBT_EXECUTION_WARN_STATUSES: frozenset[str] = frozenset({"warn", "warning"}) -DBT_EXCLUDE_FLAG: str = "--exclude" DBT_FULL_REFRESH_FLAG: str = "--full-refresh" DBT_PATH_SELECTOR_SEPARATOR: str = "~" -DBT_SELECT_FLAGS: frozenset[str] = frozenset({"--select", "-s"}) DBT_SELECT_FLAG: str = "--select" -DBT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "--exclude"}) DBT_UNIQUE_ID_SEPARATOR: str = "." -DBT_VERBOSE_FLAGS: frozenset[str] = frozenset({"--verbose", "-v"}) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_build_manifest_index.py b/src/sqlbuild/integrations/dbt/main/manifest/_build_manifest_index.py deleted file mode 100644 index 425e86a0c5..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_build_manifest_index.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public dbt manifest indexing entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - build_dbt_manifest_index as _build_dbt_manifest_index, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def build_manifest_index(*, raw_data: object) -> DbtManifestIndex: - """Build a dbt manifest index from decoded manifest JSON.""" - - return _build_dbt_manifest_index(raw_data=raw_data) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_compile_model_refs.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_compile_model_refs.py deleted file mode 100644 index 9a69f07c63..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_compile_model_refs.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public dbt compile model ref resolution entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - resolve_compile_dbt_ref_references, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def resolve_compile_model_refs(*, query_sql: str, dbt_manifest: DbtManifestIndex | None) -> str: - """Replace SQLBuild model __dbt_ref() calls with dbt relation names.""" - - return resolve_compile_dbt_ref_references( - query_sql=query_sql, - dbt_manifest=dbt_manifest, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_model.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_model.py deleted file mode 100644 index 7c2c62e227..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_model.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public dbt manifest model resolution entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - resolve_dbt_manifest_model as _resolve_dbt_manifest_model, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex, DbtManifestModel - - -def resolve_manifest_model( - *, manifest: DbtManifestIndex, name: str, package_name: str | None = None -) -> DbtManifestModel: - """Resolve a dbt model from a manifest index.""" - - return _resolve_dbt_manifest_model( - manifest=manifest, - name=name, - package_name=package_name, - ) diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 2b80317664..336184a5fe 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -103,15 +103,6 @@ class DbtCliConfigOverrides: target: str | None = None target_path: str | None = None - @property - def has_any(self) -> bool: - """Return whether any dbt CLI config flag was supplied.""" - - return any( - value is not None - for value in (self.project_dir, self.profiles_dir, self.target, self.target_path) - ) - @dataclass(frozen=True) class ResolvedDbtConfig: diff --git a/src/sqlbuild/integrations/dlt/constants.py b/src/sqlbuild/integrations/dlt/constants.py index 8bc0ab8300..bd492e9b06 100644 --- a/src/sqlbuild/integrations/dlt/constants.py +++ b/src/sqlbuild/integrations/dlt/constants.py @@ -2,19 +2,6 @@ from __future__ import annotations -from sqlbuild.adapter.contract.types import BuiltinAdapter - -DLT_DESTINATION_ADAPTERS: frozenset[BuiltinAdapter] = frozenset( - { - BuiltinAdapter.DUCKDB, - BuiltinAdapter.MOTHERDUCK, - BuiltinAdapter.SNOWFLAKE, - BuiltinAdapter.BIGQUERY, - BuiltinAdapter.DATABRICKS, - BuiltinAdapter.POSTGRES, - BuiltinAdapter.SQLSERVER, - } -) DLT_INTEGRATION_KIND: str = "dlt" DLT_SOURCE_TYPE_SQL_DATABASE: str = "sql_database" DLT_SOURCE_TYPE_REST_API: str = "rest_api" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/output.py b/src/sqlbuild/integrations/ingestr/_helpers/output.py index 47f0d2f8d2..a9eed714b8 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/output.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/output.py @@ -3,7 +3,6 @@ from __future__ import annotations from shlex import quote -from typing import TextIO from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.ingestr.models import IngestrCommandResult @@ -15,17 +14,6 @@ def format_ingestr_command(command: tuple[str, ...]) -> str: return " ".join(quote(part) for part in command) -def write_external_output(*, stream: TextIO, label: str, output: str) -> None: - """Write a labeled external command output block.""" - - stream.write(f"{label}\n") - stream.write(output) - if not output.endswith("\n"): - stream.write("\n") - stream.write("\n") - stream.flush() - - def record_ingestr_output(*, ctx: LoaderContext, result: IngestrCommandResult) -> None: """Attach captured ingestr output to the loader lifecycle.""" diff --git a/src/sqlbuild/integrations/rivers/types.py b/src/sqlbuild/integrations/rivers/types.py index 7ea63a5c99..4344d98cb1 100644 --- a/src/sqlbuild/integrations/rivers/types.py +++ b/src/sqlbuild/integrations/rivers/types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -10,4 +10,3 @@ type DagNode = Mapping[str, Any] type DagCheck = Mapping[str, Any] type DagEdge = Mapping[str, Any] -type AssetKeyParts = Sequence[str] diff --git a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py b/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py index 077045087d..034bb04a19 100644 --- a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py +++ b/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py @@ -155,7 +155,6 @@ def thread(self) -> threading.Thread: def notification_thread(self) -> threading.Thread | None: return self._notification_thread - def enqueue(self, event: LifecycleEvent) -> None: """Filter and enqueue eligible exporter attempts without waiting.""" diff --git a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py index 51d6bc66ed..ee1d73ea38 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py @@ -4,8 +4,6 @@ from dataclasses import dataclass, field -from sqlbuild.adapter.contract.types import BuiltinAdapter - @dataclass(frozen=True) class DltLoaderDiscoveryTestCase: @@ -38,12 +36,6 @@ class DltDestinationErrorTestCase: destination_config: dict[str, object] | None = None -@dataclass(frozen=True) -class DltDestinationCoverageTestCase: - description: str - expected_adapters: frozenset[BuiltinAdapter] - - @dataclass(frozen=True) class DltProgressCollectorTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py b/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py index 96b4e1a935..37f92ee27f 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py @@ -4,13 +4,10 @@ import pytest -from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.integrations.dlt._helpers.destination import build_dlt_destination -from sqlbuild.integrations.dlt.constants import DLT_DESTINATION_ADAPTERS from sqlbuild.integrations.dlt.exceptions import DltIntegrationError from sqlbuild.integrations.dlt.models import DltDestinationConfig from tests.unit.src.sqlbuild.integrations.dlt._test_types import ( - DltDestinationCoverageTestCase, DltDestinationErrorTestCase, DltDestinationTestCase, ) @@ -172,19 +169,3 @@ def test_given_real_warehouse_without_schema_when_building_dlt_destination_then_ ) assert test_case.expected_error_fragment in str(exc_info.value) - - -@pytest.mark.parametrize( - "test_case", - [ - DltDestinationCoverageTestCase( - description="covers all builtin adapters", - expected_adapters=frozenset(BuiltinAdapter), - ) - ], - ids=lambda case: case.description, -) -def test_given_builtin_adapters_when_checking_dlt_destination_support_then_all_are_explicit( - test_case: DltDestinationCoverageTestCase, -) -> None: - assert DLT_DESTINATION_ADAPTERS == test_case.expected_adapters From 62c183500d4a7eb974b09a13f473824740484195 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:29:02 +0100 Subject: [PATCH 14/32] refactor(adapters): remove unused code --- .../databricks/classes/databricks_adapter.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 1b67dc4f65..0150d073ce 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -672,33 +672,6 @@ def _get_delta_history_freshness_metadata( ) return results - def _get_table_type( - self, - *, - connection: Any, - database: str, - schema: str, - name: str, - ) -> str: - query: str = ( - f"SELECT table_type FROM {self._information_schema(database)}.tables " - f"WHERE table_schema = {self._string_literal(schema)} " - f"AND table_name = {self._string_literal(name)}" - ) - cursor: Any = connection.cursor() - try: - cursor.execute(query) - row: tuple[Any, ...] | None = cursor.fetchone() - finally: - cursor.close() - if row is None: - raise AdapterUserError( - message=( - f"Databricks table freshness metadata not found for {database}.{schema}.{name}" - ) - ) - return str(row[0]) - def maximum_identifier_length(self) -> int: """Return the maximum unqualified identifier length supported by the adapter.""" From e79d9673ae56aa7a470b93e8125b239ffb30f8fd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:30:00 +0100 Subject: [PATCH 15/32] refactor(core): remove unused code --- src/sqlbuild/cost/constants.py | 1 - src/sqlbuild/cursor_algebra/constants.py | 10 ---------- src/sqlbuild/cursor_algebra/types.py | 2 -- src/sqlbuild/diagnostics/_helpers/logging.py | 8 -------- .../diagnostics/_helpers/logging_formatters.py | 6 ------ src/sqlbuild/presentation/constants.py | 1 - src/sqlbuild/spec/contracts/types.py | 2 -- src/sqlbuild/sql_values/types.py | 1 - 8 files changed, 31 deletions(-) diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py index 5edf101319..c426667edb 100644 --- a/src/sqlbuild/cost/constants.py +++ b/src/sqlbuild/cost/constants.py @@ -15,6 +15,5 @@ NO_WAREHOUSE_COMPUTE_REASON: str = "no_warehouse_compute" MISSING_WAREHOUSE_METADATA_REASON: str = "missing_warehouse_metadata" INCOMPLETE_HISTORY_REASON: str = "incomplete_history" -OUTSIDE_RUN_WINDOW_REASON: str = "outside_run_window" LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX: str = "Statement-ledger persistence failed (" COST_TELEMETRY_HEALTH: CostTelemetryHealth = CostTelemetryHealth() diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index aa59274c65..cd243f2ef6 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -38,16 +38,6 @@ DURATION_MINUTE_UNIT: str = "m" DURATION_SECOND_UNIT: str = "s" -DURATION_UNITS: frozenset[str] = frozenset( - { - DURATION_YEAR_UNIT, - DURATION_MONTH_UNIT, - DURATION_DAY_UNIT, - DURATION_HOUR_UNIT, - DURATION_MINUTE_UNIT, - DURATION_SECOND_UNIT, - } -) MINUTE_TO_DAY_DURATION_UNITS: frozenset[str] = frozenset( {DURATION_MINUTE_UNIT, DURATION_HOUR_UNIT, DURATION_DAY_UNIT} ) diff --git a/src/sqlbuild/cursor_algebra/types.py b/src/sqlbuild/cursor_algebra/types.py index 8f6f680ce7..6d86a5f165 100644 --- a/src/sqlbuild/cursor_algebra/types.py +++ b/src/sqlbuild/cursor_algebra/types.py @@ -2,10 +2,8 @@ from enum import StrEnum -from sqlbuild.compiler.planner.types import CursorGrain from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -type Grain = CursorGrain type CursorScalar = TimestampValue | DateValue | IntegerValue diff --git a/src/sqlbuild/diagnostics/_helpers/logging.py b/src/sqlbuild/diagnostics/_helpers/logging.py index 27407a4341..a6ae2cab03 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging.py +++ b/src/sqlbuild/diagnostics/_helpers/logging.py @@ -34,14 +34,6 @@ def _current_identity_context() -> dict[str, object]: return {f"sqlbuild_{key}": value for key, value in execution_identity_to_dict(identity).items()} -def get_diagnostics_logger(name: str | None = None) -> logging.Logger: - """Return a SQLBuild diagnostics logger.""" - - if name is None: - return logging.getLogger(_LOGGER_ROOT_NAME) - return logging.getLogger(f"{_LOGGER_ROOT_NAME}.{name}") - - def log_sql( *, logger: logging.Logger, diff --git a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py index f155a69aa1..39a986cf75 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py +++ b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py @@ -11,12 +11,6 @@ _SQL_EVENT_FIELD: str = "sqlbuild_sql" -def _format_prefix(*, record: logging.LogRecord, date_text: str) -> str: - return ( - f"{date_text}.{int(record.msecs):03d} {record.levelname} {_short_logger_name(record.name)}" - ) - - def _short_logger_name(name: str) -> str: prefix: str = f"{_LOGGER_ROOT_NAME}." if name.startswith(prefix): diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py index 1ba51e8557..c79c8b6d41 100644 --- a/src/sqlbuild/presentation/constants.py +++ b/src/sqlbuild/presentation/constants.py @@ -11,5 +11,4 @@ SUCCESS_STATUS_LABELS: frozenset[str] = frozenset({"OK", "PASS", "SUCCESS"}) TREE_BRANCH_GLYPH: str = "\u251c\u2500\u2500" TREE_LAST_GLYPH: str = "\u2514\u2500\u2500" -TREE_PIPE_GLYPH: str = "\u2502" WARNING_STATUS_LABELS: frozenset[str] = frozenset({"WARN", "WARNING"}) diff --git a/src/sqlbuild/spec/contracts/types.py b/src/sqlbuild/spec/contracts/types.py index 9c174147ec..7f822ee5a0 100644 --- a/src/sqlbuild/spec/contracts/types.py +++ b/src/sqlbuild/spec/contracts/types.py @@ -4,8 +4,6 @@ from enum import StrEnum -type PathDefaultsMap = dict[str, dict[str, object]] - class EventExportSeverity(StrEnum): """Severity used to filter exported lifecycle events.""" diff --git a/src/sqlbuild/sql_values/types.py b/src/sqlbuild/sql_values/types.py index 834f74fd91..8636af335a 100644 --- a/src/sqlbuild/sql_values/types.py +++ b/src/sqlbuild/sql_values/types.py @@ -41,4 +41,3 @@ class StateSqlValueType(StrEnum): type SqlScalar = str | int | bool | float | Decimal | None type SqlValuePayload = SqlScalar | tuple[object, ...] -type SqlCollectionRendering = CollectionRendering From a3b8770173b5476db843fd4670fc1dd700b7dd34 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:32:11 +0100 Subject: [PATCH 16/32] refactor(tests): remove orphaned test case types --- .../cli/commands/main/build/_test_types.py | 8 -- .../cli/commands/main/plan/_test_types.py | 82 ------------------- .../cli/commands/main/postgres/_test_types.py | 9 -- .../adapters/snowflake/_test_types.py | 8 -- .../executor/run/incremental/_test_types.py | 13 --- .../cli/commands/_helpers/_test_types.py | 7 -- .../compiler/compile/_helpers/_test_types.py | 27 ------ .../main/shared/_helpers/_test_types.py | 8 -- .../compiler/planner/_helpers/_test_types.py | 43 ---------- .../planner/_helpers/changes/_test_types.py | 8 -- .../compiler/planner/main/_test_types.py | 10 --- .../executor/run/_helpers/_test_types.py | 58 ------------- .../sqlbuild/integrations/dbt/_test_types.py | 44 ---------- .../runtime/event_exporting/_test_types.py | 22 ----- .../virtual/executor/_helpers/_test_types.py | 31 ------- 15 files changed, 378 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 6766f3f217..03d490842f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -918,14 +918,6 @@ class SchemaBackfillBuildE2ETestCase: expected_warning_entries: tuple[tuple[str, str], ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DirectReuseFromBuildE2ETestCase: - description: str - project_name: str - expected_prod_build_exit_code: int - expected_dev_build_exit_code: int - - @dataclass(frozen=True) class MixedTimestampGrainBuildE2ETestCase: """Test case for mixed timestamp grain replay behavior.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index faa1412494..60d4fa93bd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -1,88 +1,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class DirectPlanE2ETestCase: - description: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class DirectPlanJsonE2ETestCase: - description: str - expected_selected_count: int - expected_model_count: int - expected_function_count: int - - -@dataclass(frozen=True) -class DirectChangesOnlyJsonE2ETestCase: - description: str - expected_reasons_by_name: dict[str, str] - expected_identity_status_by_name: dict[str, str] - - -@dataclass(frozen=True) -class DirectPlanIdentityJsonE2ETestCase: - description: str - project_name: str - expected_identity_status: str - expected_built_version_present: bool - - -@dataclass(frozen=True) -class DirectChangesOnlyReplayE2ETestCase: - description: str - policy_fragment: str - expected_backfill_action: str - expected_backfill_duration: str | None - - -@dataclass(frozen=True) -class DirectChangesOnlySchemaReplayE2ETestCase: - description: str - expected_reason: str - expected_backfill_action: str - expected_backfill_duration: str | None - - -@dataclass(frozen=True) -class DirectChangesOnlyFunctionReplayE2ETestCase: - description: str - expected_reason: str - expected_backfill_action: str - expected_backfill_duration: str | None - - -@dataclass(frozen=True) -class DirectChangesOnlySelectorE2ETestCase: - description: str - selector: str - expected_selected_count: int - expected_model_names: tuple[str, ...] - unexpected_model_names: tuple[str, ...] - expected_remaining_stale_names: tuple[str, ...] = () - - -@dataclass(frozen=True) -class DirectFunctionIdentityE2ETestCase: - description: str - expected_initial_count: int - expected_changed_count: int - expected_model_name: str - - -@dataclass(frozen=True) -class DirectFunctionSelectorE2ETestCase: - description: str - selector: str - expected_plan_fragment: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] - expected_remaining_stale_names: tuple[str, ...] = () - - @dataclass(frozen=True) class DirectModeVirtualFlagGuardE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index 91164c51e3..07fa4f5cc9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -66,15 +66,6 @@ class PostgresLoaderWaffleShopE2ETestCase: expected_return_code: int = 0 -@dataclass(frozen=True) -class PostgresDiffE2ETestCase: - description: str - mutation_sql: tuple[str, ...] - command: tuple[str, ...] - expected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_return_code: int = 0 - - @dataclass(frozen=True) class PostgresSnapshotE2ETestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py index f4250ed9a1..869fc34d1d 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py @@ -116,14 +116,6 @@ class SnowflakeTableTypeConversionTestCase: expected_conversion_statement_count: int -@dataclass(frozen=True) -class SnowflakeE2ETestCase: - description: str - command: tuple[str, ...] - expected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_return_code: int = 0 - - @dataclass(frozen=True) class SnowflakeCostTestCase: description: str diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py index 4f4b950d50..baef1aa160 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py @@ -95,16 +95,3 @@ class IncrementalFailureTestCase: future_cursor_config: FutureCursorsConfig | None = None invocation_time: datetime | None = None expected_has_future_cursor_safety: bool = False - - -@dataclass(frozen=True) -class IncrementalSeedReuseFailureTestCase: - description: str - origin_sql: str - input_sql: str - model_sql: str - fingerprint_version_hash: str - expected_status: ExecutionStatus - expected_failed_phase: ExecutionPhase - expected_error_fragments: tuple[str, ...] - expected_target_exists: bool diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py index 5b825cb43c..a70998a9a2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py @@ -11,10 +11,3 @@ class SourceFreshnessAppendEligibilityTestCase: model_statuses: dict[str, ExecutionStatus] expected_insert_count: int expected_lifecycle_order: tuple[str, ...] - - -@dataclass(frozen=True) -class DbtSqlbuildWorkOutputTestCase: - description: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index d5f642b92d..27631ff05d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -18,7 +18,6 @@ ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import UsageRecord -from sqlbuild.compiler.scopes.types import ScopeKind from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.sql_values.models import SqlValue @@ -188,23 +187,6 @@ class ExpandSqlMacrosErrorTestCase: macro_overrides: dict[str, str] = field(default_factory=dict) -@dataclass(frozen=True) -class ScopedMacroExpansionTestCase: - description: str - definitions: dict[str, tuple[str, ScopeKind, str | None, str]] - expected_sql: str - expected_dependencies: tuple[str, ...] - expected_usages: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True) -class ScopedMacroExpansionErrorTestCase: - description: str - definitions: dict[str, tuple[str, ScopeKind, str | None, str]] - sql: str - expected_error_fragment: str - - @dataclass(frozen=True) class FindMacroCallNamesTestCase: description: str @@ -691,15 +673,6 @@ class ExpandTemplateDataErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class VarMacroCollisionTestCase: - description: str - var_names: tuple[str, ...] - macro_names: tuple[str, ...] - expected_valid: bool - expected_error_fragment: str | None = None - - @dataclass(frozen=True) class NonIncrementalConfigValidTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py index dc3feeb51c..599e836f63 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py @@ -20,14 +20,6 @@ class BuildCreateTableSqlTestCase: transient: bool = False -@dataclass(frozen=True) -class BuildReadAllSqlTestCase: - description: str - database: str | None - schema: str - expected_contains: tuple[str, ...] - - @dataclass(frozen=True) class BuildInsertSqlTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index bd20f33618..1e8b04cec4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -21,7 +21,6 @@ MissingUpstream, ParsedSelector, PathSelector, - PlannerScope, ScenarioArtifactIdentity, ScenarioGraphPlan, ScenarioRelationMap, @@ -184,34 +183,6 @@ class SeedIdentityCsvConfigTestCase: expected_same_identity: bool -@dataclass(frozen=True) -class MarkVersionIdentityStaleActionsTestCase: - description: str - model_key: CompiledObjectKey - change_kind: ChangeKind - previous_version_hash: str - expected_version_hash: str - expected_cascade_present: bool - - -@dataclass(frozen=True) -class DirectParentRunActionTestCase: - description: str - parent_key: CompiledObjectKey - child_key: CompiledObjectKey - expected_cascade_present: bool - expected_root_cause: str | None = None - expected_root_reason: PlanReason | None = None - - -@dataclass(frozen=True) -class DirectIdentityStaleModelNamesTestCase: - description: str - expected_version_hashes: dict[str, str] - built_version_hashes: dict[str, str | None] - expected_stale_model_names: frozenset[str] - - @dataclass(frozen=True) class SelectionStalenessWarningTestCase: description: str @@ -440,13 +411,6 @@ class FindPathKeysErrorTestCase: expected_error_type: type[Exception] -@dataclass(frozen=True) -class PruneUnchangedScopeTestCase: - description: str - scope: PlannerScope - expected_selected_keys: frozenset[CompiledObjectKey] - - @dataclass(frozen=True) class ParseSelectorTestCase: description: str @@ -962,13 +926,6 @@ class RunDespiteUnchangedPlanningTestCase: expected_error_fragment: str | None = None -@dataclass(frozen=True) -class ReuseSatisfiedStalenessTestCase: - description: str - reuse_satisfied_model_names: frozenset[str] - expected_warns: bool - - @dataclass(frozen=True) class PlannerStaleWarningScopeTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index b38cfcd1f9..5031448f40 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -65,11 +65,3 @@ class ResolveBackfillTestCase: description: str raw_value: str | None expected_result: BackfillResult - - -@dataclass(frozen=True) -class ResolveSchemaBackfillTestCase: - description: str - replay_on_change: dict[str, str] - findings: tuple[SchemaFinding, ...] - expected_result: BackfillResult diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index 25ddd6b3a7..c439e4c0e4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -87,16 +87,6 @@ class ExternalBlockedPlanOutputTestCase: expected_model_names: tuple[str, ...] -@dataclass(frozen=True) -class DirectInputBaselineTestCase: - description: str - models_by_name: dict[str, str] - origin_model_names: tuple[str, ...] - selected_model_name: str - expected_baseline_names: tuple[str, ...] - unexpected_baseline_names: tuple[str, ...] - - @dataclass(frozen=True) class SelectorExpansionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 6abdb62087..2bdc500950 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -1,5 +1,4 @@ from dataclasses import dataclass, field -from datetime import datetime from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent from sqlbuild.executor.run.types import AuditGateReuseReason, AuditGateStatus, ExecutionPhase @@ -168,26 +167,6 @@ class ReportedRowsAffectedTestCase: expected_rows_affected: int | None -@dataclass(frozen=True) -class MicrobatchCursorDiscoveryTestCase: - description: str - warehouse_column_type: str - cursor_min: object - cursor_max: object - cursor_type: str - expected_start: str - expected_end: str - - -@dataclass(frozen=True) -class MicrobatchCursorDiscoveryFailureTestCase: - description: str - warehouse_column_type: str - cursor_min: object - cursor_max: object - expected_error_fragment: str - - @dataclass(frozen=True) class RuntimeTargetMaxTestCase: description: str @@ -332,43 +311,6 @@ class PythonHookInvocationTestCase: expected_recorded_events: tuple[str, ...] -@dataclass(frozen=True) -class PermanentRequirementTestCase: - description: str - source_created_at: datetime - expected_operation_kind: str - expected_retention_days: int - - -@dataclass(frozen=True) -class PermanentPromotionTestCase: - description: str - initial_state: str - operation_identity: str - expected_timeline: tuple[str, ...] - expected_completion_time: datetime | None - - -@dataclass(frozen=True) -class PermanentArchiveConflictTestCase: - description: str - archive_generation_offset_seconds: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class PermanentPersistedConflictTestCase: - description: str - expected_error_fragment: str - - -@dataclass(frozen=True) -class PermanentIdentifierFitTestCase: - description: str - identifier_limit: int - expected_prefix: str - - @dataclass(frozen=True) class PythonHookSkipTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py index 7ed8c637d8..7fa1804900 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py @@ -3,12 +3,10 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt.models import ( DbtCliOptions, DbtCommandResult, DbtInteropParsedArgs, - DbtLsNode, ) from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig @@ -100,17 +98,6 @@ class DbtSelectionErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class DbtExecutionSpacingTestCase: - description: str - expected_spacing_fragment: str - expected_no_work_spacing_fragment: str - unexpected_no_blank_fragment: str - unexpected_no_work_no_blank_fragment: str - unexpected_extra_blank_fragment: str - unexpected_no_work_extra_blank_fragment: str - - @dataclass(frozen=True) class DbtCompileFullRefreshPipelineTestCase: description: str @@ -211,22 +198,6 @@ class DbtExecutionSelectionStatusTestCase: expected_completion_fragment: str -@dataclass(frozen=True) -class DbtRunResultsFallbackRenderTestCase: - description: str - unique_id: str - status: str - message: str - expected_output_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class DbtExecutionTotalRenderTestCase: - description: str - expected_output_fragments: tuple[str, ...] - unexpected_output_fragments: tuple[str, ...] - - @dataclass(frozen=True) class DbtExecutionSummaryFooterTestCase: description: str @@ -321,21 +292,6 @@ class DbtPlanTestCase: expected_json_fragments: tuple[str, ...] -@dataclass(frozen=True) -class DbtPlanHumanFormatterTestCase: - description: str - dbt_ls_nodes: tuple[DbtLsNode, ...] - sqlbuild_model_names: tuple[str, ...] - sqlbuild_plan_model_names: tuple[str, ...] - display_limit: int | None - use_color: bool - expected_human_fragments: tuple[str, ...] - expected_human_regex_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] - sqlbuild_command_argvs: tuple[tuple[str, ...], ...] = () - sqlbuild_plan_output: PlanOutput | None = None - - @dataclass(frozen=True) class DbtPlanOrchestrationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py index 08dde06482..3210501fc8 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py @@ -1,34 +1,12 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class EventExporterDecoratorTestCase: - description: str - expected_name: str - - -@dataclass(frozen=True) -class InvalidEventExporterNameTestCase: - description: str - name: str - expected_error: str - - @dataclass(frozen=True) class SinkApiTestCase: description: str expected_name: str | None = None -@dataclass(frozen=True) -class EventExporterFilterTestCase: - description: str - event_kinds: object - min_severity: str - expected_error: str | None = None - expected_kinds: frozenset[str] | None = None - - @dataclass(frozen=True) class EventExportPolicyTestCase: description: str diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py index 14626d3566..d501162b99 100644 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py @@ -1,7 +1,5 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction - @dataclass(frozen=True) class HydrationLifecycleTestCase: @@ -44,35 +42,6 @@ class RewriteProjectTargetsTestCase: expected_rewritten_name: str -@dataclass(frozen=True) -class SeededPlanAdaptationTestCase: - description: str - incremental_strategy: str - expected_action: PlanAction - expected_sql_fragment: str - - -@dataclass(frozen=True) -class SeededPlanNoAdaptationTestCase: - description: str - incremental_strategy: str - bound_version_hash: str | None - expected_version_hash: str - materialization_type: MaterializationType - action: PlanAction - cursor_bounds_enabled: bool - expected_action: PlanAction - unexpected_sql_fragment: str - - -@dataclass(frozen=True) -class SeededPlanBoundsPrecedenceTestCase: - description: str - entry_bounds_enabled: bool - expected_sql_fragment: str - unexpected_sql_fragment: str - - @dataclass(frozen=True) class SeedingStrategyTestCase: description: str From 94955d0670002d545f2581811d6126d982155760 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:34:14 +0100 Subject: [PATCH 17/32] refactor(compiler): remove no-op cursor alias deprecation shim --- .../compile/_helpers/config/deprecation.py | 13 ---- .../compile/main/_build_compile_inputs.py | 6 +- .../compiler/compile/_helpers/_test_types.py | 7 -- .../_helpers/test_cursor_alias_deprecation.py | 68 ------------------- 4 files changed, 1 insertion(+), 93 deletions(-) delete mode 100644 src/sqlbuild/compiler/compile/_helpers/config/deprecation.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_alias_deprecation.py diff --git a/src/sqlbuild/compiler/compile/_helpers/config/deprecation.py b/src/sqlbuild/compiler/compile/_helpers/config/deprecation.py deleted file mode 100644 index 1b078c6bd1..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/deprecation.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Build compile diagnostics for deprecated model configuration.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic - - -def build_cursor_alias_diagnostics( - *, model_inputs: tuple[CompileModelInput, ...] -) -> tuple[CompilerDiagnostic, ...]: - """Build one project-level deprecation warning for the legacy cursor alias.""" - - return () diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index c70da831c5..4ec1ba2245 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -31,7 +31,6 @@ from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, ) -from sqlbuild.compiler.compile._helpers.config.deprecation import build_cursor_alias_diagnostics from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id from sqlbuild.compiler.compile._helpers.render.declarations import ( build_public_declaration_indexes, @@ -309,8 +308,5 @@ def _build_models_with_declarations( public_enums=declarations.enums, public_constants=declarations.constants, ), - diagnostics=( - *build_cursor_alias_diagnostics(model_inputs=model_inputs), - *build_audit_factory_orphan_diagnostics(discovered_inputs=discovered_inputs), - ), + diagnostics=build_audit_factory_orphan_diagnostics(discovered_inputs=discovered_inputs), ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 27631ff05d..59d679e50e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -607,13 +607,6 @@ class IncrementalConfigErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class CursorAliasWarningTestCase: - description: str - config_values: tuple[dict[str, object], ...] - expected_warning_count: int - - @dataclass(frozen=True) class ContractConfigValidTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_alias_deprecation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_alias_deprecation.py deleted file mode 100644 index 32f2e17f8b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_alias_deprecation.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.compile._helpers.config.deprecation import ( - build_cursor_alias_diagnostics, -) -from sqlbuild.compiler.compile.models import ( - CompileModelConfig, - CompileModelInput, - CompilerDiagnostic, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - CursorAliasWarningTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - CursorAliasWarningTestCase( - description="cursor inputs emit no legacy alias warning", - config_values=( - {"cursor_inputs": {"orders": "event_time"}}, - {"cursor_inputs": {"shipments": "event_time"}}, - ), - expected_warning_count=0, - ), - CursorAliasWarningTestCase( - description="removed cursor role names emit no deprecation warning", - config_values=( - {"cursor_filter_inputs": {"orders": "event_time"}}, - {"cursor_watermark_inputs": {"raw_orders": "loaded_at"}}, - ), - expected_warning_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_project_cursor_roles_when_building_diagnostics_then_warning_cardinality_is_exact( - test_case: CursorAliasWarningTestCase, -) -> None: - model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( - file_path=Path("models/orders.sql"), - relative_path=Path("models/orders.sql"), - contents="SELECT 1", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="SELECT 1", - ) - model_inputs: tuple[CompileModelInput, ...] = tuple( - CompileModelInput(model_file=model_file, config=CompileModelConfig(values=values)) - for values in test_case.config_values - ) - - diagnostics: tuple[CompilerDiagnostic, ...] = build_cursor_alias_diagnostics( - model_inputs=model_inputs - ) - - assert len(diagnostics) == test_case.expected_warning_count - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) From c53633b07ca9ad5aea8a4c7da3c4e1a23a60af51 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 03:35:43 +0100 Subject: [PATCH 18/32] refactor(compiler): remove unused legacy hook key copies --- src/sqlbuild/compiler/compile/_helpers/attachment/audits.py | 1 - src/sqlbuild/compiler/compile/_helpers/attachment/functions.py | 1 - src/sqlbuild/compiler/compile/_helpers/attachment/references.py | 1 - src/sqlbuild/compiler/compile/_helpers/attachment/sources.py | 1 - 4 files changed, 4 deletions(-) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 339fb45626..2bf965757d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -93,7 +93,6 @@ def remember_scoped_declarations( self.scoped_declarations[key] = declarations -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index cfb32eba0e..a48b3c9dfb 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -80,7 +80,6 @@ class _PythonFunctionBuildContext: _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py index 1e76b40573..6a1da58202 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py @@ -20,7 +20,6 @@ from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py index 3383ed4e17..55ca9f3889 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py @@ -39,7 +39,6 @@ ) _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} From 9222661f3e3b303f04fd0623aa80943931846c25 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:36:02 +0100 Subject: [PATCH 19/32] refactor(adapters): remove unused adapter contract methods Drop render_query_with_cursor_bounds, render_seed_select_after_cursor, get_relation_max_cursor and count_rows from BaseAdapter, StrictAdapter, DiffMixin, DuckDbBackedAdapter and every first-class adapter, plus PromotionStrategy.CREATE_NEW. No engine code called them. Tests that used count_rows for verification use a small COUNT(*) helper; the unsupported promotion strategy test now uses the catalogued but unhandled build_aside strategy. --- .../adapter/contract/classes/base_adapter.py | 103 ---------- src/sqlbuild/adapter/contract/classes/diff.py | 13 -- .../contract/classes/duckdb_backed_adapter.py | 70 ------- .../contract/classes/strict_adapter.py | 36 ---- src/sqlbuild/adapter/contract/types.py | 1 - .../bigquery/classes/bigquery_adapter.py | 72 ------- .../databricks/classes/databricks_adapter.py | 74 ------- .../postgres/classes/postgres_adapter.py | 68 ------- .../snowflake/classes/snowflake_adapter.py | 72 ------- .../sqlserver/classes/sqlserver_adapter.py | 69 ------- .../sqlbuild/adapters/duckdb/_test_types.py | 11 - .../src/sqlbuild/adapters/duckdb/helpers.py | 9 + .../sqlbuild/adapters/duckdb/test_client.py | 68 +------ .../sqlbuild/adapters/postgres/_test_types.py | 11 - .../sqlbuild/adapters/postgres/test_client.py | 45 ---- .../classes/base_adapter/_test_types.py | 10 - .../classes/base_adapter/test_base_adapter.py | 39 ---- .../unit/src/sqlbuild/adapters/_test_types.py | 23 --- .../sqlbuild/adapters/bigquery/_test_types.py | 7 - .../src/sqlbuild/adapters/bigquery/helpers.py | 27 --- .../sqlbuild/adapters/bigquery/test_client.py | 39 ---- .../sqlbuild/adapters/duckdb/_test_types.py | 7 - .../sqlbuild/adapters/duckdb/test_client.py | 41 ---- tests/unit/src/sqlbuild/adapters/helpers.py | 88 -------- .../adapters/test_relation_max_cursor.py | 192 ------------------ .../sqlbuild/executor/janitor/main/helpers.py | 22 -- .../run/_helpers/test_lifecycle_progress.py | 4 +- 27 files changed, 22 insertions(+), 1199 deletions(-) delete mode 100644 tests/unit/src/sqlbuild/adapters/helpers.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index db9a9d663b..0abd77803d 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -301,24 +301,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: return () return tuple(str(column[0]) for column in description) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -702,23 +684,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -734,45 +699,9 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) - def _render_query_with_cursor_bounds_impl( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - quoted_cursor: str = self.render_identifier(cursor_column) - start_literal: str = self.render_cursor_bound_literal( - value=cursor_start, cursor_type=cursor_type - ) - end_literal: str = self.render_cursor_bound_literal( - value=cursor_end, cursor_type=cursor_type - ) - return ( - f"SELECT * FROM ({sql}) AS __sqlbuild_cursor_bounded " - f"WHERE {quoted_cursor} >= {start_literal} AND {quoted_cursor} < {end_literal}" - ) - def _render_seed_select_before_cursor_impl( self, *, @@ -787,20 +716,6 @@ def _render_seed_select_before_cursor_impl( ) return f"SELECT * FROM {origin} WHERE {quoted_cursor} < {end_literal}" - def _render_seed_select_after_cursor_impl( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - quoted_cursor: str = self.render_identifier(cursor_column) - start_literal: str = self.render_cursor_bound_literal( - value=cursor_start_exclusive, cursor_type=cursor_type - ) - return f"SELECT * FROM {origin} WHERE {quoted_cursor} > {start_literal}" - def _relation_names_match_impl(self, *, left: str, right: str) -> bool: return left.replace('"', "") == right.replace('"', "") @@ -1445,24 +1360,6 @@ def diff_rows( ) -> RowDiffResult: raise AdapterUserError(message="diff_rows requires an engine-specific implementation") - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - where_clause: str = "" - if cursor_column and start_cursor: - where_clause = f" WHERE {cursor_column} >= '{start_cursor.value}'" - if end_cursor: - where_clause += f" AND {cursor_column} < '{end_cursor.value}'" - cursor: Any = connection.execute(f"SELECT COUNT(*) FROM {relation}{where_clause}") - result: Any = cursor.fetchone() - return int(result[0]) - def _inspect_row_diff_coverage( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/diff.py b/src/sqlbuild/adapter/contract/classes/diff.py index 852a3d438c..ed0fa770cc 100644 --- a/src/sqlbuild/adapter/contract/classes/diff.py +++ b/src/sqlbuild/adapter/contract/classes/diff.py @@ -62,19 +62,6 @@ def diff_unkeyed_rows( """Compare exact full-row multiplicities without a unique key.""" ... - @abstractmethod - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - """Return the row count for a relation, optionally bounded by cursor.""" - ... - @abstractmethod def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 889dd0ff1c..623614c702 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -923,21 +923,6 @@ def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> return f"TIMESTAMP '{value}'" return f"'{value}'" - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def connect(self, config: dict[str, Any]) -> Any: """Open a DuckDB connection from the resolved connection config.""" @@ -1006,24 +991,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: cursor: Any = self.execute(connection=connection, sql=f"DESCRIBE {sql}") return tuple(str(row[0]) for row in cursor.fetchall()) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -1524,23 +1491,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -2041,26 +1991,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: Any = self.execute(connection=connection, sql=query).fetchone() - return int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index c436c727c0..7105100d70 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -158,17 +158,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: """Return column names produced by a SQL query.""" ... - @abstractmethod - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - ... - @abstractmethod def render_max_cursor_at_or_before( self, @@ -341,19 +330,6 @@ def render_durable_clone( """Render SQL statements that clone/copy into a durable independent destination.""" ... - @abstractmethod - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - """Render a query wrapped with adapter-correct cursor bounds.""" - ... - @abstractmethod def render_seed_select_before_cursor( self, @@ -366,18 +342,6 @@ def render_seed_select_before_cursor( """Render a seed-select query that keeps rows before a cursor bound.""" ... - @abstractmethod - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - """Render a seed-select query that keeps rows after a cursor bound.""" - ... - @abstractmethod def relation_names_match(self, *, left: str, right: str) -> bool: """Return whether two relation name strings identify the same adapter relation.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 56b0f4de32..421ae860ec 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -33,7 +33,6 @@ class CursorKind(StrEnum): class PromotionStrategy(StrEnum): ATOMIC_SWAP = "atomic_swap" ATOMIC_REPLACE = "atomic_replace" - CREATE_NEW = "create_new" class TablePromotionMode(StrEnum): diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 7715a4fbde..f51606ef52 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1712,23 +1712,6 @@ def render_durable_clone( f"CLONE {self._quote_identifier_path(origin)}", ) - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -1744,21 +1727,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -2036,24 +2004,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: return () return tuple(str(column[0]) for column in description) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -2464,28 +2414,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: tuple[Any, ...] | None = self.execute(connection=connection, sql=query).fetchone() - if result is None: - raise AdapterUserError(message="BigQuery count query returned no result") - return self._to_int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 0150d073ce..69aeff958f 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -1724,23 +1724,6 @@ def render_durable_clone( del origin_is_transient return (f"CREATE TABLE {destination} DEEP CLONE {origin}",) - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -1756,21 +1739,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -2170,26 +2138,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: finally: cursor.close() - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = connection.cursor() - try: - cursor.execute(f"SELECT max({quoted_cursor}) FROM {relation}") - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - finally: - cursor.close() - def render_max_cursor_at_or_before( self, *, @@ -2444,28 +2392,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: tuple[Any, ...] | None = self.execute(connection=connection, sql=query).fetchone() - if result is None: - raise AdapterUserError(message="Databricks count query returned no result") - return self._to_int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 0e2edbcdf2..eb6036ab3d 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -190,24 +190,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: return () return tuple(str(column[0]) for column in description) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -577,23 +559,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -609,21 +574,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -994,24 +944,6 @@ def delete_insert_cursor( for stmt in statements: self.execute(connection=connection, sql=stmt) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - where_clause: str = "" - if cursor_column and start_cursor: - where_clause = f" WHERE {cursor_column} >= '{start_cursor.value}'" - if end_cursor: - where_clause += f" AND {cursor_column} < '{end_cursor.value}'" - cursor: Any = connection.execute(f"SELECT COUNT(*) FROM {relation}{where_clause}") - result: Any = cursor.fetchone() - return int(result[0]) - def validate_row_diff_keys( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 21cce1ee90..54ee1afb4c 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -2230,23 +2230,6 @@ def render_durable_clone( table_kind: str = "TRANSIENT TABLE" if origin_is_transient else "TABLE" return (f"CREATE OR REPLACE {table_kind} {destination} CLONE {origin}",) - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -2262,21 +2245,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -2404,26 +2372,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: finally: cursor.close() - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = connection.cursor() - try: - cursor.execute(f"SELECT max({quoted_cursor}) FROM {relation}") - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - finally: - cursor.close() - def render_max_cursor_at_or_before( self, *, @@ -2676,26 +2624,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: Any | None = None, - end_cursor: Any | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: Any = self.execute(connection=connection, sql=query).fetchone() - return int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 62e09380e0..95b1a5887f 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -229,24 +229,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: ) return tuple(str(row[2]) for row in cursor.fetchall() if row[2] is not None) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -1090,25 +1072,6 @@ def durable_clone( for stmt in statements: self.execute(connection=connection, sql=stmt) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - where_clause: str = f" WHERE {cursor_filter}" if cursor_filter else "" - cursor: Any = connection.execute(f"SELECT COUNT(*) FROM {relation}{where_clause}") - result: Any = cursor.fetchone() - return int(result[0]) - def create_function( self, *, @@ -1772,23 +1735,6 @@ def render_durable_clone( del origin_is_transient return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -1804,21 +1750,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index 297f591c16..abae01ff09 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -203,17 +203,6 @@ class DiffRowsErrorTestCase: tolerances: RowDiffTolerances | None = None -@dataclass(frozen=True) -class CountRowsTestCase: - description: str - setup_sql: tuple[str, ...] - relation: str - cursor_column: str | None = None - start_cursor: CursorValue | None = None - end_cursor: CursorValue | None = None - expected_count: int = 0 - - @dataclass(frozen=True) class RecorderWriteTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py b/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py index f0c7249bd9..723bbc4323 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py @@ -25,3 +25,12 @@ def _execute_sql(self, *, connection: Any, sql: str) -> Any: def _raise_insert_failure(self, *, connection: Any, sql: str) -> Any: del connection, sql raise RuntimeError("injected snapshot insert failure") + + +def count_relation_rows(*, adapter: DuckDbAdapter, connection: Any, relation: str) -> int: + """Return ``COUNT(*)`` for one relation through the adapter connection.""" + + result: Any = adapter.execute( + connection=connection, sql=f"SELECT COUNT(*) FROM {relation}" + ).fetchone() + return int(result[0]) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index da9977b703..2462ff668c 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -10,7 +10,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, - CursorValue, ExpressionInferenceProfile, QueryResult, RowDiffColumnResult, @@ -21,7 +20,7 @@ RowDiffTolerances, SchemaDiffResult, ) -from sqlbuild.adapter.contract.types import CursorKind, FunctionNullabilityRule +from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SeedCsvSettings @@ -29,7 +28,6 @@ CompositeRowDiffSamplingTestCase, ConnectSettingsTestCase, ConnectTestCase, - CountRowsTestCase, DeleteInsertTestCase, DiffRowsErrorTestCase, DiffRowsTestCase, @@ -50,6 +48,7 @@ SwapTestCase, TransactionalAtomicityTestCase, ) +from tests.integration.src.sqlbuild.adapters.duckdb.helpers import count_relation_rows @pytest.mark.parametrize( @@ -442,7 +441,7 @@ def test_given_sql_when_creating_table_as_then_table_is_writable( statement_recorder=StatementRecorder(), ) connection.execute("INSERT INTO result VALUES (4)") - count: int = adapter.count_rows(connection=connection, relation="result") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="result") assert count == test_case.expected_row_count @@ -561,7 +560,7 @@ def test_given_source_table_when_creating_view_then_view_reflects_source( sql="SELECT id FROM source", statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="result_view") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="result_view") assert count == test_case.expected_row_count @@ -750,7 +749,7 @@ def test_given_source_table_when_cloning_then_target_has_same_rows( destination="cloned_t", statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="cloned_t") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="cloned_t") assert count == test_case.expected_row_count @@ -783,7 +782,7 @@ def test_given_existing_table_when_appending_then_row_count_increases( sql="SELECT * FROM (VALUES (2), (3)) AS t(id)", statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="append_t") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="append_t") assert count == test_case.expected_row_count @@ -831,7 +830,7 @@ def test_given_target_when_delete_inserting_then_matching_rows_replaced( unique_key=test_case.unique_key, statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="di_target") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="di_target") updated_val: Any = connection.execute("SELECT val FROM di_target WHERE id = 1").fetchone() assert count == test_case.expected_row_count @@ -881,7 +880,9 @@ def test_given_target_and_source_when_merging_then_upserts_correctly( unique_key=test_case.unique_key, statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="merge_target") + count: int = count_relation_rows( + adapter=adapter, connection=connection, relation="merge_target" + ) rows: list[tuple[Any, ...]] = connection.execute( "SELECT id, name FROM merge_target ORDER BY id, name" ).fetchall() @@ -956,7 +957,7 @@ def test_given_csv_file_when_loading_seed_twice_then_table_is_replaced( infer_types=test_case.infer_types, statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="seed_table") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="seed_table") first_row: tuple[Any, ...] = connection.execute( "SELECT * FROM seed_table ORDER BY id LIMIT 1" ).fetchone() @@ -1417,53 +1418,6 @@ def test_given_invalid_tolerance_when_diffing_rows_then_raises_clear_error( ) -@pytest.mark.parametrize( - "test_case", - [ - CountRowsTestCase( - description="counts all rows without cursor filter", - setup_sql=( - "CREATE TABLE count_t (id INTEGER)", - "INSERT INTO count_t VALUES (1), (2), (3)", - ), - relation="count_t", - expected_count=3, - ), - CountRowsTestCase( - description="counts rows bounded by integer cursor", - setup_sql=( - "CREATE TABLE count_bounded (id INTEGER)", - "INSERT INTO count_bounded VALUES (1), (2), (3), (4), (5)", - ), - relation="count_bounded", - cursor_column="id", - start_cursor=CursorValue(kind=CursorKind.INTEGER, value=2), - end_cursor=CursorValue(kind=CursorKind.INTEGER, value=4), - expected_count=2, - ), - ], - ids=lambda case: case.description, -) -def test_given_table_when_counting_rows_then_returns_expected_count( - test_case: CountRowsTestCase, - adapter: DuckDbAdapter, - connection: Any, -) -> None: - statement: str - for statement in test_case.setup_sql: - connection.execute(statement) - - count: int = adapter.count_rows( - connection=connection, - relation=test_case.relation, - cursor_column=test_case.cursor_column, - start_cursor=test_case.start_cursor, - end_cursor=test_case.end_cursor, - ) - - assert count == test_case.expected_count - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py b/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py index b324fd21cd..0eb8197d28 100644 --- a/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py @@ -68,17 +68,6 @@ class PostgresRowDiffErrorTestCase: tolerances: RowDiffTolerances | None = None -@dataclass(frozen=True) -class PostgresCountRowsTestCase: - description: str - table_name: str - values_sql: str - cursor_column: str | None = None - start_cursor: CursorValue | None = None - end_cursor: CursorValue | None = None - expected_count: int = 0 - - @dataclass(frozen=True) class PostgresRowDiffSampleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py index c266718c59..18a0f9c776 100644 --- a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py @@ -23,7 +23,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from tests.integration.src.sqlbuild.adapters.postgres._test_types import ( PostgresBuildFlowTestCase, - PostgresCountRowsTestCase, PostgresMergeTestCase, PostgresQueryTestCase, PostgresRowDiffErrorTestCase, @@ -404,50 +403,6 @@ def test_given_invalid_diff_when_diffing_rows_then_postgres_raises_clear_error( ) -@pytest.mark.parametrize( - "test_case", - [ - PostgresCountRowsTestCase( - description="counts all rows without cursor filter", - table_name="count_t", - values_sql="(1), (2), (3)", - expected_count=3, - ), - PostgresCountRowsTestCase( - description="counts rows bounded by integer cursor", - table_name="count_bounded", - values_sql="(1), (2), (3), (4), (5)", - cursor_column="id", - start_cursor=CursorValue(kind=CursorKind.INTEGER, value=2), - end_cursor=CursorValue(kind=CursorKind.INTEGER, value=4), - expected_count=2, - ), - ], - ids=lambda case: case.description, -) -def test_given_table_when_counting_rows_then_postgres_returns_expected_count( - test_case: PostgresCountRowsTestCase, - adapter: PostgresAdapter, - connection: Any, - postgres_schema: str, -) -> None: - target: str = qualified_name(schema=postgres_schema, name=test_case.table_name) - adapter.execute(connection=connection, sql=f"CREATE TABLE {target} (id INTEGER)") - adapter.execute( - connection=connection, sql=f"INSERT INTO {target} VALUES {test_case.values_sql}" - ) - - count: int = adapter.count_rows( - connection=connection, - relation=target, - cursor_column=test_case.cursor_column, - start_cursor=test_case.start_cursor, - end_cursor=test_case.end_cursor, - ) - - assert count == test_case.expected_count - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py index 598b423b22..9406da96d3 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py @@ -53,13 +53,3 @@ class BaseAdapterMetadataSqlTestCase: schema: str name: str expected_sql: tuple[str, ...] - - -@dataclass(frozen=True) -class BaseAdapterRelationMaxCursorTestCase: - description: str - rows: tuple[tuple[object, ...], ...] - relation: str - cursor_column: str - expected_value: object | None - expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py index 9a4ce368ba..8b5106839c 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py @@ -16,7 +16,6 @@ BaseAdapterIdentifierLimitTestCase, BaseAdapterMetadataSqlTestCase, BaseAdapterPythonFunctionSupportTestCase, - BaseAdapterRelationMaxCursorTestCase, BaseAdapterRetentionSupportTestCase, BaseAdapterSqlAnalysisDialectTestCase, ) @@ -199,44 +198,6 @@ def test_given_base_adapter_when_rendering_durable_clone_then_uses_copy_fallback assert result == test_case.expected_statements -@pytest.mark.parametrize( - "test_case", - ( - BaseAdapterRelationMaxCursorTestCase( - description="queries max cursor with quoted identifier", - rows=((42,),), - relation="analytics.events", - cursor_column='event"time', - expected_value=42, - expected_sql='SELECT max("event""time") FROM analytics.events', - ), - BaseAdapterRelationMaxCursorTestCase( - description="returns none when query returns no rows", - rows=(), - relation="analytics.empty_events", - cursor_column="event_time", - expected_value=None, - expected_sql='SELECT max("event_time") FROM analytics.empty_events', - ), - ), - ids=lambda case: case.description, -) -def test_given_relation_when_getting_max_cursor_then_base_adapter_queries_quoted_cursor( - test_case: BaseAdapterRelationMaxCursorTestCase, -) -> None: - adapter: RecordingBaseAdapter = RecordingBaseAdapter() - connection: RecordingConnection = RecordingConnection(rows=test_case.rows) - - result: object | None = adapter.get_relation_max_cursor( - connection=connection, - relation=test_case.relation, - cursor_column=test_case.cursor_column, - ) - - assert result == test_case.expected_value - assert tuple(connection.executed_sql) == (test_case.expected_sql,) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 39275d97d8..374f0a371d 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -24,18 +24,6 @@ class AdapterCloneModeTestCase: expected_statements: tuple[str, ...] -@dataclass(frozen=True) -class AdapterRelationMaxCursorTestCase: - description: str - adapter: StrictAdapter - connection: object - relation: str - cursor_column: str - expected_value: object | None - expected_sql: tuple[str, ...] - expected_closed_cursor_count: int - - @dataclass(frozen=True) class AdapterEligibleMaxCursorSqlTestCase: description: str @@ -46,17 +34,6 @@ class AdapterEligibleMaxCursorSqlTestCase: expected_sql: str -@dataclass(frozen=True) -class AdapterSeedSelectAfterCursorTestCase: - description: str - adapter: StrictAdapter - origin: str - cursor_column: str - cursor_start_exclusive: str - cursor_type: str | None - expected_sql: str - - @dataclass(frozen=True) class AdapterManagedWriteSchemaCapabilityTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 5ed09e990c..aaceb8412d 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -173,13 +173,6 @@ class BigQuerySampleRowsTestCase: expected_side_only_samples: tuple[tuple[tuple[str, object], ...], ...] -@dataclass(frozen=True) -class BigQueryCountRowsTestCase: - description: str - expected_count: int - expected_sql: str - - @dataclass(frozen=True) class BigQueryPruneSqlTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py index 016de83d16..1305c9a277 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py @@ -37,24 +37,6 @@ def close(self) -> None: return None -def fake_schema_diff_describe_relation( - connection: object, - relation: str, -) -> tuple[ColumnInfo, ...]: - del connection - relation_columns: dict[str, tuple[ColumnInfo, ...]] = { - "left_relation": ( - ColumnInfo(name="id", type="INT64"), - ColumnInfo(name="status", type="STRING"), - ), - "right_relation": ( - ColumnInfo(name="id", type="STRING"), - ColumnInfo(name="new_col", type="DATE"), - ), - } - return relation_columns[relation] - - def fake_row_diff_describe_relation( connection: object, relation: str, @@ -102,15 +84,6 @@ def fake_execute(connection: object, sql: str) -> StubCursor: return fake_execute -def build_count_rows_execute(executed_sql: list[str]) -> Any: - def fake_execute(connection: object, sql: str) -> StubCursor: - del connection - executed_sql.append(sql) - return StubCursor(row=(2,)) - - return fake_execute - - class FakeBigQuerySchemaField: def __init__(self, name: str) -> None: self.name: str = name diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index dd9561133e..285762bc7d 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -7,7 +7,6 @@ from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( ColumnInfo, - CursorValue, ExpressionInferenceProfile, QueryResult, RelationInfo, @@ -37,7 +36,6 @@ ) from tests.unit.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryConnectErrorTestCase, - BigQueryCountRowsTestCase, BigQueryDmlLifecycleTestCase, BigQueryExecutionErrorTestCase, BigQueryExpressionInferenceProfileTestCase, @@ -65,7 +63,6 @@ FakeBigQueryClient, FakeBigQueryFailingClient, FakeBigQueryRows, - build_count_rows_execute, build_fake_bigquery_schema_client, build_row_diff_execute, build_sample_rows_execute, @@ -1072,42 +1069,6 @@ def test_given_bigquery_relations_when_sampling_rows_then_returns_expected_examp assert side_only_samples == test_case.expected_side_only_samples -@pytest.mark.parametrize( - "test_case", - [ - BigQueryCountRowsTestCase( - description="uses typed timestamp cursor filter", - expected_count=2, - expected_sql=( - "SELECT COUNT(*) FROM left_relation WHERE updated_at >= " - "TIMESTAMP '2026-04-01 00:00:00' AND updated_at < " - "TIMESTAMP '2026-04-02 00:00:00'" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_timestamp_cursor_when_counting_rows_then_bigquery_uses_typed_filter( - test_case: BigQueryCountRowsTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - adapter: BigQueryAdapter = BigQueryAdapter() - executed_sql: list[str] = [] - - monkeypatch.setattr(adapter, "execute", build_count_rows_execute(executed_sql)) - - result: int = adapter.count_rows( - connection=object(), - relation="left_relation", - cursor_column="updated_at", - start_cursor=CursorValue(kind=CursorKind.TIMESTAMP, value=datetime(2026, 4, 1, 0, 0, 0)), - end_cursor=CursorValue(kind=CursorKind.TIMESTAMP, value=datetime(2026, 4, 2, 0, 0, 0)), - ) - - assert result == test_case.expected_count - assert executed_sql == [test_case.expected_sql] - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py index b1b57026cd..7317b8a384 100644 --- a/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py @@ -47,13 +47,6 @@ class DuckDbMetadataSqlTestCase: expected_sql: tuple[str, ...] -@dataclass(frozen=True) -class DuckDbRelationMaxCursorTestCase: - description: str - expected_populated_value: object - expected_empty_value: object | None - - @dataclass(frozen=True) class DuckDbPruneSqlTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py index 5b205e8c02..ac373d6e7c 100644 --- a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py @@ -18,7 +18,6 @@ DuckDbMergeExclusionTestCase, DuckDbMetadataSqlTestCase, DuckDbPruneSqlTestCase, - DuckDbRelationMaxCursorTestCase, DuckDbRenderCursorBoundLiteralTestCase, DuckDbRenderIdentifierTestCase, DuckDbRenderSourceFreshnessQueryTestCase, @@ -108,46 +107,6 @@ def test_given_cursor_bounds_when_rendering_then_duckdb_returns_expected_literal assert result == test_case.expected_literal -@pytest.mark.parametrize( - "test_case", - [ - DuckDbRelationMaxCursorTestCase( - description="returns max cursor from populated relation and none from empty relation", - expected_populated_value=7, - expected_empty_value=None, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_relation_when_getting_max_cursor_then_returns_relation_max( - test_case: DuckDbRelationMaxCursorTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: object = adapter.connect({"database": ":memory:"}) - try: - adapter.execute(connection=connection, sql='CREATE TABLE events ("event""time" INTEGER)') - adapter.execute(connection=connection, sql="INSERT INTO events VALUES (1), (7), (3)") - adapter.execute( - connection=connection, sql='CREATE TABLE empty_events ("event""time" INTEGER)' - ) - - populated_value: object | None = adapter.get_relation_max_cursor( - connection=connection, - relation="events", - cursor_column='event"time', - ) - empty_value: object | None = adapter.get_relation_max_cursor( - connection=connection, - relation="empty_events", - cursor_column='event"time', - ) - finally: - adapter.close(connection) - - assert populated_value == test_case.expected_populated_value - assert empty_value == test_case.expected_empty_value - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/helpers.py b/tests/unit/src/sqlbuild/adapters/helpers.py deleted file mode 100644 index 1aaea5e9c5..0000000000 --- a/tests/unit/src/sqlbuild/adapters/helpers.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from types import MappingProxyType -from typing import Protocol, cast - - -class AdapterRecordingConnection(Protocol): - def recorded_sql(self) -> tuple[str, ...]: ... - - def closed_cursor_count(self) -> int: ... - - -class AdapterCursorMaxCursor: - def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: - self.rows: tuple[tuple[object, ...], ...] = rows - self.closed: bool = False - self.executed_sql: list[str] = [] - - def execute(self, sql: str) -> None: - self.executed_sql.append(sql) - - def fetchone(self) -> tuple[object, ...] | None: - return next(iter(self.rows), None) - - def close(self) -> None: - self.closed = True - - -class AdapterExecuteRecordingConnection: - def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: - self.cursor: AdapterCursorMaxCursor = AdapterCursorMaxCursor(rows=rows) - self.executed_sql: list[str] = [] - - def execute(self, sql: str) -> AdapterCursorMaxCursor: - self.executed_sql.append(sql) - return self.cursor - - def recorded_sql(self) -> tuple[str, ...]: - return tuple(self.executed_sql) - - def closed_cursor_count(self) -> int: - return 0 - - -class AdapterCursorRecordingConnection: - def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: - self.cursor_instance: AdapterCursorMaxCursor = AdapterCursorMaxCursor(rows=rows) - - def cursor(self) -> AdapterCursorMaxCursor: - return self.cursor_instance - - def recorded_sql(self) -> tuple[str, ...]: - return tuple(self.cursor_instance.executed_sql) - - def closed_cursor_count(self) -> int: - return int(self.cursor_instance.closed) - - -def adapter_cursor_executed_sql(connection: object) -> tuple[str, ...]: - _CONNECTION_VALIDATORS[ - isinstance( - connection, (AdapterExecuteRecordingConnection, AdapterCursorRecordingConnection) - ) - ](connection) - return cast(AdapterRecordingConnection, connection).recorded_sql() - - -def adapter_closed_cursor_count(connection: object) -> int: - _CONNECTION_VALIDATORS[ - isinstance( - connection, (AdapterExecuteRecordingConnection, AdapterCursorRecordingConnection) - ) - ](connection) - return cast(AdapterRecordingConnection, connection).closed_cursor_count() - - -def _accept_recording_connection(connection: object) -> None: - del connection - - -def _reject_recording_connection(connection: object) -> None: - raise TypeError(f"unsupported connection type: {type(connection).__name__}") - - -_CONNECTION_VALIDATORS: MappingProxyType[bool, Callable[[object], None]] = MappingProxyType( - {False: _reject_recording_connection, True: _accept_recording_connection} -) diff --git a/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py b/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py index 76c8521369..e702ae026b 100644 --- a/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py +++ b/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py @@ -11,107 +11,7 @@ from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from tests.unit.src.sqlbuild.adapters._test_types import ( AdapterEligibleMaxCursorSqlTestCase, - AdapterRelationMaxCursorTestCase, - AdapterSeedSelectAfterCursorTestCase, ) -from tests.unit.src.sqlbuild.adapters.helpers import ( - AdapterCursorRecordingConnection, - AdapterExecuteRecordingConnection, - adapter_closed_cursor_count, - adapter_cursor_executed_sql, -) - - -@pytest.mark.parametrize( - "test_case", - ( - AdapterRelationMaxCursorTestCase( - description="duckdb uses shared quoted cursor SQL", - adapter=DuckDbAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((9,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=9, - expected_sql=('SELECT max("event""time") FROM analytics.events',), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="motherduck uses shared quoted cursor SQL", - adapter=MotherDuckAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((10,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=10, - expected_sql=('SELECT max("event""time") FROM analytics.events',), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="postgres uses double quoted cursor SQL", - adapter=PostgresAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((11,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=11, - expected_sql=('SELECT max("event""time") FROM analytics.events',), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="bigquery uses backtick quoted cursor SQL", - adapter=BigQueryAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((12,),)), - relation="analytics.events", - cursor_column="event`time", - expected_value=12, - expected_sql=("SELECT max(`event``time`) FROM analytics.events",), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="sqlserver uses bracket quoted cursor SQL", - adapter=SqlServerAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((13,),)), - relation="analytics.events", - cursor_column="event]time", - expected_value=13, - expected_sql=("SELECT max([event]]time]) FROM analytics.events",), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="snowflake uppercases and closes cursor", - adapter=SnowflakeAdapter(), - connection=AdapterCursorRecordingConnection(rows=((14,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=14, - expected_sql=('SELECT max("EVENT""TIME") FROM analytics.events',), - expected_closed_cursor_count=1, - ), - AdapterRelationMaxCursorTestCase( - description="databricks uses backtick cursor SQL and closes cursor", - adapter=DatabricksAdapter(), - connection=AdapterCursorRecordingConnection(rows=((15,),)), - relation="analytics.events", - cursor_column="event`time", - expected_value=15, - expected_sql=("SELECT max(`event``time`) FROM analytics.events",), - expected_closed_cursor_count=1, - ), - ), - ids=lambda case: case.description, -) -def test_given_first_class_adapter_when_getting_relation_max_cursor_then_uses_adapter_sql( - test_case: AdapterRelationMaxCursorTestCase, -) -> None: - result: object | None = test_case.adapter.get_relation_max_cursor( - connection=test_case.connection, - relation=test_case.relation, - cursor_column=test_case.cursor_column, - ) - - assert result == test_case.expected_value - assert adapter_cursor_executed_sql(test_case.connection) == test_case.expected_sql - assert ( - adapter_closed_cursor_count(test_case.connection) == test_case.expected_closed_cursor_count - ) @pytest.mark.parametrize( @@ -204,95 +104,3 @@ def test_given_cursor_horizon_when_rendering_eligible_max_then_adapter_owns_port ) assert sql == test_case.expected_sql - - -@pytest.mark.parametrize( - "test_case", - ( - AdapterSeedSelectAfterCursorTestCase( - description="duckdb renders seed select after timestamp cursor", - adapter=DuckDbAdapter(), - origin="prod.events", - cursor_column='event"time', - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - 'SELECT * FROM prod.events WHERE "event""time" > TIMESTAMP \'2026-01-01 00:00:00\'' - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="motherduck renders seed select after timestamp cursor", - adapter=MotherDuckAdapter(), - origin="prod.events", - cursor_column='event"time', - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - 'SELECT * FROM prod.events WHERE "event""time" > TIMESTAMP \'2026-01-01 00:00:00\'' - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="postgres renders seed select after integer cursor", - adapter=PostgresAdapter(), - origin="prod.events", - cursor_column='event"index', - cursor_start_exclusive="10", - cursor_type="integer", - expected_sql='SELECT * FROM prod.events WHERE "event""index" > 10', - ), - AdapterSeedSelectAfterCursorTestCase( - description="bigquery renders seed select with backtick cursor", - adapter=BigQueryAdapter(), - origin="prod.events", - cursor_column="event`time", - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - "SELECT * FROM prod.events WHERE `event``time` > TIMESTAMP '2026-01-01 00:00:00'" - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="sqlserver renders seed select with bracket cursor", - adapter=SqlServerAdapter(), - origin="prod.events", - cursor_column="event]index", - cursor_start_exclusive="10", - cursor_type="integer", - expected_sql="SELECT * FROM prod.events WHERE [event]]index] > 10", - ), - AdapterSeedSelectAfterCursorTestCase( - description="snowflake renders seed select with uppercase quoted cursor", - adapter=SnowflakeAdapter(), - origin="prod.events", - cursor_column='event"time', - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - 'SELECT * FROM prod.events WHERE "EVENT""TIME" > \'2026-01-01 00:00:00\'' - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="databricks renders seed select with backtick cursor", - adapter=DatabricksAdapter(), - origin="prod.events", - cursor_column="event`time", - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - "SELECT * FROM prod.events WHERE `event``time` > TIMESTAMP '2026-01-01 00:00:00'" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_first_class_adapter_when_rendering_seed_select_after_cursor_then_uses_adapter_sql( - test_case: AdapterSeedSelectAfterCursorTestCase, -) -> None: - sql: str = test_case.adapter.render_seed_select_after_cursor( - origin=test_case.origin, - cursor_column=test_case.cursor_column, - cursor_start_exclusive=test_case.cursor_start_exclusive, - cursor_type=test_case.cursor_type, - ) - - assert sql == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 9b142958f4..aa0eb7cfbf 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -408,31 +408,9 @@ def diff_rows( ) -> RowDiffResult: raise NotImplementedError - def count_rows( - self, - connection: Any, - *, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - raise NotImplementedError - def relation_names_match(self, left: str, right: str) -> bool: return left == right - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str | None, - cursor_type: object, - ) -> str: - raise NotImplementedError - def render_seed_select_before_cursor( self, *, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py index 62cc131a12..0bf29e1032 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py @@ -181,7 +181,7 @@ def test_given_existing_destination_when_promoting_then_terminal_records_actual_ ( PromotionProgressTestCase( description="unsupported catalogued strategy fails honestly", - strategy="create_new", + strategy="build_aside", expected_method="relation_exists", expected_event_types=("operation_started", "operation_failed"), ), @@ -196,7 +196,7 @@ def test_given_unsupported_strategy_when_promoting_then_failed_terminal_records_ dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) adapter: Mock = Mock(adapter_name="snowflake") adapter.relation_exists.return_value = True - adapter.default_promotion_strategy.return_value = PromotionStrategy(test_case.strategy) + adapter.default_promotion_strategy.return_value = test_case.strategy with dispatcher_scope(dispatcher), pytest.raises(ExecutorInputError, match="Unsupported"): promote_relation_to_destination( From b81d2a36a968f2c612242ea415ad7e77dced91b1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:37:33 +0100 Subject: [PATCH 20/32] refactor(executor): remove unwired same-target audit gate reuse decision same_target_audit_gate_reuse_decision was never called by the executor. Remove it, the BINDING_SET_CHANGED reason only it produced, and its tests. The invalid-metadata diagnostic test now exercises the live reuse_from decision, which shares the parsing path. --- .../_helpers/reuse/fingerprint_metadata.py | 62 ----- src/sqlbuild/executor/run/types.py | 1 - .../executor/run/_helpers/_test_types.py | 24 -- .../run/_helpers/test_fingerprint_metadata.py | 249 +----------------- 4 files changed, 1 insertion(+), 335 deletions(-) diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index 99183ae4fd..3bfc4e2c45 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -84,68 +84,6 @@ def model_fingerprint_metadata_with_audit_gate( return json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str) -def same_target_audit_gate_reuse_decision( - *, metadata_json: str | None, model_audits: tuple[AuditPlanEntry, ...] -) -> AuditGateReuseDecision: - """Return whether prior same-target audit gate proof covers planned audits.""" - - if not model_audits: - return AuditGateReuseDecision( - reusable=True, - reason=AuditGateReuseReason.REUSABLE, - ) - if any(audit.always_run for audit in model_audits): - return AuditGateReuseDecision( - reusable=False, - reason=AuditGateReuseReason.ALWAYS_RUN, - ) - audit_gate: AuditGateMetadata | AuditGateMetadataParseFailureDetail = _read_audit_gate( - metadata_json - ) - if isinstance(audit_gate, AuditGateMetadataParseFailureDetail): - _log_parse_failure(audit_gate) - return AuditGateReuseDecision( - reusable=False, reason=_parse_failure_reuse_reason(audit_gate) - ) - if audit_gate.status != AuditGateStatus.PASSED.value: - return AuditGateReuseDecision(reusable=False, reason=AuditGateReuseReason.NON_PASSING) - identity: AuditGateIdentity = build_audit_gate_identity(audits=model_audits) - if audit_gate.binding_set_hash != identity.binding_set_hash: - return AuditGateReuseDecision( - reusable=False, - reason=AuditGateReuseReason.BINDING_SET_CHANGED, - missing_binding_keys=tuple(audit.binding_key for audit in identity.audits), - ) - prior_results: dict[str, AuditGateResultMetadata] = { - result.binding_key: result for result in audit_gate.results - } - - reusable_binding_keys: list[str] = [] - missing_binding_keys: list[str] = [] - audit: AuditIdentity - for audit in identity.audits: - prior_result: AuditGateResultMetadata | None = prior_results.get(audit.binding_key) - if ( - prior_result is None - or prior_result.execution_fingerprint != audit.execution_fingerprint - ): - missing_binding_keys.append(audit.binding_key) - continue - reusable_binding_keys.append(audit.binding_key) - if missing_binding_keys: - return AuditGateReuseDecision( - reusable=False, - reason=AuditGateReuseReason.AUDIT_CHANGED, - reusable_binding_keys=tuple(reusable_binding_keys), - missing_binding_keys=tuple(missing_binding_keys), - ) - return AuditGateReuseDecision( - reusable=True, - reason=AuditGateReuseReason.REUSABLE, - reusable_binding_keys=tuple(reusable_binding_keys), - ) - - def reuse_from_audit_gate_reuse_decision( *, metadata_json: str | None, model_audits: tuple[AuditPlanEntry, ...] ) -> AuditGateReuseDecision: diff --git a/src/sqlbuild/executor/run/types.py b/src/sqlbuild/executor/run/types.py index 5cb57e9c45..eee3e57897 100644 --- a/src/sqlbuild/executor/run/types.py +++ b/src/sqlbuild/executor/run/types.py @@ -85,7 +85,6 @@ class AuditGateReuseReason(StrEnum): MISSING = "missing" MALFORMED = "malformed" NON_PASSING = "non_passing" - BINDING_SET_CHANGED = "binding_set_changed" AUDIT_CHANGED = "audit_changed" ALWAYS_RUN = "always_run" diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 2bdc500950..26dc2ff62d 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -392,30 +392,6 @@ class TryWriteFingerprintAuditGateTestCase: expected_result_count: int -@dataclass(frozen=True) -class AuditGateReuseDecisionTestCase: - description: str - metadata_mode: str - status: AuditGateStatus - planned_attached_column_name: str | None - planned_resolved_sql: str - expected_reusable: bool - expected_reason: AuditGateReuseReason - expected_reusable_count: int - expected_missing_count: int - planned_always_run: bool = False - - -@dataclass(frozen=True) -class AuditGatePartialReuseDecisionTestCase: - description: str - changed_resolved_sql: str - expected_reusable: bool - expected_reason: AuditGateReuseReason - expected_reusable_count: int - expected_missing_count: int - - @dataclass(frozen=True) class ReuseFromAuditGateDecisionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py index b5102c1f12..1132488185 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py @@ -20,7 +20,6 @@ parse_audit_gate_metadata, render_audit_gate_metadata, reuse_from_audit_gate_reuse_decision, - same_target_audit_gate_reuse_decision, ) from sqlbuild.executor.run.models import ( AuditGateMetadata, @@ -31,8 +30,6 @@ from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( AuditGateMetadataFailureTestCase, AuditGateMetadataSchemaTestCase, - AuditGatePartialReuseDecisionTestCase, - AuditGateReuseDecisionTestCase, FingerprintAuditGateEdgeTestCase, FingerprintAuditGateMetadataTestCase, FingerprintAuditGateNoAuditsTestCase, @@ -166,7 +163,7 @@ def test_given_invalid_metadata_when_deciding_reuse_then_denies_with_debug_diagn ) -> None: caplog.set_level(logging.DEBUG, logger="sqlbuild.execution") - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( + decision: AuditGateReuseDecision = reuse_from_audit_gate_reuse_decision( metadata_json=test_case.metadata_json, model_audits=(build_fingerprint_audit_plan_entry(),), ) @@ -263,250 +260,6 @@ def test_given_edge_audit_results_when_building_fingerprint_metadata_then_status assert len(audit_gate["results"]) == test_case.expected_result_count # type: ignore[arg-type] -@pytest.mark.parametrize( - "test_case", - [ - AuditGateReuseDecisionTestCase( - description="matching passed proof is reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=True, - expected_reason=AuditGateReuseReason.REUSABLE, - expected_reusable_count=1, - expected_missing_count=0, - ), - AuditGateReuseDecisionTestCase( - description="failed proof is not reusable", - metadata_mode="written", - status=AuditGateStatus.FAILED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.NON_PASSING, - expected_reusable_count=0, - expected_missing_count=0, - ), - AuditGateReuseDecisionTestCase( - description="binding set change is not reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="customer_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.BINDING_SET_CHANGED, - expected_reusable_count=0, - expected_missing_count=1, - ), - AuditGateReuseDecisionTestCase( - description="same binding with changed execution SQL is not reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id < 0", - expected_reusable=False, - expected_reason=AuditGateReuseReason.AUDIT_CHANGED, - expected_reusable_count=0, - expected_missing_count=1, - ), - AuditGateReuseDecisionTestCase( - description="always_run audit is not reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.ALWAYS_RUN, - expected_reusable_count=0, - expected_missing_count=0, - planned_always_run=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_prior_audit_gate_when_deciding_same_target_reuse_then_returns_decision( - test_case: AuditGateReuseDecisionTestCase, -) -> None: - audit: AuditPlanEntry = build_fingerprint_audit_plan_entry() - audit_result: AuditExecutionResult = build_fingerprint_audit_result(outcome="pass") - metadata_json: str = model_fingerprint_metadata_with_audit_gate( - metadata_json="{}", - model_audits=(audit,), - audit_results=(audit_result,), - run_id="run_1", - ) - metadata: dict[str, object] = json.loads(metadata_json) - audit_gate_value: object = metadata["audit_gate"] - assert isinstance(audit_gate_value, dict) - audit_gate: dict[str, object] = audit_gate_value - audit_gate["status"] = test_case.status.value - metadata_json = json.dumps(metadata, sort_keys=True, separators=(",", ":")) - - planned_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - attached_column_name=test_case.planned_attached_column_name, - resolved_sql=test_case.planned_resolved_sql, - always_run=test_case.planned_always_run, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json=metadata_json, - model_audits=(planned_audit,), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - -@pytest.mark.parametrize( - "test_case", - [ - AuditGateReuseDecisionTestCase( - description="missing proof is not reusable", - metadata_mode="missing", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.MISSING, - expected_reusable_count=0, - expected_missing_count=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_audit_gate_when_deciding_same_target_reuse_then_returns_missing( - test_case: AuditGateReuseDecisionTestCase, -) -> None: - planned_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - attached_column_name=test_case.planned_attached_column_name, - resolved_sql=test_case.planned_resolved_sql, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json="{}", - model_audits=(planned_audit,), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - -@pytest.mark.parametrize( - "test_case", - [ - AuditGateReuseDecisionTestCase( - description="malformed results are not reusable", - metadata_mode="malformed_results", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.MALFORMED, - expected_reusable_count=0, - expected_missing_count=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_malformed_audit_gate_when_deciding_same_target_reuse_then_returns_malformed( - test_case: AuditGateReuseDecisionTestCase, -) -> None: - audit: AuditPlanEntry = build_fingerprint_audit_plan_entry() - audit_result: AuditExecutionResult = build_fingerprint_audit_result(outcome="pass") - metadata_json: str = model_fingerprint_metadata_with_audit_gate( - metadata_json="{}", - model_audits=(audit,), - audit_results=(audit_result,), - run_id="run_1", - ) - metadata: dict[str, object] = json.loads(metadata_json) - audit_gate_value: object = metadata["audit_gate"] - assert isinstance(audit_gate_value, dict) - audit_gate: dict[str, object] = audit_gate_value - audit_gate["results"] = {"not": "a list"} - metadata_json = json.dumps(metadata, sort_keys=True, separators=(",", ":")) - planned_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - attached_column_name=test_case.planned_attached_column_name, - resolved_sql=test_case.planned_resolved_sql, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json=metadata_json, - model_audits=(planned_audit,), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - -@pytest.mark.parametrize( - "test_case", - [ - AuditGatePartialReuseDecisionTestCase( - description="one changed audit reports reusable and missing binding keys", - changed_resolved_sql="SELECT customer_id FROM analytics.orders WHERE customer_id < 0", - expected_reusable=False, - expected_reason=AuditGateReuseReason.AUDIT_CHANGED, - expected_reusable_count=1, - expected_missing_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_one_changed_audit_when_deciding_same_target_reuse_then_returns_partial_keys( - test_case: AuditGatePartialReuseDecisionTestCase, -) -> None: - unchanged_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - name="not_null_orders", - attached_column_name="order_id", - resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - ) - changed_prior_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - name="not_null_customers", - attached_column_name="customer_id", - resolved_sql="SELECT customer_id FROM analytics.orders WHERE customer_id IS NULL", - ) - metadata_json: str = model_fingerprint_metadata_with_audit_gate( - metadata_json="{}", - model_audits=(unchanged_audit, changed_prior_audit), - audit_results=( - build_fingerprint_audit_result( - outcome="pass", - audit_name="not_null_orders", - attached_column_name="order_id", - ), - build_fingerprint_audit_result( - outcome="pass", - audit_name="not_null_customers", - attached_column_name="customer_id", - ), - ), - run_id="run_1", - ) - changed_current_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - name="not_null_customers", - attached_column_name="customer_id", - resolved_sql=test_case.changed_resolved_sql, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json=metadata_json, - model_audits=(unchanged_audit, changed_current_audit), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - @pytest.mark.parametrize( "test_case", [ From a33e859285378cc064c4a76a7eabf3c33638273d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:44:13 +0100 Subject: [PATCH 21/32] refactor(compiler): remove test-only header parser and state backend methods Remove the Python _ModelHeaderParser, which only served as a cross-check oracle for the native MODEL header parser, together with its corpus parity test. Remove the StateBackend methods replace_virtual_environment_function_refs, replace_virtual_environment_seed_refs and get_physical_relation_ancestry, which only tests called; tests now write refs through the node-ref APIs production uses. No state-table layout, stored format or reader used by the engine changes. append_microbatch_event is kept: retargeting its Postgres tests to append_microbatch_events exposed a type-resolution failure in the bulk path that needs a separate fix. --- .../discovery/_helpers/sql/model_files.py | 293 ------------------ .../state/classes/_sql_state_backend.py | 83 ----- .../virtual/state/classes/state_backend.py | 31 -- .../sqlbuild/virtual/state/classes/helpers.py | 12 +- .../state/classes/postgres/test_postgres.py | 177 ----------- .../virtual/state/classes/test_duckdb.py | 213 ++----------- .../compiler/discovery/_helpers/helpers.py | 145 --------- .../discovery/_helpers/test_sql_models.py | 14 - .../virtual/state/_helpers/helpers.py | 25 -- 9 files changed, 28 insertions(+), 965 deletions(-) diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index 37579a7b5f..5ba5185769 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -10,7 +10,6 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.auditing.types import ThresholdOperator from sqlbuild.compiler.discovery.exceptions import ( DiscoveryError, ModelHeaderSyntaxError, @@ -687,298 +686,6 @@ def _project_native_header_value(value: object) -> object: raise ModelHeaderSyntaxError(f"Native MODEL header parser returned invalid '{kind}' marker") -class _ModelHeaderParser: - def __init__(self, *, header: str) -> None: - self._tokens: list[_ModelHeaderToken] = _tokenize_model_header_for_spans(header) - self._index: int = 0 - - def parse(self) -> dict[str, object]: - if self._peek().kind == _MODEL_HEADER_END_TOKEN: - return {} - values: dict[str, object] = self._parse_map(end_symbol=None) - self._expect_end() - return values - - def _parse_map( - self, - *, - end_symbol: str | None, - threshold_policy: bool = False, - allow_outside_threshold: bool = False, - column_entries: bool = False, - column_metadata: bool = False, - ) -> dict[str, object]: - values: dict[str, object] = {} - while not self._is_at_end_symbol(end_symbol): - if self._match_symbol(_MODEL_HEADER_COMMA): - continue - key: str = self._consume_key() - if key in values: - raise ModelHeaderSyntaxError(f"duplicate key '{key}'") - if self._match_symbol(_MODEL_HEADER_KEY_VALUE_SEPARATOR): - raise ModelHeaderSyntaxError( - f"unexpected ':' after key '{key}'; use SQLBuild syntax '{key} value'" - ) - if self._is_at_end_symbol(end_symbol) or self._peek().kind == _MODEL_HEADER_END_TOKEN: - raise ModelHeaderSyntaxError( - f"unexpected token '{key}' without a value; quote values with spaces" - ) - if allow_outside_threshold and key == ThresholdOperator.OUTSIDE: - values[key] = self._parse_outside_threshold() - elif key in _MODEL_HEADER_HOOK_FIELD_NAMES: - values[key] = self._parse_hook_field_value(key) - elif column_metadata and key == _MODEL_HEADER_TYPE_KEY: - values[key] = self._parse_type_value() - else: - values[key] = self._parse_value( - threshold_policy=key == _MODEL_HEADER_THRESHOLDS_KEY, - allow_outside_threshold=( - threshold_policy and key in _MODEL_HEADER_THRESHOLD_BOUND_KEYS - ), - nested_column_entries=key == _MODEL_HEADER_COLUMNS_KEY, - nested_column_metadata=column_entries, - ) - self._match_symbol(_MODEL_HEADER_COMMA) - if end_symbol is not None: - self._consume_symbol(end_symbol) - return values - - def _parse_outside_threshold(self) -> tuple[object, object]: - """Parse the directional threshold shorthand ``outside LOW HIGH``.""" - - lower: object = self._parse_value() - if self._is_at_end_symbol(_MODEL_HEADER_CLOSE_PAREN): - raise ModelHeaderSyntaxError("outside threshold requires lower and upper values") - upper: object = self._parse_value() - return lower, upper - - def _parse_type_value(self) -> object: - """Parse parameterized SQL types as scalar header values.""" - - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_WORD_TOKEN: - return self._parse_value() - if not ( - self._tokens[self._index + 1].kind == _MODEL_HEADER_SYMBOL_TOKEN - and self._tokens[self._index + 1].value == _MODEL_HEADER_OPEN_PAREN - ): - return self._parse_value() - - parts: list[str] = [self._advance().value, self._advance().value] - depth: int = 1 - while depth > 0: - parameter_token: _ModelHeaderToken = self._peek() - if parameter_token.kind == _MODEL_HEADER_END_TOKEN: - raise ModelHeaderSyntaxError(f"unterminated SQL type at position {token.position}") - if parameter_token.kind == _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError( - f"quoted SQL type parameters require the whole type to be quoted " - f"at position {parameter_token.position}" - ) - self._advance() - parts.append(parameter_token.value) - if parameter_token.kind != _MODEL_HEADER_SYMBOL_TOKEN: - continue - if parameter_token.value == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - elif parameter_token.value == _MODEL_HEADER_CLOSE_PAREN: - depth -= 1 - return "".join(parts) - - def _parse_value( - self, - *, - threshold_policy: bool = False, - allow_outside_threshold: bool = False, - nested_column_entries: bool = False, - nested_column_metadata: bool = False, - ) -> object: - token: _ModelHeaderToken = self._peek() - if token.kind == _MODEL_HEADER_STRING_TOKEN: - self._advance() - return token.value - if token.kind == _MODEL_HEADER_WORD_TOKEN: - self._advance() - if ( - self._peek().kind == _MODEL_HEADER_SYMBOL_TOKEN - and self._peek().value == _MODEL_HEADER_OPEN_PAREN - ): - self._advance() - if token.value in _MODEL_HEADER_RELATION_CALL_NAMES: - return self._parse_relation_call(token.value) - if token.value in _MODEL_HEADER_HOOK_CALL_NAMES: - return self._parse_hook_call(token.value) - if token.value == _MODEL_HEADER_TYPED_CONSTANT_CALL: - return AuthoredSqlValueCall( - arguments=tuple( - self._parse_map(end_symbol=_MODEL_HEADER_CLOSE_PAREN).items() - ) - ) - return { - token.value: self._parse_map( - end_symbol=_MODEL_HEADER_CLOSE_PAREN, - threshold_policy=threshold_policy, - allow_outside_threshold=allow_outside_threshold, - ) - } - return _parse_word_value(token.value) - if self._match_symbol(_MODEL_HEADER_OPEN_BRACKET): - return self._parse_list() - if self._match_symbol(_MODEL_HEADER_OPEN_BRACE): - return AuthoredSqlSet(values=tuple(self._parse_sequence(_MODEL_HEADER_CLOSE_BRACE))) - if self._match_symbol(_MODEL_HEADER_OPEN_PAREN): - return self._parse_map( - end_symbol=_MODEL_HEADER_CLOSE_PAREN, - threshold_policy=threshold_policy, - allow_outside_threshold=allow_outside_threshold, - column_entries=nested_column_entries, - column_metadata=nested_column_metadata, - ) - raise ModelHeaderSyntaxError(f"expected value at position {token.position}") - - def _parse_list(self) -> list[object]: - return self._parse_sequence(_MODEL_HEADER_CLOSE_BRACKET) - - def _parse_sequence(self, end_symbol: str) -> list[object]: - values: list[object] = [] - while not self._is_at_end_symbol(end_symbol): - if self._match_symbol(_MODEL_HEADER_COMMA): - continue - values.append(self._parse_value()) - self._match_symbol(_MODEL_HEADER_COMMA) - self._consume_symbol(end_symbol) - return values - - def _parse_hook_field_value(self, field_name: str) -> list[object]: - if not self._match_symbol(_MODEL_HEADER_OPEN_BRACKET): - token: _ModelHeaderToken = self._peek() - raise ModelHeaderSyntaxError( - f"{field_name} must be a list of typed inline_sql(...), sql(...), or " - "python(...) hook entries " - f"at position {token.position}" - ) - values: list[object] = [] - while not self._is_at_end_symbol(_MODEL_HEADER_CLOSE_BRACKET): - if self._match_symbol(_MODEL_HEADER_COMMA): - continue - values.append(self._parse_hook_list_entry(field_name)) - self._match_symbol(_MODEL_HEADER_COMMA) - self._consume_symbol(_MODEL_HEADER_CLOSE_BRACKET) - return values - - def _parse_hook_list_entry(self, field_name: str) -> object: - token: _ModelHeaderToken = self._peek() - if token.kind == _MODEL_HEADER_STRING_TOKEN: - self._advance() - return token.value - if token.kind not in {_MODEL_HEADER_WORD_TOKEN, _MODEL_HEADER_STRING_TOKEN}: - raise ModelHeaderSyntaxError( - f"{field_name} entries must use typed inline_sql(...), sql(...), or " - "python(...) hook syntax" - ) - self._advance() - if ( - self._peek().kind != _MODEL_HEADER_SYMBOL_TOKEN - or self._peek().value != _MODEL_HEADER_OPEN_PAREN - ): - raise ModelHeaderSyntaxError( - f"{field_name} entries must use typed inline_sql(...), sql(...), or " - "python(...) hook syntax" - ) - self._advance() - if token.value not in _MODEL_HEADER_HOOK_CALL_NAMES: - raise ModelHeaderSyntaxError( - f"{field_name} entries must use typed inline_sql(...), sql(...), or " - "python(...) hook syntax" - ) - return self._parse_hook_call(token.value) - - def _parse_relation_call(self, name: str) -> str: - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError(f"{name}(...) requires a double-quoted relation name") - self._advance() - relation_name: str = token.value.replace('"', '\\"') - self._consume_symbol(_MODEL_HEADER_CLOSE_PAREN) - return f'{name}("{relation_name}")' - - def _parse_hook_call(self, name: str) -> SqlHookEntry | NamedSqlHookEntry | PythonHookEntry: - if name == _MODEL_HEADER_INLINE_SQL_HOOK_CALL: - statement_token: _ModelHeaderToken = self._peek() - if statement_token.kind != _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError("inline_sql(...) requires a quoted SQL string") - self._advance() - if self._match_symbol(_MODEL_HEADER_COMMA): - raise ModelHeaderSyntaxError("inline_sql(...) does not accept additional arguments") - self._consume_symbol(_MODEL_HEADER_CLOSE_PAREN) - return SqlHookEntry(statement=statement_token.value) - - hook_name_token: _ModelHeaderToken = self._peek() - if hook_name_token.kind != _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError(f"{name}(...) requires a quoted hook name") - self._advance() - if not hook_name_token.value.strip(): - raise ModelHeaderSyntaxError(f"{name}(...) requires a non-empty hook name") - kwargs: dict[str, object] = self._parse_hook_kwargs() - if name == _MODEL_HEADER_NAMED_SQL_HOOK_CALL: - return NamedSqlHookEntry(name=hook_name_token.value, kwargs=kwargs) - return PythonHookEntry(name=hook_name_token.value, kwargs=kwargs) - - def _parse_hook_kwargs(self) -> dict[str, object]: - kwargs: dict[str, object] = {} - while self._match_symbol(_MODEL_HEADER_COMMA): - if self._is_at_end_symbol(_MODEL_HEADER_CLOSE_PAREN): - break - key: str = self._consume_key() - if key in kwargs: - raise ModelHeaderSyntaxError(f"duplicate hook argument '{key}'") - self._consume_symbol(_MODEL_HEADER_KEY_VALUE_SEPARATOR) - kwargs[key] = self._parse_value() - self._consume_symbol(_MODEL_HEADER_CLOSE_PAREN) - return kwargs - - def _consume_key(self) -> str: - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_WORD_TOKEN: - raise ModelHeaderSyntaxError(f"expected key at position {token.position}") - self._advance() - return token.value - - def _is_at_end_symbol(self, symbol: str | None) -> bool: - token: _ModelHeaderToken = self._peek() - if symbol is None: - return token.kind == _MODEL_HEADER_END_TOKEN - return token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == symbol - - def _match_symbol(self, symbol: str) -> bool: - token: _ModelHeaderToken = self._peek() - if token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == symbol: - self._advance() - return True - return False - - def _consume_symbol(self, symbol: str) -> None: - if self._match_symbol(symbol): - return - token: _ModelHeaderToken = self._peek() - raise ModelHeaderSyntaxError(f"expected '{symbol}' at position {token.position}") - - def _expect_end(self) -> None: - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_END_TOKEN: - raise ModelHeaderSyntaxError( - f"unexpected token '{token.value}' at position {token.position}" - ) - - def _peek(self) -> _ModelHeaderToken: - return self._tokens[self._index] - - def _advance(self) -> _ModelHeaderToken: - token: _ModelHeaderToken = self._tokens[self._index] - self._index += 1 - return token - - def _model_header_parse(header: str) -> _ModelHeaderTokenization: tokenization: _ModelHeaderTokenization | None = _MODEL_HEADER_TOKEN_CACHE.get(header) if tokenization is None: diff --git a/src/sqlbuild/virtual/state/classes/_sql_state_backend.py b/src/sqlbuild/virtual/state/classes/_sql_state_backend.py index 0606be0831..c31a45cbf2 100644 --- a/src/sqlbuild/virtual/state/classes/_sql_state_backend.py +++ b/src/sqlbuild/virtual/state/classes/_sql_state_backend.py @@ -8,7 +8,6 @@ from datetime import datetime from typing import Any, ClassVar, cast -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.node_results.main.decode_json import decode_node_result_json from sqlbuild.executor.node_results.main.encode_json import encode_node_result_json from sqlbuild.executor.node_results.models import ( @@ -446,66 +445,6 @@ def replace_virtual_environment_model_refs( ), ) - def replace_virtual_environment_function_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], - ) -> None: - ref: VirtualEnvironmentFunctionRefRecord - for ref in refs: - if ref.node_type not in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - raise StateBackendConfigError("Function ref node_type must be 'udf' or 'table_fn'") - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] = {} - for node_type in ("udf", "table_fn"): - node_refs: list[VirtualEnvironmentNodeRefRecord] = [] - for ref in refs: - if ref.node_type == node_type: - node_refs.append( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - ) - refs_by_node_type[node_type] = tuple(node_refs) - self.replace_virtual_environment_node_ref_groups( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - refs_by_node_type=refs_by_node_type, - ) - - def replace_virtual_environment_seed_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - ) -> None: - self.replace_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - node_type="seed", - refs=tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="seed", - node_name=ref.seed_name, - version_hash=ref.version_hash, - ) - for ref in refs - ), - ) - def upsert_virtual_environment_python_node_ref( self, *, @@ -1421,28 +1360,6 @@ def list_physical_relations_for_artifact( for row in rows ) - def get_physical_relation_ancestry( - self, *, connection: Any, schema: str, model_name: str, version_hash: str - ) -> PhysicalRelationAncestryRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT model_name, version_hash, parent_model_name, parent_version_hash, " - "seed_strategy " - f"FROM {self._qualified_name(schema=schema, table=PHYSICAL_RELATION_ANCESTRY_TABLE)} " - f"WHERE model_name = {p} AND version_hash = {p}", - params=[model_name, version_hash], - ) - if row is None: - return None - return PhysicalRelationAncestryRecord( - model_name=row[0], - version_hash=row[1], - parent_model_name=row[2], - parent_version_hash=row[3], - seed_strategy=row[4], - ) - def get_virtual_environment( self, *, connection: Any, schema: str, virtual_environment_name: str ) -> VirtualEnvironmentRecord | None: diff --git a/src/sqlbuild/virtual/state/classes/state_backend.py b/src/sqlbuild/virtual/state/classes/state_backend.py index c8ea885ac3..199d6a661d 100644 --- a/src/sqlbuild/virtual/state/classes/state_backend.py +++ b/src/sqlbuild/virtual/state/classes/state_backend.py @@ -259,13 +259,6 @@ def upsert_physical_relation_ancestry( """Insert or replace physical relation ancestry.""" ... - @abstractmethod - def get_physical_relation_ancestry( - self, *, connection: Any, schema: str, model_name: str, version_hash: str - ) -> PhysicalRelationAncestryRecord | None: - """Return physical relation ancestry if it exists.""" - ... - @abstractmethod def upsert_virtual_environment( self, *, connection: Any, schema: str, record: VirtualEnvironmentRecord @@ -390,18 +383,6 @@ def get_virtual_environment_model_refs( """Return refs for a virtual environment.""" ... - @abstractmethod - def replace_virtual_environment_function_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], - ) -> None: - """Replace all function refs for a virtual environment.""" - ... - @abstractmethod def get_virtual_environment_function_refs( self, *, connection: Any, schema: str, virtual_environment_name: str @@ -409,18 +390,6 @@ def get_virtual_environment_function_refs( """Return function refs for a virtual environment.""" ... - @abstractmethod - def replace_virtual_environment_seed_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - ) -> None: - """Replace all seed refs for a virtual environment.""" - ... - @abstractmethod def get_virtual_environment_seed_refs( self, *, connection: Any, schema: str, virtual_environment_name: str diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py index 34300313bd..7d28f8cf16 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py @@ -348,14 +348,16 @@ def exercise_state_ref_contract( connection=connection, schema=schema, virtual_environment_name="dev", node_type="seed" ) ) - backend.replace_virtual_environment_function_refs( + backend.replace_virtual_environment_node_ref_groups( connection=connection, schema=schema, virtual_environment_name="dev", - refs=( - VirtualEnvironmentFunctionRefRecord("dev", "table_fn", "customer_orders", "fn-v1"), - VirtualEnvironmentFunctionRefRecord("dev", "udf", "normalize_email", "fn-v2"), - ), + refs_by_node_type={ + "table_fn": ( + VirtualEnvironmentNodeRefRecord("dev", "table_fn", "customer_orders", "fn-v1"), + ), + "udf": (VirtualEnvironmentNodeRefRecord("dev", "udf", "normalize_email", "fn-v2"),), + }, ) function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( backend.get_virtual_environment_function_refs( diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py index 2c97970d33..7d908c0944 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py @@ -34,7 +34,6 @@ PhysicalRelationRecord, PythonNodeVersionRecord, ReconcileEventRecord, - SeedVersionRecord, SourceFreshnessRecord, StateLockLease, StateLockRecord, @@ -43,12 +42,10 @@ StateSchemaValidationResult, VirtualEnvironmentCheckpointModelRefRecord, VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, VirtualEnvironmentModelRefRecord, VirtualEnvironmentNodeRefRecord, VirtualEnvironmentPythonNodeRefRecord, VirtualEnvironmentRecord, - VirtualEnvironmentSeedRefRecord, ) from sqlbuild.virtual.state.types import ( ModelVersionStatus, @@ -104,7 +101,6 @@ PostgresStateBackendOperationEventTestCase, PostgresStateBackendPythonNodeIdentityTestCase, PostgresStateBackendRefContractTestCase, - PostgresStateBackendSeedRefTestCase, PostgresStateBackendSourceFreshnessTestCase, PostgresStateBackendTableCreationTestCase, PostgresStateBackendTransactionRollbackTestCase, @@ -976,15 +972,6 @@ def test_given_postgres_state_backend_when_upserting_core_records_then_round_tri ) == replaced_relation_record ) - assert ( - postgres_state_backend.get_physical_relation_ancestry( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == ancestry_record - ) assert ( postgres_state_backend.get_virtual_environment( connection=postgres_state_connection, @@ -1143,170 +1130,6 @@ def test_given_postgres_backend_when_replacing_source_freshness_then_round_trips ) -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendSeedRefTestCase( - description="persists seed versions and replaces seed refs", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - seed_name="country_codes", - version_hash="seed123", - identity_metadata_hash="meta123", - identity_metadata_json_b64="e30=", - expected_ref_count_after_replace=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_backend_when_replacing_seed_refs_then_round_trips_records( - test_case: PostgresStateBackendSeedRefTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - seed_version: SeedVersionRecord = SeedVersionRecord( - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - identity_metadata_hash=test_case.identity_metadata_hash, - identity_metadata_json_b64=test_case.identity_metadata_json_b64, - status=ModelVersionStatus.READY, - ) - postgres_state_backend.upsert_seed_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=seed_version, - ) - postgres_state_backend.replace_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - - assert ( - postgres_state_backend.get_seed_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ) - == seed_version - ) - refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert refs == ( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - - postgres_state_backend.replace_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=(), - ) - assert ( - len( - postgres_state_backend.get_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - == test_case.expected_ref_count_after_replace - ) - postgres_state_backend.replace_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - postgres_state_backend.delete_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - assert ( - postgres_state_backend.get_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - == () - ) - - checkpoint: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id="chk_seed", - virtual_environment_name=test_case.virtual_environment_name, - ) - postgres_state_backend.create_virtual_environment_checkpoint( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint=checkpoint, - refs=(), - seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_checkpoint_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - assert checkpoint_seed_refs == ( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - postgres_state_backend.delete_virtual_environment_checkpoint( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - assert ( - postgres_state_backend.get_virtual_environment_checkpoint_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - == () - ) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py index 66e92b8089..cc526ca62b 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py @@ -45,7 +45,6 @@ PhysicalRelationRecord, PythonNodeVersionRecord, ReconcileEventRecord, - SeedVersionRecord, SourceFreshnessRecord, StateLockLease, StateLockRecord, @@ -55,7 +54,6 @@ VirtualEnvironmentCheckpointFunctionRefRecord, VirtualEnvironmentCheckpointModelRefRecord, VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, VirtualEnvironmentFunctionRefRecord, VirtualEnvironmentModelRefRecord, VirtualEnvironmentNodeRefRecord, @@ -92,7 +90,6 @@ DuckDbStateBackendPythonNodeIdentityTestCase, DuckDbStateBackendRefContractTestCase, DuckDbStateBackendRollbackTestCase, - DuckDbStateBackendSeedRefTestCase, DuckDbStateBackendSourceFreshnessTestCase, DuckDbStateBackendTableCreationTestCase, DuckDbStateBackendTransactionRollbackTestCase, @@ -1066,14 +1063,16 @@ def test_given_duckdb_state_backend_when_upserting_core_records_then_round_trips ), ), ) - backend.replace_virtual_environment_seed_refs( + backend.replace_virtual_environment_node_refs( connection=connection, schema=test_case.schema, virtual_environment_name=test_case.expected_virtual_environment_name, + node_type="seed", refs=( - VirtualEnvironmentSeedRefRecord( + VirtualEnvironmentNodeRefRecord( virtual_environment_name=test_case.expected_virtual_environment_name, - seed_name=test_case.expected_model_name, + node_type="seed", + node_name=test_case.expected_model_name, version_hash="seed123", ), ), @@ -1107,15 +1106,6 @@ def test_given_duckdb_state_backend_when_upserting_core_records_then_round_trips ) == replaced_relation_record ) - assert ( - backend.get_physical_relation_ancestry( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == ancestry_record - ) assert ( backend.get_virtual_environment( connection=connection, @@ -1231,14 +1221,16 @@ def test_given_duckdb_state_backend_when_atomic_vde_ref_update_fails_then_rolls_ ), ), ) - backend.replace_virtual_environment_seed_refs( + backend.replace_virtual_environment_node_refs( connection=connection, schema=test_case.schema, virtual_environment_name=test_case.virtual_environment_name, + node_type="seed", refs=( - VirtualEnvironmentSeedRefRecord( + VirtualEnvironmentNodeRefRecord( virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, + node_type="seed", + node_name=test_case.seed_name, version_hash=test_case.expected_original_seed_hash, ), ), @@ -1680,172 +1672,6 @@ def test_given_duckdb_state_backend_when_replacing_source_freshness_then_round_t backend.close(connection) -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendSeedRefTestCase( - description="persists seed versions and replaces seed refs", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - seed_name="country_codes", - version_hash="seed123", - identity_metadata_hash="meta123", - identity_metadata_json_b64="e30=", - replacement_version_hash="seed456", - expected_ref_count_after_replace=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_replacing_seed_refs_then_round_trips_records( - test_case: DuckDbStateBackendSeedRefTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - seed_version: SeedVersionRecord = SeedVersionRecord( - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - identity_metadata_hash=test_case.identity_metadata_hash, - identity_metadata_json_b64=test_case.identity_metadata_json_b64, - status=ModelVersionStatus.READY, - ) - backend.upsert_seed_version( - connection=connection, schema=test_case.schema, record=seed_version - ) - backend.replace_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - - assert ( - backend.get_seed_version( - connection=connection, - schema=test_case.schema, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ) - == seed_version - ) - refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert refs == ( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - - backend.replace_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=(), - ) - assert ( - len( - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - == test_case.expected_ref_count_after_replace - ) - backend.replace_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - backend.delete_virtual_environment( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - assert ( - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - == () - ) - - checkpoint: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id="chk_seed", - virtual_environment_name=test_case.virtual_environment_name, - ) - backend.create_virtual_environment_checkpoint( - connection=connection, - schema=test_case.schema, - checkpoint=checkpoint, - refs=(), - seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_seed_refs( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - assert checkpoint_seed_refs == ( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - backend.delete_virtual_environment_checkpoint( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - assert ( - backend.get_virtual_environment_checkpoint_seed_refs( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - == () - ) - finally: - backend.close(connection) - - @pytest.mark.parametrize( "test_case", [ @@ -2165,18 +1991,21 @@ def test_given_duckdb_state_backend_when_upserting_function_records_then_round_t ) == function_record ) - backend.replace_virtual_environment_function_refs( + backend.replace_virtual_environment_node_ref_groups( connection=connection, schema=test_case.schema, virtual_environment_name=test_case.expected_virtual_environment_name, - refs=( - VirtualEnvironmentFunctionRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - node_type="udf", - function_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, + refs_by_node_type={ + "udf": ( + VirtualEnvironmentNodeRefRecord( + virtual_environment_name=test_case.expected_virtual_environment_name, + node_type="udf", + node_name=test_case.expected_model_name, + version_hash=test_case.expected_version_hash, + ), ), - ), + "table_fn": (), + }, ) function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( backend.get_virtual_environment_function_refs( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 55ce12db8c..ffff94995c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -2,13 +2,11 @@ from pathlib import Path from typing import cast -from sqlbuild import _native from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_constant_files, discover_enum_files, discover_macro_files, ) -from sqlbuild.compiler.discovery._helpers.sql import model_files as model_file_helpers from sqlbuild.compiler.discovery.models import ( DiscoveredConstantFile, DiscoveredEnumFile, @@ -39,149 +37,6 @@ } -def _standard_model_header(index: int, suffix: str) -> str: - return ( - f"name model_{suffix}, enabled true, columns (col_{suffix} " - '(type DECIMAL(10,2), nullable false, description "Order total"))' - ) - - -def _constant_model_header(_index: int, suffix: str) -> str: - return ( - f"constants (_set_{suffix} {{FR, GB, FR}}, " - f"_array_{suffix} constant(value [1, 2], render_as array))" - ) - - -def _hook_model_header(index: int, suffix: str) -> str: - return ( - f'pre_hooks [inline_sql("select {index}"), sql("record_{suffix}", ' - 'table: "orders"), python("notify", attempts: 2, urgent: true)]' - ) - - -def _audit_model_header(_index: int, suffix: str) -> str: - return ( - f"audits [rate_{suffix} (thresholds (warn (outside -1.5 2.5)))], " - f'parent __ref("orders_{suffix}")' - ) - - -def _template_model_header(_index: int, suffix: str) -> str: - return f"schema dev_${{user_{suffix}}}, tags [core, 'daily orders'], value null" - - -def _unicode_column_model_header(_index: int, suffix: str) -> str: - return ( - f"columns (café_{suffix} (type TIMESTAMP_NTZ(9), audits " - "[accepted_values (values [placed, completed])]))" - ) - - -def _numeric_model_header(index: int, suffix: str) -> str: - numeric_headers: tuple[str, ...] = ( - f"unicode_integer_{suffix} +١٢٣", - f"unicode_integer_{suffix} -१२३", - f"unicode_float_{suffix} ١٢.Ù¥", - f"unicode_float_{suffix} -१२.५", - f"numeric_character_{suffix} ²", - f"large_{suffix} {10**40 + index}, ratio_{suffix} +.25", - ) - return numeric_headers[(index // 10) % len(numeric_headers)] - - -def _escaped_model_header(_index: int, suffix: str) -> str: - return f'description "escaped \\"value_{suffix}\\"", config (x [a, b])' - - -def _duplicate_or_control_model_header(index: int, suffix: str) -> str: - headers: tuple[str, str] = ( - 'post_hooks [python("\u001c")]', - f"duplicate_{suffix} one, duplicate_{suffix} two", - ) - return headers[index % 20 == 8] - - -def _invalid_model_header(_index: int, suffix: str) -> str: - return f"columns (col_{suffix} (type DECIMAL(10,2))" - - -_MODEL_HEADER_BUILDERS: tuple[Callable[[int, str], str], ...] = ( - _standard_model_header, - _constant_model_header, - _hook_model_header, - _audit_model_header, - _template_model_header, - _unicode_column_model_header, - _numeric_model_header, - _escaped_model_header, - _duplicate_or_control_model_header, - _invalid_model_header, -) - - -def _validate_successful_native_header( - *, - native_values: dict[str, object] | None, - native_offsets: list[tuple[str, int, int]] | None, - parent_values: dict[str, object] | None, -) -> None: - assert native_values is not None - assert model_file_helpers._project_native_header_map(native_values) == parent_values - assert native_offsets is not None - - -def _validate_failed_native_header( - *, - native_values: dict[str, object] | None, - native_offsets: list[tuple[str, int, int]] | None, - parent_values: dict[str, object] | None, -) -> None: - assert native_values is None - assert parent_values is None - assert native_offsets is None - - -_NATIVE_HEADER_VALIDATORS: tuple[Callable[..., None], ...] = ( - _validate_successful_native_header, - _validate_failed_native_header, -) - - -def assert_generated_model_header_corpus_parity() -> bool: - """Assert native MODEL-header parsing matches the Python reference corpus.""" - - headers: list[str] = [ - "config (columns (nested (type INTEGER))), columns (top (type INTEGER))", - "config (columns (nested (type INTEGER)))", - ] - for index in range(9_998): - suffix: str = str(index) - headers.append(_MODEL_HEADER_BUILDERS[index % len(_MODEL_HEADER_BUILDERS)](index, suffix)) - - native_results: list[ - tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None] - ] = _native.parse_model_headers(headers) - for header, (native_values, native_offsets, native_error) in zip( - headers, native_results, strict=True - ): - try: - parent_values: dict[str, object] | None = model_file_helpers._ModelHeaderParser( - header=header - ).parse() - parent_error: str | None = None - except model_file_helpers.ModelHeaderSyntaxError as error: - parent_values = None - parent_error = str(error) - assert native_error == parent_error - _NATIVE_HEADER_VALIDATORS[native_values is None]( - native_values=native_values, - native_offsets=native_offsets, - parent_values=parent_values, - ) - return True - - def expected_or_actual[T](expected: T | None, actual: T) -> T: return (actual, cast(T, expected))[expected is not None] diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 9f3ede17d7..aabfd31b0c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -32,9 +32,6 @@ ParseModelSqlErrorTestCase, ParseModelSqlHeaderTestCase, ) -from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( - assert_generated_model_header_corpus_parity, -) @pytest.mark.parametrize( @@ -134,17 +131,6 @@ def test_given_cached_model_header_when_parsing_twice_then_top_level_dictionarie assert "added" not in second -@pytest.mark.parametrize( - "test_case", - [ExpectedBooleanTestCase(description="generated header corpus matches", expected_result=True)], - ids=lambda case: case.description, -) -def test_given_generated_header_corpus_when_native_parsing_then_parent_behavior_is_exact( - test_case: ExpectedBooleanTestCase, -) -> None: - assert assert_generated_model_header_corpus_parity() is test_case.expected_result - - @pytest.mark.parametrize( "test_case", [ExpectedCountTestCase(description="invalid faults preserve order", expected_count=2)], diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py index 705e7093be..304f30ac97 100644 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py @@ -303,11 +303,6 @@ def upsert_physical_relation_ancestry( ) -> None: return None - def get_physical_relation_ancestry( - self, connection: Any, *, schema: str, model_name: str, version_hash: str - ) -> PhysicalRelationAncestryRecord | None: - return None - def upsert_virtual_environment( self, connection: Any, *, schema: str, record: VirtualEnvironmentRecord ) -> None: @@ -431,31 +426,11 @@ def get_virtual_environment_source_freshness( ) -> tuple[SourceFreshnessRecord, ...]: return () - def replace_virtual_environment_function_refs( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], - ) -> None: - return None - def get_virtual_environment_function_refs( self, connection: Any, *, schema: str, virtual_environment_name: str ) -> tuple[VirtualEnvironmentFunctionRefRecord, ...]: return () - def replace_virtual_environment_seed_refs( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - ) -> None: - return None - def get_virtual_environment_seed_refs( self, connection: Any, *, schema: str, virtual_environment_name: str ) -> tuple[VirtualEnvironmentSeedRefRecord, ...]: From 13417507db1fb95bc599fe8286bcc0bba7cdcdb9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:45:17 +0100 Subject: [PATCH 22/32] refactor(execution-history): remove deprecated EventLogStorage alias This removes a deprecated public alias: sqlbuild.execution_history.EventLogStorage is gone. Use LifecycleEventLogStorage, which it pointed to. docs/execution-history.md is updated accordingly. --- docs/execution-history.md | 2 +- src/sqlbuild/execution_history.py | 5 +---- .../runtime/execution_history/test_import_boundary.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/execution-history.md b/docs/execution-history.md index f382e05495..37258df52d 100644 --- a/docs/execution-history.md +++ b/docs/execution-history.md @@ -13,7 +13,7 @@ contract. The public types, filters, pages, records, errors, and helper function The concrete SQLite and PostgreSQL classes additionally provide transactional `append_and_project`, `reconcile`, and `check_health` operations. -`EventLogStorage` remains a deprecated compatibility alias for `LifecycleEventLogStorage`. Storage +The deprecated `EventLogStorage` alias has been removed; use `LifecycleEventLogStorage`. Storage tables use the lifecycle-specific names `lifecycle_event_log` in SQLite and `sqlbuild_lifecycle_event_log` in PostgreSQL; the former `event_log` and `sqlbuild_event_log` names are retired. diff --git a/src/sqlbuild/execution_history.py b/src/sqlbuild/execution_history.py index aaf1aa1c78..39f5217e07 100644 --- a/src/sqlbuild/execution_history.py +++ b/src/sqlbuild/execution_history.py @@ -1,4 +1,4 @@ -"""Typed public execution history API with a deprecated ``EventLogStorage`` alias.""" +"""Typed public execution history API.""" from collections.abc import Iterable @@ -44,8 +44,6 @@ RunStorage, ) -EventLogStorage: type[LifecycleEventLogStorage] = LifecycleEventLogStorage - __all__ = ( "DEFAULT_PAGE_LIMIT", "MAX_PAGE_LIMIT", @@ -53,7 +51,6 @@ "EventCursor", "EventFamily", "EventFilter", - "EventLogStorage", "LifecycleEventLogStorage", "EventPage", "ExecutionHistoryStorageError", diff --git a/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py b/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py index 25fd91b2d5..5c4c885051 100644 --- a/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py +++ b/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py @@ -30,7 +30,7 @@ def test_given_clean_interpreter_when_importing_execution_history_then_optional_ module_names: str = ",".join(repr(name) for name in test_case.expected_forbidden_imports) script = ( "import sys; import sqlbuild.execution_history as history; " - "assert history.EventLogStorage is history.LifecycleEventLogStorage; " + "assert not hasattr(history, 'EventLogStorage'); " f"forbidden=({module_names},); " "loaded=tuple(name for name in forbidden if name in sys.modules); print(repr(loaded))" ) From 629476e53e1ba9a53aa48a46377094c5860e4fa9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:48:10 +0100 Subject: [PATCH 23/32] refactor(lint): remove unused code Remove the unused fix-run machinery (FixRunResult, LintFixRecord, fix planning helpers and their CLI renderers), other unreferenced lint helpers and constants, and the test-only map_neutralized_offset. LINT_ENGINE_SQLBUILD is kept and now used by native.py instead of the duplicated "sqlbuild" literal. --- .../cli/commands/_helpers/lint/diagnostics.py | 6 +- .../cli/commands/_helpers/lint/runs.py | 89 ----------- src/sqlbuild/lint/_helpers/fixes.py | 147 ------------------ src/sqlbuild/lint/_helpers/native.py | 9 +- src/sqlbuild/lint/_helpers/native_format.py | 6 - src/sqlbuild/lint/_helpers/sqlbuild_tokens.py | 16 -- src/sqlbuild/lint/constants.py | 3 - src/sqlbuild/lint/exceptions.py | 6 - src/sqlbuild/lint/models.py | 40 ----- .../src/sqlbuild/lint/_helpers/_test_types.py | 10 -- .../lint/_helpers/test_sqlbuild_tokens.py | 47 ------ 11 files changed, 8 insertions(+), 371 deletions(-) diff --git a/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py b/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py index a86722a0b4..51d1f3fe00 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py @@ -8,7 +8,7 @@ from rich.cells import cell_len from sqlbuild.lint.constants import TAB_CHARACTER, VIOLATION_SEVERITY_FAULT -from sqlbuild.lint.models import FixRunResult, LintRunResult, LintViolation +from sqlbuild.lint.models import LintRunResult, LintViolation from sqlbuild.presentation.classes.cli_style import CliStyle _REPORT_LINE_WIDTH: int = 100 @@ -17,7 +17,7 @@ def format_lint_diagnostics( - *, result: LintRunResult | FixRunResult, root: Path, use_color: bool + *, result: LintRunResult, root: Path, use_color: bool ) -> tuple[str, ...]: """Format every lint violation as a source-pointing diagnostic block.""" @@ -31,7 +31,7 @@ def format_lint_diagnostics( def _format_violation( *, violation: LintViolation, - result: LintRunResult | FixRunResult, + result: LintRunResult, root: Path, style: CliStyle, ) -> str: diff --git a/src/sqlbuild/cli/commands/_helpers/lint/runs.py b/src/sqlbuild/cli/commands/_helpers/lint/runs.py index 0e04b72938..0a7b14259e 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/runs.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/runs.py @@ -13,19 +13,14 @@ ADAPTER_DIALECT_TRANSLATIONS, DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES, - FIX_STATUS_APPLIED, - FIX_STATUS_SKIPPED, FORMAT_SECTION_KEY, LINE_WIDTH_KEY, MAX_DESCRIPTION_LINES_KEY, PROJECT_CONFIG_FILENAME_KEY, ) from sqlbuild.lint.models import ( - FixRunResult, LintConfig, - LintFixRecord, LintRunResult, - LintViolation, ) from sqlbuild.presentation.classes.cli_style import CliStyle @@ -133,90 +128,6 @@ def render_lint_result_json(*, result: LintRunResult) -> None: ) -def render_fix_result(*, result: FixRunResult, root: Path, use_color: bool, preview: bool) -> None: - """Render applied/skipped repairs followed by remaining diagnostics.""" - - style: CliStyle = CliStyle(use_color=use_color) - for record in result.fixes: - label: str = f"{record.status}[{record.code}]" - styled: str = ( - style.success_strong(label) - if record.status == FIX_STATUS_APPLIED - else style.warning_strong(label) - ) - location: str = _relative_path(path=record.file_path, root=root) - suffix: str = f": {record.reason}" if record.reason is not None else "" - print(f"{styled}: {location}:{record.line}:{record.column}{suffix}") - if result.fixes: - print() - if result.violations: - print("\n\n".join(format_lint_diagnostics(result=result, root=root, use_color=use_color))) - print() - fixed_count: int = sum(record.status == FIX_STATUS_APPLIED for record in result.fixes) - skipped_count: int = sum(record.status == FIX_STATUS_SKIPPED for record in result.fixes) - action: str = "WOULD_FIX" if preview else "FIXED" - summary: str = ( - f"Completed. {action}={fixed_count} SKIPPED={skipped_count} " - f"REMAINING={len(result.violations)} FILES={result.files_checked}" - ) - if result.violations or (preview and result.changed_files): - print(style.warning_strong(summary)) - else: - print(style.success_strong(summary)) - - -def render_fix_result_json(*, result: FixRunResult, preview: bool) -> None: - """Render a stable machine-readable fix result.""" - - print( - json.dumps( - { - "changed_files": [str(path) for path in result.changed_files], - "files_checked": result.files_checked, - "fixes": [_fix_record_json(record=record) for record in result.fixes], - "mode": "check" if preview else "write", - "remaining": [_violation_json(violation=entry) for entry in result.violations], - }, - sort_keys=True, - separators=(",", ":"), - ) - ) - - -def _fix_record_json(*, record: LintFixRecord) -> dict[str, object]: - return { - "code": record.code, - "column": record.column, - "file": str(record.file_path), - "line": record.line, - "reason": record.reason, - "status": record.status, - } - - -def _violation_json(*, violation: LintViolation) -> dict[str, object]: - return { - "code": violation.code, - "column": violation.column, - "end_column": violation.end_column, - "end_line": violation.end_line, - "engine": violation.engine, - "file": str(violation.file_path), - "fix_unavailable_reason": violation.fix_unavailable_reason, - "line": violation.line, - "message": violation.message, - "remediation": violation.remediation, - "severity": violation.severity, - } - - -def _relative_path(*, path: Path, root: Path) -> str: - try: - return path.relative_to(root).as_posix() - except ValueError: - return path.as_posix() - - def _styled_summary(*, summary: str, result: LintRunResult, use_color: bool) -> str: style: CliStyle = CliStyle(use_color=use_color) if result.faults: diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index f97f085b54..caf897b212 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -6,149 +6,6 @@ import tempfile from pathlib import Path -from sqlbuild.compiler.compile.models import SqlExpansionContext -from sqlbuild.lint._helpers.expansion import prepare_lint_body -from sqlbuild.lint._helpers.headers import lint_body_ranges, scan_headers -from sqlbuild.lint._helpers.native import format_native_headers, lint_native_headers -from sqlbuild.lint._helpers.native_sql import run_native_sql_lint -from sqlbuild.lint.constants import FIX_STATUS_APPLIED -from sqlbuild.lint.models import ( - HeaderSpan, - LintBody, - LintConfig, - LintEdit, - LintFixRecord, - LintViolation, -) - - -def apply_header_repairs( - *, files: dict[Path, str], config: LintConfig -) -> tuple[dict[Path, str], list[LintFixRecord]]: - repaired: dict[Path, str] = {} - records: list[LintFixRecord] = [] - for file_path, contents in sorted(files.items()): - before: tuple[LintViolation, ...] = lint_native_headers( - contents=contents, - file_path=file_path, - headers=scan_headers(contents=contents), - config=config, - ) - updated, _faults = format_native_headers( - contents=contents, - file_path=file_path, - config=config, - ) - repaired[file_path] = updated - after_codes: set[str] = { - violation.code - for violation in lint_native_headers( - contents=updated, - file_path=file_path, - headers=scan_headers(contents=updated), - config=config, - ) - } - records.extend( - fix_record(violation=violation, status=FIX_STATUS_APPLIED, reason=None) - for violation in before - if violation.code not in after_codes - ) - return repaired, records - - -def lint_contents( - *, - files: dict[Path, str], - config: LintConfig, - context: SqlExpansionContext | None, - project_dir: Path, -) -> list[LintViolation]: - violations: list[LintViolation] = [] - bodies: list[LintBody] = [] - for file_path, contents in sorted(files.items()): - headers: tuple[HeaderSpan, ...] = scan_headers(contents=contents) - violations.extend( - lint_native_headers( - contents=contents, - file_path=file_path, - headers=headers, - config=config, - ) - ) - if context is None: - continue - for body_start, body_end in lint_body_ranges( - contents=contents, - headers=headers, - file_path=file_path, - project_dir=project_dir, - ): - bodies.append( - prepare_lint_body( - project_dir=project_dir, - file_path=file_path, - contents=contents, - body_range=(body_start, body_end), - context=context, - dialect=config.dialect, - ) - ) - if bodies: - native: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( - bodies=tuple(bodies), - contents_by_path=files, - config=config, - ) - for entries in native.values(): - violations.extend(entries) - return violations - - -def planned_edits( - *, violations: list[LintViolation] -) -> tuple[dict[Path, tuple[LintEdit, ...]], list[LintFixRecord]]: - selected: dict[Path, list[LintEdit]] = {} - records: list[LintFixRecord] = [] - for violation in violations: - if violation.fix is None: - continue - existing: list[LintEdit] = selected.setdefault(violation.file_path, []) - if any(overlaps(left=violation.fix, right=edit) for edit in existing): - continue - existing.append(violation.fix) - records.append(fix_record(violation=violation, status=FIX_STATUS_APPLIED, reason=None)) - return ( - { - path: tuple(sorted(edits, key=lambda edit: (edit.start, edit.end, edit.code))) - for path, edits in selected.items() - }, - records, - ) - - -def fix_record(*, violation: LintViolation, status: str, reason: str | None) -> LintFixRecord: - return LintFixRecord( - file_path=violation.file_path, - code=violation.code, - line=violation.line, - column=violation.column, - status=status, - reason=reason, - ) - - -def apply_edits( - *, files: dict[Path, str], edits: dict[Path, tuple[LintEdit, ...]] -) -> dict[Path, str]: - updated: dict[Path, str] = dict(files) - for path, path_edits in edits.items(): - contents: str = files[path] - for edit in reversed(path_edits): - contents = contents[: edit.start] + edit.replacement + contents[edit.end :] - updated[path] = contents - return updated - def write_atomically(*, path: Path, contents: str) -> None: descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) @@ -162,7 +19,3 @@ def write_atomically(*, path: Path, contents: str) -> None: os.replace(temporary_path, path) finally: temporary_path.unlink(missing_ok=True) - - -def overlaps(*, left: LintEdit, right: LintEdit) -> bool: - return left.start < right.end and right.start < left.end diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index 553eb60398..ed30e0bd9d 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -22,6 +22,7 @@ HEADER_KIND_MODEL, IDENTIFIER_SEPARATOR_CHARACTER, LINE_FEED, + LINT_ENGINE_SQLBUILD, RULE_DESCRIPTION_LENGTH, RULE_DESCRIPTION_PRESENT, RULE_HEADER_PARSE, @@ -214,7 +215,7 @@ def _lint_header_values( code=RULE_HEADER_PARSE, message=f"{header.kind}() header could not be parsed: {error}", severity=VIOLATION_SEVERITY_FAULT, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation=f"Correct the {header.kind}() header syntax.", ), ) @@ -317,7 +318,7 @@ def _lint_header_whitespace( code=RULE_HEADER_WHITESPACE, message=f"{header.kind}() header contains trailing whitespace", severity=VIOLATION_SEVERITY_WARNING, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation="Run sqb format to remove trailing header whitespace.", ), ) @@ -568,7 +569,7 @@ def _relocate_leading_comment( code=RULE_LEADING_COMMENT_DESCRIPTION, message=(f"Relocated leading comment exceeds {config.max_description_lines} lines"), severity=VIOLATION_SEVERITY_FAULT, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation=( f"Shorten the relocated description to {config.max_description_lines} lines." ), @@ -675,7 +676,7 @@ def _violation_for_header_start( code=code, message=message, severity=severity, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation=remediation, ) diff --git a/src/sqlbuild/lint/_helpers/native_format.py b/src/sqlbuild/lint/_helpers/native_format.py index 2f55ea9ff1..00a9d13dda 100644 --- a/src/sqlbuild/lint/_helpers/native_format.py +++ b/src/sqlbuild/lint/_helpers/native_format.py @@ -31,12 +31,6 @@ class _PreparedBody: interpolation_sites: tuple[InterpolationSite, ...] -def newline_style(*, contents: str) -> str: - """Return the newline convention used by the authored contents.""" - - return CARRIAGE_RETURN_LINE_FEED if CARRIAGE_RETURN_LINE_FEED in contents else LINE_FEED - - def with_newline_style(*, contents: str, newline: str) -> str: """Normalize generated contents back to the authored newline convention.""" diff --git a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py index bea7500304..81adcbbc2b 100644 --- a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py +++ b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py @@ -243,22 +243,6 @@ def sentinel_spans(*, sites: tuple[InterpolationSite, ...]) -> tuple[ExpansionSp return tuple(spans) -def map_neutralized_offset(*, offset: int, sites: tuple[InterpolationSite, ...]) -> int: - """Map an offset in neutralized text back onto the authored body.""" - - mapped: int = offset - site: InterpolationSite - for site in sites: - if offset < site.neutralized_start: - break - if offset < site.neutralized_end: - return site.original_start - mapped += (site.original_end - site.original_start) - ( - site.neutralized_end - site.neutralized_start - ) - return mapped - - def _interpolation_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: character: str = body[start] if character == MACRO_TOKEN: diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index cf2dcf769b..6d517783bd 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -96,6 +96,3 @@ LINE_FEED: str = "\n" CARRIAGE_RETURN_LINE_FEED: str = "\r\n" TAB_CHARACTER: str = "\t" -FIX_STATUS_APPLIED: str = "fixed" -FIX_STATUS_SKIPPED: str = "skipped" -MAX_FIX_PASSES: int = 8 diff --git a/src/sqlbuild/lint/exceptions.py b/src/sqlbuild/lint/exceptions.py index 7e8905c218..80e650904b 100644 --- a/src/sqlbuild/lint/exceptions.py +++ b/src/sqlbuild/lint/exceptions.py @@ -31,9 +31,3 @@ class ProjectCompileError(LintError): """The project could not be compiled, so its SQL cannot be linted.""" code: str = "L005" - - -class CustomLintError(LintError): - """A repository-defined statement-local lint rule is invalid or failed.""" - - code: str = "L006" diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py index c0079f988a..88563a4875 100644 --- a/src/sqlbuild/lint/models.py +++ b/src/sqlbuild/lint/models.py @@ -95,18 +95,6 @@ class FormatChange: after: str -@dataclass(frozen=True) -class LintFixRecord: - """One applied or skipped lint repair for human and machine reporting.""" - - file_path: Path - code: str - line: int - column: int - status: str - reason: str | None = None - - @dataclass(frozen=True) class LintConfig: """Resolved lint and format configuration for one run.""" @@ -149,31 +137,3 @@ def warnings(self) -> tuple[LintViolation, ...]: for violation in self.violations if violation.severity == VIOLATION_SEVERITY_WARNING ) - - -@dataclass(frozen=True) -class FixRunResult: - """Aggregated result of planning or applying native lint repairs.""" - - files_checked: int - violations: tuple[LintViolation, ...] - changed_files: tuple[Path, ...] - changes: tuple[FormatChange, ...] - fixes: tuple[LintFixRecord, ...] - source_texts: Mapping[Path, str] = field(default_factory=dict, repr=False, compare=False) - - @property - def faults(self) -> tuple[LintViolation, ...]: - return tuple( - violation - for violation in self.violations - if violation.severity == VIOLATION_SEVERITY_FAULT - ) - - @property - def warnings(self) -> tuple[LintViolation, ...]: - return tuple( - violation - for violation in self.violations - if violation.severity == VIOLATION_SEVERITY_WARNING - ) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index ff9a6dba16..c55bb96eac 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -157,16 +157,6 @@ class DialectNeutralizeInterpolationTestCase: expected_original_texts: tuple[str, ...] -@dataclass(frozen=True) -class MapOffsetTestCase: - """Test case for neutralized-to-original offset mapping.""" - - description: str - body: str - neutralized_offset: int - expected_original_offset: int - - @dataclass(frozen=True) class RestoreInterpolationTestCase: """Test case for restoring sentinels after native formatting.""" diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index 3ac410fd38..7d4ed6267d 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -6,7 +6,6 @@ import sqlbuild._native as _native from sqlbuild.lint._helpers.sqlbuild_tokens import ( - map_neutralized_offset, neutralize_interpolation, restore_interpolation, ) @@ -14,7 +13,6 @@ from sqlbuild.lint.models import InterpolationSite from tests.unit.src.sqlbuild.lint._helpers._test_types import ( DialectNeutralizeInterpolationTestCase, - MapOffsetTestCase, NeutralizeInterpolationTestCase, RestoreFailureTestCase, RestoreInterpolationTestCase, @@ -248,51 +246,6 @@ def test_given_many_sites_when_neutralizing_then_every_sentinel_is_distinct( assert len(set(sentinels)) == len(test_case.expected_original_texts) -@pytest.mark.parametrize( - "test_case", - [ - MapOffsetTestCase( - description="offset before any site is unchanged", - body="SELECT @a FROM t", - neutralized_offset=0, - expected_original_offset=0, - ), - MapOffsetTestCase( - description="offset at a sentinel start maps to the interpolation start", - body="SELECT @a FROM t", - neutralized_offset=7, - expected_original_offset=7, - ), - MapOffsetTestCase( - description="offset inside a sentinel clamps to the interpolation start", - body="SELECT @a FROM t", - neutralized_offset=12, - expected_original_offset=7, - ), - MapOffsetTestCase( - description="offset after a sentinel is shifted back by the length delta", - body="SELECT @a FROM t", - neutralized_offset=22, - expected_original_offset=10, - ), - MapOffsetTestCase( - description="offset after two sentinels accumulates both deltas", - body="SELECT @a, @b FROM t", - neutralized_offset=39, - expected_original_offset=15, - ), - ], - ids=lambda case: case.description, -) -def test_given_neutralized_offset_when_mapping_then_authored_offset_matches( - test_case: MapOffsetTestCase, -) -> None: - sites: tuple[InterpolationSite, ...] - _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") - mapped: int = map_neutralized_offset(offset=test_case.neutralized_offset, sites=sites) - assert mapped == test_case.expected_original_offset - - @pytest.mark.parametrize( "test_case", [ From e1ee05fcdf88c44d6be709397bba38487b0f05f1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:50:24 +0100 Subject: [PATCH 24/32] refactor(rule-engine): remove unused code Remove unreferenced SQL helper modules (decision signatures, model-name grammar, passthrough detection), table_parts, ModelNameParts and unused constants. The public custom-rule API is unchanged: SqlDocument.subqueries and RuleOption.integer_list stay. --- src/sqlbuild/rule_engine/_helpers/sql/ast.py | 11 -- .../rule_engine/_helpers/sql/decisions.py | 48 ------- .../rule_engine/_helpers/sql/model_name.py | 52 -------- .../rule_engine/_helpers/sql/passthrough.py | 118 ------------------ src/sqlbuild/rule_engine/constants.py | 24 ---- src/sqlbuild/rule_engine/models.py | 9 -- 6 files changed, 262 deletions(-) delete mode 100644 src/sqlbuild/rule_engine/_helpers/sql/decisions.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/sql/model_name.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/sql/passthrough.py diff --git a/src/sqlbuild/rule_engine/_helpers/sql/ast.py b/src/sqlbuild/rule_engine/_helpers/sql/ast.py index af5f37ebbb..a24a0b4ff4 100644 --- a/src/sqlbuild/rule_engine/_helpers/sql/ast.py +++ b/src/sqlbuild/rule_engine/_helpers/sql/ast.py @@ -47,17 +47,6 @@ def name_value(value: object) -> str: return "" -def table_parts(node: Any) -> tuple[str, str, str]: - """Return catalog, schema, and name for one table node.""" - - data: dict[str, object] = payload(node) - return ( - name_value(data.get("catalog")), - name_value(data.get("schema")), - name_value(data.get("name")), - ) - - def cte_name(node: Any) -> str: if isinstance(node, tuple) and len(node) == NAMED_CTE_TUPLE_SIZE: return str(node[0]) diff --git a/src/sqlbuild/rule_engine/_helpers/sql/decisions.py b/src/sqlbuild/rule_engine/_helpers/sql/decisions.py deleted file mode 100644 index d6327c2fd5..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/decisions.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Polyglot decision-site identification.""" - -from __future__ import annotations - -import json -from typing import Any - -_COMPARISON_KINDS: frozenset[str] = frozenset({"eq", "neq", "gt", "gte", "lt", "lte"}) -_DECISION_KEYS: frozenset[str] = frozenset({"having", "ifs", "on", "qualify", "where_clause"}) - - -def decision_comparison_signatures(*, ast: Any) -> frozenset[str]: - """Return stable payload signatures for comparisons under decision clauses.""" - - signatures: set[str] = set() - signatures = _collect_signatures( - value=ast.to_dict(), inside_decision=False, signatures=signatures - ) - return frozenset(signatures) - - -def comparison_signature(*, node: Any) -> str: - """Return the stable payload signature for one Polyglot comparison node.""" - - return json.dumps(node.to_dict(), sort_keys=True, separators=(",", ":")) - - -def _collect_signatures(*, value: object, inside_decision: bool, signatures: set[str]) -> set[str]: - if isinstance(value, list): - for item in value: - signatures = _collect_signatures( - value=item, - inside_decision=inside_decision, - signatures=signatures, - ) - return signatures - if not isinstance(value, dict): - return signatures - for key, child in value.items(): - child_is_decision: bool = inside_decision or key in _DECISION_KEYS - if child_is_decision and key in _COMPARISON_KINDS: - signatures.add(json.dumps({key: child}, sort_keys=True, separators=(",", ":"))) - signatures = _collect_signatures( - value=child, - inside_decision=child_is_decision, - signatures=signatures, - ) - return signatures diff --git a/src/sqlbuild/rule_engine/_helpers/sql/model_name.py b/src/sqlbuild/rule_engine/_helpers/sql/model_name.py deleted file mode 100644 index ebd163f717..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/model_name.py +++ /dev/null @@ -1,52 +0,0 @@ -"""SQLBuild model-name grammar parsing.""" - -from __future__ import annotations - -import re - -from sqlbuild.rule_engine.constants import MAX_ENTITY_PARTS, MODEL_NAME_PART_COUNT -from sqlbuild.rule_engine.models import ModelNameParts - -_LAYERS: tuple[str, ...] = ( - "int_enriched", - "int_clean", - "stg_v", - "mart_v", - "int_v", - "stg", - "mart", -) -_TOKEN_PATTERN: re.Pattern[str] = re.compile(r"^[a-z][a-z0-9_]*$") - - -def parse_model_name(name: str) -> ModelNameParts | None: - """Parse the closed rule model-name grammar.""" - - parts: list[str] = name.split("__") - if len(parts) < MODEL_NAME_PART_COUNT or not _TOKEN_PATTERN.fullmatch(parts[0]): - return None - domain: str = parts[0] - tail: str = "__".join(parts[1:]) - for layer in _LAYERS: - prefix: str = f"{layer}__" - if tail.startswith(prefix): - entity_parts: list[str] = tail[len(prefix) :].split("__") - if len(entity_parts) > MAX_ENTITY_PARTS or any( - not _TOKEN_PATTERN.fullmatch(part) for part in entity_parts - ): - return None - return ModelNameParts( - domain=domain, - layer=layer, - entity=entity_parts[0], - source=entity_parts[1] if len(entity_parts) > 1 else None, - is_view=layer.endswith("_v"), - ) - return None - - -def apparent_layer(name: str) -> str | None: - """Return the raw layer-looking token for invalid-name diagnostics.""" - - parts: list[str] = name.split("__", maxsplit=2) - return parts[1] if len(parts) == MODEL_NAME_PART_COUNT else None diff --git a/src/sqlbuild/rule_engine/_helpers/sql/passthrough.py b/src/sqlbuild/rule_engine/_helpers/sql/passthrough.py deleted file mode 100644 index bc07fb4bce..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/passthrough.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Shared shape-only passthrough classification.""" - -from __future__ import annotations - -import re -from typing import Any, cast - -from sqlbuild.rule_engine._helpers.sql.ast import kind, name_value, payload, top_level_ctes -from sqlbuild.rule_engine.constants import ( - AST_ALIAS_KIND, - AST_COLUMN_KIND, - AST_FUNCTION_KIND, - AST_SELECT_KIND, - AST_STAR_KIND, - AST_TABLE_KIND, - DEPENDENCY_FUNCTIONS, -) - -_REFERENCE_PATTERN: re.Pattern[str] = re.compile(r"__(?:ref|source)\s*\(", re.IGNORECASE) -_FORBIDDEN_KINDS: frozenset[str] = frozenset( - {"aggfunc", "case", "group", "join", "window", "windowfunc"} -) - - -def is_passthrough_ast(*, ast: Any, source: str) -> bool: - """Return whether a model has the narrow import-and-project passthrough shape.""" - - ctes: tuple[Any, ...] = top_level_ctes(ast) - if len(ctes) != 1 or len(_REFERENCE_PATTERN.findall(source)) != 1: - return False - import_body: Any = ctes[0][1] - if not is_dependency_import(node=import_body) or not is_plain_projection( - select=import_body, allow_star=True - ): - return False - terminal_source: dict[str, object] | None = _sole_from_expression(select=ast) - if terminal_source is None or AST_TABLE_KIND not in terminal_source: - return False - table_payload: object = terminal_source.get(AST_TABLE_KIND) - if not isinstance(table_payload, dict): - return False - table_name: str = name_value(cast(dict[str, object], table_payload).get("name")) - return table_name == str(ctes[0][0]) and is_plain_projection(select=ast, allow_star=True) - - -def is_dependency_import(*, node: Any) -> bool: - """Return whether one select is a projection from exactly one dependency macro.""" - - source: dict[str, object] | None = _sole_from_expression(select=node) - if source is None or AST_FUNCTION_KIND not in source: - return False - function_payload: object = source.get(AST_FUNCTION_KIND) - if not isinstance(function_payload, dict): - return False - function_name: object = cast(dict[str, object], function_payload).get("name") - data: dict[str, object] = payload(node) - has_forbidden_shape: bool = any( - ( - data.get("group_by"), - data.get("having"), - data.get("joins"), - data.get("qualify"), - data.get("with"), - ) - ) - return ( - str(function_name).lower() in DEPENDENCY_FUNCTIONS - and not has_forbidden_shape - and _has_plain_expressions(select=node, allow_star=True) - ) - - -def is_plain_projection(*, select: Any, allow_star: bool) -> bool: - """Return whether one select contains only a direct column projection.""" - - if kind(select) != AST_SELECT_KIND: - return False - data: dict[str, object] = payload(select) - if data.get("where_clause") is not None or data.get("having") is not None: - return False - if data.get("qualify") is not None or data.get("group_by") is not None: - return False - if data.get("joins"): - return False - for node in select.walk(): - if kind(node) in _FORBIDDEN_KINDS: - return False - return _has_plain_expressions(select=select, allow_star=allow_star) - - -def _has_plain_expressions(*, select: Any, allow_star: bool) -> bool: - expressions: tuple[Any, ...] = tuple(select.expressions) - if not expressions: - return False - if len(expressions) == 1 and kind(expressions[0]) == AST_STAR_KIND: - return allow_star - for expression in expressions: - expression_kind: str = kind(expression) - if expression_kind == AST_COLUMN_KIND: - continue - if expression_kind == AST_ALIAS_KIND: - children: tuple[Any, ...] = tuple(expression.children()) - if children and kind(children[0]) == AST_COLUMN_KIND: - continue - return False - return True - - -def _sole_from_expression(*, select: Any) -> dict[str, object] | None: - data: dict[str, object] = payload(select) - from_payload: object = data.get("from") - if not isinstance(from_payload, dict): - return None - raw_expressions: object = cast(dict[str, object], from_payload).get("expressions") - if not isinstance(raw_expressions, list) or len(raw_expressions) != 1: - return None - expression: object = raw_expressions[0] - return cast(dict[str, object], expression) if isinstance(expression, dict) else None diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index e435ae04b8..ed7084a29b 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -15,44 +15,21 @@ SKILL_STALE: str = "stale" REPLACEABLE_SKILL_STATES: frozenset[str] = frozenset({SKILL_FRESH, SKILL_MISSING, SKILL_STALE}) -AST_COLUMN_KIND: str = "column" -AST_ALIAS_KIND: str = "alias" -AST_LITERAL_KIND: str = "literal" AST_SELECT_KIND: str = "select" -AST_STAR_KIND: str = "star" -AST_FUNCTION_KIND: str = "function" -AST_TABLE_KIND: str = "table" AST_CTE_KIND: str = "cte" -DEPENDENCY_FUNCTIONS: frozenset[str] = frozenset({"__ref", "__source"}) -CROSS_JOIN_KIND: str = "CROSS" CONTRACT_ENFORCED: str = "enforced" CONTRACT_BOOLEAN_NAME_CODE: str = "SQBRCONTRACT102" CONTRACT_TIMESTAMP_NAME_CODE: str = "SQBRCONTRACT103" CONTRACT_DATE_NAME_CODE: str = "SQBRCONTRACT104" EMPTY_INPUT_TEST_CODE: str = "SQBRTEST203" -MATERIALIZED_VIEW: str = "view" -REFERENCE_KIND_REF: str = "ref" -LAYER_INT_CLEAN: str = "int_clean" -LAYER_INT_ENRICHED: str = "int_enriched" -LAYER_INT_VIEW: str = "int_v" BOOLEAN_TYPE: str = "BOOLEAN" DATE_TYPE: str = "DATE" TIMESTAMP_TYPE: str = "TIMESTAMP" -CANONICAL_NUMERIC_LITERALS: frozenset[str] = frozenset({"-1", "0", "1"}) -FINAL_CTE_NAMES: frozenset[str] = frozenset({"final", "final_cte"}) -SET_OPERATION_BY_NAME: str = " BY NAME" -MODEL_NAME_PART_COUNT: int = 3 NAMED_CTE_TUPLE_SIZE: int = 2 -MAX_ENTITY_PARTS: int = 2 -DECLARATION_DOMAIN_PART_COUNT: int = 3 TARGET_DIRECTORY_NAME: str = "target" -SQL_LINE_COMMENT: str = "--" -SQL_BLOCK_COMMENT: str = "/*" -EVALUATE_RULE_CALL: str = "evaluate_rule" -CANONICAL_ONE: str = "1" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" RULE_CONTEXT_PROJECT_FACT: str = "project" @@ -61,7 +38,6 @@ RULE_CONTEXT_AUDITS_FACT: str = "audits" RULE_CONTEXT_DECLARATIONS_FACT: str = "declarations" PARENT_DIRECTORY_TOKEN: str = ".." -TRACKED_RULES_FILE_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) MIN_AUDITS_PER_MODEL: str = "min_audits_per_model" MIN_TESTS_PER_MODEL: str = "min_tests_per_model" diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index f5f7df34b0..a05beb6bfe 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -351,15 +351,6 @@ class ResolvedRuleset: cacheable: bool -@dataclass(frozen=True) -class ModelNameParts: - domain: str - layer: str - entity: str - source: str | None - is_view: bool - - @dataclass(frozen=True) class RuleFile: path: str From ed88edd4ff5d73878f153f41cd9a47b50d51bc78 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:51:57 +0100 Subject: [PATCH 25/32] refactor(compiler): remove unused SQL analysis, scenario and SQL-test code Remove unused Polyglot analysis constants, the scenario _dedupe_names helper and the never-emitted S505 scenario code, the unused _LEGACY_MODEL_HOOK_KEYS copy in SQL-test attachment, and the SQL-test CTE SQLite cache module, which has had no production caller since #758. --- .../compile/_helpers/attachment/sql_tests.py | 1 - .../compile/_helpers/sql_tests/cache.py | 212 ------------------ .../planner/_helpers/scenario/graph.py | 12 - src/sqlbuild/compiler/planner/constants.py | 1 - .../compiler/sql_analysis/constants.py | 4 - .../compiler/compile/_helpers/_test_types.py | 6 - .../_helpers/test_sql_test_cte_cache.py | 88 -------- 7 files changed, 324 deletions(-) delete mode 100644 src/sqlbuild/compiler/compile/_helpers/sql_tests/cache.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_cte_cache.py diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index f11dd9b130..26a0eb28b2 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -77,7 +77,6 @@ from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind, UsageKind _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/cache.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/cache.py deleted file mode 100644 index 795fef6f12..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/cache.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Versioned project-local cache for SQL test CTE extraction.""" - -from __future__ import annotations - -import hashlib -import hmac -import json -import sqlite3 -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from pathlib import Path -from types import TracebackType -from typing import Any, cast - -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - classify_sql_test_ctes, - extract_unclassified_sql_test_ctes, -) -from sqlbuild.compiler.compile.models import CompileSqlTestCte, CompileSqlTestCtes -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.profiling.main.record import record_compile_timing - -_CACHE_VERSION: int = 1 -_CACHE_DATABASE_NAME: str = "sql-test-ctes.sqlite3" -_MAX_CACHE_ENTRY_BYTES: int = 10_000_000 -_SQLITE_TIMEOUT_SECONDS: float = 0.1 -_CACHE_CTE_ROW_FIELD_COUNT: int = 2 -_CREATE_CACHE_TABLE_SQL: str = """ -CREATE TABLE IF NOT EXISTS sql_test_cte ( - cache_key TEXT PRIMARY KEY, - payload TEXT NOT NULL -) -""" - - -class _SqlTestCteCache: - """Reuse exact CTE boundaries while retaining classification and validation.""" - - def __init__(self, *, root: Path | None) -> None: - self._root: Path | None = root - self._database_path: Path | None = None - self._connection: sqlite3.Connection | None = None - self._pending_by_key: dict[str, str] = {} - - def __enter__(self) -> _SqlTestCteCache: - if self._root is None: - return self - self._database_path = self._root / f"sql-test-ctes-v{_CACHE_VERSION}" / _CACHE_DATABASE_NAME - if not self._database_path.is_file(): - return self - try: - self._connection = sqlite3.connect( - self._database_path, - timeout=_SQLITE_TIMEOUT_SECONDS, - ) - except (OSError, sqlite3.DatabaseError): - self._disable() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - del exc_value, traceback - connection: sqlite3.Connection | None = self._connection - self._connection = None - try: - if exc_type is None: - with record_compile_timing("cache_publication_ms"): - connection = self._write_pending(connection=connection) - elif connection is not None: - connection.rollback() - except (OSError, sqlite3.DatabaseError): - pass - finally: - self._pending_by_key.clear() - if connection is not None: - try: - connection.close() - except sqlite3.DatabaseError: - pass - - def extract(self, *, sql: str, file_label: str, mode: SqlTestMode) -> CompileSqlTestCtes: - """Return cached raw CTEs or scan and record this exact expanded SQL.""" - - cache_key: str = hashlib.sha256(sql.encode()).hexdigest() - ctes: tuple[CompileSqlTestCte, ...] | None = self._pending_ctes(cache_key=cache_key) - if ctes is None: - ctes = self._read_ctes(cache_key=cache_key) - if ctes is None: - ctes = extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label) - if self._database_path is not None: - contents: str = _cache_contents(cache_key=cache_key, ctes=ctes) - if len(contents.encode()) <= _MAX_CACHE_ENTRY_BYTES: - self._pending_by_key[cache_key] = contents - return classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode) - - def _pending_ctes(self, *, cache_key: str) -> tuple[CompileSqlTestCte, ...] | None: - contents: str | None = self._pending_by_key.get(cache_key) - return ( - None - if contents is None - else _ctes_from_contents(contents=contents, cache_key=cache_key) - ) - - def _read_ctes(self, *, cache_key: str) -> tuple[CompileSqlTestCte, ...] | None: - connection: sqlite3.Connection | None = self._connection - if connection is None: - return None - try: - row: tuple[str] | None = connection.execute( - "SELECT payload FROM sql_test_cte WHERE cache_key = ?", - (cache_key,), - ).fetchone() - except sqlite3.DatabaseError: - self._disable() - return None - return None if row is None else _ctes_from_contents(contents=row[0], cache_key=cache_key) - - def _write_pending(self, *, connection: sqlite3.Connection | None) -> sqlite3.Connection | None: - if not self._pending_by_key or self._database_path is None: - return connection - if connection is None: - self._database_path.parent.mkdir(parents=True, exist_ok=True) - connection = sqlite3.connect(self._database_path, timeout=_SQLITE_TIMEOUT_SECONDS) - _ = connection.execute(_CREATE_CACHE_TABLE_SQL) - _ = connection.executemany( - "INSERT OR REPLACE INTO sql_test_cte (cache_key, payload) VALUES (?, ?)", - self._pending_by_key.items(), - ) - connection.commit() - return connection - - def _disable(self) -> None: - connection: sqlite3.Connection | None = self._connection - self._connection = None - if connection is not None: - try: - connection.close() - except sqlite3.DatabaseError: - pass - - -def _cache_contents(*, cache_key: str, ctes: tuple[CompileSqlTestCte, ...]) -> str: - serialized: str = json.dumps( - { - "v": _CACHE_VERSION, - "k": cache_key, - "c": [[cte.name, cte.sql_body] for cte in ctes], - }, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) - return f"{_cache_digest(cache_key=cache_key, serialized=serialized)}\n{serialized}" - - -def _ctes_from_contents( - *, contents: object, cache_key: str -) -> tuple[CompileSqlTestCte, ...] | None: - if not isinstance(contents, str) or len(contents.encode()) > _MAX_CACHE_ENTRY_BYTES: - return None - try: - digest, separator, serialized = contents.partition("\n") - if not separator or not hmac.compare_digest( - digest, - _cache_digest(cache_key=cache_key, serialized=serialized), - ): - return None - payload: object = json.loads(serialized) - if not isinstance(payload, dict) or payload.get("v") != _CACHE_VERSION: - return None - if payload.get("k") != cache_key or not isinstance(payload.get("c"), list): - return None - rows: list[object] = cast(list[object], payload["c"]) - if not all( - isinstance(row, list) - and len(row) == _CACHE_CTE_ROW_FIELD_COUNT - and isinstance(row[0], str) - and isinstance(row[1], str) - for row in rows - ): - return None - return tuple( - CompileSqlTestCte(name=cast(list[str], row)[0], sql_body=cast(list[str], row)[1]) - for row in rows - ) - except (ValueError, TypeError, KeyError, RecursionError, json.JSONDecodeError): - return None - - -def _cache_digest(*, cache_key: str, serialized: str) -> str: - digest: Any = hashlib.sha256(cache_key.encode()) - digest.update(b"\0") - digest.update(serialized.encode()) - return digest.hexdigest() - - -@contextmanager -def cached_sql_test_cte_extractor( - *, root: Path | None -) -> Iterator[Callable[[str, str, SqlTestMode], CompileSqlTestCtes]]: - """Yield an exact cached SQL test CTE extractor for one compile invocation.""" - - with _SqlTestCteCache(root=root) as cache: - yield lambda sql, file_label, mode: cache.extract( - sql=sql, - file_label=file_label, - mode=mode, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py b/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py index ee6d0dbaca..38ddc0e397 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py @@ -335,18 +335,6 @@ def _walk( return requirements -def _dedupe_names(names: tuple[str, ...]) -> tuple[str, ...]: - deduped: list[str] = [] - seen: set[str] = set() - name: str - for name in names: - if name in seen: - continue - seen.add(name) - deduped.append(name) - return tuple(deduped) - - def _fixture_name_for(name: str) -> str: return name.replace(".", "__") diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index f546ba1de9..b4d3f0afb4 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -48,7 +48,6 @@ SCENARIO_PLAN_HASH_COLLISION: str = "S502" SCENARIO_PLAN_RELATION_COLLISION: str = "S503" SCENARIO_PLAN_GRAPH_VALIDATION: str = "S504" -SCENARIO_PLAN_SQLGLOT_UNAVAILABLE: str = "S505" SCENARIO_PLAN_SQLGLOT_PARSE: str = "S506" SCENARIO_PLAN_UNKNOWN_SEED: str = "S507" SCENARIO_PLAN_MISSING_FIXTURE_SQL: str = "S508" diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index 74b456c87a..6724ecbc59 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -134,7 +134,6 @@ POLYGLOT_ANALYSIS_COLUMN_USES: str = "columnUses" POLYGLOT_ANALYSIS_CONTEXT: str = "context" POLYGLOT_ANALYSIS_CONTEXT_FILTER: str = "filter" -POLYGLOT_ANALYSIS_CTES: str = "ctes" POLYGLOT_ANALYSIS_CTE_FACTS: str = "cteFacts" POLYGLOT_ANALYSIS_EXPRESSION_SQL: str = "expressionSql" POLYGLOT_ANALYSIS_IS_STAR: str = "isStar" @@ -142,10 +141,8 @@ POLYGLOT_ANALYSIS_NULLABILITY: str = "nullability" POLYGLOT_ANALYSIS_NULLABILITY_NON_NULL: str = "non_null" POLYGLOT_ANALYSIS_NULLABILITY_NULLABLE: str = "nullable" -POLYGLOT_ANALYSIS_NULLABILITY_UNKNOWN: str = "unknown" POLYGLOT_ANALYSIS_PROJECTIONS: str = "projections" POLYGLOT_ANALYSIS_RELATIONS: str = "relations" -POLYGLOT_ANALYSIS_SET_OPERATIONS: str = "setOperations" POLYGLOT_ANALYSIS_SHAPE: str = "shape" POLYGLOT_ANALYSIS_SOURCE_ALIAS: str = "sourceAlias" POLYGLOT_ANALYSIS_SOURCE_CONFIDENCE: str = "confidence" @@ -163,6 +160,5 @@ POLYGLOT_ANALYSIS_TRANSFORM_CAST: str = "cast" POLYGLOT_ANALYSIS_TRANSFORM_CONSTANT: str = "constant" POLYGLOT_ANALYSIS_TRANSFORM_DIRECT: str = "direct" -POLYGLOT_ANALYSIS_TRANSFORM_EXPRESSION: str = "expression" POLYGLOT_ANALYSIS_TRANSFORM_STAR: str = "star" POLYGLOT_ANALYSIS_UNSAFE_TRANSFORMS: frozenset[str] = frozenset() diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 59d679e50e..6968bea9ab 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -66,12 +66,6 @@ class AnalysisCacheTestCase: expected_count: int -@dataclass(frozen=True) -class SqlTestCteCacheTestCase: - description: str - expected_scanner_calls: int - - @dataclass(frozen=True) class NativeSqlTestExtractionParityTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_cte_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_cte_cache.py deleted file mode 100644 index 54d935531b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_cte_cache.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import sqlite3 -from pathlib import Path -from unittest.mock import Mock - -import pytest - -from sqlbuild.compiler.compile._helpers.sql_tests import cache as cte_cache -from sqlbuild.compiler.compile._helpers.sql_tests.cache import cached_sql_test_cte_extractor -from sqlbuild.compiler.compile.models import CompileSqlTestCtes -from sqlbuild.compiler.compile.types import SqlTestMode -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - SqlTestCteCacheTestCase, -) - -_TEST_SQL: str = ( - "WITH __ref__raw_orders AS (SELECT 1 AS order_id), " - "__expected__orders AS (SELECT 1 AS order_id) SELECT 1" -) - - -@pytest.mark.parametrize( - "test_case", - (SqlTestCteCacheTestCase(description="unchanged SQL reuses CTEs", expected_scanner_calls=0),), - ids=lambda case: case.description, -) -def test_given_cached_test_ctes_when_extracting_again_then_skips_scanner( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - test_case: SqlTestCteCacheTestCase, -) -> None: - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - expected: CompileSqlTestCtes = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - scanner: Mock = Mock(wraps=cte_cache.extract_unclassified_sql_test_ctes) - monkeypatch.setattr(cte_cache, "extract_unclassified_sql_test_ctes", scanner) - - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - actual: CompileSqlTestCtes = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - - assert scanner.call_count == test_case.expected_scanner_calls - assert actual == expected - - -@pytest.mark.parametrize( - "test_case", - (SqlTestCteCacheTestCase(description="changed SQL scans CTEs", expected_scanner_calls=1),), - ids=lambda case: case.description, -) -def test_given_changed_test_sql_when_extracting_then_scans_new_ctes( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - test_case: SqlTestCteCacheTestCase, -) -> None: - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - scanner: Mock = Mock(wraps=cte_cache.extract_unclassified_sql_test_ctes) - monkeypatch.setattr(cte_cache, "extract_unclassified_sql_test_ctes", scanner) - changed_sql: str = _TEST_SQL.replace("SELECT 1 AS order_id", "SELECT 2 AS order_id") - - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(changed_sql, "test_orders.sql", SqlTestMode.MODEL) - - assert scanner.call_count == test_case.expected_scanner_calls - - -@pytest.mark.parametrize( - "test_case", - (SqlTestCteCacheTestCase(description="corrupt cache rescans", expected_scanner_calls=1),), - ids=lambda case: case.description, -) -def test_given_corrupt_test_cte_cache_when_extracting_then_rescans_safely( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - test_case: SqlTestCteCacheTestCase, -) -> None: - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - cache_path: Path = next(tmp_path.rglob("sql-test-ctes.sqlite3")) - with sqlite3.connect(cache_path) as connection: - _ = connection.execute("UPDATE sql_test_cte SET payload = 'broken'") - scanner: Mock = Mock(wraps=cte_cache.extract_unclassified_sql_test_ctes) - monkeypatch.setattr(cte_cache, "extract_unclassified_sql_test_ctes", scanner) - - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - - assert scanner.call_count == test_case.expected_scanner_calls From 526387bddf76b126a78a1c022389fbad9124aa74 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:53:55 +0100 Subject: [PATCH 26/32] refactor(python-nodes): remove unused code Remove the unused loader-to-node conversion module, the placeholder DiscoveredPython{Task,Asset,Check}Node classes, PythonSqlRunLifecyclePlan.python_phase, and fields that were set but never read: PythonNodeIdentity.object_module/object_qualname, PythonNodeDependencyEdge.upstream_function/downstream_function and PythonSqlRunLifecyclePlan.read_side_sql_keys. The identity definition payload and hashes are unchanged. --- .../python_nodes/_helpers/identity.py | 2 - .../python_nodes/_helpers/inventory.py | 2 - .../compiler/python_nodes/_helpers/loaders.py | 73 ------------ .../python_nodes/_helpers/run_lifecycle.py | 1 - src/sqlbuild/compiler/python_nodes/models.py | 37 +----- .../_helpers/ingress_execution.py | 1 - .../commands/_helpers/check/test_planning.py | 2 - .../python_nodes/_helpers/_test_types.py | 22 ---- .../python_nodes/_helpers/test_loaders.py | 112 ------------------ .../_helpers/test_run_lifecycle.py | 7 -- .../_helpers/test_fingerprinting.py | 2 - .../state/main/test_python_node_identity.py | 2 - 12 files changed, 1 insertion(+), 262 deletions(-) delete mode 100644 src/sqlbuild/compiler/python_nodes/_helpers/loaders.py delete mode 100644 tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_loaders.py diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py index 29ee0a55c6..24edafabb6 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py @@ -79,8 +79,6 @@ def build_python_identity( return PythonNodeIdentity( node_type=node_type, node_name=node_name, - object_module=function.__module__, - object_qualname=_object_qualname(function), source_path=_display_path(source_path=source_path, roots=allowed_roots), source_hash=source_hash, definition_hash=definition_hash, diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py index 5070249fad..69090a97d3 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py @@ -123,8 +123,6 @@ def build_python_node_dependency_edges( PythonNodeDependencyEdge( upstream_name=upstream_node.name, downstream_name=node.name, - upstream_function=upstream_node.function, - downstream_function=node.function, ) ) return tuple(edges) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/loaders.py b/src/sqlbuild/compiler/python_nodes/_helpers/loaders.py deleted file mode 100644 index b083bd7b94..0000000000 --- a/src/sqlbuild/compiler/python_nodes/_helpers/loaders.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Loader adapters for internal Python-node models.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonLoaderMetadata, - DiscoveredPythonNode, - PythonNodeDependencyEdge, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import SqlResourceRef - - -def build_python_loader_node(*, loader: DiscoveredLoaderFunction) -> DiscoveredPythonNode: - """Return an internal Python-node view of an existing discovered loader.""" - - return DiscoveredPythonNode( - kind=PythonNodeKind.LOADER, - file_path=loader.file_path, - relative_path=loader.relative_path, - name=loader.name, - function=loader.function, - depends_on=loader.depends_on, - loader=DiscoveredPythonLoaderMetadata( - destination=loader.destination, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - columns=loader.columns, - contract=loader.contract, - connection_mode=loader.connection_mode, - ), - ) - - -def build_python_loader_nodes( - *, loaders: tuple[DiscoveredLoaderFunction, ...] -) -> tuple[DiscoveredPythonNode, ...]: - """Return internal Python-node views of existing discovered loaders.""" - - return tuple(build_python_loader_node(loader=loader) for loader in loaders) - - -def build_python_loader_dependency_edges( - *, loaders: tuple[DiscoveredLoaderFunction, ...] -) -> tuple[PythonNodeDependencyEdge, ...]: - """Return internal dependency edges between existing discovered loaders.""" - - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in loaders - } - edges: list[PythonNodeDependencyEdge] = [] - loader: DiscoveredLoaderFunction - for loader in loaders: - dependency: Callable[..., object] | SqlResourceRef - for dependency in loader.depends_on: - if isinstance(dependency, SqlResourceRef): - continue - upstream_name: str | None = loader_name_by_function.get(dependency) - if upstream_name is None: - continue - edges.append( - PythonNodeDependencyEdge( - upstream_name=upstream_name, - downstream_name=loader.name, - upstream_function=dependency, - downstream_function=loader.function, - ) - ) - return tuple(edges) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py b/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py index f541dd0634..1532a3e18d 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py @@ -47,7 +47,6 @@ def build_python_sql_run_lifecycle_plan( return PythonSqlRunLifecyclePlan( ingress_python_node_names=ingress_names, ingress_loader_names=selected_loader_names, - read_side_sql_keys=selection.sql_keys, read_side_python_node_names=read_side_names, ) diff --git a/src/sqlbuild/compiler/python_nodes/models.py b/src/sqlbuild/compiler/python_nodes/models.py index 8bc0a575ef..532d74095f 100644 --- a/src/sqlbuild/compiler/python_nodes/models.py +++ b/src/sqlbuild/compiler/python_nodes/models.py @@ -9,7 +9,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonRunPhase +from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import ColumnLineageRef, RetryPolicy, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.spec.contracts.models import SourceColumnEntry @@ -81,8 +81,6 @@ class PythonNodeDependencyEdge: upstream_name: str downstream_name: str - upstream_function: Callable[..., object] - downstream_function: Callable[..., object] @dataclass(frozen=True) @@ -138,39 +136,8 @@ class PythonSqlRunLifecyclePlan: ingress_python_node_names: frozenset[str] ingress_loader_names: frozenset[str] - read_side_sql_keys: frozenset[CompiledObjectKey] read_side_python_node_names: frozenset[str] - def python_phase(self, node_name: str) -> PythonRunPhase | None: - """Return the assigned lifecycle phase for a Python node name.""" - - if node_name in self.ingress_python_node_names: - return PythonRunPhase.PRE_SQL_INGRESS - if node_name in self.read_side_python_node_names: - return PythonRunPhase.READ_SIDE - return None - - -@dataclass(frozen=True) -class DiscoveredPythonTaskNode: - """Placeholder for future task-specific discovered node metadata.""" - - node: DiscoveredPythonNode - - -@dataclass(frozen=True) -class DiscoveredPythonAssetNode: - """Placeholder for future asset-specific discovered node metadata.""" - - node: DiscoveredPythonNode - - -@dataclass(frozen=True) -class DiscoveredPythonCheckNode: - """Placeholder for future check-specific discovered node metadata.""" - - node: DiscoveredPythonNode - @dataclass(frozen=True) class PythonIdentityDependency: @@ -190,8 +157,6 @@ class PythonNodeIdentity: node_type: str node_name: str - object_module: str - object_qualname: str source_path: str source_hash: str definition_hash: str diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 40555505e2..23d14ecfad 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -87,7 +87,6 @@ def execute_ingress_python_loader_nodes( for name in selected_python_names if python_graph.nodes_by_name[name].kind == PythonNodeKind.LOADER ), - read_side_sql_keys=frozenset(), read_side_python_node_names=frozenset(), ), python_graph=python_graph, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 9a60750aa0..8c449efd1f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -116,7 +116,6 @@ def test_given_direct_check_sql_refs_when_preflighting_then_refs_are_validated_w lifecycle_plan=PythonSqlRunLifecyclePlan( ingress_python_node_names=frozenset(), ingress_loader_names=frozenset(), - read_side_sql_keys=frozenset(), read_side_python_node_names=frozenset(), ), relation_targets={model_ref: "analytics.orders", source_ref: "raw.orders"}, @@ -198,7 +197,6 @@ def test_given_missing_direct_check_sql_ref_when_preflighting_then_existing_rela lifecycle_plan=PythonSqlRunLifecyclePlan( ingress_python_node_names=frozenset(), ingress_loader_names=frozenset(), - read_side_sql_keys=frozenset(), read_side_python_node_names=frozenset(), ), relation_targets={ref: "analytics.orders"}, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py index 968741bc70..384066c5bb 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py @@ -1,32 +1,11 @@ -from collections.abc import Callable from dataclasses import dataclass -from pathlib import Path -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.planner.models import ParsedSelector from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import SqlResourceRef -@dataclass(frozen=True) -class PythonLoaderNodeConversionTestCase: - description: str - expected_kind: PythonNodeKind - expected_file_path: Path - expected_relative_path: Path - expected_name: str - expected_depends_on: tuple[Callable[..., object], ...] - expected_dependency_edges: tuple[tuple[str, str], ...] - expected_target: str | None - expected_write_strategy: str | None - expected_cursor_column: str | None - expected_unique_key: tuple[str, ...] - expected_column_names: tuple[str, ...] - expected_contract: str | None - expected_connection_mode: LoaderConnectionMode - - @dataclass(frozen=True) class PythonNodeGraphInventoryTestCase: description: str @@ -95,7 +74,6 @@ class PythonSqlRunLifecycleTestCase: expected_ingress_python_names: frozenset[str] expected_ingress_loader_names: frozenset[str] expected_read_side_python_names: frozenset[str] - expected_read_side_sql_names: frozenset[str] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_loaders.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_loaders.py deleted file mode 100644 index a9491ea7c6..0000000000 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_loaders.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Tests for loader conversion to internal Python-node models.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes._helpers.loaders import ( - build_python_loader_dependency_edges, - build_python_loader_node, -) -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonLoaderMetadata, - DiscoveredPythonNode, - PythonNodeDependencyEdge, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.spec.contracts.models import SourceColumnEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy -from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( - PythonLoaderNodeConversionTestCase, -) -from tests.unit.src.sqlbuild.compiler.python_nodes._helpers.helpers import ( - fetch_events, - load_events, -) - - -@pytest.mark.parametrize( - "test_case", - [ - PythonLoaderNodeConversionTestCase( - description="preserves source loader metadata in internal Python-node view", - expected_kind=PythonNodeKind.LOADER, - expected_file_path=Path("/project/loaders/events.py"), - expected_relative_path=Path("loaders/events.py"), - expected_name="load_events", - expected_depends_on=(fetch_events,), - expected_dependency_edges=(("fetch_events", "load_events"),), - expected_target="staging.events", - expected_write_strategy="merge", - expected_cursor_column="updated_at", - expected_unique_key=("event_id", "updated_at"), - expected_column_names=("event_id", "updated_at"), - expected_contract="enforced", - expected_connection_mode=LoaderConnectionMode.EXTERNAL, - ) - ], - ids=lambda case: case.description, -) -def test_given_discovered_loader_when_building_python_node_then_preserves_loader_metadata( - test_case: PythonLoaderNodeConversionTestCase, -) -> None: - loader: DiscoveredLoaderFunction = DiscoveredLoaderFunction( - file_path=test_case.expected_file_path, - relative_path=test_case.expected_relative_path, - name=test_case.expected_name, - function=load_events, - depends_on=(fetch_events,), - destination=test_case.expected_target, - write_strategy=SourceWriteStrategy(test_case.expected_write_strategy or "table"), - cursor_column=test_case.expected_cursor_column, - unique_key=test_case.expected_unique_key, - columns=( - SourceColumnEntry(name="event_id", type="BIGINT"), - SourceColumnEntry(name="updated_at", type="TIMESTAMP"), - ), - contract=test_case.expected_contract, - connection_mode=test_case.expected_connection_mode, - ) - - node: DiscoveredPythonNode = build_python_loader_node(loader=loader) - dependency_edges: tuple[PythonNodeDependencyEdge, ...] = build_python_loader_dependency_edges( - loaders=( - DiscoveredLoaderFunction( - file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), - name="fetch_events", - function=fetch_events, - ), - loader, - ) - ) - - assert node.kind == test_case.expected_kind - assert node.file_path == test_case.expected_file_path - assert node.relative_path == test_case.expected_relative_path - assert node.name == test_case.expected_name - assert node.function is load_events - assert node.depends_on == test_case.expected_depends_on - assert node.tags == () - assert node.group is None - assert node.description is None - assert node.loader == DiscoveredPythonLoaderMetadata( - destination=test_case.expected_target, - write_strategy=SourceWriteStrategy(test_case.expected_write_strategy or "table"), - cursor_column=test_case.expected_cursor_column, - unique_key=test_case.expected_unique_key, - columns=loader.columns, - contract=test_case.expected_contract, - connection_mode=test_case.expected_connection_mode, - ) - assert node.loader is not None - assert tuple(column.name for column in node.loader.columns) == test_case.expected_column_names - assert tuple((edge.upstream_name, edge.downstream_name) for edge in dependency_edges) == ( - test_case.expected_dependency_edges - ) - assert dependency_edges[0].upstream_function is fetch_events - assert dependency_edges[0].downstream_function is load_events diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 703e755973..93ac2e1b3b 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -41,7 +41,6 @@ expected_ingress_python_names=frozenset({"prepare_orders", "load_events"}), expected_ingress_loader_names=frozenset({"load_events"}), expected_read_side_python_names=frozenset(), - expected_read_side_sql_names=frozenset({"raw_orders"}), ), PythonSqlRunLifecycleTestCase( description="classifies read only asset into read-side Python", @@ -55,7 +54,6 @@ expected_ingress_python_names=frozenset(), expected_ingress_loader_names=frozenset(), expected_read_side_python_names=frozenset({"export_orders"}), - expected_read_side_sql_names=frozenset({"orders"}), ), PythonSqlRunLifecycleTestCase( description="classifies intermediate loader dependency into pre sql ingress", @@ -73,7 +71,6 @@ expected_ingress_python_names=frozenset({"fetch_pages"}), expected_ingress_loader_names=frozenset({"fetch_pages"}), expected_read_side_python_names=frozenset({"export_orders"}), - expected_read_side_sql_names=frozenset({"fetch_pages"}), ), PythonSqlRunLifecycleTestCase( description="excludes external loader from ingress so it runs pre-connection", @@ -85,7 +82,6 @@ expected_ingress_python_names=frozenset(), expected_ingress_loader_names=frozenset(), expected_read_side_python_names=frozenset(), - expected_read_side_sql_names=frozenset(), ), ], ids=lambda case: case.description, @@ -103,6 +99,3 @@ def test_given_run_selection_when_building_lifecycle_plan_then_classifies_phases assert result.ingress_python_node_names == test_case.expected_ingress_python_names assert result.ingress_loader_names == test_case.expected_ingress_loader_names assert result.read_side_python_node_names == test_case.expected_read_side_python_names - assert frozenset(key.name for key in result.read_side_sql_keys) == ( - test_case.expected_read_side_sql_names - ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py index 38d10e759f..1a1be44d0f 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py @@ -52,8 +52,6 @@ def test_given_python_identity_when_writing_fingerprint_then_uses_fingerprint_st identity=PythonNodeIdentity( node_type="task", node_name="build_orders", - object_module="tasks.orders", - object_qualname="build_orders", source_path="tasks/orders.py", source_hash="source-hash", definition_hash="definition-hash", diff --git a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py b/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py index edfd3b1ba1..f70a8f48a3 100644 --- a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py +++ b/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py @@ -40,8 +40,6 @@ def test_given_virtual_python_identity_when_recording_then_reads_previous_finger identity: PythonNodeIdentity = PythonNodeIdentity( node_type=test_case.expected_node_type, node_name=test_case.expected_node_name, - object_module="tasks.orders", - object_qualname="prepare_orders", source_path="tasks/orders.py", source_hash="source-hash", definition_hash="definition-hash", From 76139aad5a2fb168aa2441b693d55682aa2d8b55 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:54:51 +0100 Subject: [PATCH 27/32] refactor(executor): remove unread microbatch applied intervals ModelExecutionResult.microbatch_applied_intervals and MicrobatchPhaseOutcome.applied_intervals were accumulated for every batch but never read. Remove the fields and their bookkeeping; execution behaviour is unchanged. --- .../run/_helpers/materializations/microbatch.py | 14 -------------- src/sqlbuild/executor/run/models.py | 2 -- 2 files changed, 16 deletions(-) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 0c221d3a3c..69e5b0eee1 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -535,7 +535,6 @@ def execute_microbatch_entry( batch_count=batch_outcome.completed_batches, batch_size=batch_plan.effective_batch_size, rows_affected=batch_outcome.rows_affected, - microbatch_applied_intervals=batch_outcome.applied_intervals, **_microbatch_result_fields(history=history_context, succeeded=False), ) final_audit_run: FinalAuditRun = run_final_scope_audits( @@ -618,7 +617,6 @@ def execute_microbatch_entry( batch_count=batch_outcome.completed_batches, batch_size=batch_plan.effective_batch_size, rows_affected=batch_outcome.rows_affected, - microbatch_applied_intervals=batch_outcome.applied_intervals, cursor_range_start=( None if resolved_range is None else sentinel_to_token(sentinel=resolved_range.start) ), @@ -1040,7 +1038,6 @@ def _execute_microbatch_batches( ) schema_checked: bool = False completed_batches: int = 0 - applied_intervals: list[tuple[str, str]] = [] total_rows: int = 0 row_count_known: bool = False batch: BatchWindow @@ -1124,7 +1121,6 @@ def _execute_microbatch_batches( total_rows=total_rows, row_count_known=row_count_known, ), - applied_intervals=tuple(applied_intervals), ) if isinstance(dml_result, int): total_rows += dml_result @@ -1145,7 +1141,6 @@ def _execute_microbatch_batches( total_rows=total_rows, row_count_known=row_count_known, ), - applied_intervals=tuple(applied_intervals), ) completion_failure: ModelExecutionResult | None = ( None @@ -1182,7 +1177,6 @@ def _execute_microbatch_batches( raise lifecycle.completed(affected_rows=dml_result if isinstance(dml_result, int) else None) completed_batches += 1 - applied_intervals.append((render(value=batch.start), render(value=batch.end))) if on_progress is not None: batch_elapsed: float = time.monotonic() - batch_start_time on_progress( @@ -1195,7 +1189,6 @@ def _execute_microbatch_batches( total_rows=total_rows, row_count_known=row_count_known, ), - applied_intervals=tuple(applied_intervals), ) @@ -1361,7 +1354,6 @@ def _execute_microbatch_batches_concurrently( not in history_context.recovery_intervals ) completed_batches: int = 0 - applied_intervals: list[tuple[str, str]] = [] total_rows: int = 0 row_count_known: bool = False for phase_batches in (recovery_batches, ordinary_batches): @@ -1379,7 +1371,6 @@ def _execute_microbatch_batches_concurrently( ) state = phase_outcome.state completed_batches += phase_outcome.completed_batches - applied_intervals.extend(phase_outcome.applied_intervals) if phase_outcome.rows_affected is not None: total_rows += phase_outcome.rows_affected row_count_known = True @@ -1391,7 +1382,6 @@ def _execute_microbatch_batches_concurrently( rows_affected=_reported_rows_affected( total_rows=total_rows, row_count_known=row_count_known ), - applied_intervals=tuple(applied_intervals), ) return MicrobatchPhaseOutcome( state=state, @@ -1399,7 +1389,6 @@ def _execute_microbatch_batches_concurrently( rows_affected=_reported_rows_affected( total_rows=total_rows, row_count_known=row_count_known ), - applied_intervals=tuple(applied_intervals), ) @@ -1468,14 +1457,12 @@ def execute(*, batch: BatchWindow, connection: Any) -> MicrobatchPhaseOutcome: lambda batch, connection: execute(batch=batch, connection=connection), ) completed_batches: int = first_outcome.completed_batches - applied_intervals: list[tuple[str, str]] = list(first_outcome.applied_intervals) total_rows: int = first_outcome.rows_affected or 0 row_count_known: bool = first_outcome.rows_affected is not None failure: ModelExecutionResult | None = None for outcome in outcomes: aggregate_state = _merge_microbatch_states(target=aggregate_state, source=outcome.state) completed_batches += outcome.completed_batches - applied_intervals.extend(outcome.applied_intervals) if outcome.rows_affected is not None: total_rows += outcome.rows_affected row_count_known = True @@ -1488,7 +1475,6 @@ def execute(*, batch: BatchWindow, connection: Any) -> MicrobatchPhaseOutcome: rows_affected=_reported_rows_affected( total_rows=total_rows, row_count_known=row_count_known ), - applied_intervals=tuple(applied_intervals), ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 95f83baffa..52c4b70829 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -179,7 +179,6 @@ class ModelExecutionResult: microbatch_accounting_intervals: tuple[MicrobatchAccountingInterval, ...] = field( default_factory=tuple ) - microbatch_applied_intervals: tuple[tuple[str, str], ...] = field(default_factory=tuple) microbatch_limit: int | None = None microbatch_limit_count: int | None = None microbatch_limit_action: MicrobatchLimitAction | None = None @@ -554,7 +553,6 @@ class MicrobatchPhaseOutcome: failure: ModelExecutionResult | None = None completed_batches: int = 0 rows_affected: int | None = None - applied_intervals: tuple[tuple[str, str], ...] = field(default_factory=tuple) @dataclass(frozen=True) From e0221af34f276283745dba51205ff55fb319d4c2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:56:23 +0100 Subject: [PATCH 28/32] refactor(virtual): share ref model-version lookup virtual diff and promote each carried an identical private _read_model_versions. Both now use one read_ref_model_versions entry in virtual/state/main/environments. --- src/sqlbuild/virtual/diff/_helpers/diff.py | 26 ++++-------------- .../virtual/executor/_helpers/promote.py | 25 ++++------------- .../main/environments/_ref_model_versions.py | 27 +++++++++++++++++++ 3 files changed, 37 insertions(+), 41 deletions(-) create mode 100644 src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py diff --git a/src/sqlbuild/virtual/diff/_helpers/diff.py b/src/sqlbuild/virtual/diff/_helpers/diff.py index d7d5aa7b60..cc5519b35a 100644 --- a/src/sqlbuild/virtual/diff/_helpers/diff.py +++ b/src/sqlbuild/virtual/diff/_helpers/diff.py @@ -23,8 +23,10 @@ from sqlbuild.virtual.planner.main._semantics import build_virtual_plan_semantics from sqlbuild.virtual.planner.main._targets import build_virtual_destination_from_physical_relation from sqlbuild.virtual.planner.models import VirtualPlanSemantics +from sqlbuild.virtual.state.main.environments._ref_model_versions import ( + read_ref_model_versions, +) from sqlbuild.virtual.state.models import ( - ModelVersionRecord, PhysicalRelationRecord, VirtualEnvironmentModelRefRecord, VirtualEnvironmentRecord, @@ -177,7 +179,7 @@ def read_virtual_diff_state( from_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( graph=graph, bound_refs=from_refs, - bound_model_versions=_read_model_versions( + bound_model_versions=read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, @@ -192,7 +194,7 @@ def read_virtual_diff_state( to_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( graph=graph, bound_refs=to_refs, - bound_model_versions=_read_model_versions( + bound_model_versions=read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, @@ -240,24 +242,6 @@ def read_virtual_diff_state( ) -def _read_model_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, ModelVersionRecord | None]: - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - } - - def execute_virtual_diff_between_relations( *, adapter: BaseAdapter, diff --git a/src/sqlbuild/virtual/executor/_helpers/promote.py b/src/sqlbuild/virtual/executor/_helpers/promote.py index 2bde285f97..ef23e8b9d9 100644 --- a/src/sqlbuild/virtual/executor/_helpers/promote.py +++ b/src/sqlbuild/virtual/executor/_helpers/promote.py @@ -25,6 +25,9 @@ from sqlbuild.virtual.state.main.checkpoints._checkpoints import ( create_finalized_virtual_environment_checkpoint, ) +from sqlbuild.virtual.state.main.environments._ref_model_versions import ( + read_ref_model_versions, +) from sqlbuild.virtual.state.models import ( FunctionVersionRecord, ModelVersionRecord, @@ -152,13 +155,13 @@ def build_promote_semantics( ) -> PromoteSemantics: """Build source and target virtual plan semantics from bound refs.""" - source_versions: dict[str, ModelVersionRecord | None] = _read_model_versions( + source_versions: dict[str, ModelVersionRecord | None] = read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, refs=environment_state.source_refs, ) - target_versions: dict[str, ModelVersionRecord | None] = _read_model_versions( + target_versions: dict[str, ModelVersionRecord | None] = read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, @@ -510,24 +513,6 @@ def _stale_models_after_promotion( ) -def _read_model_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, ModelVersionRecord | None]: - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - } - - def _read_physical_relations( *, backend: Any, diff --git a/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py b/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py new file mode 100644 index 0000000000..5dba51746c --- /dev/null +++ b/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py @@ -0,0 +1,27 @@ +"""Public helper resolving model-version records for environment refs.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.virtual.state.models import ModelVersionRecord, VirtualEnvironmentModelRefRecord + + +def read_ref_model_versions( + *, + backend: Any, + state_connection: Any, + schema: str, + refs: tuple[VirtualEnvironmentModelRefRecord, ...], +) -> dict[str, ModelVersionRecord | None]: + """Return the model-version record bound by each ref, keyed by model name.""" + + return { + ref.model_name: backend.get_model_version( + connection=state_connection, + schema=schema, + model_name=ref.model_name, + version_hash=ref.version_hash, + ) + for ref in refs + } From 7df49f22be9bd0f80dc1c8ec9b7926d63b1bb276 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 04:57:07 +0100 Subject: [PATCH 29/32] refactor(planner): rename stale SchemaColumnSource.SQLGLOT member sqlglot was removed as a runtime dependency; the member is now SQL_ANALYSIS. Only the Python name changes: the enum value stays "sql_analysis", so nothing serialised or displayed changes. --- .../compiler/planner/_helpers/changes/schema.py | 8 +++++--- .../compiler/planner/_helpers/output/strategy.py | 2 +- src/sqlbuild/compiler/planner/types.py | 2 +- .../planner/_helpers/changes/test_schema.py | 14 +++++++------- .../planner/_helpers/test_strategy_warnings.py | 4 ++-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py index c3b405e6c4..6da7f7e8f8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py @@ -119,7 +119,9 @@ def detect_schema_changes( kind=SchemaChangeKind.COLUMN_REMOVED, column_name=col_name, source=( - SchemaColumnSource.YML if yml_columns else SchemaColumnSource.SQLGLOT + SchemaColumnSource.YML + if yml_columns + else SchemaColumnSource.SQL_ANALYSIS ), actual_type=col_type, ) @@ -216,7 +218,7 @@ def _compare_inferred_columns( SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name=col.name, - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type=col.type, ) ) @@ -227,7 +229,7 @@ def _compare_inferred_columns( SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name=col.name, - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type=col.type, actual_type=warehouse_map[col.name], ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index c124677f32..94dc76a815 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -165,7 +165,7 @@ def build_model_warnings( ), ) ) - elif finding.source == SchemaColumnSource.SQLGLOT: + elif finding.source == SchemaColumnSource.SQL_ANALYSIS: if finding.kind in ( SchemaChangeKind.COLUMN_ADDED, SchemaChangeKind.COLUMN_TYPE_CHANGED, diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index 8cf32feb3f..17138ae5ae 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -91,7 +91,7 @@ class SchemaChangeKind(StrEnum): class SchemaColumnSource(StrEnum): YML = "yml" - SQLGLOT = "sql_analysis" + SQL_ANALYSIS = "sql_analysis" class RetentionDirection(StrEnum): diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index 6ef31435d4..d28471cad1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -95,7 +95,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name="new_col", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, ), ), ), @@ -109,7 +109,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="DECIMAL(10, 2)", actual_type="INTEGER", ), @@ -149,7 +149,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="DECIMAL(10, 2)", actual_type="decimal(10,3)", ), @@ -189,7 +189,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="FLOAT", actual_type="INTEGER", ), @@ -234,7 +234,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_REMOVED, column_name="old_col", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, actual_type="BOOLEAN", ), ), @@ -256,14 +256,14 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="FLOAT", actual_type="INTEGER", ), SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name="loaded_at", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="TIMESTAMP", ), ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index f1a55b2c00..24d7d390b6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -44,7 +44,7 @@ _TYPE_CHANGED_SQLGLOT_FINDING: SchemaFinding = SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="DECIMAL", actual_type="INTEGER", ) @@ -52,7 +52,7 @@ _ADDED_SQLGLOT_FINDING: SchemaFinding = SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name="new_col", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="VARCHAR", ) From 4cc45dddc987cb114e330edea000511b69336e4c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 05:09:26 +0100 Subject: [PATCH 30/32] fix(state): type bulk microbatch event inserts on Postgres --- .../virtual/state/classes/postgres.py | 14 ++++- .../virtual/state/classes/_test_types.py | 9 +++ .../sqlbuild/virtual/state/classes/helpers.py | 56 +++++++++++++++++ .../state/classes/postgres/test_postgres.py | 60 ++++++++++++++++++- .../virtual/state/classes/test_duckdb.py | 55 ++++++++++++++++- 5 files changed, 191 insertions(+), 3 deletions(-) diff --git a/src/sqlbuild/virtual/state/classes/postgres.py b/src/sqlbuild/virtual/state/classes/postgres.py index 867e80760e..8033f0a217 100644 --- a/src/sqlbuild/virtual/state/classes/postgres.py +++ b/src/sqlbuild/virtual/state/classes/postgres.py @@ -196,7 +196,19 @@ def append_microbatch_events( event for event in events if event.event_id not in existing_ids ) if missing: - row_placeholders: str = "(" + ", ".join("%s" for _ in MICROBATCH_COLUMNS) + ")" + # VALUES resolves all-NULL columns before the INSERT target can type them. + row_placeholders: str = ( + "(" + + ", ".join( + f"%s::{ + self._state_column_sql_type( + STATE_TABLE_COLUMNS[MICROBATCH_EVENT_TABLE][column] + ) + }" + for column in MICROBATCH_COLUMNS + ) + + ")" + ) values_sql: str = ", ".join(row_placeholders for _ in missing) columns: str = ", ".join(MICROBATCH_COLUMNS) params: list[object | None] = [] diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py b/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py index f1ae320a32..78feb6ef37 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py @@ -2,6 +2,7 @@ from dataclasses import dataclass +from sqlbuild.microbatches.types import MicrobatchRecordType from sqlbuild.virtual.state.types import VirtualEnvironmentStatus from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( ConditionalPublicationPayload, @@ -272,3 +273,11 @@ class DuckDbStateBackendRefContractTestCase: expected_observation: StateRefContractObservation schema: str sqlbuild_version: str + + +@dataclass(frozen=True) +class MicrobatchBulkAppendTestCase: + """One production-shaped microbatch event batch written through the bulk append.""" + + description: str + record_type: MicrobatchRecordType diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py index 7d28f8cf16..473a56716a 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py @@ -5,6 +5,13 @@ from pathlib import Path from typing import Any, NamedTuple +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope +from sqlbuild.microbatches.types import ( + MicrobatchCompletionType, + MicrobatchFingerprintStatus, + MicrobatchRecordType, + MicrobatchRunType, +) from sqlbuild.virtual.state.classes.duckdb import DuckDbStateBackend from sqlbuild.virtual.state.classes.state_backend import StateBackend from sqlbuild.virtual.state.models import ( @@ -451,3 +458,52 @@ def exercise_state_ref_contract( deleted_model_refs=(), surviving_model_refs=(VirtualEnvironmentNodeRefRecord("qa", "model", "orders", "orders-v3"),), ) + + +def build_production_shaped_microbatch_event( + *, scope: MicrobatchScope, record_type: MicrobatchRecordType +) -> MicrobatchEvent: + """Build one event with the nullable columns the executor leaves unset for its kind. + + Replay requirements carry no completion or observation facts, and executed partition + completions carry no observation facts, so each single-event batch has timestamp and + integer columns that are NULL in every row. + """ + + run_started_at: datetime = datetime(2026, 1, 1, 9, 0, 0) + common: dict[str, Any] = { + "scope": scope, + "origin_run_id": "run-1", + "execution_run_id": "run-1", + "run_type": MicrobatchRunType.NORMAL, + "run_start": "0", + "run_end": "2", + "batch_size": "1", + "cursor_column": "batch_id", + "cursor_type": "integer", + "model_version_hash": "F2", + "definition_hash": "definition", + "fingerprint_status": MicrobatchFingerprintStatus.KNOWN, + "origin_run_started_at": run_started_at, + "execution_run_started_at": run_started_at, + "created_at": datetime(2026, 1, 1, 9, 0, 1), + } + if record_type == MicrobatchRecordType.REPLAY_REQUIREMENT: + return MicrobatchEvent( + event_id="requirement-1", + record_type=record_type, + replay_requirement_id="requirement-1", + required_model_version_hash="F2", + replay_policy="full", + **common, + ) + return MicrobatchEvent( + event_id="completion-1", + record_type=record_type, + completion_type=MicrobatchCompletionType.INITIAL, + partition_start="0", + partition_end="1", + rows_affected=3, + completed_at=datetime(2026, 1, 1, 9, 0, 2), + **common, + ) diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py index 7d908c0944..f22bbe2d47 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py @@ -14,7 +14,7 @@ NodeResultRecord, ) from sqlbuild.executor.node_results.types import NodeResultStatus -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult from sqlbuild.microbatches.types import ( MicrobatchCompletionType, MicrobatchFingerprintStatus, @@ -57,6 +57,9 @@ StateSchemaValidationIssueKind, VirtualEnvironmentStatus, ) +from tests.integration.src.sqlbuild.virtual.state.classes._test_types import ( + MicrobatchBulkAppendTestCase, +) from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( ACTIVE_CHECKPOINT_PAYLOAD, ACTIVE_PAYLOAD, @@ -80,6 +83,7 @@ VALID_PAYLOAD, StateReadContractObservation, StateRefContractObservation, + build_production_shaped_microbatch_event, exercise_state_read_contract, exercise_state_ref_contract, ) @@ -361,6 +365,60 @@ def test_given_invalid_conditional_payload_when_postgres_publishes_then_contract ) +@pytest.mark.parametrize( + "test_case", + [ + MicrobatchBulkAppendTestCase( + description="replay requirement with no completion or observation facts", + record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, + ), + MicrobatchBulkAppendTestCase( + description="executed partition completion with no observation facts", + record_type=MicrobatchRecordType.PARTITION_COMPLETION, + ), + ], + ids=lambda case: case.description, +) +def test_given_all_null_nullable_columns_when_bulk_appending_then_postgres_persists_idempotently( + test_case: MicrobatchBulkAppendTestCase, + postgres_state_backend: PostgresStateBackend, + postgres_state_connection: Any, + postgres_state_schema: str, +) -> None: + postgres_state_backend.initialize( + connection=postgres_state_connection, + schema=postgres_state_schema, + sqlbuild_version="test", + ) + scope: MicrobatchScope = MicrobatchScope( + scope_kind="virtual_physical", + scope_key="postgres:state:orders:F2:analytics.orders__F2", + model_name="orders", + target_database="warehouse", + target_schema="analytics", + target_name="orders__F2", + physical_generation_id="F2:analytics.orders__F2", + virtual_environment_name="dev", + virtual_model_version_hash="F2", + ) + event: MicrobatchEvent = build_production_shaped_microbatch_event( + scope=scope, record_type=test_case.record_type + ) + + first: MicrobatchWriteResult = postgres_state_backend.append_microbatch_events( + connection=postgres_state_connection, schema=postgres_state_schema, events=(event,) + ) + second: MicrobatchWriteResult = postgres_state_backend.append_microbatch_events( + connection=postgres_state_connection, schema=postgres_state_schema, events=(event,) + ) + + history: tuple[MicrobatchEvent, ...] = postgres_state_backend.read_microbatch_scope_history( + connection=postgres_state_connection, schema=postgres_state_schema, scope=scope + ) + assert (first.inserted, second.already_existing) == (1, 1) + assert history == (event,) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py index cc526ca62b..462e3b0036 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py @@ -22,7 +22,7 @@ NodeResultRecord, ) from sqlbuild.executor.node_results.types import NodeResultStatus -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult from sqlbuild.microbatches.types import ( MicrobatchCompletionType, MicrobatchFingerprintStatus, @@ -94,6 +94,7 @@ DuckDbStateBackendTableCreationTestCase, DuckDbStateBackendTransactionRollbackTestCase, DuckDbStateBackendValidationTestCase, + MicrobatchBulkAppendTestCase, MicrobatchStateRoundTripTestCase, StateReadContractTestCase, ) @@ -120,6 +121,7 @@ VALID_PAYLOAD, StateReadContractObservation, StateRefContractObservation, + build_production_shaped_microbatch_event, exercise_state_read_contract, exercise_state_ref_contract, fetch_all, @@ -288,6 +290,57 @@ def test_given_invalid_conditional_payload_when_duckdb_publishes_then_contract_r ) +@pytest.mark.parametrize( + "test_case", + [ + MicrobatchBulkAppendTestCase( + description="replay requirement with no completion or observation facts", + record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, + ), + MicrobatchBulkAppendTestCase( + description="executed partition completion with no observation facts", + record_type=MicrobatchRecordType.PARTITION_COMPLETION, + ), + ], + ids=lambda case: case.description, +) +def test_given_all_null_nullable_columns_when_bulk_appending_then_duckdb_persists_idempotently( + test_case: MicrobatchBulkAppendTestCase, tmp_path: Path +) -> None: + backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") + scope: MicrobatchScope = MicrobatchScope( + scope_kind="virtual_physical", + scope_key="duckdb:state:orders:F2:main.orders__F2", + model_name="orders", + target_database=None, + target_schema="main", + target_name="orders__F2", + physical_generation_id="F2:main.orders__F2", + virtual_environment_name="dev", + virtual_model_version_hash="F2", + ) + event: MicrobatchEvent = build_production_shaped_microbatch_event( + scope=scope, record_type=test_case.record_type + ) + try: + backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="test") + + first: MicrobatchWriteResult = backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) + ) + second: MicrobatchWriteResult = backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) + ) + history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_scope_history( + connection=connection, schema="sqlbuild_state", scope=scope + ) + finally: + backend.close(connection) + + assert (first.inserted, second.already_existing) == (1, 1) + assert history == (event,) + + @pytest.mark.parametrize( "test_case", [ From 90021a08f79a2bd1a751cc48e2c09fc8594a5cb6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 05:12:46 +0100 Subject: [PATCH 31/32] refactor(state): remove single microbatch event append --- .../state_storage/microbatch_events.py | 14 ---------- src/sqlbuild/virtual/state/classes/duckdb.py | 10 ------- .../virtual/state/classes/postgres.py | 15 ---------- .../virtual/state/classes/state_backend.py | 6 ---- .../virtual/state/classes/_test_types.py | 1 + .../sqlbuild/virtual/state/classes/helpers.py | 28 ++++++++++--------- .../state/classes/postgres/_test_types.py | 10 +++++++ .../state/classes/postgres/test_postgres.py | 18 ++++++------ .../virtual/state/classes/test_duckdb.py | 14 ++++++---- 9 files changed, 43 insertions(+), 73 deletions(-) diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py b/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py index 26040a051d..a9acb526ac 100644 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py +++ b/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py @@ -13,20 +13,6 @@ from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -def append_duckdb_microbatch_event( - *, connection: Any, qualified_table: str, event: MicrobatchEvent -) -> None: - """Append one idempotent logical event to DuckDB state.""" - - placeholders: str = ", ".join("?" for _ in MICROBATCH_COLUMNS) - connection.execute( - f"INSERT INTO {qualified_table} ({', '.join(MICROBATCH_COLUMNS)}) " - f"SELECT {placeholders} WHERE NOT EXISTS " - f"(SELECT 1 FROM {qualified_table} WHERE event_id = ?)", - [*MicrobatchEventCodec.values(event), event.event_id], - ) - - def append_duckdb_microbatch_events( *, connection: Any, qualified_table: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: diff --git a/src/sqlbuild/virtual/state/classes/duckdb.py b/src/sqlbuild/virtual/state/classes/duckdb.py index d4edb7945f..0f04be4598 100644 --- a/src/sqlbuild/virtual/state/classes/duckdb.py +++ b/src/sqlbuild/virtual/state/classes/duckdb.py @@ -13,7 +13,6 @@ to_naive_utc_wall_clock, ) from sqlbuild.virtual.state._helpers.state_storage.microbatch_events import ( - append_duckdb_microbatch_event, append_duckdb_microbatch_events, read_duckdb_microbatch_model_history, read_duckdb_microbatch_retention_history, @@ -140,15 +139,6 @@ def inspect_schema(self, *, connection: Any, schema: str) -> StateSchemaValidati existing_indexes_by_table=indexes_by_table, ) - def append_microbatch_event( - self, *, connection: Any, schema: str, event: MicrobatchEvent - ) -> None: - append_duckdb_microbatch_event( - connection=connection, - qualified_table=self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE), - event=event, - ) - def append_microbatch_events( self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: diff --git a/src/sqlbuild/virtual/state/classes/postgres.py b/src/sqlbuild/virtual/state/classes/postgres.py index 8033f0a217..ac11fb12ec 100644 --- a/src/sqlbuild/virtual/state/classes/postgres.py +++ b/src/sqlbuild/virtual/state/classes/postgres.py @@ -165,20 +165,6 @@ def inspect_schema(self, *, connection: Any, schema: str) -> StateSchemaValidati existing_indexes_by_table=indexes_by_table, ) - def append_microbatch_event( - self, *, connection: Any, schema: str, event: MicrobatchEvent - ) -> None: - placeholders: str = ", ".join("%s" for _ in MICROBATCH_COLUMNS) - with connection.cursor() as cursor: - cursor.execute( - f"INSERT INTO {self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - f"({', '.join(MICROBATCH_COLUMNS)}) SELECT {placeholders} " - "WHERE NOT EXISTS (SELECT 1 FROM " - f"{self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - "WHERE event_id = %s)", - [*MicrobatchEventCodec.values(event), event.event_id], - ) - def append_microbatch_events( self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: @@ -196,7 +182,6 @@ def append_microbatch_events( event for event in events if event.event_id not in existing_ids ) if missing: - # VALUES resolves all-NULL columns before the INSERT target can type them. row_placeholders: str = ( "(" + ", ".join( diff --git a/src/sqlbuild/virtual/state/classes/state_backend.py b/src/sqlbuild/virtual/state/classes/state_backend.py index 199d6a661d..45bcf63860 100644 --- a/src/sqlbuild/virtual/state/classes/state_backend.py +++ b/src/sqlbuild/virtual/state/classes/state_backend.py @@ -167,12 +167,6 @@ def read_node_results( """Read runtime node result rows for one virtual environment identity.""" ... - def append_microbatch_event( - self, *, connection: Any, schema: str, event: MicrobatchEvent - ) -> None: - """Append one logical microbatch event to virtual state.""" - raise MicrobatchStateError("virtual state backend does not support microbatch events") - def append_microbatch_events( self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py b/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py index 78feb6ef37..27c66108f4 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py @@ -281,3 +281,4 @@ class MicrobatchBulkAppendTestCase: description: str record_type: MicrobatchRecordType + expected_write_counts: tuple[int, int] = (1, 1) diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py index 473a56716a..1d0ba81788 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py @@ -488,22 +488,24 @@ def build_production_shaped_microbatch_event( "execution_run_started_at": run_started_at, "created_at": datetime(2026, 1, 1, 9, 0, 1), } - if record_type == MicrobatchRecordType.REPLAY_REQUIREMENT: - return MicrobatchEvent( + events: dict[MicrobatchRecordType, MicrobatchEvent] = { + MicrobatchRecordType.REPLAY_REQUIREMENT: MicrobatchEvent( event_id="requirement-1", record_type=record_type, replay_requirement_id="requirement-1", required_model_version_hash="F2", replay_policy="full", **common, - ) - return MicrobatchEvent( - event_id="completion-1", - record_type=record_type, - completion_type=MicrobatchCompletionType.INITIAL, - partition_start="0", - partition_end="1", - rows_affected=3, - completed_at=datetime(2026, 1, 1, 9, 0, 2), - **common, - ) + ), + MicrobatchRecordType.PARTITION_COMPLETION: MicrobatchEvent( + event_id="completion-1", + record_type=record_type, + completion_type=MicrobatchCompletionType.INITIAL, + partition_start="0", + partition_end="1", + rows_affected=3, + completed_at=datetime(2026, 1, 1, 9, 0, 2), + **common, + ), + } + return events[record_type] diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py index f993dd7f92..a77e8bf730 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py @@ -2,6 +2,7 @@ from dataclasses import dataclass +from sqlbuild.microbatches.types import MicrobatchRecordType from sqlbuild.virtual.state.types import VirtualEnvironmentStatus from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( ConditionalPublicationPayload, @@ -9,6 +10,15 @@ ) +@dataclass(frozen=True) +class MicrobatchBulkAppendTestCase: + """One production-shaped event batch and its idempotent write counts.""" + + description: str + record_type: MicrobatchRecordType + expected_write_counts: tuple[int, int] = (1, 1) + + @dataclass(frozen=True) class PostgresConditionalVirtualRefPublishTestCase: """Expected conditional virtual-ref publication results.""" diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py index f22bbe2d47..492c20ad0c 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py @@ -57,9 +57,6 @@ StateSchemaValidationIssueKind, VirtualEnvironmentStatus, ) -from tests.integration.src.sqlbuild.virtual.state.classes._test_types import ( - MicrobatchBulkAppendTestCase, -) from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( ACTIVE_CHECKPOINT_PAYLOAD, ACTIVE_PAYLOAD, @@ -88,6 +85,7 @@ exercise_state_ref_contract, ) from tests.integration.src.sqlbuild.virtual.state.classes.postgres._test_types import ( + MicrobatchBulkAppendTestCase, PostgresAtomicFinalizedVirtualPublishTestCase, PostgresConditionalPublicationPayloadContractTestCase, PostgresConditionalVirtualRefPublishTestCase, @@ -150,10 +148,10 @@ def test_given_retired_postgres_rows_when_reading_history_then_only_active_event ) scope: MicrobatchScope = postgres_microbatch_scope() event: MicrobatchEvent = postgres_microbatch_event(scope=scope, event_id="active-event") - postgres_state_backend.append_microbatch_event( + postgres_state_backend.append_microbatch_events( connection=postgres_state_connection, schema=postgres_state_schema, - event=event, + events=(event,), ) insert_raw_postgres_microbatch_record( connection=postgres_state_connection, @@ -415,7 +413,7 @@ def test_given_all_null_nullable_columns_when_bulk_appending_then_postgres_persi history: tuple[MicrobatchEvent, ...] = postgres_state_backend.read_microbatch_scope_history( connection=postgres_state_connection, schema=postgres_state_schema, scope=scope ) - assert (first.inserted, second.already_existing) == (1, 1) + assert (first.inserted, second.already_existing) == test_case.expected_write_counts assert history == (event,) @@ -473,15 +471,15 @@ def test_given_postgres_microbatch_event_when_appending_then_scope_history_round created_at=datetime(2026, 1, 1), ) - postgres_state_backend.append_microbatch_event( + postgres_state_backend.append_microbatch_events( connection=postgres_state_connection, schema=postgres_state_schema, - event=event, + events=(event,), ) - postgres_state_backend.append_microbatch_event( + postgres_state_backend.append_microbatch_events( connection=postgres_state_connection, schema=postgres_state_schema, - event=event, + events=(event,), ) history: tuple[MicrobatchEvent, ...] = postgres_state_backend.read_microbatch_scope_history( diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py index 462e3b0036..4c8e46ecbb 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py @@ -337,7 +337,7 @@ def test_given_all_null_nullable_columns_when_bulk_appending_then_duckdb_persist finally: backend.close(connection) - assert (first.inserted, second.already_existing) == (1, 1) + assert (first.inserted, second.already_existing) == test_case.expected_write_counts assert history == (event,) @@ -389,12 +389,16 @@ def test_given_virtual_microbatch_event_when_appending_then_scope_history_round_ ) try: backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="test") - backend.append_microbatch_event(connection=connection, schema="sqlbuild_state", event=event) + backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) + ) second_event: MicrobatchEvent = replace(event, event_id="event-2") - backend.append_microbatch_event( - connection=connection, schema="sqlbuild_state", event=second_event + backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(second_event,) + ) + backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) ) - backend.append_microbatch_event(connection=connection, schema="sqlbuild_state", event=event) history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_scope_history( connection=connection, schema="sqlbuild_state", scope=scope From 126c2a55a112a8e5a021718fa57dea8d20ae53f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 25 Sep 2026 05:29:10 +0100 Subject: [PATCH 32/32] test(ci): run Python node identity regression in build-core shard --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 124fe03891..9b29cb0610 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py \