From 47495ddf6104f85c5e4b5945914ed13743d6affb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:48:35 +0100 Subject: [PATCH] feat(native): report SQL test and scenario errors natively Scenario extraction, scope relationship scans and macro-test CTE scans return Python's exact scanner and classification errors from one native port of Python's top-level CTE scanner, under the adapter's lexical rules. Scenario source checks no longer fall back to Python on a reference error. Only Polyglot steps and non-ASCII classification still run in Python. --- Cargo.lock | 2 + crates/README.md | 6 +- crates/sqlbuild-analysis/Cargo.toml | 1 + .../compiler/_helpers/sql_tests/extraction.rs | 46 +--- .../_helpers/sql_tests/scenario_extraction.rs | 214 +++++++++++----- .../compiler/main/sql_scenario_extraction.rs | 12 +- .../tests/test_sql_scenario_extraction.rs | 75 +++++- .../bindings/_helpers/analysis/sql_tests.rs | 8 +- .../_helpers/scopes/relationship_names.rs | 59 ++++- crates/sqlbuild-scopes/Cargo.toml | 1 + .../relationship_names/_helpers/cte_scan.rs | 240 +++++++++++++----- .../main/expected_model_names.rs | 28 +- .../src/relationship_names/main/mod.rs | 1 + .../relationship_names/main/top_level_ctes.rs | 20 ++ .../src/relationship_names/models.rs | 28 +- .../src/relationship_names/tests/helpers.rs | 52 +++- .../tests/test_expected_model_names.rs | 171 ++++++++++--- .../relationship_names/tests/test_types.rs | 22 +- .../corpus/attachment_failure_cases.py | 102 +++++++- .../_helpers/corpus/scope_failure_cases.py | 27 ++ src/sqlbuild/_native.pyi | 11 +- .../compile/_helpers/analysis/ctes.py | 4 +- .../_helpers/attachment/declaration_scope.py | 2 +- .../attachment/scope_relationships.py | 84 +++++- .../compile/_helpers/attachment/sql_tests.py | 87 ++++--- .../compile/_helpers/scenarios/core.py | 76 +++++- .../_helpers/sql_tests/extraction_errors.py | 5 +- .../compile/_helpers/sql_tests/native.py | 21 +- .../main/_native_expected_model_names.py | 19 +- .../main/compile/sql_test_errors/__init__.py | 0 .../compile/sql_test_errors/_test_types.py | 12 + .../test_native_sql_test_errors.py | 153 +++++++++++ .../compiler/attachments/_test_types.py | 1 + .../sqlbuild/compiler/attachments/helpers.py | 50 +++- .../test_native_attachment_project.py | 71 ++++++ .../test_native_scenario_extraction.py | 23 +- .../sqlbuild/compiler/scopes/_test_types.py | 1 + .../src/sqlbuild/compiler/scopes/helpers.py | 103 ++++++-- .../scopes/test_native_relationship_names.py | 13 +- .../scopes/test_native_scope_parity.py | 45 ++++ 40 files changed, 1523 insertions(+), 373 deletions(-) create mode 100644 crates/sqlbuild-scopes/src/relationship_names/main/top_level_ctes.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/__init__.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/test_native_sql_test_errors.py diff --git a/Cargo.lock b/Cargo.lock index 60df0ccbff..dc37ec85f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -703,6 +703,7 @@ dependencies = [ "serde_json", "sha2", "sqlbuild-core", + "sqlbuild-scopes", "sqlbuild-sqltext", ] @@ -810,6 +811,7 @@ dependencies = [ name = "sqlbuild-scopes" version = "0.143.1" dependencies = [ + "sqlbuild-core", "sqlbuild-sqltext", ] diff --git a/crates/README.md b/crates/README.md index 05398296bd..2848f75818 100644 --- a/crates/README.md +++ b/crates/README.md @@ -11,11 +11,11 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. | -| `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | +| `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models and top-level CTEs, with Python's scanner errors. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | | `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events, which can carry results across compiles through the shared native store. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-attachments` | Compile attachments for tests, audits, sources, functions, scenarios and seeds: attached generic audit argument merge, raw and quoted argument rendering, and severity and run scope resolution. Anything it cannot reproduce exactly defers to Python, which also raises every attachment error. | -| `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, type normalization, and SQL-test extraction, planning and rendering. Type normalization defers to Python for anything it cannot reproduce exactly. | +| `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, type normalization, and SQL-test and scenario extraction, planning and rendering. Type normalization defers to Python for anything it cannot reproduce exactly. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -31,7 +31,7 @@ sqlbuild-python ``` `sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and -`sqlbuild-scopes` uses only `sqlbuild-sqltext`; their place in the order fixes which crates may +`sqlbuild-scopes` uses only `sqlbuild-sqltext` and `sqlbuild-core`; their place in the order fixes which crates may use them. The JSON emitter lives in `sqlbuild-core` so that any layer can produce text that must equal Python's `json.dumps` output. diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 215d01af4c..12bdc8ea41 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -21,6 +21,7 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-core.workspace = true +sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true [lints] diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index 215f896d8a..e070c2d140 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -69,15 +69,6 @@ struct TestRequest { authored: bool, } -/// How strictly top-level CTE names are read. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum NameRule { - /// Unquoted ASCII identifiers only; anything else is an error naming the CTE. - Plain, - /// Quoted and Unicode identifiers are read, and the caller decides what to do with them. - Lenient, -} - /// One extraction error, with help and the offending text where the author can act on it. #[derive(Debug, PartialEq, Eq)] pub(crate) struct Failure { @@ -198,8 +189,7 @@ fn extract_test(test: &TestRequest, syntax: &LexicalSyntax) -> Result = scan_ctes(&test.sql, &test.file_label, syntax)?.ctes; let scope = TestScope { file: &test.file_label, syntax, @@ -214,7 +204,7 @@ fn extract_test(test: &TestRequest, syntax: &LexicalSyntax) -> Result Result { - match scan_ctes(&test.sql, &test.file_label, syntax, NameRule::Plain) { + match scan_ctes(&test.sql, &test.file_label, syntax) { Ok(scanned) => Ok(Classified::Authored { ctes: scanned .ctes @@ -228,31 +218,14 @@ fn authored_ctes(test: &TestRequest, syntax: &LexicalSyntax) -> Result Result<(Vec, bool), Failure> { - scan_ctes(sql, file, syntax, rule).map(|scanned| (scanned.ctes, scanned.quoted)) -} - /// The top-level CTEs of one test, with the code-point offset of each stripped body. struct ScannedCtes { ctes: Vec, - quoted: bool, body_starts: Vec, } -fn scan_ctes( - sql: &str, - file: &str, - syntax: &LexicalSyntax, - rule: NameRule, -) -> Result { +fn scan_ctes(sql: &str, file: &str, syntax: &LexicalSyntax) -> Result { let mut body_starts: Vec = Vec::new(); - let mut quoted = false; let mut index = skip_ignorable(sql, 0, syntax)?; index = consume_keyword(sql, index, "WITH").ok_or_else(|| { format!("SQL test '{file}' must declare mock CTEs and one __expected__ CTE in a top-level WITH clause") @@ -264,12 +237,7 @@ fn scan_ctes( let mut ctes: Vec = Vec::new(); let mut seen: HashSet = HashSet::new(); loop { - quoted |= matches!(byte_at(sql, index), Some(b'"' | b'`')); - let (name, end) = match rule { - NameRule::Plain => read_cte_name(sql, index, file, syntax)?, - NameRule::Lenient => read_identifier(sql, index, syntax) - .ok_or_else(|| format!("SQL test '{file}' expected a CTE name"))?, - }; + let (name, end) = read_cte_name(sql, index, file, syntax)?; if !seen.insert(name.clone()) { return Err(format!("SQL test '{file}' defines duplicate CTE '{name}'").into()); } @@ -304,11 +272,7 @@ fn scan_ctes( "SQL test '{file}' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them" ).into()); } - Ok(ScannedCtes { - ctes, - quoted, - body_starts, - }) + Ok(ScannedCtes { ctes, body_starts }) } /// Read a top-level CTE name, which must be an unquoted ASCII identifier. diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs index 54704d9e75..d5d4f066e2 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs @@ -1,17 +1,21 @@ -//! Python's `extract_sql_scenario_ctes` for scenarios its top-level scanner reads exactly. +//! Python's `extract_sql_scenario_ctes`: the exact top-level scan, then classification and errors. + +use std::collections::{HashMap, HashSet}; use serde::Serialize; +use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; +use sqlbuild_scopes::relationship_names::models::{RelationshipSource, TopLevelCtes}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::compiler::_helpers::sql_tests::extraction::{ - Cte, NameRule, extract_ctes_with_quoting, generic_syntax, validate_independence, -}; +type Cte = (String, String); -/// Characters whose Python case mapping reaches ASCII, which Python's keyword match would accept. -const CASE_MAPPED_TO_ASCII: [char; 20] = [ - '\u{df}', '\u{130}', '\u{131}', '\u{149}', '\u{17f}', '\u{1f0}', '\u{1e96}', '\u{1e97}', - '\u{1e98}', '\u{1e99}', '\u{1e9a}', '\u{1e9e}', '\u{212a}', '\u{fb00}', '\u{fb01}', '\u{fb02}', - '\u{fb03}', '\u{fb04}', '\u{fb05}', '\u{fb06}', -]; +const SOURCE_PREFIX: &str = "__source__"; +const REF_PREFIX: &str = "__ref__"; +const SEED_PREFIX: &str = "__seed__"; +const DBT_REF_PREFIX: &str = "__dbt_ref__"; +const EXPECTED_PREFIX: &str = "__expected__"; +const ASSERT_PREFIX: &str = "__assert__"; +const MACRO_PREFIX: &str = "__macro__"; #[derive(Debug, Default, Serialize)] #[serde(rename_all = "camelCase")] @@ -27,77 +31,169 @@ struct ClassifiedScenario { assertion_names: Vec, } -pub(crate) fn extract_scenario_json(sql: &str, file: &str) -> Result, String> { - if sql.chars().any(|character| { - matches!(character, '\u{1c}'..='\u{1f}') || CASE_MAPPED_TO_ASCII.contains(&character) - }) { - return Ok(None); +/// One scenario's scanner error, or its classification after any Polyglot independence check. +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ScenarioOutcome { + #[serde(skip_serializing_if = "Option::is_none")] + scan_error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + independence: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + scenario: Option, +} + +pub(crate) fn extract_scenario_json( + sql: &str, + file: &str, + syntax: &LexicalSyntax, +) -> Result, String> { + let outcome: ScenarioOutcome = + match scan_top_level_ctes(sql, file, RelationshipSource::Scenario, syntax) { + TopLevelCtes::Deferred => return Ok(None), + TopLevelCtes::Failed(message) => ScenarioOutcome { + scan_error: Some(message), + ..ScenarioOutcome::default() + }, + TopLevelCtes::Scanned(ctes) => { + let independence: Option> = + python_checks_independence(&ctes).then(|| ctes.clone()); + match classify(ctes, file) { + Ok(scenario) => ScenarioOutcome { + independence, + scenario: Some(scenario), + ..ScenarioOutcome::default() + }, + Err(message) => ScenarioOutcome { + independence, + error: Some(message), + ..ScenarioOutcome::default() + }, + } + } + }; + serde_json::to_string(&outcome) + .map(Some) + .map_err(|error| error.to_string()) +} + +/// Whether Python's independence check parses a body; when it does not, the check passes. +fn python_checks_independence(ctes: &[Cte]) -> bool { + let mut names_by_key: HashMap = HashMap::new(); + for (name, _) in ctes { + names_by_key.insert(name.to_ascii_lowercase(), name); } - let syntax = generic_syntax(); - let Ok((ctes, quoted)) = extract_ctes_with_quoting(sql, file, &syntax, NameRule::Lenient) - else { - return Ok(None); + let keys_with_prefix = |prefix: &str| -> HashSet<&String> { + names_by_key + .iter() + .filter(|(_, name)| name.starts_with(prefix)) + .map(|(key, _)| key) + .collect() }; - if quoted || !ctes.iter().all(|cte| is_plain_identifier(&cte.0)) { - return Ok(None); + let expected: HashSet<&String> = keys_with_prefix(EXPECTED_PREFIX); + let assertions: HashSet<&String> = keys_with_prefix(ASSERT_PREFIX); + for (name, body) in ctes { + let key: String = name.to_ascii_lowercase(); + let prohibited: &str = if expected.contains(&key) { + ASSERT_PREFIX + } else if assertions.contains(&key) { + EXPECTED_PREFIX + } else { + continue; + }; + if folded_body_may_contain(body, prohibited) { + return true; + } } - if validate_independence(&ctes, file, &syntax).is_err() { - return Ok(None); + if expected.is_empty() || assertions.is_empty() { + return false; } - let Some(scenario) = classify(ctes) else { - return Ok(None); - }; - serde_json::to_string(&scenario) - .map(Some) - .map_err(|error| error.to_string()) + for (_, body) in ctes { + for key in names_by_key.keys() { + if folded_body_may_contain(body, key) { + return true; + } + } + } + false } -/// Python's `_read_identifier` reads only `[A-Za-z_][A-Za-z0-9_]*` the same way. -fn is_plain_identifier(name: &str) -> bool { - let mut bytes = name.bytes(); - bytes - .next() - .is_some_and(|first| first.is_ascii_alphabetic() || first == b'_') - && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +/// Whether the body's Python `casefold` may contain the text; non-ASCII may fold onto ASCII. +fn folded_body_may_contain(body: &str, lower_ascii: &str) -> bool { + !body.is_ascii() || body.to_ascii_lowercase().contains(lower_ascii) } -fn classify(ctes: Vec) -> Option { +fn classify(ctes: Vec, file: &str) -> Result { let mut scenario: ClassifiedScenario = ClassifiedScenario::default(); for cte in ctes { - let name: String = cte.0.clone(); - if let Some(value) = name.strip_prefix("__source__") { - scenario.source_fixtures.push(required(value)?); + let name: &str = &cte.0; + if let Some(value) = name.strip_prefix(SOURCE_PREFIX) { + scenario + .source_fixtures + .push(required(value, "__source__", file)?); scenario.authored.push(cte); - } else if let Some(value) = name.strip_prefix("__ref__") { - scenario.ref_fixtures.push(required(value)?); + } else if let Some(value) = name.strip_prefix(REF_PREFIX) { + scenario + .ref_fixtures + .push(required(value, "__ref__", file)?); scenario.authored.push(cte); - } else if let Some(value) = name.strip_prefix("__seed__") { - scenario.seed_fixtures.push(required(value)?); + } else if let Some(value) = name.strip_prefix(SEED_PREFIX) { + scenario + .seed_fixtures + .push(required(value, "__seed__", file)?); scenario.authored.push(cte); - } else if let Some(value) = name.strip_prefix("__dbt_ref__") { - scenario.dbt_ref_fixtures.push(required(value)?); + } else if let Some(value) = name.strip_prefix(DBT_REF_PREFIX) { + scenario.dbt_ref_fixtures.push(required( + value, + "__dbt_ref__ or __dbt_ref____", + file, + )?); scenario.authored.push(cte); - } else if let Some(value) = name.strip_prefix("__expected__") { - scenario.expected_models.push(required(value)?); + } else if let Some(value) = name.strip_prefix(EXPECTED_PREFIX) { + scenario + .expected_models + .push(required(value, "__expected__", file)?); scenario.expected.push(cte); - } else if let Some(value) = name.strip_prefix("__assert__") { - scenario.assertion_names.push(required(value)?); + } else if let Some(value) = name.strip_prefix(ASSERT_PREFIX) { + scenario + .assertion_names + .push(required(value, "__assert__", file)?); scenario.assertions.push(cte); - } else if name.starts_with("__macro__") { - return None; + } else if name.starts_with(MACRO_PREFIX) { + return Err(format!( + "SQL scenario '{file}' does not support macro mock CTE '{name}'. Scenarios run \ + real project macros; use SQL unit tests for macro mocks." + )); } else { scenario.authored.push(cte); } } - let has_fixture: bool = !(scenario.source_fixtures.is_empty() + if scenario.source_fixtures.is_empty() && scenario.ref_fixtures.is_empty() && scenario.seed_fixtures.is_empty() - && scenario.dbt_ref_fixtures.is_empty()); - let has_check: bool = - !(scenario.expected_models.is_empty() && scenario.assertion_names.is_empty()); - (has_fixture && has_check).then_some(scenario) + && scenario.dbt_ref_fixtures.is_empty() + { + return Err(format!( + "SQL scenario '{file}' must define at least one __source__*, __ref__*, __seed__*, or \ + __dbt_ref__* fixture CTE" + )); + } + if scenario.expected_models.is_empty() && scenario.assertion_names.is_empty() { + return Err(format!( + "SQL scenario '{file}' must define at least one __expected__ or \ + __assert__ CTE" + )); + } + Ok(scenario) } -fn required(value: &str) -> Option { - (!value.is_empty()).then(|| value.to_owned()) +fn required(value: &str, label: &str, file: &str) -> Result { + if value.is_empty() { + return Err(format!( + "SQL scenario '{file}' must use {label} to identify a target" + )); + } + Ok(value.to_owned()) } diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs index 0d19a3146a..ccf4589ca4 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs @@ -1,6 +1,12 @@ +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + use crate::compiler::_helpers; -/// Python's extracted scenario CTEs as JSON, or `None` where Python must extract the scenario. -pub fn extract_scenario_json(sql: &str, file_label: &str) -> Result, String> { - _helpers::sql_tests::scenario_extraction::extract_scenario_json(sql, file_label) +/// Python's scenario extraction outcome as JSON, or `None` where Python must extract the scenario. +pub fn extract_scenario_json( + sql: &str, + file_label: &str, + syntax: &LexicalSyntax, +) -> Result, String> { + _helpers::sql_tests::scenario_extraction::extract_scenario_json(sql, file_label, syntax) } diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs index aceb0557fc..1a3b083864 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs @@ -1,43 +1,90 @@ +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + use crate::compiler::main::sql_scenario_extraction::extract_scenario_json; use crate::compiler::tests::test_types::ScenarioExtractionTestCase; #[test] -fn given_scenarios_when_extracting_natively_then_python_payloads_or_deferrals_are_returned() { +fn given_scenarios_when_extracting_natively_then_python_outcomes_or_deferrals_are_returned() { + let syntax = LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..LexicalSyntax::default() + }; let test_cases = [ ScenarioExtractionTestCase { - description: "fixtures, helpers, expected results and assertions are classified", - sql: "WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT id FROM helper), \ + description: "fixtures, helpers and expected results without cross references", + sql: "WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT 2 AS id), \ + __expected__orders AS (SELECT 1 AS id) SELECT 1", + expected_json: Some( + r#"{"scenario":{"authored":[["helper","SELECT 1 AS id"],["__source__raw_orders","SELECT 2 AS id"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":[]}}"#, + ), + }, + ScenarioExtractionTestCase { + description: "expected results and assertions reading a CTE leave independence to Python", + sql: "WITH helper AS (SELECT 1 AS id), __source__raw AS (SELECT id FROM helper), \ __expected__orders AS (SELECT 1 AS id), __assert__positive AS (SELECT 1) SELECT 1", expected_json: Some( - r#"{"authored":[["helper","SELECT 1 AS id"],["__source__raw_orders","SELECT id FROM helper"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[["__assert__positive","SELECT 1"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":["positive"]}"#, + r#"{"independence":[["helper","SELECT 1 AS id"],["__source__raw","SELECT id FROM helper"],["__expected__orders","SELECT 1 AS id"],["__assert__positive","SELECT 1"]],"scenario":{"authored":[["helper","SELECT 1 AS id"],["__source__raw","SELECT id FROM helper"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[["__assert__positive","SELECT 1"]],"sourceFixtures":["raw"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":["positive"]}}"#, + ), + }, + ScenarioExtractionTestCase { + description: "an expected result naming the assertion prefix leaves independence to Python", + sql: "WITH __source__raw AS (SELECT 1), __expected__orders AS (SELECT '__ASSERT__' AS x) \ + SELECT 1", + expected_json: Some( + r#"{"independence":[["__source__raw","SELECT 1"],["__expected__orders","SELECT '__ASSERT__' AS x"]],"scenario":{"authored":[["__source__raw","SELECT 1"]],"expected":[["__expected__orders","SELECT '__ASSERT__' AS x"]],"assertions":[],"sourceFixtures":["raw"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":[]}}"#, ), }, ScenarioExtractionTestCase { - description: "a macro mock defers so Python raises", + description: "a macro mock raises Python's error", sql: "WITH __source__raw AS (SELECT 1), __macro__tidy AS (SELECT 'x'), \ __expected__orders AS (SELECT 1 AS id) SELECT 1", - expected_json: None, + expected_json: Some( + r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' does not support macro mock CTE '__macro__tidy'. Scenarios run real project macros; use SQL unit tests for macro mocks."}"#, + ), }, ScenarioExtractionTestCase { - description: "a quoted CTE name defers to Python's scanner fallback", + description: "a bare fixture prefix raises Python's error before later CTEs", + sql: "WITH __seed__ AS (SELECT 1), __macro__tidy AS (SELECT 'x') SELECT 1", + expected_json: Some( + r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must use __seed__ to identify a target"}"#, + ), + }, + ScenarioExtractionTestCase { + description: "a scenario without fixtures raises Python's error", + sql: "WITH __expected__orders AS (SELECT 1 AS id) SELECT 1", + expected_json: Some( + r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must define at least one __source__*, __ref__*, __seed__*, or __dbt_ref__* fixture CTE"}"#, + ), + }, + ScenarioExtractionTestCase { + description: "a scenario without checks raises Python's error", + sql: "WITH __dbt_ref__shop__orders AS (SELECT 1 AS id)", + expected_json: Some( + r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must define at least one __expected__ or __assert__ CTE"}"#, + ), + }, + ScenarioExtractionTestCase { + description: "a quoted CTE name is a scanner error Python may still accept with Polyglot", sql: "WITH \"__source__raw\" AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)", - expected_json: None, + expected_json: Some( + r#"{"scanError":"SQL scenario 'tests/scenarios/orders.sql' expected a CTE name"}"#, + ), }, ScenarioExtractionTestCase { - description: "a keyword Python would match by case mapping defers", - sql: "w\u{131}th __source__raw AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)", - expected_json: None, + description: "an unclosed body is a scanner error naming the scenario context", + sql: "WITH __source__raw AS (SELECT 1", + expected_json: Some(r#"{"scanError":"SQL scenario contains an unclosed parenthesis"}"#), }, ScenarioExtractionTestCase { - description: "a scenario without fixtures defers so Python raises", - sql: "WITH __expected__orders AS (SELECT 1 AS id) SELECT 1", + description: "a keyword Python would match by case mapping defers", + sql: "w\u{131}th __source__raw AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)", expected_json: None, }, ]; for test_case in test_cases { assert_eq!( - extract_scenario_json(test_case.sql, "tests/scenarios/orders.sql"), + extract_scenario_json(test_case.sql, "tests/scenarios/orders.sql", &syntax), Ok(test_case.expected_json.map(str::to_owned)), "{}", test_case.description diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs index 63fee6eef9..949fe854b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs @@ -2,8 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -50,16 +52,18 @@ fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult, sql: &str, file_label: &str, + syntax: LexicalSyntaxInput, ) -> PyResult> { + let syntax: LexicalSyntax = syntax.into(); py.compiler_detach(|| { sqlbuild_analysis::compiler::main::sql_scenario_extraction::extract_scenario_json( - sql, file_label, + sql, file_label, &syntax, ) }) .map_err(value_error) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs index 124a38f37b..bcf4e0593a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs @@ -1,36 +1,81 @@ -//! Expected-model relationship names of SQL tests and scenarios, scanned natively. +//! Expected-model relationship names and top-level CTEs of SQL tests and scenarios, scanned natively. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::relationship_names::main::expected_model_names::expected_model_names; -use sqlbuild_scopes::relationship_names::models::ExpectedNames; +use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; +use sqlbuild_scopes::relationship_names::models::{ + ExpectedNames, RelationshipSource, TopLevelCtes, +}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; -/// Return each body's expected-model names, or `None` where Python must extract them. +/// One scan's Python error, or its values; `None` where Python must scan the text itself. +type Scanned = Option<(Option, Vec)>; + +fn source(scenario: bool) -> RelationshipSource { + if scenario { + RelationshipSource::Scenario + } else { + RelationshipSource::Test + } +} + +/// Each `(sql, file label)` body's expected models or Python's error; `None` defers to Python. #[pyfunction] fn scope_expected_model_names( py: Python<'_>, - sqls: Vec, + texts: Vec<(String, String)>, + scenario: bool, syntax: LexicalSyntaxInput, -) -> PyResult>>> { +) -> PyResult>> { let syntax: LexicalSyntax = syntax.into(); let outcomes: Vec = py - .compiler_detach(|| Ok(expected_model_names(&sqls, &syntax))) + .compiler_detach(|| Ok(expected_model_names(&texts, source(scenario), &syntax))) .map_err(compiler_error)?; Ok(outcomes .into_iter() .map(|outcome| match outcome { - ExpectedNames::Scanned(names) => Some(names), + ExpectedNames::Scanned(names) => Some((None, names)), + ExpectedNames::Failed(message) => Some((Some(message), Vec::new())), ExpectedNames::Deferred => None, }) .collect()) } +/// Each `(sql, file label)` test body's top-level CTEs or Python's error; `None` defers. +#[pyfunction] +fn scope_test_ctes( + py: Python<'_>, + texts: Vec<(String, String)>, + syntax: LexicalSyntaxInput, +) -> PyResult>> { + let syntax: LexicalSyntax = syntax.into(); + let outcomes: Vec = py + .compiler_detach(|| { + Ok(texts + .iter() + .map(|(sql, file)| { + scan_top_level_ctes(sql, file, RelationshipSource::Test, &syntax) + }) + .collect()) + }) + .map_err(compiler_error)?; + Ok(outcomes + .into_iter() + .map(|outcome| match outcome { + TopLevelCtes::Scanned(ctes) => Some((None, ctes)), + TopLevelCtes::Failed(message) => Some((Some(message), Vec::new())), + TopLevelCtes::Deferred => None, + }) + .collect()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; + module.add_function(wrap_pyfunction!(scope_test_ctes, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-scopes/Cargo.toml b/crates/sqlbuild-scopes/Cargo.toml index 90a67d20a5..5d11915923 100644 --- a/crates/sqlbuild-scopes/Cargo.toml +++ b/crates/sqlbuild-scopes/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +sqlbuild-core.workspace = true sqlbuild-sqltext.workspace = true [lints] diff --git a/crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs b/crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs index d43bf704e9..0e758a3fde 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/_helpers/cte_scan.rs @@ -1,117 +1,210 @@ -//! The top-level CTE scan of Python `extract_sql_test_expected_model_names`, reproduced exactly. +//! The top-level CTE scan of Python `extract_top_level_ctes_with_scanner`, reproduced exactly. +use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren; use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; use std::collections::HashSet; -/// Python raises here, or a non-ASCII code character needs Python's `str` classification. +use crate::relationship_names::models::RelationshipSource; + +/// Why the scan stopped before Python's result. #[derive(Debug, PartialEq, Eq)] -pub(crate) struct Deferred; +pub(crate) enum Stop { + /// Python raises this message. + Failed(String), + /// Python's `str` classification or parenthesis matcher could read the text differently. + Deferred, +} -type Scan = Result; +type Scan = Result; -const EXPECTED_PREFIX: &[u8] = b"__expected__"; +const EXPECTED_PREFIX: &str = "__expected__"; const QUOTE_TOKENS: &[u8] = b"'\"`$"; /// Python's parenthesis matcher only looks for comments at these characters. const PAREN_SCAN_SPECIALS: &[u8] = b"-/#'\"`$()"; +/// The authored text the scan reads, with the labels Python's messages name it by. +pub(crate) struct ScanText<'sql> { + pub(crate) sql: &'sql str, + pub(crate) file: &'sql str, + pub(crate) source: RelationshipSource, +} + /// Return the expected-model names of one SQL test or scenario body. -pub(crate) fn expected_names(sql: &[u8], syntax: &LexicalSyntax) -> Scan> { - if !syntax.line_comment_prefixes.iter().all(|prefix| { - prefix.is_ascii() - && prefix - .as_bytes() - .first() - .is_some_and(|first| PAREN_SCAN_SPECIALS.contains(first)) - }) { - return Err(Deferred); - } - let start = skip_ignorable(sql, 0, syntax)?; +pub(crate) fn expected_names(text: &ScanText<'_>, syntax: &LexicalSyntax) -> Scan> { + let sql = text.sql.as_bytes(); + let start = skip_ignorable(sql, 0, text, syntax)?; if try_consume_keyword(sql, start, b"WITH")?.is_none() { return Ok(Vec::new()); } - let (names, end) = scan_top_level_ctes(sql, syntax)?; - if !(trailing_statement_end(sql, end, syntax)? || trailing_ceremonial_select(sql, end, syntax)?) - { - return Err(Deferred); - } - names - .into_iter() - .filter_map(|name| name.strip_prefix(EXPECTED_PREFIX).map(<[u8]>::to_vec)) + let ctes = top_level_ctes(text, syntax)?; + ctes.into_iter() + .filter_map(|(name, _)| name.strip_prefix(EXPECTED_PREFIX).map(str::to_owned)) .map(|model| { if model.is_empty() { - Err(Deferred) + Err(Stop::Failed(format!( + "{} '{}' must use __expected__ to identify a target", + context_label(text.source), + text.file + ))) } else { - String::from_utf8(model).map_err(|_| Deferred) + Ok(model) } }) .collect() } -fn scan_top_level_ctes<'sql>( - sql: &'sql [u8], +/// Return the top-level CTE names and stripped bodies, then check only `SELECT 1` follows. +pub(crate) fn top_level_ctes( + text: &ScanText<'_>, syntax: &LexicalSyntax, -) -> Scan<(Vec<&'sql [u8]>, usize)> { - let with_end = - try_consume_keyword(sql, skip_ignorable(sql, 0, syntax)?, b"WITH")?.ok_or(Deferred)?; - let mut index = skip_ignorable(sql, with_end, syntax)?; +) -> Scan> { + if !syntax.line_comment_prefixes.iter().all(|prefix| { + prefix.is_ascii() + && prefix + .as_bytes() + .first() + .is_some_and(|first| PAREN_SCAN_SPECIALS.contains(first)) + }) { + return Err(Stop::Deferred); + } + let sql = text.sql.as_bytes(); + let with_end = try_consume_keyword(sql, skip_ignorable(sql, 0, text, syntax)?, b"WITH")? + .ok_or_else(|| { + failed( + text, + format!( + "must declare {} in a top-level WITH clause", + with_requirement(text.source) + ), + ) + })?; + let mut index = skip_ignorable(sql, with_end, text, syntax)?; if let Some(recursive_end) = try_consume_keyword(sql, index, b"RECURSIVE")? { - index = skip_ignorable(sql, recursive_end, syntax)?; + index = skip_ignorable(sql, recursive_end, text, syntax)?; } - let mut names: Vec<&[u8]> = Vec::new(); - let mut seen: HashSet<&[u8]> = HashSet::new(); + let mut ctes: Vec<(String, String)> = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); loop { - let (name, name_end) = read_identifier(sql, index)?; + let (name, name_end) = read_identifier(text, index)?; if !seen.insert(name) { - return Err(Deferred); + return Err(failed(text, format!("defines duplicate CTE '{name}'"))); } - names.push(name); - index = skip_ignorable(sql, name_end, syntax)?; + index = skip_ignorable(sql, name_end, text, syntax)?; if sql.get(index) == Some(&b'(') { - index = matching_paren(sql, index, syntax)? + 1; - index = skip_ignorable(sql, index, syntax)?; + index = matching_paren(text, index, syntax)? + 1; + index = skip_ignorable(sql, index, text, syntax)?; } - index = try_consume_keyword(sql, index, b"AS")?.ok_or(Deferred)?; - index = skip_ignorable(sql, index, syntax)?; + index = try_consume_keyword(sql, index, b"AS")? + .ok_or_else(|| failed(text, "expected keyword AS".to_owned()))?; + index = skip_ignorable(sql, index, text, syntax)?; if sql.get(index) != Some(&b'(') { - return Err(Deferred); + return Err(failed(text, format!("CTE '{name}' must use AS (...)"))); } - index = skip_ignorable(sql, matching_paren(sql, index, syntax)? + 1, syntax)?; + let body_end = matching_paren(text, index, syntax)?; + ctes.push(( + name.to_owned(), + python_strip(&text.sql[index + 1..body_end]).to_owned(), + )); + index = skip_ignorable(sql, body_end + 1, text, syntax)?; if sql.get(index) == Some(&b',') { - index = skip_ignorable(sql, index + 1, syntax)?; + index = skip_ignorable(sql, index + 1, text, syntax)?; continue; } - return Ok((names, index)); + if trailing_statement_end(sql, index, text, syntax)? + || trailing_ceremonial_select(sql, index, text, syntax)? + { + return Ok(ctes); + } + return Err(failed( + text, + "must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow \ + them" + .to_owned(), + )); + } +} + +fn context_label(source: RelationshipSource) -> &'static str { + match source { + RelationshipSource::Test => "SQL test", + RelationshipSource::Scenario => "SQL scenario", + } +} + +fn with_requirement(source: RelationshipSource) -> &'static str { + match source { + RelationshipSource::Test => "mock CTEs and one __expected__ CTE", + RelationshipSource::Scenario => { + "fixture CTEs and at least one __expected__ or __assert__ CTE" + } } } -fn trailing_ceremonial_select(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Scan { - let index = skip_ignorable(sql, start, syntax)?; +/// Python's message for one file, after the context and file label it always starts with. +fn failed(text: &ScanText<'_>, detail: String) -> Stop { + Stop::Failed(format!( + "{} '{}' {detail}", + context_label(text.source), + text.file + )) +} + +/// Python's message for unclosed text, which names only the context. +fn unclosed(text: &ScanText<'_>, kind: Unclosed) -> Stop { + let construct = match kind { + Unclosed::BlockComment => "block comment", + Unclosed::Quote => "quoted string", + Unclosed::Parenthesis => "parenthesis", + }; + Stop::Failed(format!( + "{} contains an unclosed {construct}", + context_label(text.source) + )) +} + +fn trailing_ceremonial_select( + sql: &[u8], + start: usize, + text: &ScanText<'_>, + syntax: &LexicalSyntax, +) -> Scan { + let index = skip_ignorable(sql, start, text, syntax)?; let Some(select_end) = try_consume_keyword(sql, index, b"SELECT")? else { return Ok(false); }; - let index = skip_ignorable(sql, select_end, syntax)?; + let index = skip_ignorable(sql, select_end, text, syntax)?; if sql.get(index) != Some(&b'1') { return Ok(false); } - let index = skip_ignorable(sql, index + 1, syntax)?; - trailing_statement_end(sql, index, syntax) + let index = skip_ignorable(sql, index + 1, text, syntax)?; + trailing_statement_end(sql, index, text, syntax) } -fn trailing_statement_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Scan { +fn trailing_statement_end( + sql: &[u8], + start: usize, + text: &ScanText<'_>, + syntax: &LexicalSyntax, +) -> Scan { let mut index = start; if sql.get(index) == Some(&b';') { - index = skip_ignorable(sql, index + 1, syntax)?; + index = skip_ignorable(sql, index + 1, text, syntax)?; } Ok(index == sql.len()) } -fn skip_ignorable(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Scan { +fn skip_ignorable( + sql: &[u8], + start: usize, + text: &ScanText<'_>, + syntax: &LexicalSyntax, +) -> Scan { let mut index = start; while let Some(&byte) = sql.get(index) { if !byte.is_ascii() { - return Err(Deferred); + return Err(Stop::Deferred); } if is_python_space(byte) { index += 1; @@ -123,24 +216,29 @@ fn skip_ignorable(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Scan index = end, Ok(None) => return Ok(index), - Err(_) => return Err(Deferred), + Err(kind) => return Err(unclosed(text, kind)), } } Ok(index) } +/// Python's keyword match; text after the first non-ASCII character needs Python's case tables. fn try_consume_keyword(sql: &[u8], start: usize, keyword: &[u8]) -> Scan> { let end = start + keyword.len(); let window = &sql[start.min(sql.len())..end.min(sql.len())]; - if !window.is_ascii() { - return Err(Deferred); + if let Some(non_ascii) = window.iter().position(|byte| !byte.is_ascii()) { + return if window[..non_ascii].eq_ignore_ascii_case(&keyword[..non_ascii]) { + Err(Stop::Deferred) + } else { + Ok(None) + }; } if !window.eq_ignore_ascii_case(keyword) { return Ok(None); } if let Some(&next) = sql.get(end) { if !next.is_ascii() { - return Err(Deferred); + return Err(Stop::Deferred); } if is_identifier_character(next) { return Ok(None); @@ -148,7 +246,7 @@ fn try_consume_keyword(sql: &[u8], start: usize, keyword: &[u8]) -> Scan Scan Scan<(&[u8], usize)> { - let first = *sql.get(start).ok_or(Deferred)?; - if !(first.is_ascii_alphabetic() || first == b'_') { - return Err(Deferred); +fn read_identifier<'sql>(text: &ScanText<'sql>, start: usize) -> Scan<(&'sql str, usize)> { + let sql = text.sql.as_bytes(); + match sql.get(start) { + Some(byte) if !byte.is_ascii() => return Err(Stop::Deferred), + Some(byte) if byte.is_ascii_alphabetic() || *byte == b'_' => {} + _ => return Err(failed(text, "expected a CTE name".to_owned())), } let mut index = start + 1; while let Some(&byte) = sql.get(index) { if !byte.is_ascii() { - return Err(Deferred); + return Err(Stop::Deferred); } if !is_identifier_character(byte) { break; } index += 1; } - Ok((&sql[start..index], index)) + Ok((&text.sql[start..index], index)) } -fn matching_paren(sql: &[u8], open: usize, syntax: &LexicalSyntax) -> Scan { - dialect_matching_paren(sql, open, syntax).map_err(|_| Deferred) +fn matching_paren(text: &ScanText<'_>, open: usize, syntax: &LexicalSyntax) -> Scan { + dialect_matching_paren(text.sql.as_bytes(), open, syntax).map_err(|kind| unclosed(text, kind)) } fn is_identifier_character(byte: u8) -> bool { diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs b/crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs index 46c4be07c1..85b602400d 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/main/expected_model_names.rs @@ -1,16 +1,24 @@ -//! Expected-model relationship names of SQL tests and scenarios, or a deferral to Python. +//! Expected-model relationship names of SQL tests and scenarios, Python's error, or a deferral. use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::relationship_names::_helpers::cte_scan::{Deferred, expected_names}; -use crate::relationship_names::models::ExpectedNames; +use crate::relationship_names::_helpers::cte_scan::{ScanText, Stop, expected_names}; +use crate::relationship_names::models::{ExpectedNames, RelationshipSource}; -/// Return each SQL text's `__expected__` names in CTE order, or a deferral to Python. -pub fn expected_model_names(sqls: &[String], syntax: &LexicalSyntax) -> Vec { - sqls.iter() - .map(|sql| match expected_names(sql.as_bytes(), syntax) { - Ok(names) => ExpectedNames::Scanned(names), - Err(Deferred) => ExpectedNames::Deferred, - }) +/// Return each `(sql, file label)` text's expected-model names, Python's error, or a deferral. +pub fn expected_model_names( + texts: &[(String, String)], + source: RelationshipSource, + syntax: &LexicalSyntax, +) -> Vec { + texts + .iter() + .map( + |(sql, file)| match expected_names(&ScanText { sql, file, source }, syntax) { + Ok(names) => ExpectedNames::Scanned(names), + Err(Stop::Failed(message)) => ExpectedNames::Failed(message), + Err(Stop::Deferred) => ExpectedNames::Deferred, + }, + ) .collect() } diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs b/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs index 9ec2fdbfe4..948c16b2bf 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs @@ -1 +1,2 @@ pub mod expected_model_names; +pub mod top_level_ctes; diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/top_level_ctes.rs b/crates/sqlbuild-scopes/src/relationship_names/main/top_level_ctes.rs new file mode 100644 index 0000000000..149f4c61c3 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/main/top_level_ctes.rs @@ -0,0 +1,20 @@ +//! Top-level CTEs of SQL tests and scenarios as Python's scanner reads them. + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relationship_names::_helpers::cte_scan::{ScanText, Stop, top_level_ctes}; +use crate::relationship_names::models::{RelationshipSource, TopLevelCtes}; + +/// Return the top-level CTEs of one SQL text, the error Python's scanner raises, or a deferral. +pub fn scan_top_level_ctes( + sql: &str, + file: &str, + source: RelationshipSource, + syntax: &LexicalSyntax, +) -> TopLevelCtes { + match top_level_ctes(&ScanText { sql, file, source }, syntax) { + Ok(ctes) => TopLevelCtes::Scanned(ctes), + Err(Stop::Failed(message)) => TopLevelCtes::Failed(message), + Err(Stop::Deferred) => TopLevelCtes::Deferred, + } +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/models.rs b/crates/sqlbuild-scopes/src/relationship_names/models.rs index d25b107563..6e830197ad 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/models.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/models.rs @@ -1,10 +1,32 @@ -//! Outcomes of the native relationship-name scan. +//! Outcomes of the native relationship-name and top-level CTE scans. -/// The expected-model names of one SQL text, or a deferral to Python's extraction. +/// Which authored file kind a SQL text comes from, which names it in Python's messages. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RelationshipSource { + /// A SQL test block. + Test, + /// A SQL scenario file. + Scenario, +} + +/// The expected-model names of one SQL text, Python's error, or a deferral to Python. #[derive(Debug, PartialEq, Eq)] pub enum ExpectedNames { /// The `__expected__` names in CTE order, exactly as Python extracts them. Scanned(Vec), - /// Python raises for this text, or classifies a character the scan reached differently. + /// The message Python's extraction raises. + Failed(String), + /// Python classifies a character the scan reached differently. + Deferred, +} + +/// The top-level CTEs of one SQL text, Python's scanner error, or a deferral to Python. +#[derive(Debug, PartialEq, Eq)] +pub enum TopLevelCtes { + /// Each CTE's name and stripped body in authored order, exactly as Python's scanner reads them. + Scanned(Vec<(String, String)>), + /// The message Python's scanner raises. + Failed(String), + /// Python classifies a character the scan reached differently. Deferred, } diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs index 04f167aa10..e71769b8f2 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs @@ -1,9 +1,13 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::relationship_names::main::expected_model_names::expected_model_names; -use crate::relationship_names::models::ExpectedNames; +use crate::relationship_names::main::top_level_ctes::scan_top_level_ctes; +use crate::relationship_names::models::{ExpectedNames, RelationshipSource, TopLevelCtes}; use crate::relationship_names::tests::test_types::SyntaxTestCase; +const TEST_FILE: &str = "tests/unit/test_orders.sql"; +const SCENARIO_FILE: &str = "tests/scenarios/orders.sql"; + /// The generic lexical rules of the base adapter. pub(super) fn generic_syntax() -> LexicalSyntax { LexicalSyntax { @@ -22,16 +26,48 @@ pub(super) fn dialect_syntax(test_case: &SyntaxTestCase) -> LexicalSyntax { } } -/// The native outcome for one SQL text. +/// The native outcome for one SQL test body. pub(super) fn names(sql: &str, syntax: &LexicalSyntax) -> ExpectedNames { - expected_model_names(&[sql.to_owned()], syntax).remove(0) + expected_model_names( + &[(sql.to_owned(), TEST_FILE.to_owned())], + RelationshipSource::Test, + syntax, + ) + .remove(0) +} + +/// The native outcome for one scenario body. +pub(super) fn scenario_names(sql: &str) -> ExpectedNames { + expected_model_names( + &[(sql.to_owned(), SCENARIO_FILE.to_owned())], + RelationshipSource::Scenario, + &generic_syntax(), + ) + .remove(0) +} + +/// The top-level CTEs of one SQL test body. +pub(super) fn ctes(sql: &str) -> TopLevelCtes { + scan_top_level_ctes(sql, TEST_FILE, RelationshipSource::Test, &generic_syntax()) +} + +/// Expected-model names a case expects the scan to return. +pub(super) fn scanned(names: &[&str]) -> ExpectedNames { + ExpectedNames::Scanned(owned(names)) +} + +/// The error a case expects Python to raise. +pub(super) fn failed(message: &str) -> ExpectedNames { + ExpectedNames::Failed(message.to_owned()) } -/// The outcome a case expects: its names, or a deferral when it has none. -pub(super) fn expected(names: Option<&[&str]>) -> ExpectedNames { - names.map_or(ExpectedNames::Deferred, |names| { - ExpectedNames::Scanned(owned(names)) - }) +/// The CTE names and bodies a scan case expects. +pub(super) fn scanned_ctes(ctes: &[(&str, &str)]) -> TopLevelCtes { + TopLevelCtes::Scanned( + ctes.iter() + .map(|(name, body)| ((*name).to_owned(), (*body).to_owned())) + .collect(), + ) } fn owned(values: &[&str]) -> Vec { diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs index f9fa4eb5f1..94f30eecf4 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/test_expected_model_names.rs @@ -1,107 +1,137 @@ -use crate::relationship_names::tests::helpers::{dialect_syntax, expected, generic_syntax, names}; -use crate::relationship_names::tests::test_types::{ExpectedNamesTestCase, SyntaxTestCase}; +use crate::relationship_names::models::{ExpectedNames, TopLevelCtes}; +use crate::relationship_names::tests::helpers::{ + ctes, dialect_syntax, failed, generic_syntax, names, scanned, scanned_ctes, scenario_names, +}; +use crate::relationship_names::tests::test_types::{ + ExpectedNamesTestCase, ScenarioNamesTestCase, SyntaxTestCase, TopLevelCtesTestCase, +}; #[test] -fn given_sql_test_when_scanning_expected_models_then_names_match_python_or_defer() { +fn given_sql_test_when_scanning_expected_models_then_names_and_errors_match_python_or_defer() { let test_cases = [ ExpectedNamesTestCase { description: "expected CTEs in authored order after mocks", sql: "WITH\n__source__raw AS (SELECT 1),\n__expected__orders AS (SELECT 2),\n\ __expected__customers AS (SELECT 3)\nSELECT 1\n", - expected_names: Some(&["orders", "customers"]), + expected_outcome: scanned(&["orders", "customers"]), }, ExpectedNamesTestCase { description: "comments, column lists, RECURSIVE and an omitted SELECT 1", sql: "-- header\n/* note */ with recursive __expected__a (x, y) AS (\n\ SELECT ')' AS x, 1 AS y -- )\n)", - expected_names: Some(&["a"]), + expected_outcome: scanned(&["a"]), }, ExpectedNamesTestCase { description: "a terminated ceremonial select with trailing comments", sql: "WITH __expected__a AS (SELECT 1) select 1 ; -- done\n", - expected_names: Some(&["a"]), + expected_outcome: scanned(&["a"]), }, ExpectedNamesTestCase { description: "no top-level WITH has no expected models", sql: "SELECT 1", - expected_names: Some(&[]), + expected_outcome: scanned(&[]), }, ExpectedNamesTestCase { description: "a keyword prefix of an identifier is not WITH", sql: "WITHIN AS (SELECT 1)", - expected_names: Some(&[]), + expected_outcome: scanned(&[]), }, ExpectedNamesTestCase { description: "non-ASCII inside bodies, strings and comments is opaque", sql: "WITH __expected__a AS (SELECT 'caf\u{e9}' AS \"\u{e9}\") -- \u{e9}\nSELECT 1", - expected_names: Some(&["a"]), + expected_outcome: scanned(&["a"]), }, ExpectedNamesTestCase { description: "non-ASCII at a code position defers", sql: "WITH __expected__caf\u{e9} AS (SELECT 1) SELECT 1", - expected_names: None, + expected_outcome: ExpectedNames::Deferred, }, ExpectedNamesTestCase { description: "non-ASCII whitespace defers", sql: "WITH\u{a0}__expected__a AS (SELECT 1) SELECT 1", - expected_names: None, + expected_outcome: ExpectedNames::Deferred, }, ExpectedNamesTestCase { description: "a non-ASCII character Python upper-cases into a keyword defers", sql: "w\u{131}th __expected__a AS (SELECT 1) SELECT 1", - expected_names: None, + expected_outcome: ExpectedNames::Deferred, }, ExpectedNamesTestCase { description: "Python separators count as whitespace", sql: "WITH\u{1f}__expected__a AS (SELECT 1)\u{b}SELECT 1", - expected_names: Some(&["a"]), + expected_outcome: scanned(&["a"]), }, ExpectedNamesTestCase { - description: "a bare expected prefix defers to Python's error", + description: "a bare expected prefix raises Python's error", sql: "WITH __expected__ AS (SELECT 1) SELECT 1", - expected_names: None, + expected_outcome: failed( + "SQL test 'tests/unit/test_orders.sql' must use __expected__ to identify a target", + ), }, ExpectedNamesTestCase { - description: "a duplicate CTE defers to Python's error", + description: "a duplicate CTE raises Python's error", sql: "WITH a AS (SELECT 1), a AS (SELECT 2) SELECT 1", - expected_names: None, + expected_outcome: failed( + "SQL test 'tests/unit/test_orders.sql' defines duplicate CTE 'a'", + ), }, ExpectedNamesTestCase { - description: "a final statement other than SELECT 1 defers", + description: "a final statement other than SELECT 1 raises Python's error", sql: "WITH __expected__a AS (SELECT 1) SELECT 2", - expected_names: None, + expected_outcome: failed( + "SQL test 'tests/unit/test_orders.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them", + ), }, ExpectedNamesTestCase { - description: "a missing AS defers", + description: "a missing AS raises Python's error", sql: "WITH __expected__a (SELECT 1) SELECT 1", - expected_names: None, + expected_outcome: failed("SQL test 'tests/unit/test_orders.sql' expected keyword AS"), }, ExpectedNamesTestCase { - description: "an unclosed parenthesis defers", + description: "an unclosed parenthesis raises Python's error", sql: "WITH __expected__a AS (SELECT (1) SELECT 1", - expected_names: None, + expected_outcome: failed("SQL test contains an unclosed parenthesis"), }, ExpectedNamesTestCase { - description: "an unclosed block comment defers", + description: "an unclosed block comment raises Python's error", sql: "/* open\nWITH __expected__a AS (SELECT 1)", - expected_names: None, + expected_outcome: failed("SQL test contains an unclosed block comment"), }, ExpectedNamesTestCase { - description: "a quoted CTE name defers to Python's error", + description: "a quoted CTE name raises Python's error", sql: "WITH \"__expected__a\" AS (SELECT 1) SELECT 1", - expected_names: None, + expected_outcome: failed("SQL test 'tests/unit/test_orders.sql' expected a CTE name"), }, ExpectedNamesTestCase { - description: "a dollar sign at a code position defers", + description: "a missing CTE body raises Python's error", + sql: "WITH __expected__a AS SELECT 1", + expected_outcome: failed( + "SQL test 'tests/unit/test_orders.sql' CTE '__expected__a' must use AS (...)", + ), + }, + ExpectedNamesTestCase { + description: "an unclosed quote inside a body raises Python's error", + sql: "WITH __expected__a AS (SELECT 'open) SELECT 1", + expected_outcome: failed("SQL test contains an unclosed quoted string"), + }, + ExpectedNamesTestCase { + description: "a non-ASCII CTE name start after a valid CTE defers", + sql: "WITH __expected__a AS (SELECT 1), \u{e9}x AS (SELECT 2) SELECT 1", + expected_outcome: ExpectedNames::Deferred, + }, + ExpectedNamesTestCase { + description: "a dollar sign after the CTEs raises Python's trailing statement error", sql: "WITH __expected__a AS (SELECT 1) $x", - expected_names: None, + expected_outcome: failed( + "SQL test 'tests/unit/test_orders.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them", + ), }, ]; for test_case in test_cases { assert_eq!( names(test_case.sql, &generic_syntax()), - expected(test_case.expected_names), + test_case.expected_outcome, "{}", test_case.description ); @@ -117,7 +147,7 @@ fn given_dialect_rules_when_scanning_expected_models_then_comments_and_quotes_fo nested_block_comments: false, line_comment_prefixes: &["--", "#"], sql: "WITH __expected__a AS (SELECT 1 # )\n) SELECT 1", - expected_names: Some(&["a"]), + expected_outcome: scanned(&["a"]), }, SyntaxTestCase { description: "nested block comments close at the outer delimiter", @@ -125,7 +155,7 @@ fn given_dialect_rules_when_scanning_expected_models_then_comments_and_quotes_fo nested_block_comments: true, line_comment_prefixes: &["--"], sql: "/* a /* b */ ) */ WITH __expected__a AS (SELECT 1) SELECT 1", - expected_names: Some(&["a"]), + expected_outcome: scanned(&["a"]), }, SyntaxTestCase { description: "backslash-escaped quotes keep the string open", @@ -133,7 +163,7 @@ fn given_dialect_rules_when_scanning_expected_models_then_comments_and_quotes_fo nested_block_comments: false, line_comment_prefixes: &["--"], sql: "WITH __expected__a AS (SELECT 'it\\')' AS x) SELECT 1", - expected_names: Some(&["a"]), + expected_outcome: scanned(&["a"]), }, SyntaxTestCase { description: "a line comment prefix Python's matcher never looks for defers", @@ -141,7 +171,7 @@ fn given_dialect_rules_when_scanning_expected_models_then_comments_and_quotes_fo nested_block_comments: false, line_comment_prefixes: &["--", ";;"], sql: "WITH __expected__a AS (SELECT 1) SELECT 1", - expected_names: None, + expected_outcome: ExpectedNames::Deferred, }, ]; @@ -149,7 +179,78 @@ fn given_dialect_rules_when_scanning_expected_models_then_comments_and_quotes_fo let syntax = dialect_syntax(&test_case); assert_eq!( names(test_case.sql, &syntax), - expected(test_case.expected_names), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scenario_when_scanning_expected_models_then_errors_name_the_scenario() { + let test_cases = [ + ScenarioNamesTestCase { + description: "expected CTEs after fixtures", + sql: "WITH __source__raw AS (SELECT 1), __expected__orders AS (SELECT 2) SELECT 1", + expected_outcome: scanned(&["orders"]), + }, + ScenarioNamesTestCase { + description: "a bare expected prefix names the scenario", + sql: "WITH __source__raw AS (SELECT 1), __expected__ AS (SELECT 2) SELECT 1", + expected_outcome: failed( + "SQL scenario 'tests/scenarios/orders.sql' must use __expected__ to \ + identify a target", + ), + }, + ScenarioNamesTestCase { + description: "an unclosed parenthesis names the scenario context", + sql: "WITH __source__raw AS (SELECT 1", + expected_outcome: failed("SQL scenario contains an unclosed parenthesis"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + scenario_names(test_case.sql), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_text_when_scanning_top_level_ctes_then_bodies_and_errors_match_python() { + let test_cases = [ + TopLevelCtesTestCase { + description: "stripped bodies in authored order", + sql: "WITH helper AS (\n SELECT 1 AS id\n), __macro_actual__ AS ( SELECT @tidy(id) FROM \ + helper ) SELECT 1", + expected_outcome: scanned_ctes(&[ + ("helper", "SELECT 1 AS id"), + ("__macro_actual__", "SELECT @tidy(id) FROM helper"), + ]), + }, + TopLevelCtesTestCase { + description: "non-ASCII inside a body is kept and stripped by Python's rules", + sql: "WITH a AS (\u{3000}SELECT 'caf\u{e9}'\u{a0})", + expected_outcome: scanned_ctes(&[("a", "SELECT 'caf\u{e9}'")]), + }, + TopLevelCtesTestCase { + description: "a missing WITH names the test requirement", + sql: "SELECT 1", + expected_outcome: TopLevelCtes::Failed( + "SQL test 'tests/unit/test_orders.sql' must declare mock CTEs and one \ + __expected__ CTE in a top-level WITH clause" + .to_owned(), + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + ctes(test_case.sql), + test_case.expected_outcome, "{}", test_case.description ); diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs index 2f43ea29cb..cc7681257a 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs @@ -1,8 +1,24 @@ -/// One SQL text and the expected-model names the native scan returns, or `None` for a deferral. +use crate::relationship_names::models::{ExpectedNames, TopLevelCtes}; + +/// One SQL test body and the native expected-model outcome. pub(super) struct ExpectedNamesTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, - pub(super) expected_names: Option<&'static [&'static str]>, + pub(super) expected_outcome: ExpectedNames, +} + +/// One scenario body and the native expected-model outcome. +pub(super) struct ScenarioNamesTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_outcome: ExpectedNames, +} + +/// One SQL text and the CTEs Python's scanner reads, or its error. +pub(super) struct TopLevelCtesTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_outcome: TopLevelCtes, } /// One dialect's lexical rules as the Python adapter declares them. @@ -12,5 +28,5 @@ pub(super) struct SyntaxTestCase { pub(super) nested_block_comments: bool, pub(super) line_comment_prefixes: &'static [&'static str], pub(super) sql: &'static str, - pub(super) expected_names: Option<&'static [&'static str]>, + pub(super) expected_outcome: ExpectedNames, } diff --git a/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py index 2761d8c5e6..dd0e1ca45a 100644 --- a/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py @@ -1,4 +1,4 @@ -"""Minimal projects whose audits, functions or sources fail while attaching compile inputs.""" +"""Minimal projects whose audits, functions, sources, SQL tests or scenarios fail to attach.""" from scripts.compiler_differential._helpers.corpus.case_builder import config_files, failure_case from scripts.compiler_differential.constants import ( @@ -16,6 +16,19 @@ 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE ordered_at >= __cursor_start()\n' ) _ORDER_LABEL_PATH: str = "functions/sql/order_label.sql" +_SCENARIO_PATH: str = "tests/scenarios/orders.sql" +_SCENARIO_HEADER: str = 'SCENARIO (\n description "Order flow"\n);\n\n' +_SOURCE_FIXTURE: str = ( + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status\n" + ")" +) +_EXPECTED_STAGING: str = "__expected__stg_orders AS (\n SELECT 1 AS order_id\n)" +_TEST_PATH: str = "tests/unit/test_stg_orders.sql" + + +def _scenario(*ctes: str, tail: str = "\nSELECT 1\n") -> str: + return _SCENARIO_HEADER + "WITH\n" + ",\n".join(ctes) + tail def _staging_audits(audits: str) -> dict[str, str]: @@ -31,7 +44,7 @@ def _order_label(header: str) -> dict[str, str]: def attachment_failure_cases() -> tuple[FailureCase, ...]: - """Return audit, function and source attachment failures the native stage reports.""" + """Return audit, function, source, SQL test and scenario failures the native stage reports.""" return ( failure_case( @@ -106,4 +119,89 @@ def attachment_failure_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="scenario-fixture-without-target", + expected_code="P001", + expected_message=( + f"SQL scenario '{_SCENARIO_PATH}' must use __seed__ to identify a target" + ), + files={_SCENARIO_PATH: _scenario("__seed__ AS (SELECT 1 AS id)", _EXPECTED_STAGING)}, + ), + failure_case( + name="scenario-macro-mock", + expected_code="P001", + expected_message="does not support macro mock CTE '__macro__tidy'", + files={ + _SCENARIO_PATH: _scenario( + _SOURCE_FIXTURE, "__macro__tidy AS (SELECT 'x')", _EXPECTED_STAGING + ) + }, + ), + failure_case( + name="scenario-without-checks", + expected_code="P001", + expected_message=( + f"SQL scenario '{_SCENARIO_PATH}' must define at least one __expected__ " + "or __assert__ CTE" + ), + files={_SCENARIO_PATH: _scenario(_SOURCE_FIXTURE)}, + ), + failure_case( + name="scenario-statement-after-ctes", + expected_code="P001", + expected_message="must end after its CTEs", + files={ + _SCENARIO_PATH: _scenario( + _SOURCE_FIXTURE, _EXPECTED_STAGING, tail="\nSELECT order_id FROM orders\n" + ) + }, + ), + failure_case( + name="scenario-check-reads-source", + expected_code="P001", + expected_message="CTE '__expected__stg_orders' must not reference project source", + files={ + _SCENARIO_PATH: _scenario( + _SOURCE_FIXTURE, + '__expected__stg_orders AS (\n SELECT order_id FROM __source("raw_orders")\n)', + ) + }, + ), + failure_case( + name="scenario-dependent-checks", + expected_code="P001", + expected_message=( + "check CTE '__expected__stg_orders' must not depend on '__assert__no_rows'" + ), + files={ + _SCENARIO_PATH: _scenario( + _SOURCE_FIXTURE, + "__assert__no_rows AS (\n SELECT 1 FROM __source__raw_orders WHERE false\n)", + "__expected__stg_orders AS (\n SELECT 1 AS order_id FROM __assert__no_rows\n)", + ) + }, + ), + failure_case( + name="scenario-first-of-two-errors", + expected_code="P001", + expected_message=( + "SQL scenario 'tests/scenarios/a_orders.sql' must use __seed__ to identify " + "a target" + ), + files={ + "tests/scenarios/a_orders.sql": _scenario( + "__seed__ AS (SELECT 1 AS id)", _EXPECTED_STAGING + ), + "tests/scenarios/b_orders.sql": _scenario(_SOURCE_FIXTURE), + }, + ), + failure_case( + name="test-mocks-unknown-source", + expected_code="P001", + expected_message="mocks unknown source 'raw_payments'", + files={ + _TEST_PATH: "TEST();\n\nWITH\n__source__raw_payments AS (\n SELECT 1 AS id\n),\n" + f"{_EXPECTED_STAGING}\nSELECT 1\n" + }, + ), ) diff --git a/scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py index 82d5a3f1e0..d016d85ccd 100644 --- a/scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/scope_failure_cases.py @@ -14,6 +14,8 @@ _MART_HEADER: str = 'MODEL (\n description "Order totals per customer",\n);\n\n' _STAGING_HEADER: str = 'MODEL (\n description "Staged orders",\n);\n\n' _TEST_PATH: str = "tests/unit/test_customer_totals.sql" +_SCOPE_SCENARIO_PATH: str = "tests/scenarios/orders.sql" +_SCENARIO_HEADER: str = 'SCENARIO (\n description "Order flow"\n);\n\n' _GLOBAL_MACRO_PATH: str = "macros/amounts.py" @@ -106,6 +108,31 @@ def scope_failure_cases() -> tuple[FailureCase, ...]: "__expected__ AS (\n SELECT 10 AS customer_id\n)\nSELECT 1\n", }, ), + _case( + name="scope-scenario-expected-without-target", + expected_code="P001", + expected_message=( + f"SQL scenario '{_SCOPE_SCENARIO_PATH}' must use __expected__ to identify " + "a target" + ), + files={ + _STAGING_ENUM_PATH: _STAGING_ENUM, + _SCOPE_SCENARIO_PATH: _SCENARIO_HEADER + + "WITH\n__source__raw_orders AS (SELECT 1 AS order_id),\n" + "__expected__ AS (SELECT 1)\n", + }, + ), + _case( + name="scope-macro-test-cte-without-body", + expected_code="P001", + expected_message=f"SQL test '{_TEST_PATH}' CTE '__macro_actual__' must use AS (...)", + files={ + _STAGING_ENUM_PATH: _STAGING_ENUM, + "macros/amounts.py": "def doubled(value):\n return f'{value} * 2'\n", + _TEST_PATH: 'TEST (mode macro, name "doubled");\n\nWITH\n' + "__macro_actual__ AS SELECT @doubled('1') AS amount\n", + }, + ), ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5dc76a2ff4..d829b52b2d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -209,9 +209,14 @@ class SqlReferenceScanner: ) -> tuple[list[tuple[str, str, str | None, int | None]], None] | tuple[None, str] | None: ... def scope_expected_model_names( - sqls: list[str], syntax: dict[str, object] -) -> list[list[str] | None]: ... -def extract_sql_scenario_json(sql: str, file_label: str) -> str | None: ... + texts: list[tuple[str, str]], scenario: bool, syntax: dict[str, object] +) -> list[tuple[str | None, list[str]] | None]: ... +def scope_test_ctes( + texts: list[tuple[str, str]], syntax: dict[str, object] +) -> list[tuple[str | None, list[tuple[str, str]]] | None]: ... +def extract_sql_scenario_json( + sql: str, file_label: str, syntax: dict[str, object] +) -> str | None: ... class SqlTestTargetCatalog: def __init__( diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/ctes.py b/src/sqlbuild/compiler/compile/_helpers/analysis/ctes.py index 8e99144ae0..9433ea8e4a 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/ctes.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/ctes.py @@ -46,6 +46,8 @@ def extract_top_level_ctes_with_sql_analysis( ) -> tuple[tuple[str, str], ...] | None: """Extract top-level CTE aliases and rendered bodies with Polyglot when available.""" + if _CEREMONIAL_SELECT_PATTERN.search(sql) is None: + return None polyglot_module: Any = import_polyglot_sql() try: analysis: Any = polyglot_module.analyze_query(sql, {"dialect": "generic"}) @@ -59,8 +61,6 @@ def extract_top_level_ctes_with_sql_analysis( return None if not isinstance(analysis, dict): return None - if _CEREMONIAL_SELECT_PATTERN.search(sql) is None: - return None if not _is_ceremonial_select_analysis(analysis): return None cte_facts: Any = analysis.get(_POLYGLOT_ANALYSIS_CTE_FACTS) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 12a8478da9..48c29ad860 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -133,7 +133,7 @@ def _build_native_declaration_scope( scan_cache=scan_cache, ) if fault is not None: - raise CompileInputError(fault) + raise CompileInputError(fault, bridge_independent=True) if not native.grant(facts): return None index: ScopeIndex = native.index_with_relationships() diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index f722b7a28c..0eef949471 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -2,12 +2,15 @@ from __future__ import annotations -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence from pathlib import Path from typing import cast import orjson +from sqlbuild.compiler.compile._helpers.analysis.ctes import ( + extract_top_level_ctes_with_sql_analysis, +) from sqlbuild.compiler.compile._helpers.render.macros import ( find_macro_call_names, find_nested_macro_call_names, @@ -19,11 +22,13 @@ extract_sql_test_expected_model_names, extract_unclassified_sql_test_ctes, ) +from sqlbuild.compiler.compile._helpers.sql_tests.native import native_sql_test_ctes from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import ( MACRO_ACTUAL_TEST_CTE_NAME, SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileSqlTestCte, ScopeRelationshipBuild, @@ -58,6 +63,8 @@ type _SharedDeclarations = dict[tuple[str, str], tuple[DeclarationRecord, ...]] type _BlockNames = tuple[str, ...] | Exception +_SQL_TEST_CONTEXT: str = "SQL test" + def build_scope_relationship_grants( *, @@ -105,8 +112,21 @@ def extract_scope_relationship_facts( syntax=sql_lexical_syntax, scan_natively=True, ) + macro_blocks: list[tuple[str, str]] = [] + for test_file in discovered_inputs.test_files: + macro_blocks.extend( + (block.sql_body, str(test_file.relative_path)) + for block in test_file.blocks + if block.mode is SqlTestMode.MACRO + ) + macro_test_ctes: Iterator[tuple[tuple[str, str], ...] | str | None] = iter( + native_sql_test_ctes(texts=macro_blocks, syntax=sql_lexical_syntax) + ) for test_file, file_names in zip(discovered_inputs.test_files, names_by_file, strict=True): for block, block_names in zip(test_file.blocks, file_names, strict=True): + native_ctes: tuple[tuple[str, str], ...] | str | None = ( + next(macro_test_ctes) if block.mode is SqlTestMode.MACRO else None + ) try: expected_names: tuple[str, ...] = _names_or_raise(block_names) called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) @@ -118,7 +138,8 @@ def extract_scope_relationship_facts( expected_models=expected_names, called_macros=called_macros, tested_macros=( - _tested_macro_names( + _natively_tested_macro_names( + native_ctes=native_ctes, sql=block.sql_body, file_label=str(test_file.relative_path), syntax=sql_lexical_syntax, @@ -130,11 +151,18 @@ def extract_scope_relationship_facts( ) except Exception as error: faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) - scenario_names: list[tuple[str, ...] | None] = native_expected_model_names( - sqls=[scenario.sql_body for scenario in discovered_inputs.scenario_files], + scenario_names: list[tuple[str, ...] | str | None] = native_expected_model_names( + texts=[ + (scenario.sql_body, str(scenario.relative_path)) + for scenario in discovered_inputs.scenario_files + ], + scenario=True, syntax=sql_lexical_syntax, ) for scenario, scanned in zip(discovered_inputs.scenario_files, scenario_names, strict=True): + if isinstance(scanned, str): + faults.append(ScopeRelationshipFault(scenario.relative_path, scanned)) + continue try: facts.append( RelationshipFact( @@ -171,7 +199,7 @@ def _expected_names_by_file( ) for test_file in test_files ] - scanned: dict[tuple[int, int], tuple[str, ...]] = ( + scanned: dict[tuple[int, int], _BlockNames] = ( _scanned_expected_names(test_files=test_files, stored=stored, syntax=syntax) if scan_natively else {} @@ -184,7 +212,7 @@ def _expected_names_by_file( continue file_names: list[_BlockNames] = [] for block_index, block in enumerate(test_file.blocks): - native_names: tuple[str, ...] | None = scanned.get((file_index, block_index)) + native_names: _BlockNames | None = scanned.get((file_index, block_index)) if native_names is not None: file_names.append(native_names) continue @@ -252,8 +280,8 @@ def _scanned_expected_names( test_files: Sequence[DiscoveredSqlTestFile], stored: Sequence[tuple[tuple[str, ...], ...] | None], syntax: SqlLexicalSyntax, -) -> dict[tuple[int, int], tuple[str, ...]]: - """Scan model-mode blocks of unstored files natively, keeping only names the scan reproduces.""" +) -> dict[tuple[int, int], _BlockNames]: + """Scan model-mode blocks of unstored files natively: names or Python's error, where exact.""" pending: list[tuple[int, int]] = [] for file_index, test_file in enumerate(test_files): @@ -264,20 +292,52 @@ def _scanned_expected_names( for block_index, block in enumerate(test_file.blocks) if block.mode is SqlTestMode.MODEL ) - scanned: list[tuple[str, ...] | None] = native_expected_model_names( - sqls=[ - test_files[file_index].blocks[block_index].sql_body + scanned: list[tuple[str, ...] | str | None] = native_expected_model_names( + texts=[ + ( + test_files[file_index].blocks[block_index].sql_body, + str(test_files[file_index].relative_path), + ) for file_index, block_index in pending ], + scenario=False, syntax=syntax, ) return { - position: names + position: CompileInputError(names, bridge_independent=True) + if isinstance(names, str) + else names for position, names in zip(pending, scanned, strict=True) if names is not None } +def _natively_tested_macro_names( + *, + native_ctes: tuple[tuple[str, str], ...] | str | None, + sql: str, + file_label: str, + syntax: SqlLexicalSyntax, +) -> tuple[str, ...]: + """Macros a macro test's actual CTE calls, scanned natively where Python's scanner is exact.""" + + if native_ctes is None: + return _tested_macro_names(sql=sql, file_label=file_label, syntax=syntax) + if isinstance(native_ctes, str): + polyglot_ctes: tuple[tuple[str, str], ...] | None = ( + extract_top_level_ctes_with_sql_analysis( + sql=sql, file_label=file_label, context_label=_SQL_TEST_CONTEXT + ) + ) + if polyglot_ctes is None: + raise CompileInputError(native_ctes, bridge_independent=True) + native_ctes = polyglot_ctes + actual_sql: str | None = next( + (body for name, body in native_ctes if name == MACRO_ACTUAL_TEST_CTE_NAME), None + ) + return () if actual_sql is None else find_macro_call_names(actual_sql) + + def _tested_macro_names(*, sql: str, file_label: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: test_ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( sql=sql, file_label=file_label, syntax=syntax diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index 17b278dce8..b4c4a6ec57 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -241,7 +241,8 @@ def build_test_inputs( raise CompileInputError( f"SQL test '{test_block.name or test_file.file_path.stem}' declares " f"unused parameters: {', '.join(unused_parameters)} in case " - f"'{test_case.name}'" + f"'{test_case.name}'", + bridge_independent=True, ) expanded_test_block: DiscoveredSqlTestBlock = ( test_block @@ -543,7 +544,8 @@ def _infer_tested_direct_logic_resource_names( if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( f"SQL test file {test_file.relative_path} mode '{raw_test_ctes.mode.value}' must " - "define exactly one actual CTE and exactly one expected CTE" + "define exactly one actual CTE and exactly one expected CTE", + bridge_independent=True, ) if raw_test_ctes.mode == SqlTestMode.UDF: return _infer_tested_udf_names( @@ -568,14 +570,16 @@ def _infer_tested_direct_logic_resource_names( if not tested_macro_names: raise CompileInputError( f"SQL test file {test_file.relative_path} mode 'macro' must call at least one " - "macro in __macro_actual__" + "macro in __macro_actual__", + bridge_independent=True, ) tested_macro_name: str for tested_macro_name in tested_macro_names: if tested_macro_name not in loaded_macros: raise CompileInputError( f"SQL test file {test_file.relative_path} references unknown macro " - f"'@{tested_macro_name}'" + f"'@{tested_macro_name}'", + bridge_independent=True, ) return tested_macro_names @@ -591,7 +595,8 @@ def _infer_tested_udf_names( if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( f"SQL test file {test_file.relative_path} mode 'udf' must define exactly one " - "__udf_actual__ CTE and exactly one __udf_expected__ CTE" + "__udf_actual__ CTE and exactly one __udf_expected__ CTE", + bridge_independent=True, ) references: tuple[CompileSqlReference, ...] = extract_sql_references( sql=raw_test_ctes.payload.actual_cte.sql_body, @@ -608,20 +613,23 @@ def _infer_tested_udf_names( if not tested_udf_names: raise CompileInputError( f"SQL test file {test_file.relative_path} mode 'udf' must call at least one " - "scalar UDF in __udf_actual__" + "scalar UDF in __udf_actual__", + bridge_independent=True, ) tested_udf_name: str for tested_udf_name in tested_udf_names: if tested_udf_name not in known_function_names: raise CompileInputError( f"SQL test file {test_file.relative_path} references unknown SQL function " - f"'{tested_udf_name}'" + f"'{tested_udf_name}'", + bridge_independent=True, ) if tested_udf_name in known_table_function_names: raise CompileInputError( f"SQL test file {test_file.relative_path} references table function " f"'{tested_udf_name}' with {SqlReferenceKind.UDF.placeholder_call()}; use " - f"{SqlReferenceKind.TABLE_FUNCTION.placeholder_call()} for table functions" + f"{SqlReferenceKind.TABLE_FUNCTION.placeholder_call()} for table functions", + bridge_independent=True, ) return tested_udf_names @@ -638,7 +646,8 @@ def _infer_tested_table_function_names( if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( f"SQL test file {test_file.relative_path} mode 'table_fn' must define exactly one " - "__table_fn_actual__ CTE and exactly one __table_fn_expected__ CTE" + "__table_fn_actual__ CTE and exactly one __table_fn_expected__ CTE", + bridge_independent=True, ) references: tuple[CompileSqlReference, ...] = extract_sql_references( sql=raw_test_ctes.payload.actual_cte.sql_body, @@ -660,14 +669,16 @@ def _infer_tested_table_function_names( if not tested_table_function_names: raise CompileInputError( f"SQL test file {test_file.relative_path} mode 'table_fn' must call at least one " - "table function in __table_fn_actual__" + "table function in __table_fn_actual__", + bridge_independent=True, ) tested_table_function_name: str for tested_table_function_name in tested_table_function_names: if tested_table_function_name not in known_function_names: raise CompileInputError( f"SQL test file {test_file.relative_path} references unknown SQL function " - f"'{tested_table_function_name}'" + f"'{tested_table_function_name}'", + bridge_independent=True, ) if tested_table_function_name not in known_table_function_names: raise CompileInputError( @@ -792,19 +803,19 @@ def _validate_scenario_source_references( syntax: SqlLexicalSyntax, target_catalog: _native.SqlTestTargetCatalog | None = None, ) -> None: - native_sources: list[tuple[str, bool, list[str]]] | None = ( - _native_scenario_cte_sources( + if native_stage_enabled(NativeStage.ATTACHMENTS): + cte_sources: list[tuple[str, bool, list[str]]] + extraction_error: CompileInputError | None + cte_sources, extraction_error = _scenario_cte_sources( scenario_ctes=scenario_ctes, scenario_file=scenario_file, syntax=syntax ) - if native_stage_enabled(NativeStage.ATTACHMENTS) - else None - ) - if native_sources is not None: error: str | None = ( target_catalog or native_test_target_catalog(sources=known_source_names) - ).scenario_source_error(str(scenario_file.relative_path), native_sources) + ).scenario_source_error(str(scenario_file.relative_path), cte_sources) if error is not None: - raise CompileInputError(error) + raise CompileInputError(error, bridge_independent=True) + if extraction_error is not None: + raise extraction_error return cte: CompileSqlScenarioCte for cte in (*scenario_ctes.expected_ctes, *scenario_ctes.assertion_ctes): @@ -837,38 +848,38 @@ def _validate_scenario_source_references( ) -def _native_scenario_cte_sources( +def _scenario_cte_sources( *, scenario_ctes: CompileSqlScenarioCtes, scenario_file: DiscoveredSqlScenarioFile, syntax: SqlLexicalSyntax, -) -> list[tuple[str, bool, list[str]]] | None: - """Every CTE's source references, or None where extraction raises so Python orders errors.""" +) -> tuple[list[tuple[str, bool, list[str]]], CompileInputError | None]: + """Each check-then-fixture CTE's source references, up to the first CTE whose scan raises.""" checks: tuple[CompileSqlScenarioCte, ...] = ( *scenario_ctes.expected_ctes, *scenario_ctes.assertion_ctes, ) cte_sources: list[tuple[str, bool, list[str]]] = [] - try: - for position, cte in enumerate((*checks, *scenario_ctes.authored_ctes)): + for position, cte in enumerate((*checks, *scenario_ctes.authored_ctes)): + try: references: tuple[CompileSqlReference, ...] = extract_sql_references( sql=cte.sql_body, syntax=syntax, origin=_sql_file_reference_origin(scenario_file) ) - cte_sources.append( - ( - cte.name, - position < len(checks), - [ - reference.ref_name - for reference in references - if reference.ref_kind == SqlReferenceKind.SOURCE - ], - ) + except CompileInputError as error: + return cte_sources, error + cte_sources.append( + ( + cte.name, + position < len(checks), + [ + reference.ref_name + for reference in references + if reference.ref_kind == SqlReferenceKind.SOURCE + ], ) - except CompileInputError: - return None - return cte_sources + ) + return cte_sources, None def validate_test_ctes( @@ -910,7 +921,7 @@ def validate_test_ctes( ), ) if error is not None: - raise CompileInputError(error) + raise CompileInputError(error, bridge_independent=True) return mock_model_name: str diff --git a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py index 0f0d3388bb..ea20aa0217 100644 --- a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py @@ -48,9 +48,8 @@ def extract_sql_scenario_ctes( """Extract top-level SQL-native scenario fixture, expected, and assertion CTEs.""" native_ctes: CompileSqlScenarioCtes | None = ( - _native_scenario_ctes(sql=sql, file_label=file_label) + _native_scenario_ctes(sql=sql, file_label=file_label, syntax=syntax) if native_stage_enabled(NativeStage.ATTACHMENTS) - and not syntax.reads_differently_from_generic(sql) else None ) if native_ctes is not None: @@ -65,22 +64,77 @@ def extract_sql_scenario_ctes( syntax=syntax, ) except CompileInputError as scanner_error: - cte_values: tuple[tuple[str, str], ...] | None = extract_top_level_ctes_with_sql_analysis( - sql=sql, - file_label=file_label, - context_label="SQL scenario", + polyglot_ctes: tuple[CompileSqlScenarioCte, ...] | None = _polyglot_scenario_ctes( + sql=sql, file_label=file_label ) - if cte_values is None: + if polyglot_ctes is None: raise scanner_error from None - ctes = tuple(CompileSqlScenarioCte(name=name, sql_body=body) for name, body in cte_values) + ctes = polyglot_ctes return _classify_sql_scenario_ctes(ctes=ctes, file_label=file_label, syntax=syntax) -def _native_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenarioCtes | None: - response: str | None = _native.extract_sql_scenario_json(sql, file_label) +def _polyglot_scenario_ctes( + *, sql: str, file_label: str +) -> tuple[CompileSqlScenarioCte, ...] | None: + cte_values: tuple[tuple[str, str], ...] | None = extract_top_level_ctes_with_sql_analysis( + sql=sql, + file_label=file_label, + context_label=_CONTEXT, + ) + if cte_values is None: + return None + return tuple(CompileSqlScenarioCte(name=name, sql_body=body) for name, body in cte_values) + + +def _native_scenario_ctes( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> CompileSqlScenarioCtes | None: + """Extract natively with Python's errors, or None where Python must read the scenario.""" + + try: + response: str | None = _native.extract_sql_scenario_json( + sql, file_label, syntax.native_mapping + ) + except (TypeError, UnicodeError): + return None if response is None: return None - payload: dict[str, list[Any]] = orjson.loads(response) + try: + return _scenario_from_native_outcome( + outcome=orjson.loads(response), sql=sql, file_label=file_label, syntax=syntax + ) + except CompileInputError as error: + error.bridge_independent = True + raise + + +def _scenario_from_native_outcome( + *, outcome: dict[str, Any], sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> CompileSqlScenarioCtes: + """Build the native outcome, running only Python's Polyglot steps the native scan leaves.""" + + scan_error: str | None = outcome.get("scanError") + if scan_error is not None: + polyglot_ctes: tuple[CompileSqlScenarioCte, ...] | None = _polyglot_scenario_ctes( + sql=sql, file_label=file_label + ) + if polyglot_ctes is None: + raise CompileInputError(scan_error) + return _classify_sql_scenario_ctes(ctes=polyglot_ctes, file_label=file_label, syntax=syntax) + independence: list[list[str]] | None = outcome.get("independence") + if independence is not None: + validate_independent_expected_and_assertion_ctes( + ctes=_scenario_ctes(independence), + expected_prefix=EXPECTED_TEST_CTE_PREFIX, + assertion_prefix=ASSERT_SCENARIO_CTE_PREFIX, + file_label=file_label, + context_label=_CONTEXT, + syntax=syntax, + ) + error: str | None = outcome.get("error") + if error is not None: + raise CompileInputError(error) + payload: dict[str, list[Any]] = outcome["scenario"] return CompileSqlScenarioCtes( authored_ctes=_scenario_ctes(payload["authored"]), expected_ctes=_scenario_ctes(payload["expected"]), diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py index 17a8148199..52c5039e6b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py @@ -79,7 +79,9 @@ def located_extraction_error( else None ) if offset is None: - return CompileInputError(error.message, code=error.code, help=error.help) + return CompileInputError( + error.message, code=error.code, help=error.help, bridge_independent=True + ) contents: str = test_file.contents line: int = contents.count(_LINE_BREAK, 0, offset) + 1 column: int = offset - (contents.rfind(_LINE_BREAK, 0, offset) + 1) + 1 @@ -87,6 +89,7 @@ def located_extraction_error( f"{error.message}\n --> {test_file.relative_path.as_posix()}:{line}:{column}", code=error.code, help=error.help, + bridge_independent=True, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py index 1577857686..427ba38bdf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Sequence from typing import Protocol, cast import orjson @@ -132,6 +132,25 @@ def decode(value: bytes) -> tuple[CompileSqlTestCtes, ...] | None: return decode +def native_sql_test_ctes( + *, texts: Sequence[tuple[str, str]], syntax: SqlLexicalSyntax +) -> list[tuple[tuple[str, str], ...] | str | None]: + """Return each `(sql, file label)` test body's CTEs, Python's scanner error, or None.""" + + if not texts: + return [] + try: + outcomes: list[tuple[str | None, list[tuple[str, str]]] | None] = _native.scope_test_ctes( + list(texts), syntax.native_mapping + ) + except (TypeError, UnicodeError): + return [None] * len(texts) + return [ + None if outcome is None else outcome[0] if outcome[0] is not None else tuple(outcome[1]) + for outcome in outcomes + ] + + def extract_unexpanded_sql_test( *, sql: str, file_label: str, mode: SqlTestMode, syntax: SqlLexicalSyntax ) -> tuple[CompileSqlTestCtes, bool]: diff --git a/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py b/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py index ecc08957b8..4c42880e35 100644 --- a/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py +++ b/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py @@ -9,16 +9,19 @@ def native_expected_model_names( - *, sqls: Sequence[str], syntax: SqlLexicalSyntax -) -> list[tuple[str, ...] | None]: - """Return each body's expected-model names, or None where Python must extract them.""" + *, texts: Sequence[tuple[str, str]], scenario: bool, syntax: SqlLexicalSyntax +) -> list[tuple[str, ...] | str | None]: + """Return each `(sql, file label)` body's names, Python's error, or None for Python.""" - if not sqls: + if not texts: return [] try: - names: list[list[str] | None] = _native.scope_expected_model_names( - list(sqls), syntax.native_mapping + outcomes: list[tuple[str | None, list[str]] | None] = _native.scope_expected_model_names( + list(texts), scenario, syntax.native_mapping ) except (TypeError, UnicodeError): - return [None] * len(sqls) - return [None if model_names is None else tuple(model_names) for model_names in names] + return [None] * len(texts) + return [ + None if outcome is None else outcome[0] if outcome[0] is not None else tuple(outcome[1]) + for outcome in outcomes + ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/_test_types.py new file mode 100644 index 0000000000..ad6a486147 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/_test_types.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeSqlTestErrorTestCase: + """One failing SQL test, scenario or relationship project compiled by every engine.""" + + description: str + case_name: str + expected_distinct_macro_call_counts: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/test_native_sql_test_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/test_native_sql_test_errors.py new file mode 100644 index 0000000000..4f7c22fc3b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_errors/test_native_sql_test_errors.py @@ -0,0 +1,153 @@ +"""Every engine reports SQL test, scenario and relationship errors identically, without re-runs.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.corpus.attachment_failure_cases import ( + attachment_failure_cases, +) +from scripts.compiler_differential._helpers.corpus.scope_failure_cases import scope_failure_cases +from scripts.compiler_differential.constants import FAILURE_BASE_MART, FAILURE_MART_PATH +from scripts.compiler_differential.models import FailureCase +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + lifecycle_error_type, + report_without_engine, + run_reuse_compile, + stderr_without_durations, + write_project_file, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_test_errors._test_types import ( + NativeSqlTestErrorTestCase, +) + +_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") +_CALL_LOG: str = "macro_calls.log" +_COUNTED_MACRO: str = ( + "from pathlib import Path\n\n\n" + "def counted(expression: str) -> str:\n" + ' """Return the expression, logging each execution."""\n\n' + f" with (Path(__file__).resolve().parents[1] / {_CALL_LOG!r}).open(\n" + ' "a", encoding="utf-8"\n' + " ) as log:\n" + ' _ = log.write("call\\n")\n' + " return expression\n" +) +_CASES: dict[str, FailureCase] = { + case.name: case for case in (*attachment_failure_cases(), *scope_failure_cases()) +} + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSqlTestErrorTestCase( + description="a scenario fixture without a target name", + case_name="scenario-fixture-without-target", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="a scenario macro mock", + case_name="scenario-macro-mock", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="a scenario without checks", + case_name="scenario-without-checks", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="a scenario statement after its CTEs", + case_name="scenario-statement-after-ctes", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="a scenario check reading a project source", + case_name="scenario-check-reads-source", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="dependent scenario checks, still checked by Python's Polyglot parse", + case_name="scenario-dependent-checks", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="the first of two broken scenarios", + case_name="scenario-first-of-two-errors", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="a test mocking an unknown source", + case_name="test-mocks-unknown-source", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="a scoped scenario's expected CTE without a target", + case_name="scope-scenario-expected-without-target", + expected_distinct_macro_call_counts=1, + ), + NativeSqlTestErrorTestCase( + description="a scoped macro test whose actual CTE has no body", + case_name="scope-macro-test-cte-without-body", + expected_distinct_macro_call_counts=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_test_error_when_compiling_with_each_engine_then_errors_and_types_match( + test_case: NativeSqlTestErrorTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + case: FailureCase = _CASES[test_case.case_name] + runs: list[CompileReuseRun] = [] + macro_calls: list[int] = [] + error_types: list[str] = [] + project_dir: Path = tmp_path / "orders" + files: dict[str, str] = { + **case.files, + "macros/counted.py": _COUNTED_MACRO, + FAILURE_MART_PATH: FAILURE_BASE_MART.replace("SUM(amount)", "SUM(@counted('amount'))"), + _CALL_LOG: "", + } + for engine in _ENGINES: + shutil.rmtree(project_dir, ignore_errors=True) + for relative_path, contents in files.items(): + write_project_file(project_dir, relative_path, contents) + runs.append( + run_reuse_compile(project_dir=project_dir, global_args=("--compiler-engine", engine)) + ) + macro_calls.append(len((project_dir / _CALL_LOG).read_text(encoding="utf-8").splitlines())) + error_types.append( + lifecycle_error_type(project_dir=project_dir, engine=engine, monkeypatch=monkeypatch) + ) + + assert ( + tuple(run.returncode for run in runs), + {report_without_engine(run) for run in runs[1:]} == {report_without_engine(runs[0])}, + {stderr_without_durations(stderr=run.stderr) for run in runs[1:]} + == {stderr_without_durations(stderr=runs[0].stderr)}, + f"[{case.expected_code}]" in runs[0].report + runs[0].stderr, + str(case.expected_message) in runs[0].report + runs[0].stderr, + len(set(macro_calls)), + tuple(error_types), + ) == ( + (1, 1, 1), + True, + True, + True, + True, + test_case.expected_distinct_macro_call_counts, + ("CompileInputError", "CompileInputError", "CompileInputError"), + ), ( + runs[0].report, + runs[0].stderr, + runs[2].report, + macro_calls, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py index 8dd34cf57c..b2a4e0fd5d 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py @@ -100,6 +100,7 @@ class ScenarioParityTestCase: expected_minimum_extracted: int expected_minimum_native: int expected_minimum_python_errors: int + expected_minimum_native_errors: int @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index 0d71605bc9..92f2affef3 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import cast +import orjson import pytest import sqlbuild._native as _native @@ -845,6 +846,37 @@ def raw_test_compile_outcome( "dollar AS (SELECT $$ ) $$ AS note)", "hashed AS (SELECT 1 # )\n)", ) +_SCENARIO_ERROR_CTES: tuple[str, ...] = ( + "__ref__ AS (SELECT 1)", + "__seed__ AS (SELECT 1)", + "__dbt_ref__ AS (SELECT 1)", + "__assert__ AS (SELECT 1)", + "missing_as (SELECT 1)", + "no_body AS SELECT 1", + "columns (id, label) AS (SELECT 1, 'x')", + "unclosed AS (SELECT (1)", + "open_quote AS (SELECT 'x)", + "/* open AS (SELECT 1)", + "__assert__reads_expected AS (SELECT id FROM __expected__orders_view)", + "caf\u00e9 AS (SELECT 1)", +) +_SCENARIO_ERROR_WEIGHTS: tuple[int, int] = (3, 1) +_DEFERRED_ANSWER: str = '{"deferred": true}' +_NATIVE_SCENARIO_ANSWERS: dict[tuple[frozenset[str], bool], str] = { + (frozenset({"deferred"}), False): "deferred", + (frozenset({"deferred"}), True): "deferred", + (frozenset({"scanError"}), False): "error", + (frozenset({"scanError"}), True): "polyglot", + (frozenset({"independence", "scenario"}), False): "independence", + (frozenset({"independence", "scenario"}), True): "independence", + (frozenset({"independence", "error"}), False): "independence", + (frozenset({"independence", "error"}), True): "independence", + (frozenset({"error"}), False): "error", + (frozenset({"error"}), True): "error", + (frozenset({"scenario"}), False): "extracted", + (frozenset({"scenario"}), True): "extracted", +} +_CEREMONIAL_SELECT_TAIL: re.Pattern[str] = re.compile(r"\bSELECT\s+1\s*;?\s*$", re.IGNORECASE) _SCENARIO_TAILS: tuple[str, ...] = ("\nSELECT 1\n", "", ";", "\nSELECT 2", " -- end", ";\n") _SCENARIO_KEYWORDS: tuple[str, ...] = ( "WITH\n", @@ -859,7 +891,11 @@ def generated_scenario(*, rng: random.Random) -> str: """Return one scenario body mixing fixtures, checks, helpers and invalid CTEs.""" ctes: list[str] = rng.sample(_SCENARIO_CTES, k=rng.randint(1, 5)) - return rng.choice(_SCENARIO_KEYWORDS) + ",\n".join(ctes) + rng.choice(_SCENARIO_TAILS) + errors: list[str] = rng.sample( + _SCENARIO_ERROR_CTES, k=rng.choices((0, 1), weights=_SCENARIO_ERROR_WEIGHTS)[0] + ) + mixed: list[str] = rng.sample([*ctes, *errors], k=len(ctes) + len(errors)) + return rng.choice(_SCENARIO_KEYWORDS) + ",\n".join(mixed) + rng.choice(_SCENARIO_TAILS) def scenario_outcome( @@ -880,10 +916,16 @@ def scenario_outcome( return str(error) -def native_scenario_answered(sql: str) -> bool: - """Whether the native extraction answers instead of deferring to Python.""" +def native_scenario_answer(*, sql: str, syntax: SqlLexicalSyntax) -> str: + """How the native extraction answers: extracted, error, or the Python step it leaves.""" - return _native.extract_sql_scenario_json(sql, "tests/scenarios/orders.sql") is not None + response: str | None = _native.extract_sql_scenario_json( + sql, "tests/scenarios/orders.sql", syntax.native_mapping + ) + outcome: dict[str, object] = orjson.loads(response or _DEFERRED_ANSWER) + return _NATIVE_SCENARIO_ANSWERS[ + (frozenset(outcome), _CEREMONIAL_SELECT_TAIL.search(sql) is not None) + ] _TARGET_NAMES: tuple[str, ...] = ("orders", "customers", "channel_codes", "returns", "order_rows") diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py index 30db6f6ad9..20c66d351c 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py @@ -94,6 +94,77 @@ expected_outcome_fragment="error: SQL scenario file tests/scenarios/orders_scenario.sql " "references unknown source 'missing_events'", ), + AttachmentProjectTestCase( + description="a check CTE reading a source before a later malformed reference call", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + "WITH\n__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + + '__expected__orders AS (SELECT id, 1.5 AS amount, label FROM __source("order_events")),\n' + + "__assert__no_rows AS (SELECT 1 FROM __source(order_events))\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="error: SQL scenario file tests/scenarios/orders_scenario.sql CTE " + "'__expected__orders' must not reference project source", + ), + AttachmentProjectTestCase( + description="a malformed call in a check CTE leaves the unknown helper source first", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + 'WITH\nhelper AS (SELECT * FROM __source("missing_events")),\n' + + "__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + + "__expected__orders AS (SELECT 1 AS id FROM __source(order_events))\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="error: SQL scenario file tests/scenarios/orders_scenario.sql " + "references unknown source 'missing_events'", + ), + AttachmentProjectTestCase( + description="an unknown helper source before a later malformed fixture call", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + 'WITH\nhelper AS (SELECT * FROM __source("missing_events")),\n' + + "__seed__channel_codes AS (SELECT 1 AS id FROM __source(order_events)),\n" + + "__expected__orders AS (SELECT 1 AS id, 1.5 AS amount, 'web' AS label)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="error: SQL scenario file tests/scenarios/orders_scenario.sql " + "references unknown source 'missing_events'", + ), + AttachmentProjectTestCase( + description="a scenario fixture without a target name", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + "WITH\n__seed__ AS (SELECT 1 AS id, 'web' AS label),\n" + + "__expected__orders AS (SELECT 1 AS id, 1.5 AS amount, 'web' AS label)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="error: SQL scenario 'tests/scenarios/orders_scenario.sql' " + "must use __seed__ to identify a target", + ), + AttachmentProjectTestCase( + description="a scenario macro mock", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + "WITH\n__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + + "__macro__tidy AS (SELECT 'x'),\n" + + "__expected__orders AS (SELECT 1 AS id, 1.5 AS amount, 'web' AS label)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="does not support macro mock CTE '__macro__tidy'", + ), + AttachmentProjectTestCase( + description="a scenario whose expected result depends on an assertion", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + "WITH\n__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + + "__assert__no_rows AS (SELECT 1 FROM __seed__channel_codes WHERE id IS NULL),\n" + + "__expected__orders AS (SELECT 1 AS id, 1.5 AS amount, 'web' AS label " + + "FROM __assert__no_rows)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="check CTE '__expected__orders' must not depend on " + "'__assert__no_rows'", + ), AttachmentProjectTestCase( description="a helper CTE reading a model through __ref and read by an assertion", overrides={ diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_scenario_extraction.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_scenario_extraction.py index 823ba4607c..9ac53d537d 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/test_native_scenario_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_scenario_extraction.py @@ -1,8 +1,9 @@ -"""Scenario CTEs extract and classify identically under the preview engine.""" +"""Scenario CTEs and errors extract identically under the preview engine, mostly natively.""" from __future__ import annotations import random +from collections import Counter import pytest @@ -14,7 +15,7 @@ ) from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( generated_scenario, - native_scenario_answered, + native_scenario_answer, scenario_outcome, ) from tests.integration.src.sqlbuild.compiler.helpers import mismatches @@ -28,18 +29,20 @@ seed=20261008, count=3000, syntax=SqlLexicalSyntax(), - expected_minimum_extracted=200, - expected_minimum_native=140, + expected_minimum_extracted=150, + expected_minimum_native=80, expected_minimum_python_errors=2000, + expected_minimum_native_errors=1200, ), ScenarioParityTestCase( - description="hash comments, which keep the Python scanner", + description="hash comments under the adapter's rules", seed=20261009, count=1000, syntax=SqlLexicalSyntax(line_comment_prefixes=frozenset({"--", "#"})), - expected_minimum_extracted=60, - expected_minimum_native=30, + expected_minimum_extracted=50, + expected_minimum_native=25, expected_minimum_python_errors=600, + expected_minimum_native_errors=400, ), ], ids=lambda case: case.description, @@ -56,6 +59,7 @@ def test_given_generated_scenarios_when_extracting_with_preview_then_python_outp ) for sql in sqls ] + answers: list[str] = [native_scenario_answer(sql=sql, syntax=test_case.syntax) for sql in sqls] preview: list[object] = [ scenario_outcome( sql=sql, @@ -70,9 +74,10 @@ def test_given_generated_scenarios_when_extracting_with_preview_then_python_outp mismatches(inputs=[*sqls], expected=python, actual=preview), sum(isinstance(item, CompileSqlScenarioCtes) for item in python) >= test_case.expected_minimum_extracted, - sum(map(native_scenario_answered, sqls)) >= test_case.expected_minimum_native, + answers.count("extracted") >= test_case.expected_minimum_native, sum(isinstance(item, str) for item in python) >= test_case.expected_minimum_python_errors, - ) == ([], True, True, True), test_case.description + answers.count("error") >= test_case.expected_minimum_native_errors, + ) == ([], True, True, True, True), (test_case.description, Counter(answers)) if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py index 39506a0b8f..5cb8439868 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/_test_types.py @@ -55,6 +55,7 @@ class ExpectedNameScanTestCase: expected_minimum_scanned: int expected_minimum_deferred: int expected_minimum_python_errors: int + expected_minimum_native_errors: int @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index d672aef9f4..971f779ef7 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -15,9 +15,17 @@ from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros -from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_sql_test_expected_model_names +from sqlbuild.compiler.compile._helpers.scenarios.core import ( + extract_sql_scenario_expected_model_names, +) +from sqlbuild.compiler.compile._helpers.sql_tests.core import ( + extract_sql_test_expected_model_names, + extract_top_level_ctes_with_scanner, +) +from sqlbuild.compiler.compile._helpers.sql_tests.native import native_sql_test_ctes from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( + CompileSqlTestCte, DeclarationResolutionContext, DeclarationScopeBuild, DeclarationScopeResolver, @@ -120,6 +128,11 @@ class ScopeOutcome: error: str = "" +_BROKEN_SCENARIO_CTES: tuple[str, ...] = ( + "__expected__ AS (\n SELECT 1\n)", + "__expected__orders AS (\n SELECT (1\n)", + "__expected__orders (SELECT 1)", +) _FOLDERS: tuple[str, ...] = ( "models/sales", "models/sales/eu", @@ -187,7 +200,14 @@ def generated_scope_files(*, rng: random.Random) -> dict[str, str]: " SELECT 1 AS order_id\n)\nSELECT 1\n" ) } - files.update(rng.choice((scenario, scenario, {}, {}, {}))) + broken_scenario: dict[str, str] = { + "tests/scenarios/flow.sql": ( + 'SCENARIO (\n description "Flow"\n);\n\nWITH\n' + "__source__raw_orders AS (\n SELECT 1 AS order_id\n),\n" + f"{rng.choice(_BROKEN_SCENARIO_CTES)}\n" + ) + } + files.update(rng.choice((scenario, scenario, broken_scenario, {}, {}, {}))) return files @@ -238,6 +258,14 @@ def _broken_test(*, index: int, calls: str, expected: list[str]) -> str: ) +def _broken_macro_test(*, index: int, calls: str, expected: list[str]) -> str: + del expected + return ( + f'TEST (mode macro, name "broken_macro_{index}");\n\nWITH\n' + f"__macro_actual__ AS SELECT 1 AS id{calls}\n" + ) + + def _expected_model_test(*, index: int, calls: str, expected: list[str]) -> str: blocks: str = ",\n".join( f"__expected__{model} AS (\n SELECT 1 AS order_id{calls}\n)" for model in expected @@ -248,11 +276,12 @@ def _expected_model_test(*, index: int, calls: str, expected: list[str]) -> str: ) -_TEST_KINDS: tuple[str, ...] = ("macro", "broken", "expected_model") -_TEST_WEIGHTS: tuple[int, ...] = (3, 1, 6) +_TEST_KINDS: tuple[str, ...] = ("macro", "broken", "broken_macro", "expected_model") +_TEST_WEIGHTS: tuple[int, ...] = (3, 1, 1, 6) _TEST_WRITERS: dict[str, Callable[..., str]] = { "macro": _macro_test, "broken": _broken_test, + "broken_macro": _broken_macro_test, "expected_model": _expected_model_test, } @@ -368,6 +397,8 @@ def _lookup_shape(lookup: ScopeLookup) -> tuple[object, ...]: ) +_TEST_LABEL: str = "tests/unit/test.sql" +_SCENARIO_LABEL: str = "tests/scenarios/orders.sql" _LEADING: tuple[str, ...] = ( "", "-- header\n", @@ -442,12 +473,13 @@ def _lookup_shape(lookup: ScopeLookup) -> tuple[object, ...]: @dataclass(frozen=True) class ExpectedNameScanParity: - """How the native expected-model scan compared with Python over one corpus.""" + """How the native relationship scans compared with Python over one corpus.""" mismatches: list[tuple[object, object, object]] scanned: int deferred: int python_errors: int + native_errors: int def generated_expected_model_sqls(*, rng: random.Random, count: int) -> list[str]: @@ -467,34 +499,65 @@ def _generated_expected_model_sql(*, rng: random.Random) -> str: def expected_name_scan_parity( *, sqls: list[str], syntax: SqlLexicalSyntax ) -> ExpectedNameScanParity: - """Compare the native scan with Python wherever the native scan does not defer.""" - - native: list[tuple[str, ...] | None] = native_expected_model_names(sqls=sqls, syntax=syntax) - python: list[tuple[str, ...] | str] = [ - _python_expected_names(sql=sql, syntax=syntax) for sql in sqls + """Compare each native scan, test and scenario names and test CTEs, with Python's result.""" + + test_texts: list[tuple[str, str]] = [(sql, _TEST_LABEL) for sql in sqls] + scenario_texts: list[tuple[str, str]] = [(sql, _SCENARIO_LABEL) for sql in sqls] + native: list[object] = [ + *native_expected_model_names(texts=test_texts, scenario=False, syntax=syntax), + *native_expected_model_names(texts=scenario_texts, scenario=True, syntax=syntax), + *native_sql_test_ctes(texts=test_texts, syntax=syntax), ] - scanned: list[tuple[str, tuple[str, ...] | str, tuple[str, ...] | None]] = list( - compress(zip(sqls, python, native, strict=True), [names is not None for names in native]) - ) + python: list[object] = [ + *(_python_expected_names(sql=sql, syntax=syntax) for sql in sqls), + *(_python_scenario_names(sql=sql, syntax=syntax) for sql in sqls), + *(_python_test_ctes(sql=sql, syntax=syntax) for sql in sqls), + ] + answered: list[bool] = [outcome is not None for outcome in native] return ExpectedNameScanParity( mismatches=mismatches( - inputs=[sql for sql, _, _ in scanned], - expected=[expected for _, expected, _ in scanned], - actual=[names for _, _, names in scanned], + inputs=list(compress([*sqls, *sqls, *sqls], answered)), + expected=list(compress(python, answered)), + actual=list(compress(native, answered)), ), - scanned=sum(names is not None for names in native), - deferred=sum(names is None for names in native), - python_errors=sum(isinstance(expected, str) for expected in python), + scanned=sum(isinstance(outcome, tuple) for outcome in native), + deferred=answered.count(False), + python_errors=sum(isinstance(outcome, str) for outcome in python), + native_errors=sum(isinstance(outcome, str) for outcome in native), ) def _python_expected_names(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[str, ...] | str: try: return extract_sql_test_expected_model_names( - sql=sql, file_label="tests/unit/test.sql", syntax=syntax, mode=SqlTestMode.MODEL + sql=sql, file_label=_TEST_LABEL, syntax=syntax, mode=SqlTestMode.MODEL + ) + except CompileInputError as error: + return str(error) + + +def _python_scenario_names(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[str, ...] | str: + try: + return extract_sql_scenario_expected_model_names( + sql=sql, file_label=_SCENARIO_LABEL, syntax=syntax + ) + except CompileInputError as error: + return str(error) + + +def _python_test_ctes(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[tuple[str, str], ...] | str: + try: + ctes: tuple[CompileSqlTestCte, ...] = extract_top_level_ctes_with_scanner( + sql=sql, + file_label=_TEST_LABEL, + context_label="SQL test", + with_requirement="mock CTEs and one __expected__ CTE", + cte_type=CompileSqlTestCte, + syntax=syntax, ) except CompileInputError as error: return str(error) + return tuple((cte.name, cte.sql_body) for cte in ctes) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py index 8c35db8819..e30d978847 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_relationship_names.py @@ -1,4 +1,4 @@ -"""The native expected-model scan returns Python's names exactly or defers to Python.""" +"""The native relationship scans return Python's names, CTEs and errors exactly, or defer.""" from __future__ import annotations @@ -26,12 +26,13 @@ expected_minimum_scanned=300, expected_minimum_deferred=300, expected_minimum_python_errors=300, + expected_minimum_native_errors=3000, ) for offset, syntax in enumerate(LEXICAL_SYNTAXES) ], ids=lambda case: case.description, ) -def test_given_generated_sql_tests_when_scanning_expected_models_then_native_matches_or_defers( +def test_given_generated_sql_when_scanning_relationships_then_native_matches_python_or_defers( test_case: ExpectedNameScanTestCase, ) -> None: sqls: list[str] = generated_expected_model_sqls( @@ -47,7 +48,13 @@ def test_given_generated_sql_tests_when_scanning_expected_models_then_native_mat parity.scanned >= test_case.expected_minimum_scanned, parity.deferred >= test_case.expected_minimum_deferred, parity.python_errors >= test_case.expected_minimum_python_errors, - ) == ([], True, True, True), (parity.scanned, parity.deferred, parity.python_errors) + parity.native_errors >= test_case.expected_minimum_native_errors, + ) == ([], True, True, True, True), ( + parity.scanned, + parity.deferred, + parity.python_errors, + parity.native_errors, + ) if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py index db93b64bc9..b33839ff34 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -29,6 +29,9 @@ "__expected__ AS (\n SELECT 1\n)\nSELECT 1\n" ) +_SCENARIO_HEADER: str = 'SCENARIO (\n description "Flow"\n);\n\n' +_MACRO_TEST_HEADER: str = 'TEST (mode macro, name "eu_label");\n\n' + @pytest.mark.parametrize( "test_case", @@ -51,6 +54,48 @@ files={**SCOPED_PROJECT, "tests/unit/test_broken.sql": _BROKEN_TEST}, expected_error="__expected__", ), + ScopeEngineParityTestCase( + description="a scenario's bare expected prefix fails with Python's scenario fault", + files={ + **SCOPED_PROJECT, + "tests/scenarios/flow.sql": _SCENARIO_HEADER + + "WITH\n__source__raw_orders AS (SELECT 1),\n__expected__ AS (SELECT 1)\n", + }, + expected_error=( + "SQL scenario 'tests/scenarios/flow.sql' must use __expected__ to " + "identify a target" + ), + ), + ScopeEngineParityTestCase( + description="the first of several broken tests and scenarios is reported", + files={ + **SCOPED_PROJECT, + "tests/unit/test_a_missing_as.sql": 'TEST (name "a");\n\nWITH\n' + "__source__raw_orders (SELECT 1),\n__expected__orders AS (SELECT 1)\n", + "tests/unit/test_b_broken.sql": _BROKEN_TEST, + "tests/scenarios/flow.sql": _SCENARIO_HEADER + + "WITH\n__source__raw_orders AS (SELECT 1\n", + }, + expected_error="SQL test 'tests/unit/test_a_missing_as.sql' expected keyword AS", + ), + ScopeEngineParityTestCase( + description="a macro test whose CTE scan fails reports Python's scanner error", + files={ + **SCOPED_PROJECT, + "tests/unit/test_eu_label.sql": _MACRO_TEST_HEADER + + "WITH\n__macro_actual__ AS SELECT @eu_label('1') AS label\n", + }, + expected_error="SQL test 'tests/unit/test_eu_label.sql' CTE '__macro_actual__' must", + ), + ScopeEngineParityTestCase( + description="a macro test only Polyglot reads keeps Python's fallback", + files={ + **SCOPED_PROJECT, + "tests/unit/test_eu_label.sql": _MACRO_TEST_HEADER + + "WITH\n__macro_actual__ AS MATERIALIZED (\n SELECT @cents('1') AS label\n),\n" + "__macro_expected__ AS (\n SELECT 100 AS label\n)\nSELECT 1\n", + }, + ), ScopeEngineParityTestCase( description="non-ASCII folders keep Python's repr order of path keys", files={