From a704a5abc4b3c83b436a83d2909e164b6b423249 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:22:59 +0000 Subject: [PATCH 1/7] wip: carry SQLQuack expression ranges through native analysis Lineage rows keep the output expression's byte range in the cleaned analysis SQL, the analysis cache stores it, shared analyses rebase it onto each member's relation names, and query_expression_ranges maps it back to the model's query SQL. Not yet exposed by the snapshot or UI. --- .../_helpers/analysis/native_analysis.rs | 4 ++ .../_helpers/analysis/native_sharing.rs | 70 ++++++++++++++++++- .../src/analysis_route/main/mod.rs | 1 + .../main/query_expression_ranges.rs | 45 ++++++++++++ .../src/analysis_route/tests.rs | 2 + .../tests/test_expression_ranges.rs | 68 ++++++++++++++++++ .../_helpers/analysis_cache.rs | 10 +++ .../_helpers/compact_batch.rs | 2 + .../analysis_session/_helpers/cte_facts.rs | 1 + .../analysis_session/_helpers/enrichment.rs | 1 + .../assembly/analysis_session/constants.rs | 2 +- .../src/assembly/analysis_session/models.rs | 3 + .../_helpers/normalization.rs | 66 ++++++++++++----- crates/sqlbuild-project/src/tests/helpers.rs | 1 + .../src/compiler/main/placeholder_defaults.rs | 41 +++++++---- 15 files changed, 281 insertions(+), 36 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs index 8006361670..07c95d22ac 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs @@ -151,6 +151,10 @@ fn lineage_row( transform_code, confidence_code, sources, + expression_range: output + .expression_range + .as_ref() + .map(|range| (range.start, range.end)), } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs index efa62fbccf..5cc8fa6959 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs @@ -110,7 +110,12 @@ pub(crate) fn analysed_members( .get(leader) .filter(|template| exact_for_others(template, members[*leader].model)); match template.map(|template| mapped(template, &members[index].model.lineage_references)) { - Some(Ok(analysis)) => { + Some(Ok(mut analysis)) => { + rebase_ranges( + &mut analysis, + members[*leader].cleaned.as_deref().unwrap_or_default(), + members[index].cleaned.as_deref().unwrap_or_default(), + ); analyses[index] = Some(Ok(analysis)); counts.shared += 1; counts.unshared -= 1; @@ -208,6 +213,69 @@ fn mapped( Ok(analysis) } +/// Move each expression range from the leader's cleaned SQL to the member's, which differ only in +/// the relation names they spell; a range that cannot be moved exactly is dropped. +pub(crate) fn rebase_ranges(analysis: &mut NativeAnalysis, leader: &str, member: &str) { + if leader == member { + return; + } + let renamed: Option> = renamed_runs(leader, member); + for row in &mut analysis.lineage { + row.expression_range = match (&renamed, row.expression_range) { + (Some(runs), Some((start, end))) => rebased(runs, start).zip(rebased(runs, end)), + _ => None, + }; + } +} + +/// `(leader start, leader end, member start, member end)` of each identifier run where two SQL +/// texts differ, when they differ only in whole identifier runs. +fn renamed_runs(leader: &str, member: &str) -> Option> { + let identifier = |c: u8| c.is_ascii_alphanumeric() || c == b'_' || c == b'$'; + let (left, right): (&[u8], &[u8]) = (leader.as_bytes(), member.as_bytes()); + let (mut i, mut j): (usize, usize) = (0, 0); + let mut runs: Vec<(usize, usize, usize, usize)> = Vec::new(); + while i < left.len() || j < right.len() { + if i < left.len() && j < right.len() && left[i] == right[j] { + i += 1; + j += 1; + continue; + } + let mut back: usize = 0; + while back < i && identifier(left[i - back - 1]) { + back += 1; + } + let (mut left_end, mut right_end): (usize, usize) = (i, j); + while left_end < left.len() && identifier(left[left_end]) { + left_end += 1; + } + while right_end < right.len() && identifier(right[right_end]) { + right_end += 1; + } + if left_end == i && right_end == j { + return None; + } + runs.push((i - back, left_end, j - back, right_end)); + (i, j) = (left_end, right_end); + } + Some(runs) +} + +/// The member offset of leader `offset`, or `None` strictly inside a renamed run. +fn rebased(runs: &[(usize, usize, usize, usize)], offset: usize) -> Option { + let mut shift: isize = 0; + for &(left_start, left_end, _, right_end) in runs { + if offset <= left_start { + break; + } + if offset < left_end { + return None; + } + shift = right_end as isize - left_end as isize; + } + usize::try_from(offset as isize + shift).ok() +} + /// Whether `leader`'s `template` succeeded, found nothing and spells none of its relation names. fn exact_for_others(template: &Result, leader: &ModelRequest) -> bool { let Ok(analysis) = template else { diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 9e43ee9093..7824a37df3 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -4,6 +4,7 @@ pub mod fast_lineage; pub mod native_gap; pub(crate) mod native_outcomes; pub mod native_revalidation; +pub mod query_expression_ranges; pub mod schema_column; pub mod skipped_analysis; pub mod unapplied_nullability_rules; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs new file mode 100644 index 0000000000..8c992715fe --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs @@ -0,0 +1,45 @@ +//! Expression ranges of a model's native analysis, moved back into the query SQL it compiled. + +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped; + +/// Each byte range of `query_sql`'s cleaned native analysis SQL as the byte range of `query_sql` +/// it came from; `None` where there was none or the dialect has no native analysis. +#[must_use] +pub fn query_expression_ranges( + dialect: &str, + (query_sql, placeholders): (&str, &[(String, String)]), + ranges: &[Option<(usize, usize)>], +) -> Vec> { + let AnalysisRoute::Native(module) = analysis_route(Some(dialect)) else { + return vec![None; ranges.len()]; + }; + let Ok((cleaned, offsets)) = + normalize_native_sql_mapped(query_sql, dialect, module.stub_prefix(), placeholders) + else { + return vec![None; ranges.len()]; + }; + let query_bytes: Vec = query_sql + .char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(query_sql.len())) + .collect(); + let mut characters: Vec> = vec![None; cleaned.len() + 1]; + for (character, (byte, _)) in cleaned.char_indices().enumerate() { + characters[byte] = Some(character); + } + characters[cleaned.len()] = Some(cleaned.chars().count()); + let query_byte = |cleaned_byte: usize| -> Option { + let character: usize = (*characters.get(cleaned_byte)?)?; + query_bytes.get(*offsets.get(character)?).copied() + }; + ranges + .iter() + .map(|range| { + let (start, end) = (*range)?; + let (start, end) = (query_byte(start)?, query_byte(end)?); + (start <= end).then_some((start, end)) + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index 3cf359ed56..660e64b4ed 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -10,6 +10,8 @@ mod test_clickhouse_type_match; mod test_databricks_dialect; #[path = "tests/test_engine_types.rs"] mod test_engine_types; +#[path = "tests/test_expression_ranges.rs"] +mod test_expression_ranges; #[path = "tests/test_native_arms.rs"] mod test_native_arms; #[path = "tests/test_native_dialect.rs"] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs new file mode 100644 index 0000000000..82ef12baa9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs @@ -0,0 +1,68 @@ +use crate::analysis_route::_helpers::analysis::native_sharing::rebase_ranges; +use crate::analysis_route::main::query_expression_ranges::query_expression_ranges; +use crate::analysis_route::models::{DuckDbDialect, NativeAnalysis}; +use crate::analysis_route::tests::helpers::analysed; +use crate::analysis_route::types::NativeDialect; +use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped; + +/// Each output column with the text of `query_sql` its mapped expression range covers. +fn mapped_texts<'a>( + query_sql: &'a str, + placeholders: &[(String, String)], +) -> Vec<(String, Option<&'a str>)> { + let dialect: &DuckDbDialect = &DuckDbDialect; + let (cleaned, _) = + normalize_native_sql_mapped(query_sql, "duckdb", dialect.stub_prefix(), placeholders) + .expect("the test query normalises"); + let analysis: NativeAnalysis = analysed(dialect, &cleaned, &[]); + let ranges: Vec> = analysis + .lineage + .iter() + .map(|row| row.expression_range) + .collect(); + let mapped = query_expression_ranges("duckdb", (query_sql, placeholders), &ranges); + analysis + .lineage + .iter() + .zip(mapped) + .map(|(row, range)| { + ( + row.output_column.clone(), + range.map(|(start, end)| &query_sql[start..end]), + ) + }) + .collect() +} + +#[test] +fn given_compiled_sql_that_cleaning_rewrites_when_mapping_ranges_then_they_cover_its_own_text() { + let placeholders: Vec<(String, String)> = vec![("floor".to_owned(), "100000".to_owned())]; + let query_sql: &str = "-- café ünïcode\nWITH x AS (SELECT * FROM __ref(\"orders\") WHERE id > @@@floor \ + AND id < __cursor_end())\nSELECT x.id AS id, UPPER(x.status) AS loud, 'é' AS mark FROM x"; + assert_eq!( + mapped_texts(query_sql, &placeholders), + vec![ + ("id".to_owned(), Some("x.id AS id")), + ("loud".to_owned(), Some("UPPER(x.status) AS loud")), + ("mark".to_owned(), Some("'é' AS mark")), + ] + ); +} + +#[test] +fn given_a_shared_analysis_when_rebasing_onto_a_member_then_ranges_follow_its_relation_names() { + let leader: &str = + "WITH x AS (SELECT * FROM orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x"; + let member: &str = "WITH x AS (SELECT * FROM customer_orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x"; + let mut analysis: NativeAnalysis = analysed(&DuckDbDialect, leader, &[]); + rebase_ranges(&mut analysis, leader, member); + let texts: Vec> = analysis + .lineage + .iter() + .map(|row| row.expression_range.map(|(start, end)| &member[start..end])) + .collect(); + assert_eq!( + texts, + vec![Some("x.id AS id"), Some("UPPER(x.status) AS loud")] + ); +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index a814d3bfaf..d265c3985e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -451,6 +451,9 @@ pub(crate) fn encode_outcome( encoder.text(name); encoder.text(column); } + let (start, end) = row.expression_range.unzip(); + encoder.optional_uint(start); + encoder.optional_uint(end); } encoder.flag(analysis.has_star); encoder.flag(analysis.star_resolved); @@ -522,11 +525,18 @@ pub(crate) fn decode_outcome( for _ in 0..source_count { sources.push((decoder.text()?, decoder.text()?, decoder.text()?)); } + let expression_range: Option<(usize, usize)> = + match (decoder.optional_index()?, decoder.optional_index()?) { + (Some(start), Some(end)) if start <= end => Some((start, end)), + (None, None) => None, + _ => return Err(StoreDecodeError), + }; rows.push(LineageRow { output_column, transform_code, confidence_code, sources, + expression_range, }); } let lineage: LineageFacts = match tag { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 8679f4480d..ef4957e07b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -864,6 +864,7 @@ impl SharedResult { transform_code: row.transform_code, confidence_code: row.confidence_code, sources: named_sources(&row.sources, &names), + expression_range: None, }) .collect(); Some(MemberResult { @@ -1361,6 +1362,7 @@ fn project_fact( transform_code: u8::try_from(transform).map_err(|error| error.to_string())?, confidence_code: u8::try_from(confidence).map_err(|error| error.to_string())?, sources: resolved, + expression_range: None, }, )) } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs index dfdaceb69f..2968a2c3ba 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs @@ -221,6 +221,7 @@ pub(crate) fn legacy_analysis(input: &LegacyInput<'_>) -> Fact { confidence }, sources, + expression_range: None, }); } Ok(LegacyAnalysis { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index 2d69ab47a5..12bab7d107 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -256,6 +256,7 @@ fn native_projection( }, transform_code: transform, sources, + expression_range: None, }); } let has_star: bool = has_star_projection(analysis); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 51d79f0389..968ee107ea 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -199,7 +199,7 @@ pub(crate) const PYTHON_RULE: &str = "python"; pub(crate) const NON_NULL_NULLABILITY: &str = "non_null"; pub(crate) const NULLABLE_NULLABILITY: &str = "nullable"; /// Bumped whenever the stored outcome or a key's fields change; older entries never match. -pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v4"; +pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v5"; /// A stored varint byte at or above this value continues into the next byte. pub(crate) const VARINT_CONTINUATION: u64 = 0x80; /// The value bits each stored varint byte carries. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 85f5ad7013..de0f5cb4c1 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -237,6 +237,9 @@ pub struct LineageRow { pub transform_code: u8, pub confidence_code: u8, pub sources: Vec<(String, String, String)>, + /// UTF-8 byte range of the column's defining projection item in the model's cleaned + /// analysis SQL, when the native analysis gives one. + pub expression_range: Option<(usize, usize)>, } /// Where a model's lineage facts live. diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index 90a595f015..522da9404d 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -3,6 +3,7 @@ use crate::relation_markers::main::lateral_start::lateral_start; use crate::semantic_checks::constants::UDF_NAME_PREFIX; use regex::Regex; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_default_edits; use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; @@ -201,41 +202,68 @@ pub(crate) fn normalize_native_sql( stub_prefix: &str, relations: &HashMap, ) -> Result { + native_mapped(sql, dialect, stub_prefix, relations).map(|mapped| mapped.sql) +} + +/// [`normalize_native_sql`] with placeholder defaults filled, and the character of `sql` each +/// character of it came from. +pub(crate) fn normalize_native_sql_mapped( + sql: &str, + dialect: &str, + stub_prefix: &str, + placeholders: &[(String, String)], +) -> Result<(String, Vec), String> { + let native: MappedSql = native_mapped(sql, dialect, stub_prefix, &HashMap::new())?; + let edits: Vec = placeholder_default_edits(&native.sql, placeholders) + .into_iter() + .map(|(start, end, default)| { + let anchor: usize = native.offsets[native.sql[..start].chars().count()]; + (start, end, MappedSql::generated(default.to_owned(), anchor)) + }) + .collect(); + let filled: MappedSql = native.edits(edits); + Ok((filled.sql, filled.offsets)) +} + +fn native_mapped( + sql: &str, + dialect: &str, + stub_prefix: &str, + relations: &HashMap, +) -> Result { let none: HashMap = HashMap::new(); let udf_prefix: String = format!("{stub_prefix}udf_"); - let normalized: String = normalize(sql, dialect, (relations, &none), &udf_prefix)?.sql; + let normalized: MappedSql = normalize(sql, dialect, (relations, &none), &udf_prefix)?; let stubs: &Regex = NATIVE_STUBS.as_ref()?; let non_code: Vec<(usize, usize)> = - dialect_non_code_ranges(&normalized, &dialect_lexical_syntax(dialect)); - let mut native: String = String::with_capacity(normalized.len()); - let mut copied_until: usize = 0; - for captures in stubs.captures_iter(&normalized) { + dialect_non_code_ranges(&normalized.sql, &dialect_lexical_syntax(dialect)); + let mut edits: Vec = Vec::new(); + for captures in stubs.captures_iter(&normalized.sql) { let whole = captures.get(0).ok_or("missing stub match")?; let quoted: bool = non_code .iter() .any(|&(start, end)| start <= whole.start() && whole.start() < end); let glued: bool = captures.get(1).is_some() - && normalized[..whole.start()] + && normalized.sql[..whole.start()] .chars() .next_back() .is_some_and(|character| character.is_alphanumeric() || character == '_'); if quoted || glued { continue; } - native.push_str(&normalized[copied_until..whole.start()]); - match (captures.get(1), captures.get(2)) { - (Some(bound), _) => { - native.push_str(&format!("{stub_prefix}cursor_{}()", bound.as_str())); - } - (None, Some(parameter)) => { - native.push_str(&format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str())); - } - (None, None) => native.push_str(whole.as_str()), - } - copied_until = whole.end(); + let text: String = match (captures.get(1), captures.get(2)) { + (Some(bound), _) => format!("{stub_prefix}cursor_{}()", bound.as_str()), + (None, Some(parameter)) => format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str()), + (None, None) => continue, + }; + let anchor: usize = normalized.offsets[normalized.sql[..whole.start()].chars().count()]; + edits.push(( + whole.start(), + whole.end(), + MappedSql::generated(text, anchor), + )); } - native.push_str(&normalized[copied_until..]); - Ok(native) + Ok(normalized.edits(edits)) } pub(crate) fn normalize_dialect_sql_mapped( diff --git a/crates/sqlbuild-project/src/tests/helpers.rs b/crates/sqlbuild-project/src/tests/helpers.rs index dfb8ee6c7e..efb87f1b6d 100644 --- a/crates/sqlbuild-project/src/tests/helpers.rs +++ b/crates/sqlbuild-project/src/tests/helpers.rs @@ -22,6 +22,7 @@ pub(super) fn lineage_row(output: &str, sources: &[&str]) -> LineageRow { ) }) .collect(), + expression_range: None, } } diff --git a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs index 7772bd42ea..f47bd01a2b 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs @@ -7,33 +7,44 @@ use crate::constants::PLACEHOLDER_PREFIX; /// `substitute_placeholder_defaults`: `@@@name` becomes its default; unknown names stay. pub fn placeholder_defaults(sql: &str, placeholders: &[(String, String)]) -> String { let mut result: String = String::with_capacity(sql.len()); - let mut rest: &str = sql; - while let Some(start) = rest.find(PLACEHOLDER_PREFIX) { - result.push_str(&rest[..start]); - let after: &str = &rest[start + PLACEHOLDER_PREFIX.len()..]; + let mut copied: usize = 0; + for (start, end, default) in placeholder_default_edits(sql, placeholders) { + result.push_str(&sql[copied..start]); + result.push_str(default); + copied = end; + } + result.push_str(&sql[copied..]); + result +} + +/// The byte range of each `@@@name` with a default and that default, in order. +pub fn placeholder_default_edits<'a>( + sql: &str, + placeholders: &'a [(String, String)], +) -> Vec<(usize, usize, &'a str)> { + let mut edits: Vec<(usize, usize, &'a str)> = Vec::new(); + let mut position: usize = 0; + while let Some(found) = sql[position..].find(PLACEHOLDER_PREFIX) { + let start: usize = position + found; + let after: &str = &sql[start + PLACEHOLDER_PREFIX.len()..]; let length: usize = after .find(|character: char| !is_word(character)) .unwrap_or(after.len()); if length == 0 { - result.push_str(&rest[start..=start]); - rest = &rest[start + 1..]; + position = start + 1; continue; } + let end: usize = start + PLACEHOLDER_PREFIX.len() + length; let name: &str = &after[..length]; - match placeholders + if let Some((_, default)) = placeholders .iter() .find(|(placeholder, _)| placeholder == name) { - Some((_, default)) => result.push_str(default), - None => { - result.push_str(PLACEHOLDER_PREFIX); - result.push_str(name); - } + edits.push((start, end, default.as_str())); } - rest = &after[length..]; + position = end; } - result.push_str(rest); - result + edits } /// A character Python's `\w` matches. From 06bfd2b72fc906bc7dbca8cf5425b3fa9edf1d51 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:49:42 +0000 Subject: [PATCH 2/7] wip: publish column expression ranges with the node SQL read Snapshot model nodes carry each output column's expression range as a byte range of compiled_sql, mapped back through typed SQL macro markers and cursor bounds; the node SQL read returns them. DuckDB e2e passes; the Snowflake DuckShop edge model and DuckDB qualified columns are strict xfails until SQLQuack's strict parsers record projection spans. --- benchmarks/duckshop/README.md | 6 +- .../edge_cases/orchestration/node_sql.json | 3 +- .../edge_cases/int_edge_expression_spans.sql | 37 ++++ .../duckshop/project/sqlbuild_project.toml | 9 + .../test_int_edge_expression_spans.sql | 37 ++++ clients/typescript/src/schema.d.ts | 7 + .../main/query_expression_ranges.rs | 43 +++- .../tests/test_expression_ranges.rs | 56 +++++- .../snapshots/main/read_person_node_sql.rs | 7 + .../src/snapshots/models.rs | 4 + .../tests/api/node_sql/helpers.rs | 6 +- .../tests/api/test_person_sessions.rs | 1 + .../_helpers/attachments/cursor_intrinsics.rs | 31 ++- .../compiled_project/snapshot_models.rs | 88 ++++++++- .../compiler/compiled_project/_test_types.py | 11 ++ .../publication/test_person_sql.py | 16 +- .../test_orchestration_snapshot.py | 184 ++++++++++++++++++ 17 files changed, 520 insertions(+), 26 deletions(-) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index ac5fd5faa5..58afe6e872 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -124,8 +124,8 @@ project and does not change it: `edge_running_peaks`, that a pre-hook of `int_edge_basket_running_peaks` creates (`models/intermediate/edge_cases/_sqlbuild/_hooks/sql/`) and SQLBuild does not declare. -It has 164 models (13 staging, 137 intermediate, 14 marts), 5 Python functions, a SQL table -function, a hook-created JavaScript function, 189 unit tests and 4 scenarios, in nine groups: +It has 165 models (13 staging, 138 intermediate, 14 marts), 5 Python functions, a SQL table +function, a hook-created JavaScript function, 190 unit tests and 4 scenarios, in nine groups: | Group | Covers | |-------|--------| @@ -134,7 +134,7 @@ function, a hook-created JavaScript function, 189 unit tests and 4 scenarios, in | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` | | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day | | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | -| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query | +| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | | Translation refusals | one model per construct that translation tools refuse or guard, most putting a construct refused only under strict checking before one refused by default, so a model shows both: `FLOAT` values through sums, averages, running totals, arrays, objects, `HASH`, `TO_CHAR` and `::NUMBER(38, 10)`; `VARIANT` dates, decimals and XML; `ARRAY_AGG` without `WITHIN GROUP`, sparse arrays and `ARRAY(NUMBER)` casts; `REDUCE`, `FILTER` and `TRANSFORM` lambdas; patterns and `REGEXP_REPLACE` replacements built from columns, and lazy patterns; `TRY_TO_DATE` with a format from a column; zoned literals, `CONVERT_TIMEZONE` from `CURRENT_TIMESTAMP`, `SEQ8`, `GENERATOR` with an expression `ROWCOUNT`; `PIVOT`, `ANY_VALUE`, `IGNORE NULLS` around `IFF`, unaliased columns read by Snowflake's name, `UNION` branches whose column is always NULL; `TO_NUMBER`, `TO_DECIMAL` and `TRY_BASE64_DECODE_BINARY`; an open (uncontracted) source `edge__supplier_notes`, the table function, read for one basket and after `LATERAL` for every basket, and the hook-created function. `rpt_edge_case_model_keys` and `rpt_edge_case_array_model_keys` read every model in the group, so a refusal there also refuses a downstream model | diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json index 1e8b77124a..cf559d82f1 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json +++ b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json @@ -3,5 +3,6 @@ "compiled_sql": "SELECT 23 AS amount -- Orders 🧾\n", "python_source": "def order_amount():\n return 23\n", "function_sql": "43", - "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n" + "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n", + "expression_ranges": {"amount": [7, 19]} } diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql new file mode 100644 index 0000000000..651ff5b9a9 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql @@ -0,0 +1,37 @@ +MODEL ( + description """ + One row per edge-case price window loaded in the cursor window, with its price converted by a + typed SQL macro and rounded by a Python macro, the reporting currency from a variable and a + non-ASCII label, so the compiled SQL differs from the authored text before every output column; + the acceptance case for column expression ranges that point into the compiled SQL. + """, + tags [edge_case], + materialized incremental, + incremental_strategy delete_insert, + cursor loaded_at, + cursor_type timestamp, + cursor_grain second, + cursor_inputs ( + edge__price_windows _loaded_at, + ), + columns ( + window_id (type VARCHAR, nullable false, audits [not_null, unique]), + price_amount (type NUMBER(38, 6)), + rounded_price (type NUMBER(18, 2)), + currency (type VARCHAR), + label (type VARCHAR), + loaded_at (type TIMESTAMP_NTZ, nullable false), + ), +); + +SELECT + w.window_id, + @sql.money_from_minor(w.price_minor, 2) AS price_amount, + @py.round_half_even("w.price_minor / 100", 2)::NUMBER(18, 2) AS rounded_price, + '@@reporting_currency'::VARCHAR AS currency, + 'Prix à jour €'::VARCHAR AS label, + w._loaded_at AS loaded_at +FROM __source("edge__price_windows") AS w +WHERE + w._loaded_at >= __cursor_start() + AND w._loaded_at < __cursor_end() diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index fca5a4ea59..6272acbeb8 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -337,6 +337,15 @@ The edge case reads the line through a derived table nested in a comma-joined LA that shape is the case under test. """ +[[rules.rule_ignores]] +rules = ["SQLCLEAN034"] +paths = ["models/intermediate/edge_cases/int_edge_expression_spans.sql"] +reason = """ +The edge case keeps its macro calls, variable and literals in the terminal select on purpose: +column expression ranges point at the terminal select's items, so that is where the compiled SQL +must differ from the authored text. +""" + [[rules.rule_ignores]] rules = ["SQLCLEAN040"] paths = ["models/intermediate/edge_cases/int_edge_amount_bit_reads.sql"] diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql new file mode 100644 index 0000000000..b3a0a373dd --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql @@ -0,0 +1,37 @@ +TEST ( + name "int_edge_expression_spans__converts_prices_loaded_in_the_window", + cursor_start "2026-03-02 00:00:00", + cursor_end "2026-03-03 00:00:00", +); + +WITH __source__edge__price_windows AS ( + SELECT + v.window_id, + v.sku, + v.price_minor::NUMBER(38, 0) AS price_minor, + v.valid_from::TIMESTAMP_NTZ(6) AS valid_from, + v.valid_to::TIMESTAMP_NTZ(6) AS valid_to, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PW-01', 'SKU-1', 1234, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 09:00:00'), + ('PW-02', 'SKU-2', NULL, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 18:30:00'), + ('PW-03', 'SKU-3', 999, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-01 23:59:59') AS v(window_id, sku, price_minor, valid_from, valid_to, _loaded_at) +), + +__expected__int_edge_expression_spans AS ( + SELECT + 'PW-01' AS window_id, + 12.34::NUMBER(38, 6) AS price_amount, + 12.34::NUMBER(18, 2) AS rounded_price, + 'USD' AS currency, + 'Prix à jour €' AS label, + '2026-03-02 09:00:00'::TIMESTAMP_NTZ AS loaded_at + UNION ALL + SELECT + 'PW-02' AS window_id, + CAST(NULL AS NUMBER(38, 6)) AS price_amount, + CAST(NULL AS NUMBER(18, 2)) AS rounded_price, + 'USD' AS currency, + 'Prix à jour €' AS label, + '2026-03-02 18:30:00'::TIMESTAMP_NTZ AS loaded_at +) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 041ea0880a..b592225e27 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2009,6 +2009,13 @@ export interface components { /** @description SQL inspection never returns Python source, execution artifacts or unrelated node metadata. */ NodeSql: { compiled_sql?: string | null; + /** + * @description Each output column's defining expression as a `[start, end)` UTF-8 byte range of + * `compiled_sql`; absent for columns the analysis could not place. + */ + expression_ranges?: { + [key: string]: number[]; + }; node_key: string; raw_sql?: string | null; /** Format: uuid */ diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs index 8c992715fe..2a2f1c6288 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs @@ -3,27 +3,30 @@ use crate::analysis_route::main::analysis_route::analysis_route; use crate::analysis_route::models::AnalysisRoute; use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped; +use crate::sql_macros::models::TextSplice; -/// Each byte range of `query_sql`'s cleaned native analysis SQL as the byte range of `query_sql` -/// it came from; `None` where there was none or the dialect has no native analysis. +/// Each byte range of `analysis_sql`'s cleaned native analysis SQL as the byte range of the query +/// SQL that `passes` rewrote, in order, into `analysis_sql`; `None` where there was none, where an +/// end falls inside rewritten text, or where the dialect has no native analysis. #[must_use] pub fn query_expression_ranges( dialect: &str, - (query_sql, placeholders): (&str, &[(String, String)]), + (analysis_sql, placeholders): (&str, &[(String, String)]), + passes: &[Vec], ranges: &[Option<(usize, usize)>], ) -> Vec> { let AnalysisRoute::Native(module) = analysis_route(Some(dialect)) else { return vec![None; ranges.len()]; }; let Ok((cleaned, offsets)) = - normalize_native_sql_mapped(query_sql, dialect, module.stub_prefix(), placeholders) + normalize_native_sql_mapped(analysis_sql, dialect, module.stub_prefix(), placeholders) else { return vec![None; ranges.len()]; }; - let query_bytes: Vec = query_sql + let analysis_bytes: Vec = analysis_sql .char_indices() .map(|(byte, _)| byte) - .chain(std::iter::once(query_sql.len())) + .chain(std::iter::once(analysis_sql.len())) .collect(); let mut characters: Vec> = vec![None; cleaned.len() + 1]; for (character, (byte, _)) in cleaned.char_indices().enumerate() { @@ -32,7 +35,11 @@ pub fn query_expression_ranges( characters[cleaned.len()] = Some(cleaned.chars().count()); let query_byte = |cleaned_byte: usize| -> Option { let character: usize = (*characters.get(cleaned_byte)?)?; - query_bytes.get(*offsets.get(character)?).copied() + let byte: usize = *analysis_bytes.get(*offsets.get(character)?)?; + passes + .iter() + .rev() + .try_fold(byte, |byte, splices| spliced_back(byte, splices)) }; ranges .iter() @@ -43,3 +50,25 @@ pub fn query_expression_ranges( }) .collect() } + +/// The source byte of output byte `byte` of a pass whose `splices` are in text order; a byte at +/// a splice's edge maps to the same edge of its source, and one inside rewritten text has none. +fn spliced_back(byte: usize, splices: &[TextSplice]) -> Option { + let (mut source_end, mut output_end): (usize, usize) = (0, 0); + for splice in splices { + if byte < splice.output.0 { + break; + } + if byte == splice.output.0 { + return Some(splice.source.0); + } + if byte < splice.output.1 { + return None; + } + if byte == splice.output.1 { + return Some(splice.source.1); + } + (source_end, output_end) = (splice.source.1, splice.output.1); + } + Some(byte - output_end + source_end) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs index 82ef12baa9..e2c3b941c1 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs @@ -4,15 +4,20 @@ use crate::analysis_route::models::{DuckDbDialect, NativeAnalysis}; use crate::analysis_route::tests::helpers::analysed; use crate::analysis_route::types::NativeDialect; use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped; +use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use crate::sql_macros::models::{TextSplice, TypedCall}; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; -/// Each output column with the text of `query_sql` its mapped expression range covers. +/// Each output column with the text of `query_sql` its mapped expression range covers, after +/// `passes` rewrote `query_sql` into `analysis_sql`. fn mapped_texts<'a>( - query_sql: &'a str, + (query_sql, analysis_sql): (&'a str, &str), placeholders: &[(String, String)], + passes: &[Vec], ) -> Vec<(String, Option<&'a str>)> { let dialect: &DuckDbDialect = &DuckDbDialect; let (cleaned, _) = - normalize_native_sql_mapped(query_sql, "duckdb", dialect.stub_prefix(), placeholders) + normalize_native_sql_mapped(analysis_sql, "duckdb", dialect.stub_prefix(), placeholders) .expect("the test query normalises"); let analysis: NativeAnalysis = analysed(dialect, &cleaned, &[]); let ranges: Vec> = analysis @@ -20,7 +25,7 @@ fn mapped_texts<'a>( .iter() .map(|row| row.expression_range) .collect(); - let mapped = query_expression_ranges("duckdb", (query_sql, placeholders), &ranges); + let mapped = query_expression_ranges("duckdb", (analysis_sql, placeholders), passes, &ranges); analysis .lineage .iter() @@ -40,7 +45,7 @@ fn given_compiled_sql_that_cleaning_rewrites_when_mapping_ranges_then_they_cover let query_sql: &str = "-- café ünïcode\nWITH x AS (SELECT * FROM __ref(\"orders\") WHERE id > @@@floor \ AND id < __cursor_end())\nSELECT x.id AS id, UPPER(x.status) AS loud, 'é' AS mark FROM x"; assert_eq!( - mapped_texts(query_sql, &placeholders), + mapped_texts((query_sql, query_sql), &placeholders, &[]), vec![ ("id".to_owned(), Some("x.id AS id")), ("loud".to_owned(), Some("UPPER(x.status) AS loud")), @@ -49,6 +54,47 @@ fn given_compiled_sql_that_cleaning_rewrites_when_mapping_ranges_then_they_cover ); } +#[test] +fn given_typed_calls_and_cursor_bounds_when_mapping_ranges_then_they_cover_the_compiled_text() { + let query_sql: &str = "SELECT ROUND(amount * 1.2, 2) AS gross, id AS id FROM orders \ + WHERE updated_at < __cursor_end()"; + let call_start: usize = query_sql.find("ROUND").expect("the call is in the query"); + let call_end: usize = query_sql + .find(" AS gross") + .expect("the alias is in the query"); + let calls: Vec = vec![TypedCall { + start: call_start, + end: call_end, + macro_name: "gross_amount".to_owned(), + returns: Some("DECIMAL(18, 2)".to_owned()), + returns_like: None, + }]; + let (typed, typed_splices) = typed_analysis_sql(query_sql, &calls, "duckdb"); + assert_ne!(typed, query_sql, "the call is marked for analysis"); + let (analysis_sql, spans) = replace_intrinsics_with_spans(&typed, "Model SQL", |_| { + "CAST('2000-01-01' AS TIMESTAMP)".to_owned() + }) + .expect("the cursor call is replaced"); + let cursor_splices: Vec = spans + .iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + assert_eq!( + mapped_texts( + (query_sql, &analysis_sql), + &[], + &[typed_splices, cursor_splices] + ), + vec![ + ("gross".to_owned(), Some("ROUND(amount * 1.2, 2) AS gross")), + ("id".to_owned(), Some("id AS id")), + ] + ); +} + #[test] fn given_a_shared_analysis_when_rebasing_onto_a_member_then_ranges_follow_its_relation_names() { let leader: &str = diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs index db85b62338..6b3962c8ae 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs @@ -40,11 +40,18 @@ pub async fn read_person_node_sql( WHERE s.snapshot_id=$1 AND s.node_key=$2 AND s.kind IN ('raw_sql','compiled_sql')", vec![Parameter::Id(record.snapshot_id), Parameter::Text(node_key.into())]).await?; if let Some(node) = attach_sources(nodes, rows)?.remove(node_key) { + let expression_ranges = node + .definition + .get("expression_ranges") + .cloned() + .and_then(|ranges| serde_json::from_value(ranges).ok()) + .unwrap_or_default(); result = Some(NodeSql { snapshot_id: record.snapshot_id, node_key: node_key.into(), raw_sql: node.raw_sql, compiled_sql: node.compiled_sql, + expression_ranges, }); } } diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 77c3558329..c4c4db6e69 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -24,6 +24,10 @@ pub struct NodeSql { pub node_key: String, pub raw_sql: Option, pub compiled_sql: Option, + /// Each output column's defining expression as a `[start, end)` UTF-8 byte range of + /// `compiled_sql`; absent for columns the analysis could not place. + #[serde(default)] + pub expression_ranges: BTreeMap, } /// A location's successful compile for one target. diff --git a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs index 9c08fe2846..2940a4a3b5 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs @@ -35,7 +35,7 @@ pub(crate) async fn node_sql_boundaries( for (key, expected) in [ ( "udf:orders", - json!({"snapshot_id":snapshot_id,"node_key":"udf:orders","raw_sql":null,"compiled_sql":data["function_sql"]}), + json!({"snapshot_id":snapshot_id,"node_key":"udf:orders","raw_sql":null,"compiled_sql":data["function_sql"],"expression_ranges":{}}), ), ("missing-orders", Value::Null), ("orders+", Value::Null), @@ -61,7 +61,7 @@ pub(crate) async fn node_sql_boundaries( assert_eq!(read.status(), StatusCode::OK); assert_eq!( read.json::().await.expect("current SQL"), - json!({"snapshot_id":snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["compiled_sql"]}) + json!({"snapshot_id":snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["compiled_sql"],"expression_ranges":data["expression_ranges"]}) ); let mut after = logical_state(store).await; assert_ne!( @@ -125,6 +125,6 @@ pub(crate) async fn node_sql_boundaries( assert_eq!(response.status(), StatusCode::OK); assert_eq!( response.json::().await.expect("latest SQL"), - json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"]}) + json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"],"expression_ranges":data["expression_ranges"]}) ); } diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index f5558bc4cb..982507bac8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -115,6 +115,7 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ .expect("compiled SQL") .into(), ); + orders.definition["expression_ranges"] = sql_input["expression_ranges"].clone(); orders.python_source = Some( sql_input["python_source"] .as_str() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index d75b591a5d..acaa44693f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{Python, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::sql_macros::models::TextSplice; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; @@ -99,12 +101,35 @@ pub(crate) fn analysis_cursor_sql( sql: &str, cursor_type: &Bound<'_, PyAny>, ) -> PyResult { - let literal: &str = if text_value(cursor_type).as_deref() == Some(INTEGER_CURSOR_TYPE) { + let literal: &str = analysis_bound(text_value(cursor_type).as_deref()); + Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) +} + +/// `analysis_cursor_sql` with each replaced call as a splice of `sql`; `None` when a call is invalid. +pub(crate) fn analysis_cursor_splices( + sql: &str, + cursor_type: Option<&str>, +) -> Option<(String, Vec)> { + let literal: &str = analysis_bound(cursor_type); + let (replaced, spans) = + replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned()).ok()?; + let splices: Vec = spans + .iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + Some((replaced, splices)) +} + +/// The literal static analysis reads for every cursor intrinsic of a model with `cursor_type`. +fn analysis_bound(cursor_type: Option<&str>) -> &'static str { + if cursor_type == Some(INTEGER_CURSOR_TYPE) { INTEGER_ANALYSIS_BOUND } else { TIMESTAMP_ANALYSIS_BOUND - }; - Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) + } } /// One model query with canonical intrinsic calls, raising `CompileInputError` when invalid. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs index 13fa7aca8c..ed299cc5f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -5,15 +5,21 @@ use std::collections::BTreeMap; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; use pyo3::types::PyTupleMethods; -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; +use sqlbuild_analysis::analysis_route::main::query_expression_ranges::query_expression_ranges; +use sqlbuild_analysis::assembly::analysis_session::models::{LineageFacts, LineageRow}; use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; +use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_orchestration::snapshots::models::SnapshotNode; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_splices; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::snapshot::{ dependency_keys, destination, insert_node, }; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; pub(crate) fn model_nodes( @@ -22,6 +28,7 @@ pub(crate) fn model_nodes( owner: &NativeCompiledProject, mut nodes: BTreeMap, ) -> PyResult> { + let dialect: Option = project.sql_analysis_dialect(); for model in project.models(py).bind(py).iter() { let model = model.cast::()?.get(); let (name, _, _) = model.file_texts(py)?; @@ -47,14 +54,23 @@ pub(crate) fn model_nodes( json!({"name": name, "type": data_type, "nullability": nullability}) }) .collect(); - Ok(json!({ + let mut definition: Value = json!({ "name": facts.name, "path": facts.relative_path, "config": facts.config.values, "materialization": facts.config.materialized, "deps": dependency_keys(&record.deps), "columns": columns, - })) + }); + let lineage: Option<&LineageFacts> = record.analysis.lineage.as_ref(); + let ranges: Map = match (dialect.as_deref(), lineage) { + (Some(dialect), Some(lineage)) => expression_ranges(dialect, input, facts, lineage), + _ => Map::new(), + }; + if !ranges.is_empty() { + definition["expression_ranges"] = Value::Object(ranges); + } + Ok(definition) })?; definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?; nodes = insert_node( @@ -71,3 +87,69 @@ pub(crate) fn model_nodes( } Ok(nodes) } + +/// Each output column's defining expression as a `[start, end)` byte range of the compiled SQL +/// the node SQL read returns; empty when the analysis SQL cannot be rebuilt exactly. +fn expression_ranges( + dialect: &str, + input: &ModelInputRecord, + facts: &CompiledModelFacts, + lineage: &LineageFacts, +) -> Map { + let mut ranges: Map = Map::new(); + let (LineageFacts::Native(rows) | LineageFacts::NativeFacts(rows)) = lineage; + if *input.query_sql != *facts.query_sql || rows.iter().all(|row| row.expression_range.is_none()) + { + return ranges; + } + let Some(placeholders) = placeholder_texts(facts.config.values.get("placeholders")) else { + return ranges; + }; + let (typed, typed_splices) = typed_analysis_sql(&input.query_sql, &input.typed_calls, dialect); + let cursor_type: Option<&str> = facts + .config + .values + .get("cursor_type") + .and_then(Value::as_str); + let Some((analysis_sql, cursor_splices)) = analysis_cursor_splices(&typed, cursor_type) else { + return ranges; + }; + let cleaned: Vec> = + rows.iter().map(|row| row.expression_range).collect(); + let mapped = query_expression_ranges( + dialect, + (&analysis_sql, &placeholders), + &[typed_splices, cursor_splices], + &cleaned, + ); + for (row, range) in rows.iter().zip(mapped) { + let LineageRow { output_column, .. } = row; + if let Some((start, end)) = range { + ranges + .entry(output_column.clone()) + .or_insert_with(|| json!([start, end])); + } + } + ranges +} + +/// The `placeholders` config as the `(name, text)` pairs analysis read, the way Python's `str` +/// renders each value; `None` when a value's text could differ from what analysis read. +fn placeholder_texts(value: Option<&Value>) -> Option> { + let Some(Value::Object(values)) = value else { + return Some(Vec::new()); + }; + values + .iter() + .map(|(name, value)| { + let text: String = match value { + Value::String(text) => text.clone(), + Value::Number(number) if number.is_i64() || number.is_u64() => number.to_string(), + Value::Bool(true) => "True".to_owned(), + Value::Bool(false) => "False".to_owned(), + _ => return None, + }; + Some((name.clone(), text)) + }) + .collect() +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 12c96c457a..503ec468ec 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -233,6 +233,17 @@ class SnapshotEnvironmentTestCase: expected_environment_count: int +@dataclass(frozen=True) +class SnapshotExpressionRangeTestCase: + """A published model whose compiled SQL differs from its source, and each column's text.""" + + description: str + adapter: str + files: dict[str, str] + model_name: str + expected_texts: dict[str, str] + + @dataclass(frozen=True) class SnapshotLineageTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py index d73478e099..093f0997e0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -177,7 +177,21 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori inspected: subprocess.CompletedProcess[str] = invoke(command) assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr sql: dict[str, Any] = json.loads(inspected.stdout) - assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert set(sql) == { + "snapshot_id", + "node_key", + "raw_sql", + "compiled_sql", + "expression_ranges", + } + compiled: bytes = sql["compiled_sql"].encode() + assert { + column: compiled[start:end].decode() + for column, (start, end) in sql["expression_ranges"].items() + } == { + "amount": "23 AS amount", + "description": "'" + "orders " * 400 + "' AS description", + } assert ( sql["snapshot_id"] == current_snapshot(worker.api, "development")["record"]["snapshot_id"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index c6e520bb1f..e12382ef1e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -20,6 +20,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.orchestration import ( RunEventKind, @@ -37,6 +38,7 @@ SnapshotDiscoveryRejectionTestCase, SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, + SnapshotExpressionRangeTestCase, SnapshotIdentityTestCase, SnapshotLineageTestCase, SnapshotRejectionTestCase, @@ -399,6 +401,188 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s assert sources | targets <= set(publication["nodes"]) +_DUCKSHOP: Path = Path("benchmarks/duckshop/project") +_STRICT_SPANS: str = ( + "SQLQuack's strict parsers do not yet record projection-item spans: Snowflake gives aliased " + "items none, and DuckDB's column span leaves out the qualifier" +) +_PRICE_MODELS: dict[str, str] = { + "models/stg_prices.sql": ( + 'MODEL (description "Staged prices");\n' + "SELECT 'W-1' AS window_id, CAST(12345 AS BIGINT) AS price_minor, " + "CAST('2026-03-30 08:00:00' AS TIMESTAMP) AS loaded_at\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + pytest.param( + SnapshotExpressionRangeTestCase( + description="SQL and Python macros, a variable and cursor bounds rewrite the SQL", + adapter="duckdb", + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n' + '[vars]\nreporting_currency = "USD"\n' + ), + "models/_macros/sql/minor_to_major.sql": ( + "MACRO (\n" + ' description "Minor units to major units",\n' + " arguments (amount BIGINT, places INT CONST DEFAULT 2),\n" + " returns DECIMAL(18, 2),\n" + ");\n\n" + "ROUND(@amount / 100, @places)::DECIMAL(18, 2)\n" + ), + "models/_macros/python/rounding.py": ( + "def round_half_even(ctx, expression: str, places: int = 2) -> str:\n" + " return f'ROUND_EVEN({expression}, {places})'\n" + ), + **_PRICE_MODELS, + "models/price_spans.sql": ( + "MODEL (\n" + ' description "Prices with rewritten expressions",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor loaded_at,\n" + " cursor_type timestamp,\n" + " cursor_grain second,\n" + ");\n\n" + "-- Prix à jour €\n" + "SELECT\n" + " p.window_id AS window_id,\n" + " @sql.minor_to_major(p.price_minor) AS price_amount,\n" + ' @py.round_half_even("p.price_minor / 100", 2) AS rounded_price,\n' + " '@@reporting_currency' AS currency,\n" + " 'Prix à jour €' AS label,\n" + " p.loaded_at AS loaded_at\n" + 'FROM __ref("stg_prices") AS p\n' + "WHERE p.loaded_at >= __cursor_start() AND p.loaded_at < __cursor_end()\n" + ), + }, + model_name="price_spans", + expected_texts={ + "window_id": "p.window_id AS window_id", + "price_amount": ( + "ROUND(p.price_minor / 100, 2)::DECIMAL(18, 2) AS price_amount" + ), + "rounded_price": "ROUND_EVEN(p.price_minor / 100, 2) AS rounded_price", + "currency": "'USD' AS currency", + "label": "'Prix à jour €' AS label", + "loaded_at": "p.loaded_at AS loaded_at", + }, + ), + id="DuckDB macros, variable and cursor bounds", + ), + pytest.param( + SnapshotExpressionRangeTestCase( + description="an unaliased qualified column covers its qualifier", + adapter="duckdb", + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + **_PRICE_MODELS, + "models/price_columns.sql": ( + 'MODEL (description "Price columns");\n' + 'SELECT p.window_id, p.price_minor FROM __ref("stg_prices") AS p\n' + ), + }, + model_name="price_columns", + expected_texts={ + "window_id": "p.window_id", + "price_minor": "p.price_minor", + }, + ), + id="DuckDB qualified columns", + marks=pytest.mark.xfail(strict=True, raises=AssertionError, reason=_STRICT_SPANS), + ), + pytest.param( + SnapshotExpressionRangeTestCase( + description="DuckShop macros, variable and cursor bounds rewrite the SQL", + adapter="snowflake", + files={ + "sqlbuild_project.toml": ( + 'name = "duckshop"\nadapter = "snowflake"\n\n' + '[defaults]\ndatabase = "WAREHOUSE"\nschema = "MAIN"\n\n' + '[vars]\nreporting_currency = "USD"\n' + ), + "sources/edge_cases.yml": ( + "sources:\n" + " - name: edge__price_windows\n" + " description: Price windows\n" + " columns:\n" + " - name: window_id\n" + " type: VARCHAR\n" + " - name: price_minor\n" + " type: NUMBER(38,0)\n" + " - name: _loaded_at\n" + " type: TIMESTAMP_NTZ\n" + ), + # A second consumer keeps the DuckShop macros project-wide, as they are there. + "models/price_checks.sql": ( + 'MODEL (description "Price checks");\n' + "SELECT @sql.money_from_minor(150, 2) AS amount, " + '@py.round_half_even("1.25", 1) AS rounded\n' + ), + **{ + path: (_DUCKSHOP / path).read_text() + for path in ( + "macros/sql/money_from_minor.sql", + "macros/sql/safe_divide.sql", + "macros/python/rounding.py", + "models/intermediate/edge_cases/int_edge_expression_spans.sql", + ) + }, + }, + model_name="int_edge_expression_spans", + expected_texts={ + "window_id": "w.window_id", + "price_amount": ( + "ROUND((\n w.price_minor / NULLIF((POWER(10, 2)::NUMBER(38, 0)), 0)\n)" + "::NUMBER(38, 6), 2)::NUMBER(38, 6) AS price_amount" + ), + "rounded_price": ( + "ROUND(w.price_minor / 100, 2, 'HALF_TO_EVEN')::NUMBER(18, 2)" + " AS rounded_price" + ), + "currency": "'USD'::VARCHAR AS currency", + "label": "'Prix à jour €'::VARCHAR AS label", + "loaded_at": "w._loaded_at AS loaded_at", + }, + ), + id="Snowflake DuckShop edge model", + marks=pytest.mark.xfail(strict=True, raises=AssertionError, reason=_STRICT_SPANS), + ), + ], +) +def test_given_rewritten_model_sql_when_publishing_then_expression_ranges_slice_the_compiled_sql( + test_case: SnapshotExpressionRangeTestCase, + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=SnowflakeAdapter() if test_case.adapter == "snowflake" else DuckDbAdapter(), + resolved_connection={"database": "WAREHOUSE", "schema": "MAIN"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics), [ + item.message for item in project.diagnostics + ] + publication: dict[str, Any] = json.loads( + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() + ) + node: dict[str, Any] = publication["nodes"][f"model:{test_case.model_name}"] + assert node["compiled_sql"] != node["raw_sql"] + compiled: bytes = node["compiled_sql"].encode() + assert { + column: compiled[start:end].decode() + for column, (start, end) in node["definition"].get("expression_ranges", {}).items() + } == test_case.expected_texts + + @pytest.mark.parametrize( "test_case", [ From 28645e8738756a3a898775f5ef403c2442e18fbe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:51:58 +0000 Subject: [PATCH 3/7] wip: keep the person SQL test within the statement limit --- .../publication/test_person_sql.py | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py index 093f0997e0..e6e33f0a39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -177,21 +177,16 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori inspected: subprocess.CompletedProcess[str] = invoke(command) assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr sql: dict[str, Any] = json.loads(inspected.stdout) - assert set(sql) == { - "snapshot_id", - "node_key", - "raw_sql", - "compiled_sql", - "expression_ranges", - } - compiled: bytes = sql["compiled_sql"].encode() - assert { - column: compiled[start:end].decode() - for column, (start, end) in sql["expression_ranges"].items() - } == { - "amount": "23 AS amount", - "description": "'" + "orders " * 400 + "' AS description", - } + assert ( + set(sql), + { + column: sql["compiled_sql"].encode()[start:end].decode() + for column, (start, end) in sql["expression_ranges"].items() + }, + ) == ( + {"snapshot_id", "node_key", "raw_sql", "compiled_sql", "expression_ranges"}, + {"amount": "23 AS amount", "description": "'" + "orders " * 400 + "' AS description"}, + ) assert ( sql["snapshot_id"] == current_snapshot(worker.api, "development")["record"]["snapshot_id"] From 4698f1808652f84e2342d09749baca3476de06b9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:07:26 +0000 Subject: [PATCH 4/7] wip: keep expression ranges in the snapshot columns document Ranges move out of the node definition into the per-snapshot columns document, so they never change a node hash or record a node change; the node SQL read ignores unknown per-column keys and drops malformed or misplaced ranges. --- .../edge_cases/orchestration/node_sql.json | 10 ++++- .../src/snapshots/_helpers/preparation.rs | 5 +++ .../snapshots/main/read_person_node_sql.rs | 44 ++++++++++++++++--- .../tests/api/node_sql/helpers.rs | 2 +- .../tests/api/test_person_sessions.rs | 5 ++- .../tests/snapshots/helpers.rs | 34 +++++++++++--- .../tests/snapshots/test_contract.rs | 4 +- .../compiled_project/snapshot_models.rs | 26 ++++++----- .../_helpers/location_command/snapshot.rs | 8 +++- .../test_orchestration_snapshot.py | 9 ++-- 10 files changed, 115 insertions(+), 32 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json index cf559d82f1..75868daefd 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json +++ b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json @@ -4,5 +4,13 @@ "python_source": "def order_amount():\n return 23\n", "function_sql": "43", "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n", - "expression_ranges": {"amount": [7, 19]} + "columns_document": { + "amount": {"expression_range": [7, 19], "described_as": "a later per-column key readers ignore"}, + "past_the_end": {"expression_range": [7, 4096]}, + "split_character": {"expression_range": [7, 32]}, + "reversed": {"expression_range": [19, 7]}, + "no_range": {} + }, + "expression_ranges": {"amount": [7, 19]}, + "updated_expression_ranges": {"amount": [7, 19], "split_character": [7, 32]} } diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs index a697f16dd4..33a2d0d2d5 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -48,6 +48,11 @@ pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result Result)>, ControlError> { diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs index 6b3962c8ae..253438a1ac 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs @@ -1,6 +1,8 @@ //! Live SQL permission, current publication and sensitive-read audit share one writer fence. -use serde_json::json; +use std::collections::BTreeMap; + +use serde_json::{Value, json}; use uuid::Uuid; use crate::authorization::main::begin_authorized::begin_authorized; @@ -8,6 +10,7 @@ use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::snapshots::_helpers::reading::attach_sources; use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::NodeSql; use crate::store::errors::ControlError; @@ -40,12 +43,16 @@ pub async fn read_person_node_sql( WHERE s.snapshot_id=$1 AND s.node_key=$2 AND s.kind IN ('raw_sql','compiled_sql')", vec![Parameter::Id(record.snapshot_id), Parameter::Text(node_key.into())]).await?; if let Some(node) = attach_sources(nodes, rows)?.remove(node_key) { - let expression_ranges = node - .definition - .get("expression_ranges") - .cloned() - .and_then(|ranges| serde_json::from_value(ranges).ok()) - .unwrap_or_default(); + let documents; + (transaction, documents) = + read_snapshot_documents(transaction, record.snapshot_id, Some(&["columns"])) + .await?; + let expression_ranges = expression_ranges( + documents + .get("columns") + .and_then(|columns| columns.get(node_key)), + node.compiled_sql.as_deref().unwrap_or_default(), + ); result = Some(NodeSql { snapshot_id: record.snapshot_id, node_key: node_key.into(), @@ -66,3 +73,26 @@ pub async fn read_person_node_sql( commit(transaction).await?; Ok(result) } + +/// Each column's expression range from the node's `columns` document entry; a column whose +/// range is missing, malformed or off the compiled SQL's character boundaries has none. +fn expression_ranges(columns: Option<&Value>, compiled_sql: &str) -> BTreeMap { + let Some(columns) = columns.and_then(Value::as_object) else { + return BTreeMap::new(); + }; + columns + .iter() + .filter_map(|(column, metadata)| { + let range = metadata.get("expression_range")?.as_array()?; + let [start, end] = range.as_slice() else { + return None; + }; + let start = usize::try_from(start.as_u64()?).ok()?; + let end = usize::try_from(end.as_u64()?).ok()?; + compiled_sql + .get(start..end) + .filter(|text| !text.is_empty()) + .map(|_| (column.clone(), [start, end])) + }) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs index 2940a4a3b5..d34c00e80e 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs @@ -125,6 +125,6 @@ pub(crate) async fn node_sql_boundaries( assert_eq!(response.status(), StatusCode::OK); assert_eq!( response.json::().await.expect("latest SQL"), - json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"],"expression_ranges":data["expression_ranges"]}) + json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"],"expression_ranges":data["updated_expression_ranges"]}) ); } diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index 982507bac8..7c90f2ffbf 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -115,13 +115,16 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ .expect("compiled SQL") .into(), ); - orders.definition["expression_ranges"] = sql_input["expression_ranges"].clone(); orders.python_source = Some( sql_input["python_source"] .as_str() .expect("Python source") .into(), ); + input.documents.insert( + "columns".into(), + json!({"orders": sql_input["columns_document"].clone()}), + ); input.nodes.insert( "udf:orders".into(), SnapshotNode { diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 406a79fc5e..eb96893902 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; +use crate::snapshots::_helpers::preparation::prepare_snapshot; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; @@ -113,7 +114,30 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { Err(ControlError::StaleGeneration) )); + // Column metadata lives outside node identity: new expression ranges alone keep every node + // hash and record no node change. + let before = input.clone(); input.generation = 3; + input.documents.insert( + "columns".into(), + json!({"orders": {"customer_id": {"expression_range": [7, 18]}}}), + ); + assert_eq!( + prepare_snapshot(&input).expect("prepare ranged").hashes, + prepare_snapshot(&before).expect("prepare unranged").hashes + ); + let ranged = publish_snapshot(store, &input) + .await + .expect("publish expression ranges only"); + assert_ne!(ranged.manifest_digest, unchanged.manifest_digest); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read ranged snapshot") + .expect("ranged snapshot exists"); + assert!(loaded.changes.is_empty()); + assert_eq!(loaded.documents, input.documents); + + input.generation = 4; input .nodes .get_mut("orders") @@ -146,7 +170,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { .expect("development target exists") .record .generation, - 3 + 4 ); assert_eq!( current_snapshot(store, location, "testing") @@ -159,7 +183,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { ); input.target = "development".into(); - input.generation = 4; + input.generation = 5; input.nodes.remove("orders"); publish_snapshot(store, &input) .await @@ -175,8 +199,8 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { // The newer compiler wins regardless of which publisher takes the transaction lock first. let mut older = input.clone(); - older.generation = 5; - input.generation = 6; + older.generation = 6; + input.generation = 7; input .nodes .get_mut("customers") @@ -192,7 +216,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { .await .expect("read winning snapshot") .expect("winning snapshot exists"); - assert_eq!(loaded.record.generation, 6); + assert_eq!(loaded.record.generation, 7); assert_eq!(loaded.nodes, input.nodes); close_store(second).await; loaded.record.generation diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs index 708a6aff32..b9e990c5e0 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -10,12 +10,12 @@ fn given_control_store_when_snapshots_publish_then_generations_sources_and_chang SnapshotTestCase { description: "SQLite snapshot publication", backend: Backend::Sqlite, - expected_generation: 6, + expected_generation: 7, }, SnapshotTestCase { description: "PostgreSQL snapshot publication", backend: Backend::Postgres, - expected_generation: 6, + expected_generation: 7, }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs index ed299cc5f6..6639e73518 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -22,13 +22,16 @@ use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompil use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +/// Each model's snapshot node, and the `columns` document: per node key, each column's +/// expression range. The ranges stay out of the node, so they never change a node's identity. pub(crate) fn model_nodes( py: Python<'_>, project: &NativeProjectView, owner: &NativeCompiledProject, mut nodes: BTreeMap, -) -> PyResult> { +) -> PyResult<(BTreeMap, Map)> { let dialect: Option = project.sql_analysis_dialect(); + let mut columns_document: Map = Map::new(); for model in project.models(py).bind(py).iter() { let model = model.cast::()?.get(); let (name, _, _) = model.file_texts(py)?; @@ -39,7 +42,7 @@ pub(crate) fn model_nodes( "model config cannot be represented in snapshot metadata", )); } - let mut definition = model.read(py, |input, _| { + let (mut definition, ranges) = model.read(py, |input, _| { let record = input .compiled .as_ref() @@ -54,7 +57,7 @@ pub(crate) fn model_nodes( json!({"name": name, "type": data_type, "nullability": nullability}) }) .collect(); - let mut definition: Value = json!({ + let definition: Value = json!({ "name": facts.name, "path": facts.relative_path, "config": facts.config.values, @@ -67,11 +70,11 @@ pub(crate) fn model_nodes( (Some(dialect), Some(lineage)) => expression_ranges(dialect, input, facts, lineage), _ => Map::new(), }; - if !ranges.is_empty() { - definition["expression_ranges"] = Value::Object(ranges); - } - Ok(definition) + Ok((definition, ranges)) })?; + if !ranges.is_empty() { + columns_document.insert(format!("model:{}", facts.name), Value::Object(ranges)); + } definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?; nodes = insert_node( nodes, @@ -85,11 +88,12 @@ pub(crate) fn model_nodes( }, )?; } - Ok(nodes) + Ok((nodes, columns_document)) } -/// Each output column's defining expression as a `[start, end)` byte range of the compiled SQL -/// the node SQL read returns; empty when the analysis SQL cannot be rebuilt exactly. +/// Each output column's defining expression as `{"expression_range": [start, end)}`, byte +/// offsets in the compiled SQL the node SQL read returns; empty when the analysis SQL cannot be +/// rebuilt exactly. fn expression_ranges( dialect: &str, input: &ModelInputRecord, @@ -127,7 +131,7 @@ fn expression_ranges( if let Some((start, end)) = range { ranges .entry(output_column.clone()) - .or_insert_with(|| json!([start, end])); + .or_insert_with(|| json!({"expression_range": [start, end]})); } } ranges diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 89ce2a430d..bf15b867e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -98,7 +98,13 @@ pub(crate) fn orchestration_snapshot<'py>( input.sqb_version = env!("CARGO_PKG_VERSION").into(); input.environment_hash = environment_hash(py, &project_dir)?; let owner = project.owner().bind(py).borrow(); - input.nodes = model_nodes(py, &project, &owner, input.nodes)?; + let (nodes, columns) = model_nodes(py, &project, &owner, input.nodes)?; + input.nodes = nodes; + if !columns.is_empty() { + input + .documents + .insert("columns".into(), serde_json::Value::Object(columns)); + } input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); let declarations = match declarations { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index e12382ef1e..4c95d00325 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -574,12 +574,15 @@ def test_given_rewritten_model_sql_when_publishing_then_expression_ranges_slice_ project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered ).to_json() ) - node: dict[str, Any] = publication["nodes"][f"model:{test_case.model_name}"] + key: str = f"model:{test_case.model_name}" + node: dict[str, Any] = publication["nodes"][key] assert node["compiled_sql"] != node["raw_sql"] + assert "expression_range" not in json.dumps(node), "ranges stay out of node identity" compiled: bytes = node["compiled_sql"].encode() + columns: dict[str, Any] = publication["documents"].get("columns", {}).get(key, {}) assert { - column: compiled[start:end].decode() - for column, (start, end) in node["definition"].get("expression_ranges", {}).items() + column: compiled[slice(*metadata["expression_range"])].decode() + for column, metadata in columns.items() } == test_case.expected_texts From 7bfcee268a1796d50cf9a589da35d62d973d6412 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:38:29 +0000 Subject: [PATCH 5/7] wip: drop expression ranges at rewritten edges and meet layout rules Map a cleaned range edge only between source-adjacent characters, so a range beside a removed reference call or inside generated text is dropped instead of moved. Skip unnamed output columns. Split helpers into single-function entry modules, make the tests table-driven, and add a DuckShop edge model qualified by its reference call. --- benchmarks/duckshop/README.md | 6 +- .../int_edge_reference_qualified_spans.sql | 17 + .../duckshop/project/sqlbuild_project.toml | 11 +- ...est_int_edge_reference_qualified_spans.sql | 26 ++ clients/typescript/src/schema.d.ts | 5 +- .../_helpers/analysis/native_sharing.rs | 24 +- .../main/query_expression_ranges.rs | 25 +- .../src/analysis_route/tests/helpers.rs | 75 ++++ .../tests/test_expression_ranges.rs | 193 +++++----- .../src/analysis_route/tests/test_types.rs | 18 + .../src/assembly/analysis_session/models.rs | 3 +- .../_helpers/normalization.rs | 5 +- .../src/semantic_validation/main.rs | 1 + .../main/normalize_native_filled.rs | 14 + .../src/snapshots/_helpers/preparation.rs | 19 +- .../src/snapshots/constants.rs | 13 + .../snapshots/main/read_person_node_sql.rs | 57 +-- .../src/snapshots/models.rs | 3 +- .../tests/snapshots/helpers.rs | 2 +- .../_helpers/attachments/cursor_intrinsics.rs | 8 +- .../compiled_project/snapshot_models.rs | 14 +- .../sqlbuild-sqltext/src/compiler/main/mod.rs | 1 + .../main/placeholder_default_edits.rs | 40 +++ .../src/compiler/main/placeholder_defaults.rs | 39 +-- .../compiler/compiled_project/_test_types.py | 3 +- .../compiler/compiled_project/helpers.py | 33 ++ .../test_orchestration_snapshot.py | 331 +++++++++--------- 27 files changed, 581 insertions(+), 405 deletions(-) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql create mode 100644 crates/sqlbuild-analysis/src/semantic_validation/main/normalize_native_filled.rs create mode 100644 crates/sqlbuild-sqltext/src/compiler/main/placeholder_default_edits.rs diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 58afe6e872..bbb29e1669 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -124,8 +124,8 @@ project and does not change it: `edge_running_peaks`, that a pre-hook of `int_edge_basket_running_peaks` creates (`models/intermediate/edge_cases/_sqlbuild/_hooks/sql/`) and SQLBuild does not declare. -It has 165 models (13 staging, 138 intermediate, 14 marts), 5 Python functions, a SQL table -function, a hook-created JavaScript function, 190 unit tests and 4 scenarios, in nine groups: +It has 166 models (13 staging, 139 intermediate, 14 marts), 5 Python functions, a SQL table +function, a hook-created JavaScript function, 191 unit tests and 4 scenarios, in nine groups: | Group | Covers | |-------|--------| @@ -134,7 +134,7 @@ function, a hook-created JavaScript function, 190 unit tests and 4 scenarios, in | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` | | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day | | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | -| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote | +| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | | Translation refusals | one model per construct that translation tools refuse or guard, most putting a construct refused only under strict checking before one refused by default, so a model shows both: `FLOAT` values through sums, averages, running totals, arrays, objects, `HASH`, `TO_CHAR` and `::NUMBER(38, 10)`; `VARIANT` dates, decimals and XML; `ARRAY_AGG` without `WITHIN GROUP`, sparse arrays and `ARRAY(NUMBER)` casts; `REDUCE`, `FILTER` and `TRANSFORM` lambdas; patterns and `REGEXP_REPLACE` replacements built from columns, and lazy patterns; `TRY_TO_DATE` with a format from a column; zoned literals, `CONVERT_TIMEZONE` from `CURRENT_TIMESTAMP`, `SEQ8`, `GENERATOR` with an expression `ROWCOUNT`; `PIVOT`, `ANY_VALUE`, `IGNORE NULLS` around `IFF`, unaliased columns read by Snowflake's name, `UNION` branches whose column is always NULL; `TO_NUMBER`, `TO_DECIMAL` and `TRY_BASE64_DECODE_BINARY`; an open (uncontracted) source `edge__supplier_notes`, the table function, read for one basket and after `LATERAL` for every basket, and the hook-created function. `rpt_edge_case_model_keys` and `rpt_edge_case_array_model_keys` read every model in the group, so a refusal there also refuses a downstream model | diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql new file mode 100644 index 0000000000..26bd286d57 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql @@ -0,0 +1,17 @@ +MODEL ( + description """ + One row per edge-case price window, each column qualified by the source reference call itself, + so every output column's expression starts at text the compiler rewrites; the case for column + expression ranges that must be dropped rather than moved onto the relation name. + """, + tags [edge_case], + columns ( + window_id (type VARCHAR, nullable false, audits [not_null, unique]), + sku (type VARCHAR), + ), +); + +SELECT + __source("edge__price_windows").window_id, + __source("edge__price_windows").sku +FROM __source("edge__price_windows") diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 6272acbeb8..4b9a37a0bc 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -339,11 +339,14 @@ that shape is the case under test. [[rules.rule_ignores]] rules = ["SQLCLEAN034"] -paths = ["models/intermediate/edge_cases/int_edge_expression_spans.sql"] +paths = [ + "models/intermediate/edge_cases/int_edge_expression_spans.sql", + "models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql", +] reason = """ -The edge case keeps its macro calls, variable and literals in the terminal select on purpose: -column expression ranges point at the terminal select's items, so that is where the compiled SQL -must differ from the authored text. +These edge cases keep their macro calls, variable, literals and reference calls in the terminal +select on purpose: column expression ranges point at the terminal select's items, so that is where +the compiled SQL must differ from the authored text. """ [[rules.rule_ignores]] diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql new file mode 100644 index 0000000000..51ae9b9a9b --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql @@ -0,0 +1,26 @@ +TEST ( + name "int_edge_reference_qualified_spans__keeps_every_window", +); + +WITH __source__edge__price_windows AS ( + SELECT + v.window_id, + v.sku, + v.price_minor::NUMBER(38, 0) AS price_minor, + v.valid_from::TIMESTAMP_NTZ(6) AS valid_from, + v.valid_to::TIMESTAMP_NTZ(6) AS valid_to, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PW-01', 'SKU-1', 1234, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 09:00:00'), + ('PW-02', NULL, NULL, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 18:30:00') AS v(window_id, sku, price_minor, valid_from, valid_to, _loaded_at) +), + +__expected__int_edge_reference_qualified_spans AS ( + SELECT + 'PW-01' AS window_id, + 'SKU-1' AS sku + UNION ALL + SELECT + 'PW-02' AS window_id, + CAST(NULL AS VARCHAR) AS sku +) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index b592225e27..ccc5e5cc8a 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2009,10 +2009,7 @@ export interface components { /** @description SQL inspection never returns Python source, execution artifacts or unrelated node metadata. */ NodeSql: { compiled_sql?: string | null; - /** - * @description Each output column's defining expression as a `[start, end)` UTF-8 byte range of - * `compiled_sql`; absent for columns the analysis could not place. - */ + /** @description Each placed column's defining expression as a `[start, end)` byte range of `compiled_sql`. */ expression_ranges?: { [key: string]: number[]; }; diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs index 5cc8fa6959..c8ce6429b5 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs @@ -213,13 +213,12 @@ fn mapped( Ok(analysis) } -/// Move each expression range from the leader's cleaned SQL to the member's, which differ only in -/// the relation names they spell; a range that cannot be moved exactly is dropped. +/// Each expression range moved onto `member`'s relation names, or dropped where it cannot move. pub(crate) fn rebase_ranges(analysis: &mut NativeAnalysis, leader: &str, member: &str) { if leader == member { return; } - let renamed: Option> = renamed_runs(leader, member); + let renamed: Option> = renamed_runs(leader, member); for row in &mut analysis.lineage { row.expression_range = match (&renamed, row.expression_range) { (Some(runs), Some((start, end))) => rebased(runs, start).zip(rebased(runs, end)), @@ -228,13 +227,12 @@ pub(crate) fn rebase_ranges(analysis: &mut NativeAnalysis, leader: &str, member: } } -/// `(leader start, leader end, member start, member end)` of each identifier run where two SQL -/// texts differ, when they differ only in whole identifier runs. -fn renamed_runs(leader: &str, member: &str) -> Option> { +/// `(leader start, leader end, member end)` of each differing identifier run, if only those differ. +fn renamed_runs(leader: &str, member: &str) -> Option> { let identifier = |c: u8| c.is_ascii_alphanumeric() || c == b'_' || c == b'$'; let (left, right): (&[u8], &[u8]) = (leader.as_bytes(), member.as_bytes()); let (mut i, mut j): (usize, usize) = (0, 0); - let mut runs: Vec<(usize, usize, usize, usize)> = Vec::new(); + let mut runs: Vec<(usize, usize, usize)> = Vec::new(); while i < left.len() || j < right.len() { if i < left.len() && j < right.len() && left[i] == right[j] { i += 1; @@ -255,25 +253,25 @@ fn renamed_runs(leader: &str, member: &str) -> Option Option { - let mut shift: isize = 0; - for &(left_start, left_end, _, right_end) in runs { +fn rebased(runs: &[(usize, usize, usize)], offset: usize) -> Option { + let mut moved: usize = offset; + for &(left_start, left_end, right_end) in runs { if offset <= left_start { break; } if offset < left_end { return None; } - shift = right_end as isize - left_end as isize; + moved = offset - left_end + right_end; } - usize::try_from(offset as isize + shift).ok() + Some(moved) } /// Whether `leader`'s `template` succeeded, found nothing and spells none of its relation names. diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs index 2a2f1c6288..6e6808aa02 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs @@ -2,12 +2,10 @@ use crate::analysis_route::main::analysis_route::analysis_route; use crate::analysis_route::models::AnalysisRoute; -use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped; +use crate::semantic_validation::main::normalize_native_filled::normalize_native_filled; use crate::sql_macros::models::TextSplice; -/// Each byte range of `analysis_sql`'s cleaned native analysis SQL as the byte range of the query -/// SQL that `passes` rewrote, in order, into `analysis_sql`; `None` where there was none, where an -/// end falls inside rewritten text, or where the dialect has no native analysis. +/// Each cleaned-SQL byte range as a range of the SQL `passes` rewrote into `analysis_sql`. #[must_use] pub fn query_expression_ranges( dialect: &str, @@ -19,7 +17,7 @@ pub fn query_expression_ranges( return vec![None; ranges.len()]; }; let Ok((cleaned, offsets)) = - normalize_native_sql_mapped(analysis_sql, dialect, module.stub_prefix(), placeholders) + normalize_native_filled(analysis_sql, dialect, module.stub_prefix(), placeholders) else { return vec![None; ranges.len()]; }; @@ -35,6 +33,9 @@ pub fn query_expression_ranges( characters[cleaned.len()] = Some(cleaned.chars().count()); let query_byte = |cleaned_byte: usize| -> Option { let character: usize = (*characters.get(cleaned_byte)?)?; + if !copied_boundary(&offsets, character) { + return None; + } let byte: usize = *analysis_bytes.get(*offsets.get(character)?)?; passes .iter() @@ -51,8 +52,18 @@ pub fn query_expression_ranges( .collect() } -/// The source byte of output byte `byte` of a pass whose `splices` are in text order; a byte at -/// a splice's edge maps to the same edge of its source, and one inside rewritten text has none. +/// Whether cleaned boundary `character` lies between source-adjacent characters, so maps exactly. +fn copied_boundary(offsets: &[usize], character: usize) -> bool { + let Some(previous) = character.checked_sub(1) else { + return offsets.first() == Some(&0); + }; + offsets + .get(previous) + .zip(offsets.get(character)) + .is_some_and(|(before, after)| before + 1 == *after) +} + +/// The source byte of output `byte`; splice edges map to source edges, inner bytes to `None`. fn spliced_back(byte: usize, splices: &[TextSplice]) -> Option { let (mut source_end, mut output_end): (usize, usize) = (0, 0); for splice in splices { diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs index 3909552f77..84fededf40 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -1,11 +1,14 @@ use std::collections::HashMap; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; use crate::analysis_route::_helpers::analysis::native_outcome::{ analysed_sql, native_outcome, native_sql, }; +use crate::analysis_route::main::query_expression_ranges::query_expression_ranges; use crate::analysis_route::main::schema_column::schema_column as stored_column; +use crate::analysis_route::models::DuckDbDialect; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, StandInSchema}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::models::{ @@ -15,6 +18,9 @@ use crate::assembly::analysis_session::models::{ use crate::assembly::analysis_session::types::Shapes; use crate::lineage::models::{LineageColumn, LineageSource}; use crate::semantic_checks::types::RevisedBinding; +use crate::semantic_validation::main::normalize_native_filled::normalize_native_filled; +use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use crate::sql_macros::models::{TextSplice, TypedCall}; /// `orders` with a non-null INTEGER id and a nullable VARCHAR status. pub(super) fn orders() -> StandInSchema { @@ -168,3 +174,72 @@ pub(crate) fn model_outcome( }); native_outcome(model, schema, cleaned.unwrap_or_default(), analysed) } + +/// Each output column with the `query_sql` text its range covers after the compile passes. +pub(super) fn mapped_texts( + query_sql: &str, + placeholders: &[(&str, &str)], + typed_calls: &[(&str, &str)], +) -> Vec<(String, Option)> { + let placeholders: Vec<(String, String)> = placeholders + .iter() + .map(|(name, text)| ((*name).to_owned(), (*text).to_owned())) + .collect(); + let calls: Vec = typed_calls + .iter() + .map(|(call, returns)| typed_call(query_sql, call, returns)) + .collect(); + let (typed, typed_splices) = typed_analysis_sql(query_sql, &calls, "duckdb"); + let (analysis_sql, spans) = replace_intrinsics_with_spans(&typed, "Model SQL", |_| { + "CAST('2000-01-01' AS TIMESTAMP)".to_owned() + }) + .expect("the cursor calls are replaced"); + let cursor_splices: Vec = spans + .iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + let dialect: &DuckDbDialect = &DuckDbDialect; + let (cleaned, _) = normalize_native_filled( + &analysis_sql, + "duckdb", + dialect.stub_prefix(), + &placeholders, + ) + .expect("the test query normalises"); + let analysis: NativeAnalysis = analysed(dialect, &cleaned, &[]); + let ranges: Vec> = analysis + .lineage + .iter() + .map(|row| row.expression_range) + .collect(); + let mapped = query_expression_ranges( + "duckdb", + (&analysis_sql, &placeholders), + &[typed_splices, cursor_splices], + &ranges, + ); + analysis + .lineage + .iter() + .zip(mapped) + .map(|(row, range)| { + let text: Option = range.map(|(start, end)| query_sql[start..end].to_owned()); + (row.output_column.clone(), text) + }) + .collect() +} + +/// The typed call of the expression macro `call` in `query_sql`, returning `returns`. +fn typed_call(query_sql: &str, call: &str, returns: &str) -> TypedCall { + let start: usize = query_sql.find(call).expect("the call is in the query"); + TypedCall { + start, + end: start + call.len(), + macro_name: "typed_call".to_owned(), + returns: Some(returns.to_owned()), + returns_like: None, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs index e2c3b941c1..42273c7771 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs @@ -1,114 +1,95 @@ use crate::analysis_route::_helpers::analysis::native_sharing::rebase_ranges; -use crate::analysis_route::main::query_expression_ranges::query_expression_ranges; use crate::analysis_route::models::{DuckDbDialect, NativeAnalysis}; -use crate::analysis_route::tests::helpers::analysed; -use crate::analysis_route::types::NativeDialect; -use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped; -use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; -use crate::sql_macros::models::{TextSplice, TypedCall}; -use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; - -/// Each output column with the text of `query_sql` its mapped expression range covers, after -/// `passes` rewrote `query_sql` into `analysis_sql`. -fn mapped_texts<'a>( - (query_sql, analysis_sql): (&'a str, &str), - placeholders: &[(String, String)], - passes: &[Vec], -) -> Vec<(String, Option<&'a str>)> { - let dialect: &DuckDbDialect = &DuckDbDialect; - let (cleaned, _) = - normalize_native_sql_mapped(analysis_sql, "duckdb", dialect.stub_prefix(), placeholders) - .expect("the test query normalises"); - let analysis: NativeAnalysis = analysed(dialect, &cleaned, &[]); - let ranges: Vec> = analysis - .lineage - .iter() - .map(|row| row.expression_range) - .collect(); - let mapped = query_expression_ranges("duckdb", (analysis_sql, placeholders), passes, &ranges); - analysis - .lineage - .iter() - .zip(mapped) - .map(|(row, range)| { - ( - row.output_column.clone(), - range.map(|(start, end)| &query_sql[start..end]), - ) - }) - .collect() -} - -#[test] -fn given_compiled_sql_that_cleaning_rewrites_when_mapping_ranges_then_they_cover_its_own_text() { - let placeholders: Vec<(String, String)> = vec![("floor".to_owned(), "100000".to_owned())]; - let query_sql: &str = "-- café ünïcode\nWITH x AS (SELECT * FROM __ref(\"orders\") WHERE id > @@@floor \ - AND id < __cursor_end())\nSELECT x.id AS id, UPPER(x.status) AS loud, 'é' AS mark FROM x"; - assert_eq!( - mapped_texts((query_sql, query_sql), &placeholders, &[]), - vec![ - ("id".to_owned(), Some("x.id AS id")), - ("loud".to_owned(), Some("UPPER(x.status) AS loud")), - ("mark".to_owned(), Some("'é' AS mark")), - ] - ); -} +use crate::analysis_route::tests::helpers::{analysed, mapped_texts}; +use crate::analysis_route::tests::test_types::{ExpressionRangeTestCase, RebaseRangesTestCase}; #[test] -fn given_typed_calls_and_cursor_bounds_when_mapping_ranges_then_they_cover_the_compiled_text() { - let query_sql: &str = "SELECT ROUND(amount * 1.2, 2) AS gross, id AS id FROM orders \ - WHERE updated_at < __cursor_end()"; - let call_start: usize = query_sql.find("ROUND").expect("the call is in the query"); - let call_end: usize = query_sql - .find(" AS gross") - .expect("the alias is in the query"); - let calls: Vec = vec![TypedCall { - start: call_start, - end: call_end, - macro_name: "gross_amount".to_owned(), - returns: Some("DECIMAL(18, 2)".to_owned()), - returns_like: None, - }]; - let (typed, typed_splices) = typed_analysis_sql(query_sql, &calls, "duckdb"); - assert_ne!(typed, query_sql, "the call is marked for analysis"); - let (analysis_sql, spans) = replace_intrinsics_with_spans(&typed, "Model SQL", |_| { - "CAST('2000-01-01' AS TIMESTAMP)".to_owned() - }) - .expect("the cursor call is replaced"); - let cursor_splices: Vec = spans - .iter() - .map(|span| TextSplice { - source: (span.source_start, span.source_end), - output: (span.output_start, span.output_end), - }) - .collect(); - assert_eq!( - mapped_texts( - (query_sql, &analysis_sql), - &[], - &[typed_splices, cursor_splices] - ), - vec![ - ("gross".to_owned(), Some("ROUND(amount * 1.2, 2) AS gross")), - ("id".to_owned(), Some("id AS id")), - ] - ); +fn given_compiled_sql_when_mapping_expression_ranges_then_they_cover_its_own_text() { + let test_cases = [ + ExpressionRangeTestCase { + description: "cleaning rewrites multi-byte comments, references, placeholders and cursors", + query_sql: "-- café ünïcode\nWITH x AS (SELECT * FROM __ref(\"orders\") WHERE id > @@@floor \ + AND id < __cursor_end())\nSELECT x.id AS id, UPPER(x.status) AS loud, 'é' AS mark FROM x", + placeholders: &[("floor", "100000")], + typed_calls: &[], + expected_texts: &[ + ("id", Some("x.id AS id")), + ("loud", Some("UPPER(x.status) AS loud")), + ("mark", Some("'é' AS mark")), + ], + }, + ExpressionRangeTestCase { + description: "a typed macro call is marked and a cursor bound replaced before analysis", + query_sql: "SELECT ROUND(amount * 1.2, 2) AS gross, id AS id FROM orders \ + WHERE updated_at < __cursor_end()", + placeholders: &[], + typed_calls: &[("ROUND(amount * 1.2, 2)", "DECIMAL(18, 2)")], + expected_texts: &[ + ("gross", Some("ROUND(amount * 1.2, 2) AS gross")), + ("id", Some("id AS id")), + ], + }, + ExpressionRangeTestCase { + description: "an edge beside a removed reference call is dropped, not moved onto the name", + query_sql: "SELECT __ref(\"orders\").id AS id, UPPER(status) AS loud FROM __ref(\"orders\")", + placeholders: &[], + typed_calls: &[], + expected_texts: &[("id", None), ("loud", Some("UPPER(status) AS loud"))], + }, + ExpressionRangeTestCase { + description: "an edge inside a filled placeholder is dropped; one at its start maps to it", + query_sql: "SELECT @@@cols AS two, @@@floor + 1 AS above, id AS id FROM orders", + placeholders: &[("cols", "1 AS one, 2"), ("floor", "10")], + typed_calls: &[], + expected_texts: &[ + ("one", None), + ("two", None), + ("above", Some("@@@floor + 1 AS above")), + ("id", Some("id AS id")), + ], + }, + ]; + for test_case in test_cases { + let mapped: Vec<(String, Option)> = mapped_texts( + test_case.query_sql, + test_case.placeholders, + test_case.typed_calls, + ); + let texts: Vec<(&str, Option<&str>)> = mapped + .iter() + .map(|(column, text)| (column.as_str(), text.as_deref())) + .collect(); + assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); + } } #[test] -fn given_a_shared_analysis_when_rebasing_onto_a_member_then_ranges_follow_its_relation_names() { - let leader: &str = - "WITH x AS (SELECT * FROM orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x"; - let member: &str = "WITH x AS (SELECT * FROM customer_orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x"; - let mut analysis: NativeAnalysis = analysed(&DuckDbDialect, leader, &[]); - rebase_ranges(&mut analysis, leader, member); - let texts: Vec> = analysis - .lineage - .iter() - .map(|row| row.expression_range.map(|(start, end)| &member[start..end])) - .collect(); - assert_eq!( - texts, - vec![Some("x.id AS id"), Some("UPPER(x.status) AS loud")] - ); +fn given_a_shared_analysis_when_rebasing_onto_a_member_then_ranges_follow_or_drop() { + let test_cases = [ + RebaseRangesTestCase { + description: "a member that only renames a relation keeps every range", + leader: "WITH x AS (SELECT * FROM orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x", + member: "WITH x AS (SELECT * FROM customer_orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x", + expected_texts: &[Some("x.id AS id"), Some("UPPER(x.status) AS loud")], + }, + RebaseRangesTestCase { + description: "a member that differs outside identifiers drops every range", + leader: "SELECT x.id AS id FROM orders AS x", + member: "SELECT x.id AS id FROM orders AS x WHERE 1 = 1", + expected_texts: &[None], + }, + ]; + for test_case in test_cases { + let mut analysis: NativeAnalysis = analysed(&DuckDbDialect, test_case.leader, &[]); + rebase_ranges(&mut analysis, test_case.leader, test_case.member); + let texts: Vec> = analysis + .lineage + .iter() + .map(|row| { + row.expression_range + .map(|(start, end)| &test_case.member[start..end]) + }) + .collect(); + assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); + } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 9c15fcf1c3..b677fdd610 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -353,3 +353,21 @@ pub(super) struct TsqlTypeTestCase { pub(super) spelling: &'static str, pub(super) expected_type: Option<(&'static str, TypeFamily)>, } + +pub(super) struct ExpressionRangeTestCase { + pub(super) description: &'static str, + pub(super) query_sql: &'static str, + pub(super) placeholders: &'static [(&'static str, &'static str)], + /// Each typed expression macro call as `(call text, returned type)`. + pub(super) typed_calls: &'static [(&'static str, &'static str)], + /// Each output column and the text of `query_sql` its mapped range covers. + pub(super) expected_texts: &'static [(&'static str, Option<&'static str>)], +} + +pub(super) struct RebaseRangesTestCase { + pub(super) description: &'static str, + pub(super) leader: &'static str, + pub(super) member: &'static str, + /// The text of `member` each lineage row's rebased range covers. + pub(super) expected_texts: &'static [Option<&'static str>], +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index de0f5cb4c1..b8a426f348 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -237,8 +237,7 @@ pub struct LineageRow { pub transform_code: u8, pub confidence_code: u8, pub sources: Vec<(String, String, String)>, - /// UTF-8 byte range of the column's defining projection item in the model's cleaned - /// analysis SQL, when the native analysis gives one. + /// UTF-8 byte range of the column's projection item in the cleaned analysis SQL, if known. pub expression_range: Option<(usize, usize)>, } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index 522da9404d..efe0a2f7d4 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -3,7 +3,7 @@ use crate::relation_markers::main::lateral_start::lateral_start; use crate::semantic_checks::constants::UDF_NAME_PREFIX; use regex::Regex; -use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_default_edits; +use sqlbuild_sqltext::compiler::main::placeholder_default_edits::placeholder_default_edits; use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; @@ -205,8 +205,7 @@ pub(crate) fn normalize_native_sql( native_mapped(sql, dialect, stub_prefix, relations).map(|mapped| mapped.sql) } -/// [`normalize_native_sql`] with placeholder defaults filled, and the character of `sql` each -/// character of it came from. +/// [`normalize_native_sql`] with placeholder defaults filled, and each character's source in `sql`. pub(crate) fn normalize_native_sql_mapped( sql: &str, dialect: &str, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index 6ffe8340db..fb9ca0d11e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -9,6 +9,7 @@ pub mod normalize_batch; pub mod normalize_dialect; pub mod normalize_mapped; pub mod normalize_native; +pub mod normalize_native_filled; pub mod normalize_native_renamed; pub mod sqlbuild_code; pub mod text_findings; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_native_filled.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_native_filled.rs new file mode 100644 index 0000000000..b48b3f97e3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_native_filled.rs @@ -0,0 +1,14 @@ +/// Native-normalized analysis SQL with placeholder defaults filled and each character's source. +pub fn normalize_native_filled( + sql: &str, + dialect: &str, + stub_prefix: &str, + placeholders: &[(String, String)], +) -> Result<(String, Vec), String> { + crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped( + sql, + dialect, + stub_prefix, + placeholders, + ) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs index 33a2d0d2d5..ec70d9fdc7 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use crate::snapshots::_helpers::content::{json_bytes, prepare_node}; +use crate::snapshots::constants::SNAPSHOT_DOCUMENT_KINDS; use crate::snapshots::models::{PreparedNode, PreparedSnapshot, SnapshotInput}; use crate::store::errors::ControlError; use crate::store::main::compress::compress; @@ -48,27 +49,13 @@ pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result Result)>, ControlError> { let mut documents: BTreeMap)> = BTreeMap::new(); for (kind, value) in &input.documents { - if ![ - "jobs", - "groups", - "triggers", - "roles", - "lineage", - "columns", - "providers", - ] - .contains(&kind.as_str()) - { + if !SNAPSHOT_DOCUMENT_KINDS.contains(&kind.as_str()) { return Err(ControlError::Invalid(format!( "unknown snapshot document {kind}" ))); diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs index a995563fc0..f931b29c58 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/constants.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -4,6 +4,19 @@ pub(crate) const DEFAULT_SNAPSHOT_RETENTION_COUNT: u16 = 10; pub(crate) const SNAPSHOT_RETENTION_BATCH: usize = 10; pub(crate) const CONTENT_HASH_CHUNK: usize = 500; +/// `{node_key: {column: {"expression_range": [start, end]}}}`; readers skip unknown column keys. +pub(crate) const COLUMNS_DOCUMENT: &str = "columns"; +/// Snapshot document kinds: per-snapshot metadata that never changes a node hash or node change. +pub(crate) const SNAPSHOT_DOCUMENT_KINDS: [&str; 7] = [ + "jobs", + "groups", + "triggers", + "roles", + "lineage", + COLUMNS_DOCUMENT, + "providers", +]; + /// Beyond the newest generations, any retained reference keeps a snapshot. pub(crate) const EXPIRED_SNAPSHOTS: &str = "SELECT s.snapshot_id FROM sqb_snapshots s \ WHERE (SELECT COUNT(*) FROM sqb_snapshots newer WHERE newer.location_id=s.location_id \ diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs index 253438a1ac..b73be257f8 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs @@ -9,6 +9,7 @@ use crate::authorization::main::begin_authorized::begin_authorized; use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::snapshots::_helpers::reading::attach_sources; +use crate::snapshots::constants::COLUMNS_DOCUMENT; use crate::snapshots::main::read_current_record::read_current_record; use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; @@ -44,12 +45,15 @@ pub async fn read_person_node_sql( vec![Parameter::Id(record.snapshot_id), Parameter::Text(node_key.into())]).await?; if let Some(node) = attach_sources(nodes, rows)?.remove(node_key) { let documents; - (transaction, documents) = - read_snapshot_documents(transaction, record.snapshot_id, Some(&["columns"])) - .await?; + (transaction, documents) = read_snapshot_documents( + transaction, + record.snapshot_id, + Some(&[COLUMNS_DOCUMENT]), + ) + .await?; let expression_ranges = expression_ranges( documents - .get("columns") + .get(COLUMNS_DOCUMENT) .and_then(|columns| columns.get(node_key)), node.compiled_sql.as_deref().unwrap_or_default(), ); @@ -74,25 +78,34 @@ pub async fn read_person_node_sql( Ok(result) } -/// Each column's expression range from the node's `columns` document entry; a column whose -/// range is missing, malformed or off the compiled SQL's character boundaries has none. +/// Each column's well-formed expression range from the node's `columns` document entry. fn expression_ranges(columns: Option<&Value>, compiled_sql: &str) -> BTreeMap { + let mut ranges: BTreeMap = BTreeMap::new(); let Some(columns) = columns.and_then(Value::as_object) else { - return BTreeMap::new(); + return ranges; }; - columns - .iter() - .filter_map(|(column, metadata)| { - let range = metadata.get("expression_range")?.as_array()?; - let [start, end] = range.as_slice() else { - return None; - }; - let start = usize::try_from(start.as_u64()?).ok()?; - let end = usize::try_from(end.as_u64()?).ok()?; - compiled_sql - .get(start..end) - .filter(|text| !text.is_empty()) - .map(|_| (column.clone(), [start, end])) - }) - .collect() + for (column, metadata) in columns { + if let Some(range) = column_range(metadata, compiled_sql) { + ranges.insert(column.clone(), range); + } + } + ranges +} + +/// A column's `[start, end)` compiled-SQL byte range, or `None` when missing or malformed. +fn column_range(metadata: &Value, compiled_sql: &str) -> Option<[usize; 2]> { + let Some([start, end]) = metadata + .get("expression_range") + .and_then(Value::as_array) + .map(Vec::as_slice) + else { + return None; + }; + let (start, end): (u64, u64) = (start.as_u64()?, end.as_u64()?); + let length: u64 = u64::try_from(compiled_sql.len()).unwrap_or(u64::MAX); + if start >= end || end > length { + return None; + } + let range: [usize; 2] = [start as usize, end as usize]; + compiled_sql.get(range[0]..range[1]).map(|_| range) } diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index c4c4db6e69..3e035cc258 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -24,8 +24,7 @@ pub struct NodeSql { pub node_key: String, pub raw_sql: Option, pub compiled_sql: Option, - /// Each output column's defining expression as a `[start, end)` UTF-8 byte range of - /// `compiled_sql`; absent for columns the analysis could not place. + /// Each placed column's defining expression as a `[start, end)` byte range of `compiled_sql`. #[serde(default)] pub expression_ranges: BTreeMap, } diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index eb96893902..778cfb2028 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -146,7 +146,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { let changed = publish_snapshot(store, &input) .await .expect("publish changed SQL"); - assert_ne!(changed.manifest_digest, unchanged.manifest_digest); + assert_ne!(changed.manifest_digest, ranged.manifest_digest); let loaded = current_snapshot(store, location, "development") .await .expect("read changed snapshot") diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index acaa44693f..cbe9af7460 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -105,14 +105,14 @@ pub(crate) fn analysis_cursor_sql( Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) } -/// `analysis_cursor_sql` with each replaced call as a splice of `sql`; `None` when a call is invalid. +/// `analysis_cursor_sql` with each replaced call as a splice of `sql`. pub(crate) fn analysis_cursor_splices( sql: &str, cursor_type: Option<&str>, -) -> Option<(String, Vec)> { +) -> Result<(String, Vec), String> { let literal: &str = analysis_bound(cursor_type); let (replaced, spans) = - replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned()).ok()?; + replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned())?; let splices: Vec = spans .iter() .map(|span| TextSplice { @@ -120,7 +120,7 @@ pub(crate) fn analysis_cursor_splices( output: (span.output_start, span.output_end), }) .collect(); - Some((replaced, splices)) + Ok((replaced, splices)) } /// The literal static analysis reads for every cursor intrinsic of a model with `cursor_type`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs index 6639e73518..1ee1ad8cc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -22,8 +22,7 @@ use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompil use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -/// Each model's snapshot node, and the `columns` document: per node key, each column's -/// expression range. The ranges stay out of the node, so they never change a node's identity. +/// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. pub(crate) fn model_nodes( py: Python<'_>, project: &NativeProjectView, @@ -91,9 +90,7 @@ pub(crate) fn model_nodes( Ok((nodes, columns_document)) } -/// Each output column's defining expression as `{"expression_range": [start, end)}`, byte -/// offsets in the compiled SQL the node SQL read returns; empty when the analysis SQL cannot be -/// rebuilt exactly. +/// Each column's `{"expression_range": [start, end]}`, when the analysis SQL rebuilds exactly. fn expression_ranges( dialect: &str, input: &ModelInputRecord, @@ -115,7 +112,7 @@ fn expression_ranges( .values .get("cursor_type") .and_then(Value::as_str); - let Some((analysis_sql, cursor_splices)) = analysis_cursor_splices(&typed, cursor_type) else { + let Ok((analysis_sql, cursor_splices)) = analysis_cursor_splices(&typed, cursor_type) else { return ranges; }; let cleaned: Vec> = @@ -128,7 +125,7 @@ fn expression_ranges( ); for (row, range) in rows.iter().zip(mapped) { let LineageRow { output_column, .. } = row; - if let Some((start, end)) = range { + if let (Some((start, end)), false) = (range, output_column.is_empty()) { ranges .entry(output_column.clone()) .or_insert_with(|| json!({"expression_range": [start, end]})); @@ -137,8 +134,7 @@ fn expression_ranges( ranges } -/// The `placeholders` config as the `(name, text)` pairs analysis read, the way Python's `str` -/// renders each value; `None` when a value's text could differ from what analysis read. +/// The `placeholders` config as analysis read it, as Python's `str` renders each value, if exact. fn placeholder_texts(value: Option<&Value>) -> Option> { let Some(Value::Object(values)) = value else { return Some(Vec::new()); diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index 967e7982a8..a5bdd448d6 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -10,6 +10,7 @@ pub mod model_header_parsing; pub mod model_header_single_parsing; pub mod model_header_tokenizing; pub mod model_output_columns; +pub mod placeholder_default_edits; pub mod placeholder_defaults; pub mod sql_interpolation; pub mod statement_header_matching; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_default_edits.rs b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_default_edits.rs new file mode 100644 index 0000000000..0334ace5a5 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_default_edits.rs @@ -0,0 +1,40 @@ +//! Where Python's `substitute_placeholder_defaults` substitutes. + +use sqlbuild_core::text::main::is_word_char::is_word_char; + +use crate::constants::PLACEHOLDER_PREFIX; + +/// The byte range of each `@@@name` with a default and that default, in order. +pub fn placeholder_default_edits<'a>( + sql: &str, + placeholders: &'a [(String, String)], +) -> Vec<(usize, usize, &'a str)> { + let mut edits: Vec<(usize, usize, &'a str)> = Vec::new(); + let mut position: usize = 0; + while let Some(found) = sql[position..].find(PLACEHOLDER_PREFIX) { + let start: usize = position + found; + let after: &str = &sql[start + PLACEHOLDER_PREFIX.len()..]; + let length: usize = after + .find(|character: char| !is_word(character)) + .unwrap_or(after.len()); + if length == 0 { + position = start + 1; + continue; + } + let end: usize = start + PLACEHOLDER_PREFIX.len() + length; + let name: &str = &after[..length]; + if let Some((_, default)) = placeholders + .iter() + .find(|(placeholder, _)| placeholder == name) + { + edits.push((start, end, default.as_str())); + } + position = end; + } + edits +} + +/// A character Python's `\w` matches. +fn is_word(character: char) -> bool { + is_word_char(character) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs index f47bd01a2b..f03b667704 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs @@ -1,8 +1,6 @@ //! Python's `substitute_placeholder_defaults`. -use sqlbuild_core::text::main::is_word_char::is_word_char; - -use crate::constants::PLACEHOLDER_PREFIX; +use crate::compiler::main::placeholder_default_edits::placeholder_default_edits; /// `substitute_placeholder_defaults`: `@@@name` becomes its default; unknown names stay. pub fn placeholder_defaults(sql: &str, placeholders: &[(String, String)]) -> String { @@ -16,38 +14,3 @@ pub fn placeholder_defaults(sql: &str, placeholders: &[(String, String)]) -> Str result.push_str(&sql[copied..]); result } - -/// The byte range of each `@@@name` with a default and that default, in order. -pub fn placeholder_default_edits<'a>( - sql: &str, - placeholders: &'a [(String, String)], -) -> Vec<(usize, usize, &'a str)> { - let mut edits: Vec<(usize, usize, &'a str)> = Vec::new(); - let mut position: usize = 0; - while let Some(found) = sql[position..].find(PLACEHOLDER_PREFIX) { - let start: usize = position + found; - let after: &str = &sql[start + PLACEHOLDER_PREFIX.len()..]; - let length: usize = after - .find(|character: char| !is_word(character)) - .unwrap_or(after.len()); - if length == 0 { - position = start + 1; - continue; - } - let end: usize = start + PLACEHOLDER_PREFIX.len() + length; - let name: &str = &after[..length]; - if let Some((_, default)) = placeholders - .iter() - .find(|(placeholder, _)| placeholder == name) - { - edits.push((start, end, default.as_str())); - } - position = end; - } - edits -} - -/// A character Python's `\w` matches. -fn is_word(character: char) -> bool { - is_word_char(character) -} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 503ec468ec..a11bcbf230 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule, Job, @@ -238,7 +239,7 @@ class SnapshotExpressionRangeTestCase: """A published model whose compiled SQL differs from its source, and each column's text.""" description: str - adapter: str + adapter: type[BaseAdapter] files: dict[str, str] model_name: str expected_texts: dict[str, str] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 841241dd2a..22a57cf538 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -32,6 +32,7 @@ assemble_project, build_compile_inputs, build_compiled_project, + build_orchestration_snapshot, discover_project_inputs, extract_sql_references, ) @@ -47,6 +48,9 @@ if TYPE_CHECKING: from sqlbuild._native import CompileInputsView + from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + SnapshotExpressionRangeTestCase, + ) METADATA: dict[str, Any] = { "location_id": str(UUID(int=1)), @@ -695,3 +699,32 @@ def _located_binding(diagnostic: CompilerDiagnostic) -> tuple[object, ...]: diagnostic.column, getattr(diagnostic.location, "end_column", None), ) + + +def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) -> dict[str, str]: + """Publish the case's project and return each ranged column's compiled-SQL text.""" + write_files(project_dir=tmp_path, files=test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=test_case.adapter(), + resolved_connection={"database": "WAREHOUSE", "schema": "MAIN"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics), [ + item.message for item in project.diagnostics + ] + publication: dict[str, Any] = json.loads( + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() + ) + key: str = f"model:{test_case.model_name}" + node: dict[str, Any] = publication["nodes"][key] + assert node["compiled_sql"] != node["raw_sql"] + assert "expression_range" not in json.dumps(node), "ranges stay out of node identity" + compiled: bytes = node["compiled_sql"].encode() + columns: dict[str, Any] = publication["documents"].get("columns", {}).get(key, {}) + return { + column: compiled[slice(*metadata["expression_range"])].decode() + for column, metadata in columns.items() + } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 4c95d00325..33ad1bfd57 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -49,6 +49,7 @@ METADATA, PARITY_FILES, TYPE_PROOF_FILES, + published_texts, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -59,6 +60,19 @@ "a calendar schedule from the function it decorates)" ) +_DUCKSHOP: Path = Path("benchmarks/duckshop/project") +_STRICT_SPANS: str = ( + "SQLQuack's strict parsers do not yet record projection-item spans: Snowflake gives aliased " + "items none, and DuckDB's column span leaves out the qualifier" +) +_PRICE_MODELS: dict[str, str] = { + "models/stg_prices.sql": ( + 'MODEL (description "Staged prices");\n' + "SELECT 'W-1' AS window_id, CAST(12345 AS BIGINT) AS price_minor, " + "CAST('2026-03-30 08:00:00' AS TIMESTAMP) AS loaded_at\n" + ), +} + @pytest.mark.parametrize( "test_case", @@ -401,189 +415,166 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s assert sources | targets <= set(publication["nodes"]) -_DUCKSHOP: Path = Path("benchmarks/duckshop/project") -_STRICT_SPANS: str = ( - "SQLQuack's strict parsers do not yet record projection-item spans: Snowflake gives aliased " - "items none, and DuckDB's column span leaves out the qualifier" +@pytest.mark.parametrize( + "test_case", + [ + SnapshotExpressionRangeTestCase( + description="SQL and Python macros, a variable and cursor bounds rewrite the SQL", + adapter=DuckDbAdapter, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nreporting_currency = "USD"\n' + ), + "models/_macros/sql/minor_to_major.sql": ( + "MACRO (\n" + ' description "Minor units to major units",\n' + " arguments (amount BIGINT, places INT CONST DEFAULT 2),\n" + " returns DECIMAL(18, 2),\n" + ");\n\n" + "ROUND(@amount / 100, @places)::DECIMAL(18, 2)\n" + ), + "models/_macros/python/rounding.py": ( + "def round_half_even(ctx, expression: str, places: int = 2) -> str:\n" + " return f'ROUND_EVEN({expression}, {places})'\n" + ), + **_PRICE_MODELS, + "models/price_spans.sql": ( + "MODEL (\n" + ' description "Prices with rewritten expressions",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor loaded_at,\n" + " cursor_type timestamp,\n" + " cursor_grain second,\n" + ");\n\n" + "-- Prix à jour €\n" + "SELECT\n" + " p.window_id AS window_id,\n" + " @sql.minor_to_major(p.price_minor) AS price_amount,\n" + ' @py.round_half_even("p.price_minor / 100", 2) AS rounded_price,\n' + " '@@reporting_currency' AS currency,\n" + " 'Prix à jour €' AS label,\n" + " p.loaded_at AS loaded_at\n" + 'FROM __ref("stg_prices") AS p\n' + "WHERE p.loaded_at >= __cursor_start() AND p.loaded_at < __cursor_end()\n" + ), + }, + model_name="price_spans", + expected_texts={ + "window_id": "p.window_id AS window_id", + "price_amount": ("ROUND(p.price_minor / 100, 2)::DECIMAL(18, 2) AS price_amount"), + "rounded_price": "ROUND_EVEN(p.price_minor / 100, 2) AS rounded_price", + "currency": "'USD' AS currency", + "label": "'Prix à jour €' AS label", + "loaded_at": "p.loaded_at AS loaded_at", + }, + ), + SnapshotExpressionRangeTestCase( + description="a column qualified by its reference call has no range rather than a moved one", + adapter=DuckDbAdapter, + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + **_PRICE_MODELS, + "models/price_refs.sql": ( + 'MODEL (description "Price references");\n' + 'SELECT __ref("stg_prices").window_id AS window_id, ' + "'EUR' AS currency FROM __ref(\"stg_prices\")\n" + ), + }, + model_name="price_refs", + expected_texts={"currency": "'EUR' AS currency"}, + ), + ], + ids=lambda case: case.description, ) -_PRICE_MODELS: dict[str, str] = { - "models/stg_prices.sql": ( - 'MODEL (description "Staged prices");\n' - "SELECT 'W-1' AS window_id, CAST(12345 AS BIGINT) AS price_minor, " - "CAST('2026-03-30 08:00:00' AS TIMESTAMP) AS loaded_at\n" - ), -} +def test_given_rewritten_model_sql_when_publishing_then_expression_ranges_slice_the_compiled_sql( + test_case: SnapshotExpressionRangeTestCase, + tmp_path: Path, +) -> None: + assert published_texts(test_case, tmp_path) == test_case.expected_texts, test_case.description +@pytest.mark.xfail(strict=True, raises=AssertionError, reason=_STRICT_SPANS) @pytest.mark.parametrize( "test_case", [ - pytest.param( - SnapshotExpressionRangeTestCase( - description="SQL and Python macros, a variable and cursor bounds rewrite the SQL", - adapter="duckdb", - files={ - "sqlbuild_project.toml": ( - 'name = "orders"\nadapter = "duckdb"\n\n' - '[vars]\nreporting_currency = "USD"\n' - ), - "models/_macros/sql/minor_to_major.sql": ( - "MACRO (\n" - ' description "Minor units to major units",\n' - " arguments (amount BIGINT, places INT CONST DEFAULT 2),\n" - " returns DECIMAL(18, 2),\n" - ");\n\n" - "ROUND(@amount / 100, @places)::DECIMAL(18, 2)\n" - ), - "models/_macros/python/rounding.py": ( - "def round_half_even(ctx, expression: str, places: int = 2) -> str:\n" - " return f'ROUND_EVEN({expression}, {places})'\n" - ), - **_PRICE_MODELS, - "models/price_spans.sql": ( - "MODEL (\n" - ' description "Prices with rewritten expressions",\n' - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor loaded_at,\n" - " cursor_type timestamp,\n" - " cursor_grain second,\n" - ");\n\n" - "-- Prix à jour €\n" - "SELECT\n" - " p.window_id AS window_id,\n" - " @sql.minor_to_major(p.price_minor) AS price_amount,\n" - ' @py.round_half_even("p.price_minor / 100", 2) AS rounded_price,\n' - " '@@reporting_currency' AS currency,\n" - " 'Prix à jour €' AS label,\n" - " p.loaded_at AS loaded_at\n" - 'FROM __ref("stg_prices") AS p\n' - "WHERE p.loaded_at >= __cursor_start() AND p.loaded_at < __cursor_end()\n" - ), - }, - model_name="price_spans", - expected_texts={ - "window_id": "p.window_id AS window_id", - "price_amount": ( - "ROUND(p.price_minor / 100, 2)::DECIMAL(18, 2) AS price_amount" - ), - "rounded_price": "ROUND_EVEN(p.price_minor / 100, 2) AS rounded_price", - "currency": "'USD' AS currency", - "label": "'Prix à jour €' AS label", - "loaded_at": "p.loaded_at AS loaded_at", - }, - ), - id="DuckDB macros, variable and cursor bounds", - ), - pytest.param( - SnapshotExpressionRangeTestCase( - description="an unaliased qualified column covers its qualifier", - adapter="duckdb", - files={ - "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - **_PRICE_MODELS, - "models/price_columns.sql": ( - 'MODEL (description "Price columns");\n' - 'SELECT p.window_id, p.price_minor FROM __ref("stg_prices") AS p\n' - ), - }, - model_name="price_columns", - expected_texts={ - "window_id": "p.window_id", - "price_minor": "p.price_minor", - }, - ), - id="DuckDB qualified columns", - marks=pytest.mark.xfail(strict=True, raises=AssertionError, reason=_STRICT_SPANS), - ), - pytest.param( - SnapshotExpressionRangeTestCase( - description="DuckShop macros, variable and cursor bounds rewrite the SQL", - adapter="snowflake", - files={ - "sqlbuild_project.toml": ( - 'name = "duckshop"\nadapter = "snowflake"\n\n' - '[defaults]\ndatabase = "WAREHOUSE"\nschema = "MAIN"\n\n' - '[vars]\nreporting_currency = "USD"\n' - ), - "sources/edge_cases.yml": ( - "sources:\n" - " - name: edge__price_windows\n" - " description: Price windows\n" - " columns:\n" - " - name: window_id\n" - " type: VARCHAR\n" - " - name: price_minor\n" - " type: NUMBER(38,0)\n" - " - name: _loaded_at\n" - " type: TIMESTAMP_NTZ\n" - ), - # A second consumer keeps the DuckShop macros project-wide, as they are there. - "models/price_checks.sql": ( - 'MODEL (description "Price checks");\n' - "SELECT @sql.money_from_minor(150, 2) AS amount, " - '@py.round_half_even("1.25", 1) AS rounded\n' - ), - **{ - path: (_DUCKSHOP / path).read_text() - for path in ( - "macros/sql/money_from_minor.sql", - "macros/sql/safe_divide.sql", - "macros/python/rounding.py", - "models/intermediate/edge_cases/int_edge_expression_spans.sql", - ) - }, - }, - model_name="int_edge_expression_spans", - expected_texts={ - "window_id": "w.window_id", - "price_amount": ( - "ROUND((\n w.price_minor / NULLIF((POWER(10, 2)::NUMBER(38, 0)), 0)\n)" - "::NUMBER(38, 6), 2)::NUMBER(38, 6) AS price_amount" - ), - "rounded_price": ( - "ROUND(w.price_minor / 100, 2, 'HALF_TO_EVEN')::NUMBER(18, 2)" - " AS rounded_price" - ), - "currency": "'USD'::VARCHAR AS currency", - "label": "'Prix à jour €'::VARCHAR AS label", - "loaded_at": "w._loaded_at AS loaded_at", + SnapshotExpressionRangeTestCase( + description="an unaliased qualified column covers its qualifier", + adapter=DuckDbAdapter, + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + **_PRICE_MODELS, + "models/price_columns.sql": ( + 'MODEL (description "Price columns");\n' + 'SELECT p.window_id, p.price_minor FROM __ref("stg_prices") AS p\n' + ), + }, + model_name="price_columns", + expected_texts={ + "window_id": "p.window_id", + "price_minor": "p.price_minor", + }, + ), + SnapshotExpressionRangeTestCase( + description="DuckShop macros, variable and cursor bounds rewrite the SQL", + adapter=SnowflakeAdapter, + files={ + "sqlbuild_project.toml": ( + 'name = "duckshop"\nadapter = "snowflake"\n\n' + '[defaults]\ndatabase = "WAREHOUSE"\nschema = "MAIN"\n\n' + '[vars]\nreporting_currency = "USD"\n' + ), + "sources/edge_cases.yml": ( + "sources:\n" + " - name: edge__price_windows\n" + " description: Price windows\n" + " columns:\n" + " - name: window_id\n" + " type: VARCHAR\n" + " - name: price_minor\n" + " type: NUMBER(38,0)\n" + " - name: _loaded_at\n" + " type: TIMESTAMP_NTZ\n" + ), + # A second consumer keeps the DuckShop macros project-wide, as they are there. + "models/price_checks.sql": ( + 'MODEL (description "Price checks");\n' + "SELECT @sql.money_from_minor(150, 2) AS amount, " + '@py.round_half_even("1.25", 1) AS rounded\n' + ), + **{ + path: (_DUCKSHOP / path).read_text() + for path in ( + "macros/sql/money_from_minor.sql", + "macros/sql/safe_divide.sql", + "macros/python/rounding.py", + "models/intermediate/edge_cases/int_edge_expression_spans.sql", + ) }, - ), - id="Snowflake DuckShop edge model", - marks=pytest.mark.xfail(strict=True, raises=AssertionError, reason=_STRICT_SPANS), + }, + model_name="int_edge_expression_spans", + expected_texts={ + "window_id": "w.window_id", + "price_amount": ( + "ROUND((\n w.price_minor / NULLIF((POWER(10, 2)::NUMBER(38, 0)), 0)\n)" + "::NUMBER(38, 6), 2)::NUMBER(38, 6) AS price_amount" + ), + "rounded_price": ( + "ROUND(w.price_minor / 100, 2, 'HALF_TO_EVEN')::NUMBER(18, 2) AS rounded_price" + ), + "currency": "'USD'::VARCHAR AS currency", + "label": "'Prix à jour €'::VARCHAR AS label", + "loaded_at": "w._loaded_at AS loaded_at", + }, ), ], + ids=lambda case: case.description, ) -def test_given_rewritten_model_sql_when_publishing_then_expression_ranges_slice_the_compiled_sql( +def test_given_spans_strict_parsers_omit_when_publishing_then_expression_ranges_slice_the_compiled_sql( test_case: SnapshotExpressionRangeTestCase, tmp_path: Path, ) -> None: - write_files(project_dir=tmp_path, files=test_case.files) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=SnowflakeAdapter() if test_case.adapter == "snowflake" else DuckDbAdapter(), - resolved_connection={"database": "WAREHOUSE", "schema": "MAIN"}, - ) - assert not any(item.severity == "error" for item in project.diagnostics), [ - item.message for item in project.diagnostics - ] - publication: dict[str, Any] = json.loads( - build_orchestration_snapshot( - project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered - ).to_json() - ) - key: str = f"model:{test_case.model_name}" - node: dict[str, Any] = publication["nodes"][key] - assert node["compiled_sql"] != node["raw_sql"] - assert "expression_range" not in json.dumps(node), "ranges stay out of node identity" - compiled: bytes = node["compiled_sql"].encode() - columns: dict[str, Any] = publication["documents"].get("columns", {}).get(key, {}) - assert { - column: compiled[slice(*metadata["expression_range"])].decode() - for column, metadata in columns.items() - } == test_case.expected_texts + assert published_texts(test_case, tmp_path) == test_case.expected_texts, test_case.description @pytest.mark.parametrize( From 19633cfa412129a5792b51cd741437a7d2ffb4a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:42:17 +0000 Subject: [PATCH 6/7] wip: capture the expression-span edge expectations on Snowflake --- .../edge_cases/test_int_edge_expression_spans.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql index b3a0a373dd..cf0a299292 100644 --- a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql @@ -21,17 +21,17 @@ WITH __source__edge__price_windows AS ( __expected__int_edge_expression_spans AS ( SELECT 'PW-01' AS window_id, - 12.34::NUMBER(38, 6) AS price_amount, - 12.34::NUMBER(18, 2) AS rounded_price, + 12.340000::NUMBER(38,6) AS price_amount, + 12.34::NUMBER(18,2) AS rounded_price, 'USD' AS currency, 'Prix à jour €' AS label, - '2026-03-02 09:00:00'::TIMESTAMP_NTZ AS loaded_at + '2026-03-02 09:00:00'::TIMESTAMP_NTZ(9) AS loaded_at UNION ALL SELECT 'PW-02' AS window_id, - CAST(NULL AS NUMBER(38, 6)) AS price_amount, - CAST(NULL AS NUMBER(18, 2)) AS rounded_price, + CAST(NULL AS NUMBER(38,6)) AS price_amount, + CAST(NULL AS NUMBER(18,2)) AS rounded_price, 'USD' AS currency, 'Prix à jour €' AS label, - '2026-03-02 18:30:00'::TIMESTAMP_NTZ AS loaded_at + '2026-03-02 18:30:00'::TIMESTAMP_NTZ(9) AS loaded_at ) From 386208edbc70b0762e541fd4c82c536b90fa2c86 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:20:27 +0000 Subject: [PATCH 7/7] wip: expect expression ranges in the person journey's node SQL read --- .../compiler/compiled_project/publication/person/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index f6b5ebe58d..ea206b7b33 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -229,7 +229,7 @@ def verify_sql_inspection( "applied": True } sql: dict[str, Any] = person_read(api, path, session=session) - assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql", "expression_ranges"} assert sql["snapshot_id"] == snapshot_id assert sql["node_key"] == "model:orders" assert sql["raw_sql"] == model.read_text()