diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index ac5fd5faa5..bbb29e1669 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -124,8 +124,8 @@ project and does not change it: `edge_running_peaks`, that a pre-hook of `int_edge_basket_running_peaks` creates (`models/intermediate/edge_cases/_sqlbuild/_hooks/sql/`) and SQLBuild does not declare. -It has 164 models (13 staging, 137 intermediate, 14 marts), 5 Python functions, a SQL table -function, a hook-created JavaScript function, 189 unit tests and 4 scenarios, in nine groups: +It has 166 models (13 staging, 139 intermediate, 14 marts), 5 Python functions, a SQL table +function, a hook-created JavaScript function, 191 unit tests and 4 scenarios, in nine groups: | Group | Covers | |-------|--------| @@ -134,7 +134,7 @@ function, a hook-created JavaScript function, 189 unit tests and 4 scenarios, in | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` | | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day | | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | -| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query | +| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | | Translation refusals | one model per construct that translation tools refuse or guard, most putting a construct refused only under strict checking before one refused by default, so a model shows both: `FLOAT` values through sums, averages, running totals, arrays, objects, `HASH`, `TO_CHAR` and `::NUMBER(38, 10)`; `VARIANT` dates, decimals and XML; `ARRAY_AGG` without `WITHIN GROUP`, sparse arrays and `ARRAY(NUMBER)` casts; `REDUCE`, `FILTER` and `TRANSFORM` lambdas; patterns and `REGEXP_REPLACE` replacements built from columns, and lazy patterns; `TRY_TO_DATE` with a format from a column; zoned literals, `CONVERT_TIMEZONE` from `CURRENT_TIMESTAMP`, `SEQ8`, `GENERATOR` with an expression `ROWCOUNT`; `PIVOT`, `ANY_VALUE`, `IGNORE NULLS` around `IFF`, unaliased columns read by Snowflake's name, `UNION` branches whose column is always NULL; `TO_NUMBER`, `TO_DECIMAL` and `TRY_BASE64_DECODE_BINARY`; an open (uncontracted) source `edge__supplier_notes`, the table function, read for one basket and after `LATERAL` for every basket, and the hook-created function. `rpt_edge_case_model_keys` and `rpt_edge_case_array_model_keys` read every model in the group, so a refusal there also refuses a downstream model | diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json index 1e8b77124a..75868daefd 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json +++ b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json @@ -3,5 +3,14 @@ "compiled_sql": "SELECT 23 AS amount -- Orders 🧾\n", "python_source": "def order_amount():\n return 23\n", "function_sql": "43", - "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n" + "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n", + "columns_document": { + "amount": {"expression_range": [7, 19], "described_as": "a later per-column key readers ignore"}, + "past_the_end": {"expression_range": [7, 4096]}, + "split_character": {"expression_range": [7, 32]}, + "reversed": {"expression_range": [19, 7]}, + "no_range": {} + }, + "expression_ranges": {"amount": [7, 19]}, + "updated_expression_ranges": {"amount": [7, 19], "split_character": [7, 32]} } diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql new file mode 100644 index 0000000000..651ff5b9a9 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql @@ -0,0 +1,37 @@ +MODEL ( + description """ + One row per edge-case price window loaded in the cursor window, with its price converted by a + typed SQL macro and rounded by a Python macro, the reporting currency from a variable and a + non-ASCII label, so the compiled SQL differs from the authored text before every output column; + the acceptance case for column expression ranges that point into the compiled SQL. + """, + tags [edge_case], + materialized incremental, + incremental_strategy delete_insert, + cursor loaded_at, + cursor_type timestamp, + cursor_grain second, + cursor_inputs ( + edge__price_windows _loaded_at, + ), + columns ( + window_id (type VARCHAR, nullable false, audits [not_null, unique]), + price_amount (type NUMBER(38, 6)), + rounded_price (type NUMBER(18, 2)), + currency (type VARCHAR), + label (type VARCHAR), + loaded_at (type TIMESTAMP_NTZ, nullable false), + ), +); + +SELECT + w.window_id, + @sql.money_from_minor(w.price_minor, 2) AS price_amount, + @py.round_half_even("w.price_minor / 100", 2)::NUMBER(18, 2) AS rounded_price, + '@@reporting_currency'::VARCHAR AS currency, + 'Prix à jour €'::VARCHAR AS label, + w._loaded_at AS loaded_at +FROM __source("edge__price_windows") AS w +WHERE + w._loaded_at >= __cursor_start() + AND w._loaded_at < __cursor_end() diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql new file mode 100644 index 0000000000..26bd286d57 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql @@ -0,0 +1,17 @@ +MODEL ( + description """ + One row per edge-case price window, each column qualified by the source reference call itself, + so every output column's expression starts at text the compiler rewrites; the case for column + expression ranges that must be dropped rather than moved onto the relation name. + """, + tags [edge_case], + columns ( + window_id (type VARCHAR, nullable false, audits [not_null, unique]), + sku (type VARCHAR), + ), +); + +SELECT + __source("edge__price_windows").window_id, + __source("edge__price_windows").sku +FROM __source("edge__price_windows") diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index fca5a4ea59..4b9a37a0bc 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -337,6 +337,18 @@ The edge case reads the line through a derived table nested in a comma-joined LA that shape is the case under test. """ +[[rules.rule_ignores]] +rules = ["SQLCLEAN034"] +paths = [ + "models/intermediate/edge_cases/int_edge_expression_spans.sql", + "models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql", +] +reason = """ +These edge cases keep their macro calls, variable, literals and reference calls in the terminal +select on purpose: column expression ranges point at the terminal select's items, so that is where +the compiled SQL must differ from the authored text. +""" + [[rules.rule_ignores]] rules = ["SQLCLEAN040"] paths = ["models/intermediate/edge_cases/int_edge_amount_bit_reads.sql"] diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql new file mode 100644 index 0000000000..cf0a299292 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql @@ -0,0 +1,37 @@ +TEST ( + name "int_edge_expression_spans__converts_prices_loaded_in_the_window", + cursor_start "2026-03-02 00:00:00", + cursor_end "2026-03-03 00:00:00", +); + +WITH __source__edge__price_windows AS ( + SELECT + v.window_id, + v.sku, + v.price_minor::NUMBER(38, 0) AS price_minor, + v.valid_from::TIMESTAMP_NTZ(6) AS valid_from, + v.valid_to::TIMESTAMP_NTZ(6) AS valid_to, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PW-01', 'SKU-1', 1234, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 09:00:00'), + ('PW-02', 'SKU-2', NULL, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 18:30:00'), + ('PW-03', 'SKU-3', 999, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-01 23:59:59') AS v(window_id, sku, price_minor, valid_from, valid_to, _loaded_at) +), + +__expected__int_edge_expression_spans AS ( + SELECT + 'PW-01' AS window_id, + 12.340000::NUMBER(38,6) AS price_amount, + 12.34::NUMBER(18,2) AS rounded_price, + 'USD' AS currency, + 'Prix à jour €' AS label, + '2026-03-02 09:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'PW-02' AS window_id, + CAST(NULL AS NUMBER(38,6)) AS price_amount, + CAST(NULL AS NUMBER(18,2)) AS rounded_price, + 'USD' AS currency, + 'Prix à jour €' AS label, + '2026-03-02 18:30:00'::TIMESTAMP_NTZ(9) AS loaded_at +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql new file mode 100644 index 0000000000..51ae9b9a9b --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql @@ -0,0 +1,26 @@ +TEST ( + name "int_edge_reference_qualified_spans__keeps_every_window", +); + +WITH __source__edge__price_windows AS ( + SELECT + v.window_id, + v.sku, + v.price_minor::NUMBER(38, 0) AS price_minor, + v.valid_from::TIMESTAMP_NTZ(6) AS valid_from, + v.valid_to::TIMESTAMP_NTZ(6) AS valid_to, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PW-01', 'SKU-1', 1234, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 09:00:00'), + ('PW-02', NULL, NULL, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 18:30:00') AS v(window_id, sku, price_minor, valid_from, valid_to, _loaded_at) +), + +__expected__int_edge_reference_qualified_spans AS ( + SELECT + 'PW-01' AS window_id, + 'SKU-1' AS sku + UNION ALL + SELECT + 'PW-02' AS window_id, + CAST(NULL AS VARCHAR) AS sku +) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 041ea0880a..ccc5e5cc8a 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2009,6 +2009,10 @@ export interface components { /** @description SQL inspection never returns Python source, execution artifacts or unrelated node metadata. */ NodeSql: { compiled_sql?: string | null; + /** @description Each placed column's defining expression as a `[start, end)` byte range of `compiled_sql`. */ + expression_ranges?: { + [key: string]: number[]; + }; node_key: string; raw_sql?: string | null; /** Format: uuid */ diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs index 8006361670..07c95d22ac 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs @@ -151,6 +151,10 @@ fn lineage_row( transform_code, confidence_code, sources, + expression_range: output + .expression_range + .as_ref() + .map(|range| (range.start, range.end)), } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs index efa62fbccf..c8ce6429b5 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs @@ -110,7 +110,12 @@ pub(crate) fn analysed_members( .get(leader) .filter(|template| exact_for_others(template, members[*leader].model)); match template.map(|template| mapped(template, &members[index].model.lineage_references)) { - Some(Ok(analysis)) => { + Some(Ok(mut analysis)) => { + rebase_ranges( + &mut analysis, + members[*leader].cleaned.as_deref().unwrap_or_default(), + members[index].cleaned.as_deref().unwrap_or_default(), + ); analyses[index] = Some(Ok(analysis)); counts.shared += 1; counts.unshared -= 1; @@ -208,6 +213,67 @@ fn mapped( Ok(analysis) } +/// Each expression range moved onto `member`'s relation names, or dropped where it cannot move. +pub(crate) fn rebase_ranges(analysis: &mut NativeAnalysis, leader: &str, member: &str) { + if leader == member { + return; + } + let renamed: Option> = renamed_runs(leader, member); + for row in &mut analysis.lineage { + row.expression_range = match (&renamed, row.expression_range) { + (Some(runs), Some((start, end))) => rebased(runs, start).zip(rebased(runs, end)), + _ => None, + }; + } +} + +/// `(leader start, leader end, member end)` of each differing identifier run, if only those differ. +fn renamed_runs(leader: &str, member: &str) -> Option> { + let identifier = |c: u8| c.is_ascii_alphanumeric() || c == b'_' || c == b'$'; + let (left, right): (&[u8], &[u8]) = (leader.as_bytes(), member.as_bytes()); + let (mut i, mut j): (usize, usize) = (0, 0); + let mut runs: Vec<(usize, usize, usize)> = Vec::new(); + while i < left.len() || j < right.len() { + if i < left.len() && j < right.len() && left[i] == right[j] { + i += 1; + j += 1; + continue; + } + let mut back: usize = 0; + while back < i && identifier(left[i - back - 1]) { + back += 1; + } + let (mut left_end, mut right_end): (usize, usize) = (i, j); + while left_end < left.len() && identifier(left[left_end]) { + left_end += 1; + } + while right_end < right.len() && identifier(right[right_end]) { + right_end += 1; + } + if left_end == i && right_end == j { + return None; + } + runs.push((i - back, left_end, right_end)); + (i, j) = (left_end, right_end); + } + Some(runs) +} + +/// The member offset of leader `offset`, or `None` strictly inside a renamed run. +fn rebased(runs: &[(usize, usize, usize)], offset: usize) -> Option { + let mut moved: usize = offset; + for &(left_start, left_end, right_end) in runs { + if offset <= left_start { + break; + } + if offset < left_end { + return None; + } + moved = offset - left_end + right_end; + } + Some(moved) +} + /// Whether `leader`'s `template` succeeded, found nothing and spells none of its relation names. fn exact_for_others(template: &Result, leader: &ModelRequest) -> bool { let Ok(analysis) = template else { diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 9e43ee9093..7824a37df3 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -4,6 +4,7 @@ pub mod fast_lineage; pub mod native_gap; pub(crate) mod native_outcomes; pub mod native_revalidation; +pub mod query_expression_ranges; pub mod schema_column; pub mod skipped_analysis; pub mod unapplied_nullability_rules; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs new file mode 100644 index 0000000000..6e6808aa02 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/query_expression_ranges.rs @@ -0,0 +1,85 @@ +//! Expression ranges of a model's native analysis, moved back into the query SQL it compiled. + +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::semantic_validation::main::normalize_native_filled::normalize_native_filled; +use crate::sql_macros::models::TextSplice; + +/// Each cleaned-SQL byte range as a range of the SQL `passes` rewrote into `analysis_sql`. +#[must_use] +pub fn query_expression_ranges( + dialect: &str, + (analysis_sql, placeholders): (&str, &[(String, String)]), + passes: &[Vec], + ranges: &[Option<(usize, usize)>], +) -> Vec> { + let AnalysisRoute::Native(module) = analysis_route(Some(dialect)) else { + return vec![None; ranges.len()]; + }; + let Ok((cleaned, offsets)) = + normalize_native_filled(analysis_sql, dialect, module.stub_prefix(), placeholders) + else { + return vec![None; ranges.len()]; + }; + let analysis_bytes: Vec = analysis_sql + .char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(analysis_sql.len())) + .collect(); + let mut characters: Vec> = vec![None; cleaned.len() + 1]; + for (character, (byte, _)) in cleaned.char_indices().enumerate() { + characters[byte] = Some(character); + } + characters[cleaned.len()] = Some(cleaned.chars().count()); + let query_byte = |cleaned_byte: usize| -> Option { + let character: usize = (*characters.get(cleaned_byte)?)?; + if !copied_boundary(&offsets, character) { + return None; + } + let byte: usize = *analysis_bytes.get(*offsets.get(character)?)?; + passes + .iter() + .rev() + .try_fold(byte, |byte, splices| spliced_back(byte, splices)) + }; + ranges + .iter() + .map(|range| { + let (start, end) = (*range)?; + let (start, end) = (query_byte(start)?, query_byte(end)?); + (start <= end).then_some((start, end)) + }) + .collect() +} + +/// Whether cleaned boundary `character` lies between source-adjacent characters, so maps exactly. +fn copied_boundary(offsets: &[usize], character: usize) -> bool { + let Some(previous) = character.checked_sub(1) else { + return offsets.first() == Some(&0); + }; + offsets + .get(previous) + .zip(offsets.get(character)) + .is_some_and(|(before, after)| before + 1 == *after) +} + +/// The source byte of output `byte`; splice edges map to source edges, inner bytes to `None`. +fn spliced_back(byte: usize, splices: &[TextSplice]) -> Option { + let (mut source_end, mut output_end): (usize, usize) = (0, 0); + for splice in splices { + if byte < splice.output.0 { + break; + } + if byte == splice.output.0 { + return Some(splice.source.0); + } + if byte < splice.output.1 { + return None; + } + if byte == splice.output.1 { + return Some(splice.source.1); + } + (source_end, output_end) = (splice.source.1, splice.output.1); + } + Some(byte - output_end + source_end) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index 3cf359ed56..660e64b4ed 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -10,6 +10,8 @@ mod test_clickhouse_type_match; mod test_databricks_dialect; #[path = "tests/test_engine_types.rs"] mod test_engine_types; +#[path = "tests/test_expression_ranges.rs"] +mod test_expression_ranges; #[path = "tests/test_native_arms.rs"] mod test_native_arms; #[path = "tests/test_native_dialect.rs"] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs index 3909552f77..84fededf40 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -1,11 +1,14 @@ use std::collections::HashMap; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; use sqlquack_semantics::lineage::models::{Nullability, SchemaColumn}; use crate::analysis_route::_helpers::analysis::native_outcome::{ analysed_sql, native_outcome, native_sql, }; +use crate::analysis_route::main::query_expression_ranges::query_expression_ranges; use crate::analysis_route::main::schema_column::schema_column as stored_column; +use crate::analysis_route::models::DuckDbDialect; use crate::analysis_route::models::{AnalysisInput, NativeAnalysis, StandInSchema}; use crate::analysis_route::types::NativeDialect; use crate::assembly::analysis_session::models::{ @@ -15,6 +18,9 @@ use crate::assembly::analysis_session::models::{ use crate::assembly::analysis_session::types::Shapes; use crate::lineage::models::{LineageColumn, LineageSource}; use crate::semantic_checks::types::RevisedBinding; +use crate::semantic_validation::main::normalize_native_filled::normalize_native_filled; +use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use crate::sql_macros::models::{TextSplice, TypedCall}; /// `orders` with a non-null INTEGER id and a nullable VARCHAR status. pub(super) fn orders() -> StandInSchema { @@ -168,3 +174,72 @@ pub(crate) fn model_outcome( }); native_outcome(model, schema, cleaned.unwrap_or_default(), analysed) } + +/// Each output column with the `query_sql` text its range covers after the compile passes. +pub(super) fn mapped_texts( + query_sql: &str, + placeholders: &[(&str, &str)], + typed_calls: &[(&str, &str)], +) -> Vec<(String, Option)> { + let placeholders: Vec<(String, String)> = placeholders + .iter() + .map(|(name, text)| ((*name).to_owned(), (*text).to_owned())) + .collect(); + let calls: Vec = typed_calls + .iter() + .map(|(call, returns)| typed_call(query_sql, call, returns)) + .collect(); + let (typed, typed_splices) = typed_analysis_sql(query_sql, &calls, "duckdb"); + let (analysis_sql, spans) = replace_intrinsics_with_spans(&typed, "Model SQL", |_| { + "CAST('2000-01-01' AS TIMESTAMP)".to_owned() + }) + .expect("the cursor calls are replaced"); + let cursor_splices: Vec = spans + .iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + let dialect: &DuckDbDialect = &DuckDbDialect; + let (cleaned, _) = normalize_native_filled( + &analysis_sql, + "duckdb", + dialect.stub_prefix(), + &placeholders, + ) + .expect("the test query normalises"); + let analysis: NativeAnalysis = analysed(dialect, &cleaned, &[]); + let ranges: Vec> = analysis + .lineage + .iter() + .map(|row| row.expression_range) + .collect(); + let mapped = query_expression_ranges( + "duckdb", + (&analysis_sql, &placeholders), + &[typed_splices, cursor_splices], + &ranges, + ); + analysis + .lineage + .iter() + .zip(mapped) + .map(|(row, range)| { + let text: Option = range.map(|(start, end)| query_sql[start..end].to_owned()); + (row.output_column.clone(), text) + }) + .collect() +} + +/// The typed call of the expression macro `call` in `query_sql`, returning `returns`. +fn typed_call(query_sql: &str, call: &str, returns: &str) -> TypedCall { + let start: usize = query_sql.find(call).expect("the call is in the query"); + TypedCall { + start, + end: start + call.len(), + macro_name: "typed_call".to_owned(), + returns: Some(returns.to_owned()), + returns_like: None, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs new file mode 100644 index 0000000000..42273c7771 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_expression_ranges.rs @@ -0,0 +1,95 @@ +use crate::analysis_route::_helpers::analysis::native_sharing::rebase_ranges; +use crate::analysis_route::models::{DuckDbDialect, NativeAnalysis}; +use crate::analysis_route::tests::helpers::{analysed, mapped_texts}; +use crate::analysis_route::tests::test_types::{ExpressionRangeTestCase, RebaseRangesTestCase}; + +#[test] +fn given_compiled_sql_when_mapping_expression_ranges_then_they_cover_its_own_text() { + let test_cases = [ + ExpressionRangeTestCase { + description: "cleaning rewrites multi-byte comments, references, placeholders and cursors", + query_sql: "-- café ünïcode\nWITH x AS (SELECT * FROM __ref(\"orders\") WHERE id > @@@floor \ + AND id < __cursor_end())\nSELECT x.id AS id, UPPER(x.status) AS loud, 'é' AS mark FROM x", + placeholders: &[("floor", "100000")], + typed_calls: &[], + expected_texts: &[ + ("id", Some("x.id AS id")), + ("loud", Some("UPPER(x.status) AS loud")), + ("mark", Some("'é' AS mark")), + ], + }, + ExpressionRangeTestCase { + description: "a typed macro call is marked and a cursor bound replaced before analysis", + query_sql: "SELECT ROUND(amount * 1.2, 2) AS gross, id AS id FROM orders \ + WHERE updated_at < __cursor_end()", + placeholders: &[], + typed_calls: &[("ROUND(amount * 1.2, 2)", "DECIMAL(18, 2)")], + expected_texts: &[ + ("gross", Some("ROUND(amount * 1.2, 2) AS gross")), + ("id", Some("id AS id")), + ], + }, + ExpressionRangeTestCase { + description: "an edge beside a removed reference call is dropped, not moved onto the name", + query_sql: "SELECT __ref(\"orders\").id AS id, UPPER(status) AS loud FROM __ref(\"orders\")", + placeholders: &[], + typed_calls: &[], + expected_texts: &[("id", None), ("loud", Some("UPPER(status) AS loud"))], + }, + ExpressionRangeTestCase { + description: "an edge inside a filled placeholder is dropped; one at its start maps to it", + query_sql: "SELECT @@@cols AS two, @@@floor + 1 AS above, id AS id FROM orders", + placeholders: &[("cols", "1 AS one, 2"), ("floor", "10")], + typed_calls: &[], + expected_texts: &[ + ("one", None), + ("two", None), + ("above", Some("@@@floor + 1 AS above")), + ("id", Some("id AS id")), + ], + }, + ]; + for test_case in test_cases { + let mapped: Vec<(String, Option)> = mapped_texts( + test_case.query_sql, + test_case.placeholders, + test_case.typed_calls, + ); + let texts: Vec<(&str, Option<&str>)> = mapped + .iter() + .map(|(column, text)| (column.as_str(), text.as_deref())) + .collect(); + assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); + } +} + +#[test] +fn given_a_shared_analysis_when_rebasing_onto_a_member_then_ranges_follow_or_drop() { + let test_cases = [ + RebaseRangesTestCase { + description: "a member that only renames a relation keeps every range", + leader: "WITH x AS (SELECT * FROM orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x", + member: "WITH x AS (SELECT * FROM customer_orders) SELECT x.id AS id, UPPER(x.status) AS loud FROM x", + expected_texts: &[Some("x.id AS id"), Some("UPPER(x.status) AS loud")], + }, + RebaseRangesTestCase { + description: "a member that differs outside identifiers drops every range", + leader: "SELECT x.id AS id FROM orders AS x", + member: "SELECT x.id AS id FROM orders AS x WHERE 1 = 1", + expected_texts: &[None], + }, + ]; + for test_case in test_cases { + let mut analysis: NativeAnalysis = analysed(&DuckDbDialect, test_case.leader, &[]); + rebase_ranges(&mut analysis, test_case.leader, test_case.member); + let texts: Vec> = analysis + .lineage + .iter() + .map(|row| { + row.expression_range + .map(|(start, end)| &test_case.member[start..end]) + }) + .collect(); + assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 9c15fcf1c3..b677fdd610 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -353,3 +353,21 @@ pub(super) struct TsqlTypeTestCase { pub(super) spelling: &'static str, pub(super) expected_type: Option<(&'static str, TypeFamily)>, } + +pub(super) struct ExpressionRangeTestCase { + pub(super) description: &'static str, + pub(super) query_sql: &'static str, + pub(super) placeholders: &'static [(&'static str, &'static str)], + /// Each typed expression macro call as `(call text, returned type)`. + pub(super) typed_calls: &'static [(&'static str, &'static str)], + /// Each output column and the text of `query_sql` its mapped range covers. + pub(super) expected_texts: &'static [(&'static str, Option<&'static str>)], +} + +pub(super) struct RebaseRangesTestCase { + pub(super) description: &'static str, + pub(super) leader: &'static str, + pub(super) member: &'static str, + /// The text of `member` each lineage row's rebased range covers. + pub(super) expected_texts: &'static [Option<&'static str>], +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index a814d3bfaf..d265c3985e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -451,6 +451,9 @@ pub(crate) fn encode_outcome( encoder.text(name); encoder.text(column); } + let (start, end) = row.expression_range.unzip(); + encoder.optional_uint(start); + encoder.optional_uint(end); } encoder.flag(analysis.has_star); encoder.flag(analysis.star_resolved); @@ -522,11 +525,18 @@ pub(crate) fn decode_outcome( for _ in 0..source_count { sources.push((decoder.text()?, decoder.text()?, decoder.text()?)); } + let expression_range: Option<(usize, usize)> = + match (decoder.optional_index()?, decoder.optional_index()?) { + (Some(start), Some(end)) if start <= end => Some((start, end)), + (None, None) => None, + _ => return Err(StoreDecodeError), + }; rows.push(LineageRow { output_column, transform_code, confidence_code, sources, + expression_range, }); } let lineage: LineageFacts = match tag { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 8679f4480d..ef4957e07b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -864,6 +864,7 @@ impl SharedResult { transform_code: row.transform_code, confidence_code: row.confidence_code, sources: named_sources(&row.sources, &names), + expression_range: None, }) .collect(); Some(MemberResult { @@ -1361,6 +1362,7 @@ fn project_fact( transform_code: u8::try_from(transform).map_err(|error| error.to_string())?, confidence_code: u8::try_from(confidence).map_err(|error| error.to_string())?, sources: resolved, + expression_range: None, }, )) } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs index dfdaceb69f..2968a2c3ba 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs @@ -221,6 +221,7 @@ pub(crate) fn legacy_analysis(input: &LegacyInput<'_>) -> Fact { confidence }, sources, + expression_range: None, }); } Ok(LegacyAnalysis { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index 2d69ab47a5..12bab7d107 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -256,6 +256,7 @@ fn native_projection( }, transform_code: transform, sources, + expression_range: None, }); } let has_star: bool = has_star_projection(analysis); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 51d79f0389..968ee107ea 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -199,7 +199,7 @@ pub(crate) const PYTHON_RULE: &str = "python"; pub(crate) const NON_NULL_NULLABILITY: &str = "non_null"; pub(crate) const NULLABLE_NULLABILITY: &str = "nullable"; /// Bumped whenever the stored outcome or a key's fields change; older entries never match. -pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v4"; +pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v5"; /// A stored varint byte at or above this value continues into the next byte. pub(crate) const VARINT_CONTINUATION: u64 = 0x80; /// The value bits each stored varint byte carries. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 85f5ad7013..b8a426f348 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -237,6 +237,8 @@ pub struct LineageRow { pub transform_code: u8, pub confidence_code: u8, pub sources: Vec<(String, String, String)>, + /// UTF-8 byte range of the column's projection item in the cleaned analysis SQL, if known. + pub expression_range: Option<(usize, usize)>, } /// Where a model's lineage facts live. diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index 90a595f015..efe0a2f7d4 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -3,6 +3,7 @@ use crate::relation_markers::main::lateral_start::lateral_start; use crate::semantic_checks::constants::UDF_NAME_PREFIX; use regex::Regex; +use sqlbuild_sqltext::compiler::main::placeholder_default_edits::placeholder_default_edits; use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; @@ -201,41 +202,67 @@ pub(crate) fn normalize_native_sql( stub_prefix: &str, relations: &HashMap, ) -> Result { + native_mapped(sql, dialect, stub_prefix, relations).map(|mapped| mapped.sql) +} + +/// [`normalize_native_sql`] with placeholder defaults filled, and each character's source in `sql`. +pub(crate) fn normalize_native_sql_mapped( + sql: &str, + dialect: &str, + stub_prefix: &str, + placeholders: &[(String, String)], +) -> Result<(String, Vec), String> { + let native: MappedSql = native_mapped(sql, dialect, stub_prefix, &HashMap::new())?; + let edits: Vec = placeholder_default_edits(&native.sql, placeholders) + .into_iter() + .map(|(start, end, default)| { + let anchor: usize = native.offsets[native.sql[..start].chars().count()]; + (start, end, MappedSql::generated(default.to_owned(), anchor)) + }) + .collect(); + let filled: MappedSql = native.edits(edits); + Ok((filled.sql, filled.offsets)) +} + +fn native_mapped( + sql: &str, + dialect: &str, + stub_prefix: &str, + relations: &HashMap, +) -> Result { let none: HashMap = HashMap::new(); let udf_prefix: String = format!("{stub_prefix}udf_"); - let normalized: String = normalize(sql, dialect, (relations, &none), &udf_prefix)?.sql; + let normalized: MappedSql = normalize(sql, dialect, (relations, &none), &udf_prefix)?; let stubs: &Regex = NATIVE_STUBS.as_ref()?; let non_code: Vec<(usize, usize)> = - dialect_non_code_ranges(&normalized, &dialect_lexical_syntax(dialect)); - let mut native: String = String::with_capacity(normalized.len()); - let mut copied_until: usize = 0; - for captures in stubs.captures_iter(&normalized) { + dialect_non_code_ranges(&normalized.sql, &dialect_lexical_syntax(dialect)); + let mut edits: Vec = Vec::new(); + for captures in stubs.captures_iter(&normalized.sql) { let whole = captures.get(0).ok_or("missing stub match")?; let quoted: bool = non_code .iter() .any(|&(start, end)| start <= whole.start() && whole.start() < end); let glued: bool = captures.get(1).is_some() - && normalized[..whole.start()] + && normalized.sql[..whole.start()] .chars() .next_back() .is_some_and(|character| character.is_alphanumeric() || character == '_'); if quoted || glued { continue; } - native.push_str(&normalized[copied_until..whole.start()]); - match (captures.get(1), captures.get(2)) { - (Some(bound), _) => { - native.push_str(&format!("{stub_prefix}cursor_{}()", bound.as_str())); - } - (None, Some(parameter)) => { - native.push_str(&format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str())); - } - (None, None) => native.push_str(whole.as_str()), - } - copied_until = whole.end(); + let text: String = match (captures.get(1), captures.get(2)) { + (Some(bound), _) => format!("{stub_prefix}cursor_{}()", bound.as_str()), + (None, Some(parameter)) => format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str()), + (None, None) => continue, + }; + let anchor: usize = normalized.offsets[normalized.sql[..whole.start()].chars().count()]; + edits.push(( + whole.start(), + whole.end(), + MappedSql::generated(text, anchor), + )); } - native.push_str(&normalized[copied_until..]); - Ok(native) + Ok(normalized.edits(edits)) } pub(crate) fn normalize_dialect_sql_mapped( diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index 6ffe8340db..fb9ca0d11e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -9,6 +9,7 @@ pub mod normalize_batch; pub mod normalize_dialect; pub mod normalize_mapped; pub mod normalize_native; +pub mod normalize_native_filled; pub mod normalize_native_renamed; pub mod sqlbuild_code; pub mod text_findings; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_native_filled.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_native_filled.rs new file mode 100644 index 0000000000..b48b3f97e3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_native_filled.rs @@ -0,0 +1,14 @@ +/// Native-normalized analysis SQL with placeholder defaults filled and each character's source. +pub fn normalize_native_filled( + sql: &str, + dialect: &str, + stub_prefix: &str, + placeholders: &[(String, String)], +) -> Result<(String, Vec), String> { + crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped( + sql, + dialect, + stub_prefix, + placeholders, + ) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs index a697f16dd4..ec70d9fdc7 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -5,6 +5,7 @@ use std::collections::BTreeMap; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use crate::snapshots::_helpers::content::{json_bytes, prepare_node}; +use crate::snapshots::constants::SNAPSHOT_DOCUMENT_KINDS; use crate::snapshots::models::{PreparedNode, PreparedSnapshot, SnapshotInput}; use crate::store::errors::ControlError; use crate::store::main::compress::compress; @@ -48,22 +49,13 @@ pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result Result)>, ControlError> { let mut documents: BTreeMap)> = BTreeMap::new(); for (kind, value) in &input.documents { - if ![ - "jobs", - "groups", - "triggers", - "roles", - "lineage", - "columns", - "providers", - ] - .contains(&kind.as_str()) - { + if !SNAPSHOT_DOCUMENT_KINDS.contains(&kind.as_str()) { return Err(ControlError::Invalid(format!( "unknown snapshot document {kind}" ))); diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs index a995563fc0..f931b29c58 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/constants.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -4,6 +4,19 @@ pub(crate) const DEFAULT_SNAPSHOT_RETENTION_COUNT: u16 = 10; pub(crate) const SNAPSHOT_RETENTION_BATCH: usize = 10; pub(crate) const CONTENT_HASH_CHUNK: usize = 500; +/// `{node_key: {column: {"expression_range": [start, end]}}}`; readers skip unknown column keys. +pub(crate) const COLUMNS_DOCUMENT: &str = "columns"; +/// Snapshot document kinds: per-snapshot metadata that never changes a node hash or node change. +pub(crate) const SNAPSHOT_DOCUMENT_KINDS: [&str; 7] = [ + "jobs", + "groups", + "triggers", + "roles", + "lineage", + COLUMNS_DOCUMENT, + "providers", +]; + /// Beyond the newest generations, any retained reference keeps a snapshot. pub(crate) const EXPIRED_SNAPSHOTS: &str = "SELECT s.snapshot_id FROM sqb_snapshots s \ WHERE (SELECT COUNT(*) FROM sqb_snapshots newer WHERE newer.location_id=s.location_id \ diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs index db85b62338..b73be257f8 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs @@ -1,13 +1,17 @@ //! Live SQL permission, current publication and sensitive-read audit share one writer fence. -use serde_json::json; +use std::collections::BTreeMap; + +use serde_json::{Value, json}; use uuid::Uuid; use crate::authorization::main::begin_authorized::begin_authorized; use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::snapshots::_helpers::reading::attach_sources; +use crate::snapshots::constants::COLUMNS_DOCUMENT; use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::NodeSql; use crate::store::errors::ControlError; @@ -40,11 +44,25 @@ pub async fn read_person_node_sql( WHERE s.snapshot_id=$1 AND s.node_key=$2 AND s.kind IN ('raw_sql','compiled_sql')", vec![Parameter::Id(record.snapshot_id), Parameter::Text(node_key.into())]).await?; if let Some(node) = attach_sources(nodes, rows)?.remove(node_key) { + let documents; + (transaction, documents) = read_snapshot_documents( + transaction, + record.snapshot_id, + Some(&[COLUMNS_DOCUMENT]), + ) + .await?; + let expression_ranges = expression_ranges( + documents + .get(COLUMNS_DOCUMENT) + .and_then(|columns| columns.get(node_key)), + node.compiled_sql.as_deref().unwrap_or_default(), + ); result = Some(NodeSql { snapshot_id: record.snapshot_id, node_key: node_key.into(), raw_sql: node.raw_sql, compiled_sql: node.compiled_sql, + expression_ranges, }); } } @@ -59,3 +77,35 @@ pub async fn read_person_node_sql( commit(transaction).await?; Ok(result) } + +/// Each column's well-formed expression range from the node's `columns` document entry. +fn expression_ranges(columns: Option<&Value>, compiled_sql: &str) -> BTreeMap { + let mut ranges: BTreeMap = BTreeMap::new(); + let Some(columns) = columns.and_then(Value::as_object) else { + return ranges; + }; + for (column, metadata) in columns { + if let Some(range) = column_range(metadata, compiled_sql) { + ranges.insert(column.clone(), range); + } + } + ranges +} + +/// A column's `[start, end)` compiled-SQL byte range, or `None` when missing or malformed. +fn column_range(metadata: &Value, compiled_sql: &str) -> Option<[usize; 2]> { + let Some([start, end]) = metadata + .get("expression_range") + .and_then(Value::as_array) + .map(Vec::as_slice) + else { + return None; + }; + let (start, end): (u64, u64) = (start.as_u64()?, end.as_u64()?); + let length: u64 = u64::try_from(compiled_sql.len()).unwrap_or(u64::MAX); + if start >= end || end > length { + return None; + } + let range: [usize; 2] = [start as usize, end as usize]; + compiled_sql.get(range[0]..range[1]).map(|_| range) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 77c3558329..3e035cc258 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -24,6 +24,9 @@ pub struct NodeSql { pub node_key: String, pub raw_sql: Option, pub compiled_sql: Option, + /// Each placed column's defining expression as a `[start, end)` byte range of `compiled_sql`. + #[serde(default)] + pub expression_ranges: BTreeMap, } /// A location's successful compile for one target. diff --git a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs index 9c08fe2846..d34c00e80e 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs @@ -35,7 +35,7 @@ pub(crate) async fn node_sql_boundaries( for (key, expected) in [ ( "udf:orders", - json!({"snapshot_id":snapshot_id,"node_key":"udf:orders","raw_sql":null,"compiled_sql":data["function_sql"]}), + json!({"snapshot_id":snapshot_id,"node_key":"udf:orders","raw_sql":null,"compiled_sql":data["function_sql"],"expression_ranges":{}}), ), ("missing-orders", Value::Null), ("orders+", Value::Null), @@ -61,7 +61,7 @@ pub(crate) async fn node_sql_boundaries( assert_eq!(read.status(), StatusCode::OK); assert_eq!( read.json::().await.expect("current SQL"), - json!({"snapshot_id":snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["compiled_sql"]}) + json!({"snapshot_id":snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["compiled_sql"],"expression_ranges":data["expression_ranges"]}) ); let mut after = logical_state(store).await; assert_ne!( @@ -125,6 +125,6 @@ pub(crate) async fn node_sql_boundaries( assert_eq!(response.status(), StatusCode::OK); assert_eq!( response.json::().await.expect("latest SQL"), - json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"]}) + json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"],"expression_ranges":data["updated_expression_ranges"]}) ); } diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index f5558bc4cb..7c90f2ffbf 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -121,6 +121,10 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ .expect("Python source") .into(), ); + input.documents.insert( + "columns".into(), + json!({"orders": sql_input["columns_document"].clone()}), + ); input.nodes.insert( "udf:orders".into(), SnapshotNode { diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs index 406a79fc5e..778cfb2028 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; +use crate::snapshots::_helpers::preparation::prepare_snapshot; use crate::snapshots::main::current_snapshot::current_snapshot; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::{SnapshotInput, SnapshotNode}; @@ -113,7 +114,30 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { Err(ControlError::StaleGeneration) )); + // Column metadata lives outside node identity: new expression ranges alone keep every node + // hash and record no node change. + let before = input.clone(); input.generation = 3; + input.documents.insert( + "columns".into(), + json!({"orders": {"customer_id": {"expression_range": [7, 18]}}}), + ); + assert_eq!( + prepare_snapshot(&input).expect("prepare ranged").hashes, + prepare_snapshot(&before).expect("prepare unranged").hashes + ); + let ranged = publish_snapshot(store, &input) + .await + .expect("publish expression ranges only"); + assert_ne!(ranged.manifest_digest, unchanged.manifest_digest); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read ranged snapshot") + .expect("ranged snapshot exists"); + assert!(loaded.changes.is_empty()); + assert_eq!(loaded.documents, input.documents); + + input.generation = 4; input .nodes .get_mut("orders") @@ -122,7 +146,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { let changed = publish_snapshot(store, &input) .await .expect("publish changed SQL"); - assert_ne!(changed.manifest_digest, unchanged.manifest_digest); + assert_ne!(changed.manifest_digest, ranged.manifest_digest); let loaded = current_snapshot(store, location, "development") .await .expect("read changed snapshot") @@ -146,7 +170,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { .expect("development target exists") .record .generation, - 3 + 4 ); assert_eq!( current_snapshot(store, location, "testing") @@ -159,7 +183,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { ); input.target = "development".into(); - input.generation = 4; + input.generation = 5; input.nodes.remove("orders"); publish_snapshot(store, &input) .await @@ -175,8 +199,8 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { // The newer compiler wins regardless of which publisher takes the transaction lock first. let mut older = input.clone(); - older.generation = 5; - input.generation = 6; + older.generation = 6; + input.generation = 7; input .nodes .get_mut("customers") @@ -192,7 +216,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { .await .expect("read winning snapshot") .expect("winning snapshot exists"); - assert_eq!(loaded.record.generation, 6); + assert_eq!(loaded.record.generation, 7); assert_eq!(loaded.nodes, input.nodes); close_store(second).await; loaded.record.generation diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs index 708a6aff32..b9e990c5e0 100644 --- a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -10,12 +10,12 @@ fn given_control_store_when_snapshots_publish_then_generations_sources_and_chang SnapshotTestCase { description: "SQLite snapshot publication", backend: Backend::Sqlite, - expected_generation: 6, + expected_generation: 7, }, SnapshotTestCase { description: "PostgreSQL snapshot publication", backend: Backend::Postgres, - expected_generation: 6, + expected_generation: 7, }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-project/src/tests/helpers.rs b/crates/sqlbuild-project/src/tests/helpers.rs index dfb8ee6c7e..efb87f1b6d 100644 --- a/crates/sqlbuild-project/src/tests/helpers.rs +++ b/crates/sqlbuild-project/src/tests/helpers.rs @@ -22,6 +22,7 @@ pub(super) fn lineage_row(output: &str, sources: &[&str]) -> LineageRow { ) }) .collect(), + expression_range: None, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index d75b591a5d..cbe9af7460 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{Python, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::sql_macros::models::TextSplice; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; @@ -99,12 +101,35 @@ pub(crate) fn analysis_cursor_sql( sql: &str, cursor_type: &Bound<'_, PyAny>, ) -> PyResult { - let literal: &str = if text_value(cursor_type).as_deref() == Some(INTEGER_CURSOR_TYPE) { + let literal: &str = analysis_bound(text_value(cursor_type).as_deref()); + Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) +} + +/// `analysis_cursor_sql` with each replaced call as a splice of `sql`. +pub(crate) fn analysis_cursor_splices( + sql: &str, + cursor_type: Option<&str>, +) -> Result<(String, Vec), String> { + let literal: &str = analysis_bound(cursor_type); + let (replaced, spans) = + replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned())?; + let splices: Vec = spans + .iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + Ok((replaced, splices)) +} + +/// The literal static analysis reads for every cursor intrinsic of a model with `cursor_type`. +fn analysis_bound(cursor_type: Option<&str>) -> &'static str { + if cursor_type == Some(INTEGER_CURSOR_TYPE) { INTEGER_ANALYSIS_BOUND } else { TIMESTAMP_ANALYSIS_BOUND - }; - Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) + } } /// One model query with canonical intrinsic calls, raising `CompileInputError` when invalid. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs index 13fa7aca8c..1ee1ad8cc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -5,23 +5,32 @@ use std::collections::BTreeMap; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; use pyo3::types::PyTupleMethods; -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; +use sqlbuild_analysis::analysis_route::main::query_expression_ranges::query_expression_ranges; +use sqlbuild_analysis::assembly::analysis_session::models::{LineageFacts, LineageRow}; use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; +use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_orchestration::snapshots::models::SnapshotNode; +use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_splices; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::snapshot::{ dependency_keys, destination, insert_node, }; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +/// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. pub(crate) fn model_nodes( py: Python<'_>, project: &NativeProjectView, owner: &NativeCompiledProject, mut nodes: BTreeMap, -) -> PyResult> { +) -> PyResult<(BTreeMap, Map)> { + let dialect: Option = project.sql_analysis_dialect(); + let mut columns_document: Map = Map::new(); for model in project.models(py).bind(py).iter() { let model = model.cast::()?.get(); let (name, _, _) = model.file_texts(py)?; @@ -32,7 +41,7 @@ pub(crate) fn model_nodes( "model config cannot be represented in snapshot metadata", )); } - let mut definition = model.read(py, |input, _| { + let (mut definition, ranges) = model.read(py, |input, _| { let record = input .compiled .as_ref() @@ -47,15 +56,24 @@ pub(crate) fn model_nodes( json!({"name": name, "type": data_type, "nullability": nullability}) }) .collect(); - Ok(json!({ + let definition: Value = json!({ "name": facts.name, "path": facts.relative_path, "config": facts.config.values, "materialization": facts.config.materialized, "deps": dependency_keys(&record.deps), "columns": columns, - })) + }); + let lineage: Option<&LineageFacts> = record.analysis.lineage.as_ref(); + let ranges: Map = match (dialect.as_deref(), lineage) { + (Some(dialect), Some(lineage)) => expression_ranges(dialect, input, facts, lineage), + _ => Map::new(), + }; + Ok((definition, ranges)) })?; + if !ranges.is_empty() { + columns_document.insert(format!("model:{}", facts.name), Value::Object(ranges)); + } definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?; nodes = insert_node( nodes, @@ -69,5 +87,69 @@ pub(crate) fn model_nodes( }, )?; } - Ok(nodes) + Ok((nodes, columns_document)) +} + +/// Each column's `{"expression_range": [start, end]}`, when the analysis SQL rebuilds exactly. +fn expression_ranges( + dialect: &str, + input: &ModelInputRecord, + facts: &CompiledModelFacts, + lineage: &LineageFacts, +) -> Map { + let mut ranges: Map = Map::new(); + let (LineageFacts::Native(rows) | LineageFacts::NativeFacts(rows)) = lineage; + if *input.query_sql != *facts.query_sql || rows.iter().all(|row| row.expression_range.is_none()) + { + return ranges; + } + let Some(placeholders) = placeholder_texts(facts.config.values.get("placeholders")) else { + return ranges; + }; + let (typed, typed_splices) = typed_analysis_sql(&input.query_sql, &input.typed_calls, dialect); + let cursor_type: Option<&str> = facts + .config + .values + .get("cursor_type") + .and_then(Value::as_str); + let Ok((analysis_sql, cursor_splices)) = analysis_cursor_splices(&typed, cursor_type) else { + return ranges; + }; + let cleaned: Vec> = + rows.iter().map(|row| row.expression_range).collect(); + let mapped = query_expression_ranges( + dialect, + (&analysis_sql, &placeholders), + &[typed_splices, cursor_splices], + &cleaned, + ); + for (row, range) in rows.iter().zip(mapped) { + let LineageRow { output_column, .. } = row; + if let (Some((start, end)), false) = (range, output_column.is_empty()) { + ranges + .entry(output_column.clone()) + .or_insert_with(|| json!({"expression_range": [start, end]})); + } + } + ranges +} + +/// The `placeholders` config as analysis read it, as Python's `str` renders each value, if exact. +fn placeholder_texts(value: Option<&Value>) -> Option> { + let Some(Value::Object(values)) = value else { + return Some(Vec::new()); + }; + values + .iter() + .map(|(name, value)| { + let text: String = match value { + Value::String(text) => text.clone(), + Value::Number(number) if number.is_i64() || number.is_u64() => number.to_string(), + Value::Bool(true) => "True".to_owned(), + Value::Bool(false) => "False".to_owned(), + _ => return None, + }; + Some((name.clone(), text)) + }) + .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 89ce2a430d..bf15b867e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -98,7 +98,13 @@ pub(crate) fn orchestration_snapshot<'py>( input.sqb_version = env!("CARGO_PKG_VERSION").into(); input.environment_hash = environment_hash(py, &project_dir)?; let owner = project.owner().bind(py).borrow(); - input.nodes = model_nodes(py, &project, &owner, input.nodes)?; + let (nodes, columns) = model_nodes(py, &project, &owner, input.nodes)?; + input.nodes = nodes; + if !columns.is_empty() { + input + .documents + .insert("columns".into(), serde_json::Value::Object(columns)); + } input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); let declarations = match declarations { diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index 967e7982a8..a5bdd448d6 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -10,6 +10,7 @@ pub mod model_header_parsing; pub mod model_header_single_parsing; pub mod model_header_tokenizing; pub mod model_output_columns; +pub mod placeholder_default_edits; pub mod placeholder_defaults; pub mod sql_interpolation; pub mod statement_header_matching; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_default_edits.rs b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_default_edits.rs new file mode 100644 index 0000000000..0334ace5a5 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_default_edits.rs @@ -0,0 +1,40 @@ +//! Where Python's `substitute_placeholder_defaults` substitutes. + +use sqlbuild_core::text::main::is_word_char::is_word_char; + +use crate::constants::PLACEHOLDER_PREFIX; + +/// The byte range of each `@@@name` with a default and that default, in order. +pub fn placeholder_default_edits<'a>( + sql: &str, + placeholders: &'a [(String, String)], +) -> Vec<(usize, usize, &'a str)> { + let mut edits: Vec<(usize, usize, &'a str)> = Vec::new(); + let mut position: usize = 0; + while let Some(found) = sql[position..].find(PLACEHOLDER_PREFIX) { + let start: usize = position + found; + let after: &str = &sql[start + PLACEHOLDER_PREFIX.len()..]; + let length: usize = after + .find(|character: char| !is_word(character)) + .unwrap_or(after.len()); + if length == 0 { + position = start + 1; + continue; + } + let end: usize = start + PLACEHOLDER_PREFIX.len() + length; + let name: &str = &after[..length]; + if let Some((_, default)) = placeholders + .iter() + .find(|(placeholder, _)| placeholder == name) + { + edits.push((start, end, default.as_str())); + } + position = end; + } + edits +} + +/// A character Python's `\w` matches. +fn is_word(character: char) -> bool { + is_word_char(character) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs index 7772bd42ea..f03b667704 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/placeholder_defaults.rs @@ -1,42 +1,16 @@ //! Python's `substitute_placeholder_defaults`. -use sqlbuild_core::text::main::is_word_char::is_word_char; - -use crate::constants::PLACEHOLDER_PREFIX; +use crate::compiler::main::placeholder_default_edits::placeholder_default_edits; /// `substitute_placeholder_defaults`: `@@@name` becomes its default; unknown names stay. pub fn placeholder_defaults(sql: &str, placeholders: &[(String, String)]) -> String { let mut result: String = String::with_capacity(sql.len()); - let mut rest: &str = sql; - while let Some(start) = rest.find(PLACEHOLDER_PREFIX) { - result.push_str(&rest[..start]); - let after: &str = &rest[start + PLACEHOLDER_PREFIX.len()..]; - let length: usize = after - .find(|character: char| !is_word(character)) - .unwrap_or(after.len()); - if length == 0 { - result.push_str(&rest[start..=start]); - rest = &rest[start + 1..]; - continue; - } - let name: &str = &after[..length]; - match placeholders - .iter() - .find(|(placeholder, _)| placeholder == name) - { - Some((_, default)) => result.push_str(default), - None => { - result.push_str(PLACEHOLDER_PREFIX); - result.push_str(name); - } - } - rest = &after[length..]; + let mut copied: usize = 0; + for (start, end, default) in placeholder_default_edits(sql, placeholders) { + result.push_str(&sql[copied..start]); + result.push_str(default); + copied = end; } - result.push_str(rest); + result.push_str(&sql[copied..]); result } - -/// A character Python's `\w` matches. -fn is_word(character: char) -> bool { - is_word_char(character) -} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 12c96c457a..a11bcbf230 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule, Job, @@ -233,6 +234,17 @@ class SnapshotEnvironmentTestCase: expected_environment_count: int +@dataclass(frozen=True) +class SnapshotExpressionRangeTestCase: + """A published model whose compiled SQL differs from its source, and each column's text.""" + + description: str + adapter: type[BaseAdapter] + files: dict[str, str] + model_name: str + expected_texts: dict[str, str] + + @dataclass(frozen=True) class SnapshotLineageTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 841241dd2a..22a57cf538 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -32,6 +32,7 @@ assemble_project, build_compile_inputs, build_compiled_project, + build_orchestration_snapshot, discover_project_inputs, extract_sql_references, ) @@ -47,6 +48,9 @@ if TYPE_CHECKING: from sqlbuild._native import CompileInputsView + from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + SnapshotExpressionRangeTestCase, + ) METADATA: dict[str, Any] = { "location_id": str(UUID(int=1)), @@ -695,3 +699,32 @@ def _located_binding(diagnostic: CompilerDiagnostic) -> tuple[object, ...]: diagnostic.column, getattr(diagnostic.location, "end_column", None), ) + + +def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) -> dict[str, str]: + """Publish the case's project and return each ranged column's compiled-SQL text.""" + write_files(project_dir=tmp_path, files=test_case.files) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=test_case.adapter(), + resolved_connection={"database": "WAREHOUSE", "schema": "MAIN"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics), [ + item.message for item in project.diagnostics + ] + publication: dict[str, Any] = json.loads( + build_orchestration_snapshot( + project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered + ).to_json() + ) + key: str = f"model:{test_case.model_name}" + node: dict[str, Any] = publication["nodes"][key] + assert node["compiled_sql"] != node["raw_sql"] + assert "expression_range" not in json.dumps(node), "ranges stay out of node identity" + compiled: bytes = node["compiled_sql"].encode() + columns: dict[str, Any] = publication["documents"].get("columns", {}).get(key, {}) + return { + column: compiled[slice(*metadata["expression_range"])].decode() + for column, metadata in columns.items() + } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index f6b5ebe58d..ea206b7b33 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -229,7 +229,7 @@ def verify_sql_inspection( "applied": True } sql: dict[str, Any] = person_read(api, path, session=session) - assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql", "expression_ranges"} assert sql["snapshot_id"] == snapshot_id assert sql["node_key"] == "model:orders" assert sql["raw_sql"] == model.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py index d73478e099..e6e33f0a39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -177,7 +177,16 @@ def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authori inspected: subprocess.CompletedProcess[str] = invoke(command) assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr sql: dict[str, Any] = json.loads(inspected.stdout) - assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert ( + set(sql), + { + column: sql["compiled_sql"].encode()[start:end].decode() + for column, (start, end) in sql["expression_ranges"].items() + }, + ) == ( + {"snapshot_id", "node_key", "raw_sql", "compiled_sql", "expression_ranges"}, + {"amount": "23 AS amount", "description": "'" + "orders " * 400 + "' AS description"}, + ) assert ( sql["snapshot_id"] == current_snapshot(worker.api, "development")["record"]["snapshot_id"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index c6e520bb1f..33ad1bfd57 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -20,6 +20,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.orchestration import ( RunEventKind, @@ -37,6 +38,7 @@ SnapshotDiscoveryRejectionTestCase, SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, + SnapshotExpressionRangeTestCase, SnapshotIdentityTestCase, SnapshotLineageTestCase, SnapshotRejectionTestCase, @@ -47,6 +49,7 @@ METADATA, PARITY_FILES, TYPE_PROOF_FILES, + published_texts, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -57,6 +60,19 @@ "a calendar schedule from the function it decorates)" ) +_DUCKSHOP: Path = Path("benchmarks/duckshop/project") +_STRICT_SPANS: str = ( + "SQLQuack's strict parsers do not yet record projection-item spans: Snowflake gives aliased " + "items none, and DuckDB's column span leaves out the qualifier" +) +_PRICE_MODELS: dict[str, str] = { + "models/stg_prices.sql": ( + 'MODEL (description "Staged prices");\n' + "SELECT 'W-1' AS window_id, CAST(12345 AS BIGINT) AS price_minor, " + "CAST('2026-03-30 08:00:00' AS TIMESTAMP) AS loaded_at\n" + ), +} + @pytest.mark.parametrize( "test_case", @@ -399,6 +415,168 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s assert sources | targets <= set(publication["nodes"]) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotExpressionRangeTestCase( + description="SQL and Python macros, a variable and cursor bounds rewrite the SQL", + adapter=DuckDbAdapter, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nreporting_currency = "USD"\n' + ), + "models/_macros/sql/minor_to_major.sql": ( + "MACRO (\n" + ' description "Minor units to major units",\n' + " arguments (amount BIGINT, places INT CONST DEFAULT 2),\n" + " returns DECIMAL(18, 2),\n" + ");\n\n" + "ROUND(@amount / 100, @places)::DECIMAL(18, 2)\n" + ), + "models/_macros/python/rounding.py": ( + "def round_half_even(ctx, expression: str, places: int = 2) -> str:\n" + " return f'ROUND_EVEN({expression}, {places})'\n" + ), + **_PRICE_MODELS, + "models/price_spans.sql": ( + "MODEL (\n" + ' description "Prices with rewritten expressions",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor loaded_at,\n" + " cursor_type timestamp,\n" + " cursor_grain second,\n" + ");\n\n" + "-- Prix à jour €\n" + "SELECT\n" + " p.window_id AS window_id,\n" + " @sql.minor_to_major(p.price_minor) AS price_amount,\n" + ' @py.round_half_even("p.price_minor / 100", 2) AS rounded_price,\n' + " '@@reporting_currency' AS currency,\n" + " 'Prix à jour €' AS label,\n" + " p.loaded_at AS loaded_at\n" + 'FROM __ref("stg_prices") AS p\n' + "WHERE p.loaded_at >= __cursor_start() AND p.loaded_at < __cursor_end()\n" + ), + }, + model_name="price_spans", + expected_texts={ + "window_id": "p.window_id AS window_id", + "price_amount": ("ROUND(p.price_minor / 100, 2)::DECIMAL(18, 2) AS price_amount"), + "rounded_price": "ROUND_EVEN(p.price_minor / 100, 2) AS rounded_price", + "currency": "'USD' AS currency", + "label": "'Prix à jour €' AS label", + "loaded_at": "p.loaded_at AS loaded_at", + }, + ), + SnapshotExpressionRangeTestCase( + description="a column qualified by its reference call has no range rather than a moved one", + adapter=DuckDbAdapter, + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + **_PRICE_MODELS, + "models/price_refs.sql": ( + 'MODEL (description "Price references");\n' + 'SELECT __ref("stg_prices").window_id AS window_id, ' + "'EUR' AS currency FROM __ref(\"stg_prices\")\n" + ), + }, + model_name="price_refs", + expected_texts={"currency": "'EUR' AS currency"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_rewritten_model_sql_when_publishing_then_expression_ranges_slice_the_compiled_sql( + test_case: SnapshotExpressionRangeTestCase, + tmp_path: Path, +) -> None: + assert published_texts(test_case, tmp_path) == test_case.expected_texts, test_case.description + + +@pytest.mark.xfail(strict=True, raises=AssertionError, reason=_STRICT_SPANS) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotExpressionRangeTestCase( + description="an unaliased qualified column covers its qualifier", + adapter=DuckDbAdapter, + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + **_PRICE_MODELS, + "models/price_columns.sql": ( + 'MODEL (description "Price columns");\n' + 'SELECT p.window_id, p.price_minor FROM __ref("stg_prices") AS p\n' + ), + }, + model_name="price_columns", + expected_texts={ + "window_id": "p.window_id", + "price_minor": "p.price_minor", + }, + ), + SnapshotExpressionRangeTestCase( + description="DuckShop macros, variable and cursor bounds rewrite the SQL", + adapter=SnowflakeAdapter, + files={ + "sqlbuild_project.toml": ( + 'name = "duckshop"\nadapter = "snowflake"\n\n' + '[defaults]\ndatabase = "WAREHOUSE"\nschema = "MAIN"\n\n' + '[vars]\nreporting_currency = "USD"\n' + ), + "sources/edge_cases.yml": ( + "sources:\n" + " - name: edge__price_windows\n" + " description: Price windows\n" + " columns:\n" + " - name: window_id\n" + " type: VARCHAR\n" + " - name: price_minor\n" + " type: NUMBER(38,0)\n" + " - name: _loaded_at\n" + " type: TIMESTAMP_NTZ\n" + ), + # A second consumer keeps the DuckShop macros project-wide, as they are there. + "models/price_checks.sql": ( + 'MODEL (description "Price checks");\n' + "SELECT @sql.money_from_minor(150, 2) AS amount, " + '@py.round_half_even("1.25", 1) AS rounded\n' + ), + **{ + path: (_DUCKSHOP / path).read_text() + for path in ( + "macros/sql/money_from_minor.sql", + "macros/sql/safe_divide.sql", + "macros/python/rounding.py", + "models/intermediate/edge_cases/int_edge_expression_spans.sql", + ) + }, + }, + model_name="int_edge_expression_spans", + expected_texts={ + "window_id": "w.window_id", + "price_amount": ( + "ROUND((\n w.price_minor / NULLIF((POWER(10, 2)::NUMBER(38, 0)), 0)\n)" + "::NUMBER(38, 6), 2)::NUMBER(38, 6) AS price_amount" + ), + "rounded_price": ( + "ROUND(w.price_minor / 100, 2, 'HALF_TO_EVEN')::NUMBER(18, 2) AS rounded_price" + ), + "currency": "'USD'::VARCHAR AS currency", + "label": "'Prix à jour €'::VARCHAR AS label", + "loaded_at": "w._loaded_at AS loaded_at", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_spans_strict_parsers_omit_when_publishing_then_expression_ranges_slice_the_compiled_sql( + test_case: SnapshotExpressionRangeTestCase, + tmp_path: Path, +) -> None: + assert published_texts(test_case, tmp_path) == test_case.expected_texts, test_case.description + + @pytest.mark.parametrize( "test_case", [