feat: return column expression ranges with the node SQL read

benchmarks/duckshop/README.md

@@ -124,8 +124,8 @@ project and does not change it:
   `edge_running_peaks`, that a pre-hook of `int_edge_basket_running_peaks` creates
   (`models/intermediate/edge_cases/_sqlbuild/_hooks/sql/`) and SQLBuild does not declare.
 
-It has 164 models (13 staging, 137 intermediate, 14 marts), 5 Python functions, a SQL table
-function, a hook-created JavaScript function, 189 unit tests and 4 scenarios, in nine groups:
+It has 166 models (13 staging, 139 intermediate, 14 marts), 5 Python functions, a SQL table
+function, a hook-created JavaScript function, 191 unit tests and 4 scenarios, in nine groups:
 
 | Group | Covers |
 |-------|--------|
@@ -134,7 +134,7 @@ function, a hook-created JavaScript function, 189 unit tests and 4 scenarios, in
 | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` |
 | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day |
 | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders |
-| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query |
+| Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped |
 | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results |
 | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild |
 | Translation refusals | one model per construct that translation tools refuse or guard, most putting a construct refused only under strict checking before one refused by default, so a model shows both: `FLOAT` values through sums, averages, running totals, arrays, objects, `HASH`, `TO_CHAR` and `::NUMBER(38, 10)`; `VARIANT` dates, decimals and XML; `ARRAY_AGG` without `WITHIN GROUP`, sparse arrays and `ARRAY(NUMBER)` casts; `REDUCE`, `FILTER` and `TRANSFORM` lambdas; patterns and `REGEXP_REPLACE` replacements built from columns, and lazy patterns; `TRY_TO_DATE` with a format from a column; zoned literals, `CONVERT_TIMEZONE` from `CURRENT_TIMESTAMP`, `SEQ8`, `GENERATOR` with an expression `ROWCOUNT`; `PIVOT`, `ANY_VALUE`, `IGNORE NULLS` around `IFF`, unaliased columns read by Snowflake's name, `UNION` branches whose column is always NULL; `TO_NUMBER`, `TO_DECIMAL` and `TRY_BASE64_DECODE_BINARY`; an open (uncontracted) source `edge__supplier_notes`, the table function, read for one basket and after `LATERAL` for every basket, and the hook-created function. `rpt_edge_case_model_keys` and `rpt_edge_case_array_model_keys` read every model in the group, so a refusal there also refuses a downstream model |

benchmarks/duckshop/edge_cases/orchestration/node_sql.json

@@ -3,5 +3,14 @@
   "compiled_sql": "SELECT 23 AS amount -- Orders 🧾\n",
   "python_source": "def order_amount():\n    return 23\n",
   "function_sql": "43",
-  "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n"
+  "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n",
+  "columns_document": {
+    "amount": {"expression_range": [7, 19], "described_as": "a later per-column key readers ignore"},
+    "past_the_end": {"expression_range": [7, 4096]},
+    "split_character": {"expression_range": [7, 32]},
+    "reversed": {"expression_range": [19, 7]},
+    "no_range": {}
+  },
+  "expression_ranges": {"amount": [7, 19]},
+  "updated_expression_ranges": {"amount": [7, 19], "split_character": [7, 32]}
 }

benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_expression_spans.sql

@@ -0,0 +1,37 @@
+MODEL (
+  description """
+    One row per edge-case price window loaded in the cursor window, with its price converted by a
+    typed SQL macro and rounded by a Python macro, the reporting currency from a variable and a
+    non-ASCII label, so the compiled SQL differs from the authored text before every output column;
+    the acceptance case for column expression ranges that point into the compiled SQL.
+  """,
+  tags [edge_case],
+  materialized incremental,
+  incremental_strategy delete_insert,
+  cursor loaded_at,
+  cursor_type timestamp,
+  cursor_grain second,
+  cursor_inputs (
+    edge__price_windows _loaded_at,
+  ),
+  columns (
+    window_id (type VARCHAR, nullable false, audits [not_null, unique]),
+    price_amount (type NUMBER(38, 6)),
+    rounded_price (type NUMBER(18, 2)),
+    currency (type VARCHAR),
+    label (type VARCHAR),
+    loaded_at (type TIMESTAMP_NTZ, nullable false),
+  ),
+);
+
+SELECT
+  w.window_id,
+  @sql.money_from_minor(w.price_minor, 2) AS price_amount,
+  @py.round_half_even("w.price_minor / 100", 2)::NUMBER(18, 2) AS rounded_price,
+  '@@reporting_currency'::VARCHAR AS currency,
+  'Prix à jour €'::VARCHAR AS label,
+  w._loaded_at AS loaded_at
+FROM __source("edge__price_windows") AS w
+WHERE
+  w._loaded_at >= __cursor_start()
+  AND w._loaded_at < __cursor_end()

benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql

@@ -0,0 +1,17 @@
+MODEL (
+  description """
+    One row per edge-case price window, each column qualified by the source reference call itself,
+    so every output column's expression starts at text the compiler rewrites; the case for column
+    expression ranges that must be dropped rather than moved onto the relation name.
+  """,
+  tags [edge_case],
+  columns (
+    window_id (type VARCHAR, nullable false, audits [not_null, unique]),
+    sku (type VARCHAR),
+  ),
+);
+
+SELECT
+  __source("edge__price_windows").window_id,
+  __source("edge__price_windows").sku
+FROM __source("edge__price_windows")

benchmarks/duckshop/project/sqlbuild_project.toml

@@ -337,6 +337,18 @@ The edge case reads the line through a derived table nested in a comma-joined LA
 that shape is the case under test.
 """
 
+[[rules.rule_ignores]]
+rules = ["SQLCLEAN034"]
+paths = [
+  "models/intermediate/edge_cases/int_edge_expression_spans.sql",
+  "models/intermediate/edge_cases/int_edge_reference_qualified_spans.sql",
+]
+reason = """
+These edge cases keep their macro calls, variable, literals and reference calls in the terminal
+select on purpose: column expression ranges point at the terminal select's items, so that is where
+the compiled SQL must differ from the authored text.
+"""
+
 [[rules.rule_ignores]]
 rules = ["SQLCLEAN040"]
 paths = ["models/intermediate/edge_cases/int_edge_amount_bit_reads.sql"]

benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_expression_spans.sql

@@ -0,0 +1,37 @@
+TEST (
+  name "int_edge_expression_spans__converts_prices_loaded_in_the_window",
+  cursor_start "2026-03-02 00:00:00",
+  cursor_end "2026-03-03 00:00:00",
+);
+
+WITH __source__edge__price_windows AS (
+  SELECT
+    v.window_id,
+    v.sku,
+    v.price_minor::NUMBER(38, 0) AS price_minor,
+    v.valid_from::TIMESTAMP_NTZ(6) AS valid_from,
+    v.valid_to::TIMESTAMP_NTZ(6) AS valid_to,
+    v._loaded_at::TIMESTAMP_NTZ AS _loaded_at
+  FROM VALUES
+    ('PW-01', 'SKU-1', 1234, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 09:00:00'),
+    ('PW-02', 'SKU-2', NULL, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 18:30:00'),
+    ('PW-03', 'SKU-3', 999, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-01 23:59:59') AS v(window_id, sku, price_minor, valid_from, valid_to, _loaded_at)
+),
+
+__expected__int_edge_expression_spans AS (
+  SELECT
+    'PW-01' AS window_id,
+    12.340000::NUMBER(38,6) AS price_amount,
+    12.34::NUMBER(18,2) AS rounded_price,
+    'USD' AS currency,
+    'Prix à jour €' AS label,
+    '2026-03-02 09:00:00'::TIMESTAMP_NTZ(9) AS loaded_at
+  UNION ALL
+  SELECT
+    'PW-02' AS window_id,
+    CAST(NULL AS NUMBER(38,6)) AS price_amount,
+    CAST(NULL AS NUMBER(18,2)) AS rounded_price,
+    'USD' AS currency,
+    'Prix à jour €' AS label,
+    '2026-03-02 18:30:00'::TIMESTAMP_NTZ(9) AS loaded_at
+)

benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_reference_qualified_spans.sql

@@ -0,0 +1,26 @@
+TEST (
+  name "int_edge_reference_qualified_spans__keeps_every_window",
+);
+
+WITH __source__edge__price_windows AS (
+  SELECT
+    v.window_id,
+    v.sku,
+    v.price_minor::NUMBER(38, 0) AS price_minor,
+    v.valid_from::TIMESTAMP_NTZ(6) AS valid_from,
+    v.valid_to::TIMESTAMP_NTZ(6) AS valid_to,
+    v._loaded_at::TIMESTAMP_NTZ AS _loaded_at
+  FROM VALUES
+    ('PW-01', 'SKU-1', 1234, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 09:00:00'),
+    ('PW-02', NULL, NULL, '2026-01-01 00:00:00', '9999-12-31 00:00:00', '2026-03-02 18:30:00') AS v(window_id, sku, price_minor, valid_from, valid_to, _loaded_at)
+),
+
+__expected__int_edge_reference_qualified_spans AS (
+  SELECT
+    'PW-01' AS window_id,
+    'SKU-1' AS sku
+  UNION ALL
+  SELECT
+    'PW-02' AS window_id,
+    CAST(NULL AS VARCHAR) AS sku
+)

clients/typescript/src/schema.d.ts

@@ -2009,6 +2009,10 @@ export interface components {
         /** @description SQL inspection never returns Python source, execution artifacts or unrelated node metadata. */
         NodeSql: {
             compiled_sql?: string | null;
+            /** @description Each placed column's defining expression as a `[start, end)` byte range of `compiled_sql`. */
+            expression_ranges?: {
+                [key: string]: number[];
+            };
             node_key: string;
             raw_sql?: string | null;
             /** Format: uuid */

crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs

@@ -151,6 +151,10 @@ fn lineage_row(
         transform_code,
         confidence_code,
         sources,
+        expression_range: output
+            .expression_range
+            .as_ref()
+            .map(|range| (range.start, range.end)),
     }
 }
 

crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs

@@ -110,7 +110,12 @@ pub(crate) fn analysed_members(
             .get(leader)
             .filter(|template| exact_for_others(template, members[*leader].model));
         match template.map(|template| mapped(template, &members[index].model.lineage_references)) {
-            Some(Ok(analysis)) => {
+            Some(Ok(mut analysis)) => {
+                rebase_ranges(
+                    &mut analysis,
+                    members[*leader].cleaned.as_deref().unwrap_or_default(),
+                    members[index].cleaned.as_deref().unwrap_or_default(),
+                );
                 analyses[index] = Some(Ok(analysis));
                 counts.shared += 1;
                 counts.unshared -= 1;
@@ -208,6 +213,67 @@ fn mapped(
     Ok(analysis)
 }
 
+/// Each expression range moved onto `member`'s relation names, or dropped where it cannot move.
+pub(crate) fn rebase_ranges(analysis: &mut NativeAnalysis, leader: &str, member: &str) {
+    if leader == member {
+        return;
+    }
+    let renamed: Option<Vec<(usize, usize, usize)>> = renamed_runs(leader, member);
+    for row in &mut analysis.lineage {
+        row.expression_range = match (&renamed, row.expression_range) {
+            (Some(runs), Some((start, end))) => rebased(runs, start).zip(rebased(runs, end)),
+            _ => None,
+        };
+    }
+}
+
+/// `(leader start, leader end, member end)` of each differing identifier run, if only those differ.
+fn renamed_runs(leader: &str, member: &str) -> Option<Vec<(usize, usize, usize)>> {
+    let identifier = |c: u8| c.is_ascii_alphanumeric() || c == b'_' || c == b'$';
+    let (left, right): (&[u8], &[u8]) = (leader.as_bytes(), member.as_bytes());
+    let (mut i, mut j): (usize, usize) = (0, 0);
+    let mut runs: Vec<(usize, usize, usize)> = Vec::new();
+    while i < left.len() || j < right.len() {
+        if i < left.len() && j < right.len() && left[i] == right[j] {
+            i += 1;
+            j += 1;
+            continue;
+        }
+        let mut back: usize = 0;
+        while back < i && identifier(left[i - back - 1]) {
+            back += 1;
+        }
+        let (mut left_end, mut right_end): (usize, usize) = (i, j);
+        while left_end < left.len() && identifier(left[left_end]) {
+            left_end += 1;
+        }
+        while right_end < right.len() && identifier(right[right_end]) {
+            right_end += 1;
+        }
+        if left_end == i && right_end == j {
+            return None;
+        }
+        runs.push((i - back, left_end, right_end));
+        (i, j) = (left_end, right_end);
+    }
+    Some(runs)
+}
+
+/// The member offset of leader `offset`, or `None` strictly inside a renamed run.
+fn rebased(runs: &[(usize, usize, usize)], offset: usize) -> Option<usize> {
+    let mut moved: usize = offset;
+    for &(left_start, left_end, right_end) in runs {
+        if offset <= left_start {
+            break;
+        }
+        if offset < left_end {
+            return None;
+        }
+        moved = offset - left_end + right_end;
+    }
+    Some(moved)
+}
+
 /// Whether `leader`'s `template` succeeded, found nothing and spells none of its relation names.
 fn exact_for_others(template: &Result<NativeAnalysis, String>, leader: &ModelRequest) -> bool {
     let Ok(analysis) = template else {

crates/sqlbuild-analysis/src/analysis_route/main/mod.rs

@@ -4,6 +4,7 @@ pub mod fast_lineage;
 pub mod native_gap;
 pub(crate) mod native_outcomes;
 pub mod native_revalidation;
+pub mod query_expression_ranges;
 pub mod schema_column;
 pub mod skipped_analysis;
 pub mod unapplied_nullability_rules;