feat(semantics): give each output column its expression's byte range

crates/sqlquack-semantics/src/duckdb/profile/tests/test_checks.rs

@@ -145,7 +145,7 @@ fn given_a_duckdb_select_when_checking_its_scope_rules_then_broken_rules_are_fou
             expected_findings: &[(
                 "E201",
                 "Star modifier references unknown column 'quantity'",
-                0,
+                60,
             )],
         },
         QueryFindingsTestCase {

crates/sqlquack-semantics/src/lineage/_helpers/checks/issues.rs

@@ -40,7 +40,7 @@ fn own_location(node: &Expression) -> Option<Span> {
     match node {
         Expression::Column(column) => column.span.or(column.name.span),
         Expression::Identifier(identifier) => identifier.span,
-        Expression::Star(star) => star.span,
+        Expression::Star(star) => star.span.0,
         Expression::Count(count) => count.span,
         Expression::Function(function) => function.span.0,
         _ => None,

crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs

@@ -150,6 +150,7 @@ pub(crate) fn renamed(
                 name: alias.name.clone(),
                 key: key(alias, context),
                 lineage: column.lineage,
+                expression: column.expression,
             },
             None => column,
         })
@@ -193,7 +194,24 @@ fn set_operation(
         SetSides::ByName => by_name(left, right, context),
         SetSides::Intersect => positional(vec![left, right], context),
     };
-    ResolvedQuery { rows, ..query }
+    ResolvedQuery {
+        rows,
+        columns: without_expressions(query.columns),
+        ..query
+    }
+}
+
+/// `columns` without the projection items that defined them, as a set operation's outputs.
+fn without_expressions(columns: Rc<Vec<ScopedColumn>>) -> Rc<Vec<ScopedColumn>> {
+    Rc::new(
+        Rc::unwrap_or_clone(columns)
+            .into_iter()
+            .map(|column| ScopedColumn {
+                expression: None,
+                ..column
+            })
+            .collect(),
+    )
 }
 
 /// EXCEPT's columns: values and rows from the left, each typed as both branches combine.
@@ -258,6 +276,7 @@ fn positional(queries: Vec<ResolvedQuery>, context: LineageContext<'_>) -> Resol
         }
         columns.push(ScopedColumn {
             lineage: traced_if(set_merged(parts, context.profile), complete),
+            expression: None,
             ..column
         });
     }
@@ -543,19 +562,28 @@ fn select_query(
         let projection = unannotated(projection);
         if let Expression::Star(star) = projection {
             let expanded = expanded_star(star, &scope, context);
+            let star_range = projection.source_span();
             query.open |= expanded.open;
             query.open_relation = query.open_relation.or(expanded.open_relation);
             query.open_reads = query.open_reads.or(expanded.open_reads);
             query.outer_nullable |= expanded.outer_nullable;
             ungrouped.extend(expanded.columns.iter().map(|column| column.lineage.clone()));
             aggregated.extend(expanded.columns.iter().map(|_| false));
-            Rc::make_mut(&mut query.columns).extend(Rc::unwrap_or_clone(expanded.columns));
+            Rc::make_mut(&mut query.columns).extend(
+                Rc::unwrap_or_clone(expanded.columns)
+                    .into_iter()
+                    .map(|column| ScopedColumn {
+                        expression: column.expression.or(star_range),
+                        ..column
+                    }),
+            );
             continue;
         }
         let column = ScopedColumn {
             name: output_name(projection, context),
             key: output_key(projection, context),
             lineage: expression_lineage(projection, &scope, context),
+            expression: projection.source_span(),
         };
         query.rows.rows_of(&column.lineage);
         if !holds_aggregate(projection) {
@@ -860,6 +888,7 @@ fn keys_in_place(
                     name: column.name.clone(),
                     key: column.key.clone(),
                     lineage: merged.lineage.clone(),
+                    expression: None,
                 });
             }
             None => columns.push(column.clone()),
@@ -899,6 +928,7 @@ fn star_column(
     {
         return ScopedColumn {
             lineage: expression_lineage(&replacement.this, scope, context),
+            expression: replacement.this.source_span(),
             ..column.clone()
         };
     }
@@ -907,8 +937,12 @@ fn star_column(
             name: new.name.clone(),
             key: key(new, context),
             lineage: column.lineage.clone(),
+            expression: None,
+        },
+        None => ScopedColumn {
+            expression: None,
+            ..column.clone()
         },
-        None => column.clone(),
     }
 }
 

crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs

@@ -216,6 +216,7 @@ fn joined(
             name: name.clone(),
             key: name,
             lineage,
+            expression: None,
         };
         match merged_keys
             .iter_mut()
@@ -588,6 +589,7 @@ fn stored_columns(
             key: context
                 .profile
                 .resolved_identifier(&column.name, column.quoted),
+            expression: None,
             lineage: Lineage {
                 data_type: stored_type(column, context),
                 nullability: column.nullability,
@@ -641,6 +643,7 @@ fn table_function_source(
                         name: (*column).to_owned(),
                         key: plain_key(column, context),
                         lineage: input.clone(),
+                        expression: None,
                     })
                     .collect(),
             ),
@@ -701,6 +704,7 @@ pub(crate) fn values_query(
                 key: plain_key(&name, context),
                 name,
                 lineage: values_column(values, position, scope, context),
+                expression: None,
             }
         })
         .collect();
@@ -790,6 +794,7 @@ pub(crate) fn pivot_sources(
         key: context.profile.resolved_identifier(&name, true),
         name,
         lineage: aggregated.clone(),
+        expression: None,
     });
     let query = ResolvedQuery {
         columns: Rc::new(match dynamic {
@@ -826,7 +831,10 @@ fn grouped_columns(
             _ => None,
         };
         match found {
-            Some(column) => columns.push(column.clone()),
+            Some(column) => columns.push(ScopedColumn {
+                expression: None,
+                ..column.clone()
+            }),
             None => known = false,
         }
     }
@@ -890,6 +898,7 @@ fn unpivot_source(
         name: name.name.clone(),
         key: key(name, context),
         lineage,
+        expression: None,
     });
     let query = ResolvedQuery {
         columns: Rc::new(passed.into_iter().chain(generated).collect()),
@@ -945,7 +954,10 @@ fn passed_columns(
         .flat_map(|source| source.query.columns.iter())
     {
         if !listed(consumed, &column.key, context) {
-            passed.push(column.clone());
+            passed.push(ScopedColumn {
+                expression: None,
+                ..column.clone()
+            });
         }
     }
     passed

crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs

@@ -1,8 +1,10 @@
 //! The column lineage of one parsed statement, within an analysis already running.
 
+use std::ops::Range;
 use std::rc::Rc;
 
 use sqlquack_syntax::Expression;
+use sqlquack_syntax::tokens::models::Span;
 
 use crate::lineage::_helpers::tracing::queries::{projects_star, resolved_query};
 use crate::lineage::models::{LineageContext, OutputLineage, QueryLineage};
@@ -13,8 +15,14 @@ pub(crate) fn analysed_lineage(
     context: LineageContext<'_>,
 ) -> QueryLineage {
     let resolved = resolved_query(statement, &[], None, context);
+    let columns = Rc::unwrap_or_clone(resolved.columns);
+    let mut ranges = byte_ranges(
+        context.sql,
+        columns.iter().map(|column| column.expression).collect(),
+    )
+    .into_iter();
     QueryLineage {
-        outputs: Rc::unwrap_or_clone(resolved.columns)
+        outputs: columns
             .into_iter()
             .map(|column| OutputLineage {
                 name: column.name,
@@ -24,6 +32,7 @@ pub(crate) fn analysed_lineage(
                 resolved: column.lineage.resolved,
                 data_type: column.lineage.data_type.as_deref().map(str::to_owned),
                 nullability: column.lineage.nullability,
+                expression_range: ranges.next().flatten(),
             })
             .collect(),
         open_star: resolved.open,
@@ -32,3 +41,55 @@ pub(crate) fn analysed_lineage(
         rows_resolved: !resolved.rows.lost,
     }
 }
+
+/// `spans`, Unicode character ranges in `sql`, as UTF-8 byte ranges, in one pass over `sql`.
+fn byte_ranges(sql: Option<&str>, spans: Vec<Option<Span>>) -> Vec<Option<Range<usize>>> {
+    let Some(sql) = sql else {
+        return vec![None; spans.len()];
+    };
+    if sql.is_ascii() {
+        return spans
+            .into_iter()
+            .map(|span| {
+                let span = span?;
+                (span.start < span.end && span.end <= sql.len()).then_some(span.start..span.end)
+            })
+            .collect();
+    }
+    let mut offsets: Vec<usize> = spans
+        .iter()
+        .flatten()
+        .flat_map(|span| [span.start, span.end])
+        .collect();
+    offsets.sort_unstable();
+    offsets.dedup();
+    let mut bytes: Vec<(usize, usize)> = Vec::with_capacity(offsets.len());
+    let mut wanted = offsets.iter().copied().peekable();
+    for (position, (byte, _)) in sql
+        .char_indices()
+        .chain(std::iter::once((sql.len(), ' ')))
+        .enumerate()
+    {
+        while wanted.next_if_eq(&position).is_some() {
+            bytes.push((position, byte));
+        }
+        if wanted.peek().is_none() {
+            break;
+        }
+    }
+    let byte = |offset: usize| -> Option<usize> {
+        let index = bytes.partition_point(|&(position, _)| position < offset);
+        bytes
+            .get(index)
+            .filter(|&&(position, _)| position == offset)
+            .map(|&(_, byte)| byte)
+    };
+    spans
+        .into_iter()
+        .map(|span| {
+            let span = span?;
+            let (start, end) = (byte(span.start)?, byte(span.end)?);
+            (start < end).then_some(start..end)
+        })
+        .collect()
+}

crates/sqlquack-semantics/src/lineage/models.rs

@@ -2,6 +2,7 @@
 
 use std::cell::RefCell;
 use std::collections::BTreeSet;
+use std::ops::Range;
 use std::rc::Rc;
 
 use rustc_hash::FxHashMap;
@@ -92,6 +93,8 @@ pub struct OutputLineage {
     pub data_type: Option<String>,
     /// Whether it may be NULL.
     pub nullability: Nullability,
+    /// The UTF-8 byte range in the analysed SQL of the projection item or star defining it.
+    pub expression_range: Option<Range<usize>>,
 }
 
 /// The lineage of every output column of one query.
@@ -146,6 +149,8 @@ pub(crate) struct ScopedColumn {
     pub(crate) name: String,
     pub(crate) key: String,
     pub(crate) lineage: Lineage,
+    /// The source range of the projection item that defines it, when it has one.
+    pub(crate) expression: Option<Span>,
 }
 
 /// A FROM item: its key, its known columns, and whether it offers others of unknown name.

crates/sqlquack-semantics/src/lineage/tests/test_types.rs

@@ -28,6 +28,16 @@ pub(crate) struct LineageDetailTestCase {
     pub(crate) expected_rows_resolved: bool,
 }
 
+/// A query and, per output, the SQL text its expression range covers, or none.
+pub(crate) struct OutputRangeTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) sql: &'static str,
+    pub(crate) dialect: Dialect,
+    pub(crate) profile: &'static dyn DialectProfile,
+    pub(crate) schema: fn() -> ShopSchema,
+    pub(crate) expected_ranges: &'static [(&'static str, Option<&'static str>)],
+}
+
 /// A synthetic shop schema: relation name to its columns.
 #[derive(Debug)]
 pub(crate) struct ShopSchema(pub(crate) HashMap<String, Vec<SchemaColumn>>);

crates/sqlquack-semantics/src/postgres/profile/tests/test_checks.rs

@@ -154,7 +154,7 @@ fn given_a_postgres_query_when_checking_types_and_names_then_findings_follow_pos
         QueryFindingsTestCase {
             description: "two CTEs of one name (18.6: 42712)",
             sql: "WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c",
-            expected_findings: &[("E233", "Duplicate CTE name 'c'", 0)],
+            expected_findings: &[("E233", "Duplicate CTE name 'c'", 45)],
         },
         QueryFindingsTestCase {
             description: "an output name inside an ORDER BY expression (18.6: 42703)",